Related
I want to print what I just deleted by pymongo
the main function
def del_data(del_values_json ,one_or_many_bool ):
if one_or_many_bool:
x = mycol.delete_many(del_values_json)
else:
x = mycol.delete_one(del_values_json)
retrun x
number_of_delete_data = int(x.deleted_count)
modified_data_list = []
for modified_data in mycol.find().sort("_id", -1).limit(number_of_delete_data):
print(modified_data)
modified_data_list.append(modified_data)
the output
I want to print what I just deleted, I delete two { "name": "Joy22" }data, but I just print the top of 2 data in the collection, MongoDB has some examples, however, is hard to find pymongo call for the same case
Enter the IP: localhost.173
Enter exist DB name: (practice_10_14)-0002
Enter exist collection name: collection_new02cp
U are deleted one or many values? (ex:1 for many , 0 for one): 1
Enter delete values: { "name" : "Joy22" }
<pymongo.results.DeleteResult object at 0x00000282A37C6890>
2 documents delete.
{'_id': ObjectId('6350c8a3a1fa85dd0cfe590a'), 'name': 'Joy', 'ID': 100998, 'Age': 23, 'time': DateTime.datetime(2022, 10, 17, 9, 11, 54)}
{'_id': ObjectId('6350c8a3a1fa85dd0cfe5909'), 'name': 'Joy', 'ID': 100998, 'Age': 23, 'time': DateTime.datetime(2022, 10, 17, 9, 11, 54)}
{'ok': 1, 'msg': 'no error occur', 'count': 2}
<class 'str'>
the whole code in case someone wants to take a look
# Delete collection/table
import pymongo
import datetime
import json
from bson.objectid import ObjectId
from bson import json_util
def init_db(ip, db, coll):
try:
myclient = pymongo.MongoClient('mongodb://' + ip + '/')
mydb = myclient[db]
mycol = mydb[coll]
except Exception as e:
msg_fail_reason = "error in init_db function"
return msg_fail_reason
return mydb, mycol
# del_data = delete_db_data
# one_or_many_bool: input 1 means True; input 0 is False
def del_data(del_values_json ,one_or_many_bool ):
try:
if one_or_many_bool:
x = mycol.delete_many(del_values_json)
else:
x = mycol.delete_one(del_values_json)
except Exception as e:
msg_fail_reason = "error in del_data function"
return msg_fail_reason
return x
msg_fail_reason = "no error occur"
ip_input = input("Enter the ip: ")
exist_DB_name = input("Enter exist DB name: ")
exist_coll_name = input("Enter exist collection name: ")
mydb, mycol = init_db(ip_input, exist_DB_name, exist_coll_name)
insert_one_or_many = input("U are delete one or many values? (ex:1 for many , 0 for one): ")
del_values_str = input("Enter delete values: ")
one_or_many_bool = bool(int(insert_one_or_many))
del_values_json =json.loads(del_values_str)
x = del_data(del_values_json ,one_or_many_bool )
print(x)
print(x.deleted_count, "documents delete.")
number_of_delete_data = int(x.deleted_count)
modified_data_list = []
for modified_data in mycol.find().sort("_id", -1).limit(number_of_delete_data):
print(modified_data)
modified_data_list.append(modified_data)
def parse_json(data):
return json.loads(json_util.dumps(data))
# if someone wants data in JSON
# modified_data_json = parse_json(modified_data_list)
# 1 means success
return_status_str = { "ok" : 1 , "msg" : msg_fail_reason , "count" : number_of_delete_data}
print(return_status_str)
# return_status_json is json data, dump
return_status_json = (json.dumps(return_status_str))
print(type(return_status_json))
The DeleteResult object only returns the number of items deleted. Once an item is deleted it will no longer occur in the collection so this fragment of code:
modified_data_list = []
for modified_data in mycol.find().sort("_id", -1).limit(number_of_delete_data):
print(modified_data)
modified_data_list.append(modified_data)
Will return the first two remaining items which implies there were two documents with the name joy22 (hence a modified count of 2). You will note that the two documents returned do not have the name joy22. Therefore they are not the deleted documents.
To achieve your goal first find your documents, then delete_many then print the results of the find.
I have this problem with my code. When Iinsert three or more params in the body request, I get this Error "POST Error: or_ expected 2 arguments, got 3."
I can only pass one or two parameters in the body, in this case it works fine. But I don't understand where is the mistake. Can someone help me?
def read_uptime(logid, filteredData, dateStart, dateEnd, timeStart, timeEnd, elementsForPage, currentPage, filterUptime):
log.info(f"{logid} read_uptime: Started")
try:
# Check Timeframe Correct
startDateTime, endDateTime = _checkDataInput(timeStart, timeEnd, dateStart, dateEnd)
# Create Filter
filters = _createFilter(filteredData, startDateTime, endDateTime, filterUptime)
# Query
dataFiltered = uptime_model_db.query.with_entities(
uptime_model_db.projectId.label('projectId'),
uptime_model_db.url.label('url'),
uptime_model_db.timeStamp.label('timeStamp'),
uptime_model_db.uptime.label('uptime'),
uptime_model_db.latency.label('latency')
).filter(*filters).paginate(per_page=int(elementsForPage + 1), page=int(currentPage), error_out=True)
# Checking more pages
nextPage = {
"currentPage": currentPage,
"totalElements": len(dataFiltered.items)
}
if (len(dataFiltered.items) > elementsForPage):
nextPage["nextPage"] = True
else:
nextPage["nextPage"] = False
# Format and return JSON
return _createJson(dataFiltered.items, nextPage)
except Exception as e:
log.error(f"{logid} read_uptime: function read_uptime returned {e}")
raise e
i get in this code the mistake: "array.Filter.append(and_(uptime_model.db.porjectId == projectId, or_(*arrayUrl))"
def filterAppend(arrayFilter, urls, projectId, arrayUrl):
if(len(urls) == 1):
arrayFilter.append(and_(uptime_model_db.projectId == projectId, uptime_model_db.url == urls[0]))
if(len(urls) > 1):
for url in urls:
arrayUrl.append(uptime_model_db.url == url)
arrayFilter.append(and_(uptime_model_db.projectId == projectId, or_(*arrayUrl)))
i get in this code the mistake:
"filters.append(or_(*arrayFilter))"
def _createFilter(filteredData, startDateTime, endDateTime, filterUptime):
filters = []
if filteredData is not None:
arrayFilter = []
for data in filteredData:
projectId = data["projectId"]
urls = data["uptimeUrls"]
arrayUrl = []
if (len(filteredData) == 1):
filterAppend(filters, urls, projectId, arrayUrl)
else:
filterAppend(arrayFilter, urls, projectId, arrayUrl)
if(len(filteredData) > 1 or len(arrayFilter) > 1):
filters.append(or_(*arrayFilter))
if startDateTime is not None:
filters.append(str(startDateTime) <= uptime_model_db.timeStamp)
if startDateTime is not None:
filters.append(str(endDateTime) >= uptime_model_db.timeStamp)
if filterUptime == "True":
filters.append(uptime_model_db.uptime < 100)
return filters
import or_ from sqlalchemy instead of operators:
from sqlalchemy import or_
I've written a script to pipe through data from the Kustomer API to our database, and although it works fine its a bit messy, was wondering if there's a more elegant solution to this. I'm defining the row of results im pushing through as a dictionary then pushing to MySQL, but the messy part comes when some of these values aren't available in the JSON all the time.
This has resulted in a try / except statements for each data point that may or may not be missing.
Is there a better way of doing this? Code below.
try:
record_data = {
'id': record['id'],
'created_at': str(datetime.strptime(record['attributes']['createdAt'], '%Y-%m-%dT%H:%M:%S.%fZ'))[:-7],
'last_activity_at': str(datetime.strptime(record['attributes']['lastActivityAt'], '%Y-%m-%dT%H:%M:%S.%fZ'))[:-7],
'first_marked_done': None,
'last_marked_done': None,
'assigned_team': record['attributes']['assignedTeams'][0] if record['attributes']['assignedTeams'] != [] else None,
'conversation_type': None,
'conversation_category': None,
'conversation_subcategory': None,
'message_count': record['attributes']['messageCount'],
'note_count': record['attributes']['noteCount'],
'satisfaction': record['attributes']['satisfaction'],
'status': None,
'email': 1 if len(list(filter(lambda x: x == 'email', record['attributes']['channels']))) > 0 else 0,
'chat': 1 if len(list(filter(lambda x: x == 'chat', record['attributes']['channels']))) > 0 else 0,
'priority': record['attributes']['priority'],
'direction': 'outbound' if record['attributes']['direction'] == 'out' else 'in',
'nlp_score': None,
'nlp_sentiment': None,
'waiting_for': None,
'sla_breach': None,
'sla_status': None,
'breached_sla': None,
'breached_at': None
}
try:
record_data['status'] = record['attributes']['status']
except KeyError:
pass
try:
record_data['conversation_type'] = record['attributes']['custom']['typeStr']
record_data['conversation_category'] = str(record['attributes']['custom']['categoryTree']).split('.')[0]
record_data['conversation_subcategory'] = str(record['attributes']['custom']['categoryTree']).split('.')[1] if len(str(record['attributes']['custom']['categoryTree']).split('.')) > 1 else None
except KeyError:
pass
try:
record_data['waiting_for'] = record['attributes']['custom']['typeStr']
except KeyError:
pass
try:
record_data['first_marked_done'] = str(datetime.strptime(record['attributes']['firstDone']['createdAt'], '%Y-%m-%dT%H:%M:%S.%fZ'))[:-7]
record_data['last_marked_done'] = str(datetime.strptime(record['attributes']['lastDone']['createdAt'], '%Y-%m-%dT%H:%M:%S.%fZ'))[:-7]
except KeyError:
pass
try:
record_data['sla_breach'] = 0 if record['attributes']['sla']['breached'] is False else 1
record_data['sla_status'] = record['attributes']['sla']['status']
if record_data['sla_breach'] == 1:
try:
record_data['breached_sla'] = record['attributes']['sla']['breach']['metric']
record_data['breached_at'] = record['attributes']['sla']['breach']['at']
except KeyError:
for m in record['attributes']['sla']['metrics']:
try:
if record['attributes']['sla']['metrics'][m]['breachAt'] == record['attributes']['sla']['summary']['firstBreachAt']:
record_data['breached_sla'] = m
record_data['breached_at'] = str(datetime.strptime(record['attributes']['sla']['summary']['firstBreachAt'], '%Y-%m-%dT%H:%M:%S.%fZ'))[:-7]
except KeyError:
pass
except KeyError:
record_data['sla_breach'] = 0
print(record_data)
self.db.insert_update(KustomerConversations(**record_data))
except KeyError:
pass
First you should try, where possible, to use dict.get with a default value specified. Next you can consider contextmanager to make your code significantly cleaner. Consider this:
try:
record_data['status'] = record['attributes']['status']
except KeyError:
pass
try:
record_data['conversation_type'] = record['attributes']['custom']['typeStr']
except KeyError:
pass
try:
record_data['waiting_for'] = record['attributes']['custom']['typeStr']
except KeyError:
pass
try:
record_data['first_marked_done'] = record['attributes']['firstDone']['createdAt']
except KeyError:
pass
Now rewritten, you can ensure consistent error handling without repeating logic:
from contextlib import contextmanager
#contextmanager
def error_handling():
try:
yield
except KeyError:
pass
with error_handling():
record_data['status'] = record['attributes']['status']
with error_handling():
record_data['conversation_type'] = record['attributes']['custom']['typeStr']
with error_handling():
record_data['waiting_for'] = record['attributes']['custom']['typeStr']
with error_handling():
record_data['first_marked_done'] = record['attributes']['firstDone']['createdAt']
You can define an arbitrary number of functions like error_handling for various rules you wish to apply.
You can use function, that give you element from nested dicts, and doesn't raise an exception, if it doesnt' exists.
Like this quick draft:
def get_nested_dict_value(src_dict, *nested_keys, **kwargs):
"""
Get value of some nested dict by series of keys with default value.
Example:
instead of:
x = data['a']['b']['c']['d']
use
x = get_nested_dict_value(data, 'a', 'b', 'c', 'd')
or, if you need some non-None default value, add default=xxx kwarg:
x = get_nested_dict_value(data, 'a', 'b', 'c', 'd', default=0)
"""
default = kwargs.get("default", None)
pointer = src_dict
i = 0
for key in nested_keys:
i += 1
if key in pointer:
pointer = pointer[key]
if i == len(nested_keys):
return pointer
else:
return default
So, instead of:
try:
record_data['conversation_type'] = record['attributes']['custom']['typeStr']
except Exception:
pass
You just type:
record_data['conversation_type'] = get_nested_dict_value(record, 'attributes', 'custom', 'typeStr')
The different naming conventions on the input and output sides make it hard to beat the clarity of explicit assignments. Preserving the exact semantics of your version (e.g., that it doesn't assign conversation_category in the absence of a typeStr even if categoryTree is available) excludes certain choices (like making a data structure to loop over with a try/except on each access); you might be able to do better with more assumptions about your input data.
Nonetheless, in addition to the dict.get already mentioned, you can use builtins (any, or, and dict) and introduce a helper function and a few temporary variables to make the code much more readable:
# this gives one digit of the hour for me...?
def ptime(s): return str(datetime.strptime(s,'%Y-%m-%dT%H:%M:%S.%fZ'))[:-7]
try:
attr=record['attributes']
cust=attr.get('custom',{}) # defer KeyErrors into the below
record_data = dict(
id = record['id'],
created_at = ptime(attr['createdAt']),
last_activity_at = ptime(attr['lastActivityAt']),
first_marked_done = None,
last_marked_done = None,
assigned_team = attr['assignedTeams'][0] or None,
conversation_type = None,
conversation_category = None,
conversation_subcategory = None,
message_count = attr['messageCount'],
note_count = attr['noteCount'],
satisfaction = attr['satisfaction'],
status = attr.get('status'),
email = int(any(x == 'email' for x in attr['channels'])),
chat = int(any(x == 'chat' for x in attr['channels'])),
priority = attr['priority'],
direction = 'outbound' if attr['direction'] == 'out' else 'in',
nlp_score = None,
nlp_sentiment = None,
waiting_for = cust.get('typeStr'),
sla_breach = 0,
sla_status = None,
breached_sla = None,
breached_at = None
)
try:
record_data['conversation_type'] = cust['typeStr']
cat=str(cust['categoryTree']).split('.')
record_data['conversation_category'] = cat[0]
record_data['conversation_subcategory'] = cat[1] if len(cat) > 1 else None
except KeyError: pass
try:
record_data['first_marked_done'] = ptime(attr['firstDone']['createdAt'])
record_data['last_marked_done'] = ptime(attr['lastDone']['createdAt'])
except KeyError: pass
try:
sla=attr['sla']
record_data['sla_breach'] = 0 if sla['breached'] is False else 1
record_data['sla_status'] = sla['status']
if record_data['sla_breach'] == 1:
try:
record_data['breached_sla'] = sla['breach']['metric']
record_data['breached_at'] = sla['breach']['at']
except KeyError:
for m,v in sla['metrics'].items():
try:
v=v['breachAt']
if v == sla['summary']['firstBreachAt']:
record_data['breached_sla'] = m
record_data['breached_at'] = ptime(v)
except KeyError: pass
except KeyError: pass
print(record_data)
self.db.insert_update(KustomerConversations(**record_data))
except KeyError: pass
While you might have a policy against it, in this case I recommend writing the remaining except KeyError: pass clauses on one line each: it helps the visual bracketing of the tentative code.
I have a problem that:
when testing on flask there is not exception
after put online by gunicorn+gevent, they raise an exception at following code
-
try:
req_dict = request.args.to_dict()
req_desc = copy.copy(req_dict)
temp_card_pwd = str(req_desc['cardpwd'])
req_desc['cardpwd'] = temp_card_pwd.replace(temp_card_pwd[5:9], "****")
log.info('接收到一条请求信息(charge_in##) %s' % req_desc)
request_type = str(req_dict['command'])
charge_acct = str(req_dict['telephone'])
card_pwd = str(req_dict['cardpwd'])
res_dict = req_validate(request_type, card_pwd, charge_acct)
if res_dict['errcode'] == 0:
log.info('准备充值(charge_begin##)...%s '% req_desc)
time_start = time.time()
acct_charge = True
charge_res = charge(req_desc, card_pwd, charge_acct)
res_dict['body'] = charge_res
log.info('充值结果(charge_end##)：请求：%s, 结果：%s 耗时：%s' % (req_desc, res_dict,
int((time.time() - time_start)*1000)))
else:
log.warning('返回异常请求结果（charge_out##）：请求：%s 结果：%s' % (req_desc, res_dict))
except Exception as e:
if acct_charge:
code = 1007
msg = '充值异常！%s' % e
else:
code = 1006
msg = '传入参数错误! %s' % e
res_dict = {'errcode': code, 'msg': msg, 'body': None}
log.warning('返回异常请求结果（charge_exception##）：请求参数异常！请核对后再发！%s' % res_dict)
the exception is
2017-08-04 06:47:50,441 yidongserver.py[line:49] WARNING: [channel_name:yidongApp] 返回异常请求结果（charge_exception##）：请求参数异常！请核对后再发！{'errcode': 1006, 'msg': '传入参数错误', 'body': None}
2017-08-04 06:47:50,445 yidongserver.py[line:25] INFO: [channel_name:yidongApp] 接收到一条请求信息(charge_in##) {'request_id': 'HF111708041450260032', 'card_id': '17356104229493840', 'telephone': '152***94608', 'cardpwd': '42381****349361850', 'money': '100', 'callbackurl': '', 'command': 'charge', 'channel_id': 'LTAppRY'}
Although raise this exception, this charge request still SUCCESS in charge, and return ONCE
pls help to solve this warning exception! thanks!
In playbook I'm using variable {{excluded_service}}. I want to run ansible playbook from python and provide this variable. And I can't use external inventory script to provide this variable.
I am using to create inventory:
hosts = ["127.0.0.1"]
inventory=ansible.inventory.Inventory(hosts)
but I don't understand where I can add value of variable?
My code, that works with external inventory script:
import sys
import os
import stat
import json
import ansible.playbook
import ansible.constants as C
import ansible.utils.template
from ansible import errors
from ansible import callbacks
from ansible import utils
from ansible.color import ANSIBLE_COLOR, stringc
from ansible.callbacks import display
playbook="/opt/RDE/3p/ansible/loop/testloop.yml"
inventory="/opt/RDE/3p/ansible/loop/lxc.py"
connection="local"
timeout=10
subset=None
options={'subset': None, 'ask_pass': False, 'sudo': False, 'private_key_file': None, 'syntax': None, 'skip_tags': None, 'diff': False, 'check': False, 'remote_user': 'root', 'listtasks': None, 'inventory': '/opt/RDE/3p/ansible/loop/lxc.py', 'forks': 5, 'listhosts': None, 'start_at': None, 'tags': 'all', 'step': None, 'sudo_user': None, 'ask_sudo_pass': False, 'extra_vars': [], 'connection': 'smart', 'timeout': 10, 'module_path': None}
sshpass = None
sudopass = None
extra_vars = {}
def colorize(lead, num, color):
""" Print 'lead' = 'num' in 'color' """
if num != 0 and ANSIBLE_COLOR and color is not None:
return "%s%s%-15s" % (stringc(lead, color), stringc("=", color), stringc(str(num), color))
else:
return "%s=%-4s" % (lead, str(num))
def hostcolor(host, stats, color=True):
if ANSIBLE_COLOR and color:
if stats['failures'] != 0 or stats['unreachable'] != 0:
return "%-37s" % stringc(host, 'red')
elif stats['changed'] != 0:
return "%-37s" % stringc(host, 'yellow')
else:
return "%-37s" % stringc(host, 'green')
return "%-26s" % host
inventory = ansible.inventory.Inventory(options['inventory'])
hosts = ["127.0.0.1"]
#inventory=ansible.inventory.Inventory(hosts)
inventory.subset(options['subset'])
if len(inventory.list_hosts()) == 0:
raise errors.AnsibleError("provided hosts list is empty")
inventory.set_playbook_basedir(os.path.dirname(playbook))
stats = callbacks.AggregateStats()
playbook_cb = callbacks.PlaybookCallbacks(verbose=utils.VERBOSITY)
if options['step']:
playbook_cb.step = options['step']
if options['start_at']:
playbook_cb.start_at = options['start_at']
runner_cb = callbacks.PlaybookRunnerCallbacks(stats, verbose=utils.VERBOSITY)
pb = ansible.playbook.PlayBook(
playbook=playbook,
module_path=None,
inventory=inventory,
forks=options['forks'],
remote_user=options['remote_user'],
remote_pass=sshpass,
callbacks=playbook_cb,
runner_callbacks=runner_cb,
stats=stats,
timeout=options['timeout'],
transport=options['connection'],
sudo=options['sudo'],
sudo_user=options['sudo_user'],
extra_vars=extra_vars,
private_key_file=options['private_key_file'],
check=options['check'],
diff=options['diff']
)
playnum = 0
failed_hosts = []
unreachable_hosts = []
try:
print pb.run()
hosts = sorted(pb.stats.processed.keys())
print hosts
display(callbacks.banner("PLAY RECAP"))
playbook_cb.on_stats(pb.stats)
for h in hosts:
t = pb.stats.summarize(h)
if t['failures'] > 0:
failed_hosts.append(h)
if t['unreachable'] > 0:
unreachable_hosts.append(h)
retries = failed_hosts + unreachable_hosts
if len(retries) > 0:
filename = pb.generate_retry_inventory(retries)
if filename:
display(" to retry, use: --limit #%s\n" % filename)
for h in hosts:
t = pb.stats.summarize(h)
display("%s : %s %s %s %s" % (
hostcolor(h, t),
colorize('ok', t['ok'], 'green'),
colorize('changed', t['changed'], 'yellow'),
colorize('unreachable', t['unreachable'], 'red'),
colorize('failed', t['failures'], 'red')),
screen_only=True
)
display("%s : %s %s %s %s" % (
hostcolor(h, t, False),
colorize('ok', t['ok'], None),
colorize('changed', t['changed'], None),
colorize('unreachable', t['unreachable'], None),
colorize('failed', t['failures'], None)),
log_only=True
)
except Exception as e:
print ("!!!!!!!ERROR: %s" % e)
Specify the vars in a host_vars file?
E.g. create a YAML file named /etc/ansible/host_vars/localhost with the vars you wanna put there.
I don't know, yet, how to specify it in Python code itself.
--- update ---
After a quick look the code, I don't think ansible supports specifying host variables when you specify hosts via host_list parameter. Hacks I can think of (if you must do this in python code) are:
[mis]use extra_vars parameter.
Write an inventory file (YAML or executable that just prints required JSON) from your python code and pass it's path as parameter inventory.
HTH