Sorry if this has been asked, I've searched a ton and not finding what I'm looking for. Lots of ways to combine dictionaries, but haven't found anything for combining in a loop that applies.
I have a function (masternodes()) that will return a filtered queryset. With each object returned from that queryset, I loop through and query another function (masternodeDetail()) to get some details about that object. The function (masternodeDetail()) returns a dictionary. I'd like to combine the dictionaries returned in to a single dictionary so I can pass that to my template.
Here's my code I've been working with:
def masternodes(request):
user = request.user
mn = {}
queryset = Masternode.objects.filter(created_by=user)
for n in queryset:
mn = masternodeDetail(n.wallet_address, n.coin_symbol)
print(mn)
args = {'masternodes': queryset, 'masternodetail': mn}
return render(request, 'monitor.html', args)
def masternodeDetail(wallet, ticker):
monitorinfo = MonitorInfo.objects.get(coin_symbol=ticker)
coin_symbol = monitorinfo.coin_symbol
daemon = monitorinfo.daemon
rpc_user = monitorinfo.rpc_user
rpc_password = monitorinfo.rpc_password
rpc_port = monitorinfo.rpc_port
coin_name = monitorinfo.coin_name
notes = monitorinfo.notes
masternode_reward = monitorinfo.masternode_reward
coingecko_id = monitorinfo.coingecko_id
json_output = monitorinfo.json_output
headers = {'content-type': 'text/plain'}
payload = {"method": "masternodelist", "params": ['full', wallet], "jsonrpc": "1.0", "id": "MN Monitoring v0.1"}
response = requests.post("http://" + daemon + ":" + rpc_port, json=payload, headers=headers, auth=(rpc_user,rpc_password)).json()
res = response["result"]
if json_output == False:
for k,v in res.items():
list = v.split()
## status protocol payee lastseen activeseconds lastpaidtime lastpaidblock IP ##
status = list[0]
protocol = list[1]
payee = list[2]
lastseen = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(float(list[3])))
activeseconds = datetime.timedelta(seconds=float(list[4]))
lastpaidtime = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(float(list[5])))
lastpaidblock = list[6]
nodeip = list[7].split(':', 1)[0]
masternode_result = {payee: {'coin':coin_symbol, 'coin_name':coin_name, 'notes':notes, 'masternode_reward':masternode_reward,
'coingecko_id':coingecko_id, 'status':status, 'protocol':protocol, 'payee':payee, 'lastseen': lastseen,
'activeseconds': activeseconds, 'lastpaidtime': lastpaidtime, 'lastpaidblock': lastpaidblock, 'nodeip': nodeip}}
masternodes_result = {**masternode_result, **masternode_result}
return masternodes_result
else:
for txid in res:
status = res[txid]['status']
nodeip = res[txid]['address'].split(':', 1)[0]
protocol = res[txid]['protocol']
payee = res[txid]['payee']
lastseen = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(float(res[txid]['lastseen'])))
activeseconds = datetime.timedelta(seconds=float(res[txid]['activeseconds']))
lastpaidtime = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(float(res[txid]['lastpaidtime'])))
lastpaidblock = res[txid]['lastpaidblock']
masternode_result = {payee: {'coin':coin_symbol, 'coin_name':coin_name, 'notes':notes, 'masternode_reward':masternode_reward,
'coingecko_id':coingecko_id, 'status':status, 'protocol':protocol, 'payee':payee, 'lastseen': lastseen,
'activeseconds': activeseconds, 'lastpaidtime': lastpaidtime, 'lastpaidblock': lastpaidblock, 'nodeip': nodeip}}
masternodes_result = {**masternode_result, **masternode_result}
return masternodes_result
How can I combine this in to a single dictionary or get all the info I need back to my template? You can see one of my many attempts using the {,} method I've used before but since it's not two different named dictionaries, I'guessing that's why it's not working. Right now, the last dictionary returned from my loop is what's getting passed. Maybe I'm approaching this completely wrong and there's a better way to get this info? Suggestions?
I don't think you want a "single dictionary", I can't imagine what that would look like. Instead you probably want a list of dictionaries - in which case you could do:
mn = []
for n in queryset:
mn.append(masternodeDetail(n.wallet_address, n.coin_symbol))
or even shorter:
mn = [masternodeDetail(n.wallet_address, n.coin_symbol) for n in queryset]
Start with an empty dictionary full_dict and use full_dict.update(d) to add dictionary d to full_dict.
Be aware that if there are commons keys between full_dict and d, they will be replaced.
The following example code starts with an empty dict and updates it with each dict passed in a for loop. Hopefully you can adapt this idea into your code:
full_dict = {}
dict_a = {"a": 10, "b": 20}
dict_b = {"b": 100, "c": 120}
for d in (dict_a, dict_b):
full_dict.update(d)
print(full_dict) # {"a": 10, "b": 100, "c": 120}
Related
So this is a somewhat of a continuation from a previous post of mine except now I have API data to work with. I am trying to get keys Type and Email as columns in a data frame to come up with a final number. My code:
jsp_full=[]
for p in payloads:
payload = {"payload": {"segmentId":p}}
r = requests.post(url,headers = header, json = payload)
#print(r, r.reason)
time.sleep(r.elapsed.total_seconds())
json_data = r.json() if r and r.status_code == 200 else None
json_keys = json_data['payload']['supporters']
json_package = []
jsp_full.append(json_package)
for row in json_keys:
SID = row['supporterId']
Handle = row['contacts']
a_key = 'value'
list_values = [a_list[a_key] for a_list in Handle]
string = str(list_values).split(",")
data = {
'SupporterID' : SID,
'Email' : strip_characters(string[-1]),
'Type' : labels(p)
}
json_package.append(data)
t2 = round(time.perf_counter(),2)
b_key = "Email"
e = len([b_list[b_key] for b_list in json_package])
t = str(labels(p))
#print(json_package)
print(f'There are {e} emails in the {t} segment')
print(f'Finished in {t2 - t1} seconds')
excel = pd.DataFrame(json_package)
excel.to_excel(r'C:\Users\am\Desktop\email parsing\{0} segment {1}.xlsx'.format(t, str(today)), sheet_name=t)
This part works all well and good. Each payload in the API represents a different segment of people so I split them out into different files. However, I am at a point where I need to combine all records into a single data frame hence why I append out to jsp_full. This is a list of a list of dictionaries.
Once I have that I would run the balance of my code which is like this:
S= pd.DataFrame(jsp_full[0], index = {0})
Advocacy_Supporters = S.sort_values("Type").groupby("Type", as_index=False)["Email"].first()
print(Advocacy_Supporters['Email'].count())
print("The number of Unique Advocacy Supporters is :")
Advocacy_Supporters_Group = Advocacy_Supporters.groupby("Type")["Email"].nunique()
print(Advocacy_Supporters_Group)
Some sample data:
[{'SupporterID': '565f6a2f-c7fd-4f1b-bac2-e33976ef4306', 'Email': 'somebody#somewhere.edu', 'Type': 'd_Student Ambassadors'}, {'SupporterID': '7508dc12-7647-4e95-a8b8-bcb067861faf', 'Email': 'someoneelse#email.somewhere.edu', 'Type': 'd_Student Ambassadors'},...`
My desired output is a dataframe that looks like so:
SupporterID Email Type
565f6a2f-c7fd-4f1b-bac2-e33976ef4306 somebody#somewhere.edu d_Student Ambassadors
7508dc12-7647-4e95-a8b8-bcb067861faf someoneelse#email.somewhere.edu d_Student Ambassadors
Any help is greatly appreciated!!
So because this code creates an excel file for each segment, all I did was read back in the excels via a for loop like so:
filesnames = ['e_S Donors', 'b_Contributors', 'c_Activists', 'd_Student Ambassadors', 'a_Volunteers', 'f_Offline Action Takers']
S= pd.DataFrame()
for i in filesnames:
data = pd.read_excel(r'C:\Users\am\Desktop\email parsing\{0} segment {1}.xlsx'.format(i, str(today)),sheet_name= i, engine = 'openpyxl')
S= S.append(data)
This did the trick since it was in a format I already wanted.
got some functions with sqlstatements. My first func is fine because i get only 1 result.
My second function returns a large list of errorcodes and i dont know how to get them back for response.
TypeError: <sqlalchemy.engine.result.ResultProxy object at 0x7f98b85ef910> is not JSON serializable
Tried everything need help.
My Code:
def topalarms():
customer_name = request.args.get('customer_name')
machine_serial = request.args.get('machine_serial')
#ts = request.args.get('ts')
#ts_start = request.args.get('ts')
if (customer_name is None) or (machine_serial is None):
return missing_param()
# def form_response(response, session):
# response['customer'] = customer_name
# response['serial'] = machine_serial
# return do_response(customer_name, form_response)
def form_response(response, session):
result_machine_id = machine_id(session, machine_serial)
if not result_machine_id:
response['Error'] = 'Seriennummer nicht vorhanden/gefunden'
return
#response[''] = result_machine_id[0]["id"]
machineid = result_machine_id[0]["id"]
result_errorcodes = error_codes(session, machineid)
response['ErrorCodes'] = result_errorcodes
return do_response(customer_name, form_response)
def machine_id(session, machine_serial):
stmt_raw = '''
SELECT
id
FROM
machine
WHERE
machine.serial = :machine_serial_arg
'''
utc_now = datetime.datetime.utcnow()
utc_now_iso = pytz.utc.localize(utc_now).isoformat()
utc_start = datetime.datetime.utcnow() - datetime.timedelta(days = 30)
utc_start_iso = pytz.utc.localize(utc_start).isoformat()
stmt_args = {
'machine_serial_arg': machine_serial,
}
stmt = text(stmt_raw).columns(
#ts_insert = ISODateTime
)
result = session.execute(stmt, stmt_args)
ts = utc_now_iso
ts_start = utc_start_iso
ID = []
for row in result:
ID.append({
'id': row[0],
'ts': ts,
'ts_start': ts_start,
})
return ID
def error_codes(session, machineid):
stmt_raw = '''
SELECT
name
FROM
identifier
WHERE
identifier.machine_id = :machineid_arg
'''
stmt_args = {
'machineid_arg': machineid,
}
stmt = text(stmt_raw).columns(
#ts_insert = ISODateTime
)
result = session.execute(stmt, stmt_args)
errors = []
for row in result:
errors.append(result)
#({'result': [dict(row) for row in result]})
#errors = {i: result[i] for i in range(0, len(result))}
#errors = dict(result)
return errors
My problem is func error_codes somethiing is wrong with my result.
my Output should be like this:
ABCNormal
ABCSafety
Alarm_G01N01
Alarm_G01N02
Alarm_G01N03
Alarm_G01N04
Alarm_G01N05
I think you need to take a closer look at what you are doing correctly with your working function and compare that to your non-working function.
Firstly, what do you think this code does?
for row in result:
errors.append(result)
This adds to errors one copy of the result object for each row in result. So if you have six rows in result, errors contains six copies of result. I suspect this isn't what you are looking for. You want to be doing something with the row variable.
Taking a closer look at your working function, you are taking the first value out of the row, using row[0]. So, you probably want to do the same in your non-working function:
for row in result:
errors.append(row[0])
I don't have SQLAlchemy set up so I haven't tested this: I have provided this answer based solely on the differences between your working function and your non-working function.
You need a json serializer. I suggest using Marshmallow: https://marshmallow.readthedocs.io/en/stable/
There are some great tutorials online on how to do this.
I am trying to create JSON file. using json.dumps and success printing.
I have a question.
The format I wanted was
channel_info = OrderedDict()
table = OrderedDict()
table2 = OrderedDict()
channel_info["KIND1"] = pkind[2].text
table[ptime[10].text] = pnk[11].text
table[ptime[11].text] = pnk[12].text
channel_info["TABLE1"] = table
channel_info["KIND2"] = pkind[2].text
table2[ptime[10].text] = pnk[11].text
table2[ptime[11].text] = pnk[12].text
channel_info["TABLE2"] = table2
result:
{
"KIND1": "xxxx",
"TABLE1": {
"09:10": "aaaa",
"10:10": "bbbb"
},
"KIND2": "yyyy",
"TABLE2": {
"09:10": "cccc",
"10:10": "dddd"
}
}
How to output the same format using a while loop?
The names of the JSON objects? KIND1, TABLE1, KIND2, TABLE2 and so on ...
I wonder how you can change these names dynamically using a while loop.
thank you.
You could do something like this (assuming table dictionary is static over each loop, as it seems in the example you give):
channel_info = dict()
# n_tables is the number of iterations you need
for i in range(n_tables):
table = dict()
channel_info["KIND%s" % (i+1)] = pkind[1].text
table[ptime[10].text] = pnk[11].text
table[ptime[11].text] = pnk[12].text
channel_info["TABLE%s" % (i+1)] = table
You don't need the table name dynamically since you assign it to a dictionary key.
Basically, if I understood your question right:
...
i=0
no_of_tables = 4
while i<=no_of_tables:
table_counter = i+1
table_counter = str(table_counter)
kind = 'KIND' + table_counter
table = 'TABLE' + table_counter
channel_info[kind] = pkind[2].text
table[ptime[10].text] = pnk[11].text
table[ptime[11].text] = pnk[12].text
channel_info[table] = table
Note: I know it can be optimized, but for the sake of simplicity I left it as is.
I have a flask application which is receiving a request from dataTables Editor. Upon receipt at the server, request.form looks like (e.g.)
ImmutableMultiDict([('data[59282][gender]', u'M'), ('data[59282][hometown]', u''),
('data[59282][disposition]', u''), ('data[59282][id]', u'59282'),
('data[59282][resultname]', u'Joe Doe'), ('data[59282][confirm]', 'true'),
('data[59282][age]', u'27'), ('data[59282][place]', u'3'), ('action', u'remove'),
('data[59282][runnerid]', u''), ('data[59282][time]', u'29:49'),
('data[59282][club]', u'')])
I am thinking to use something similar to this really ugly code to decode it. Is there a better way?
from collections import defaultdict
# request.form comes in multidict [('data[id][field]',value), ...]
# so we need to exec this string to turn into python data structure
data = defaultdict(lambda: {}) # default is empty dict
# need to define text for each field to be received in data[id][field]
age = 'age'
club = 'club'
confirm = 'confirm'
disposition = 'disposition'
gender = 'gender'
hometown = 'hometown'
id = 'id'
place = 'place'
resultname = 'resultname'
runnerid = 'runnerid'
time = 'time'
# fill in data[id][field] = value
for formkey in request.form.keys():
exec '{} = {}'.format(d,repr(request.form[formkey]))
This question has an accepted answer and is a bit old but since the DataTable module seems being pretty popular among jQuery community still, I believe this approach may be useful for someone else. I've just wrote a simple parsing function based on regular expression and dpath module, though it appears not to be quite reliable module. The snippet may be not very straightforward due to an exception-relied fragment, but it was only one way to prevent dpath from trying to resolve strings as integer indices I found.
import re, dpath.util
rxsKey = r'(?P<key>[^\W\[\]]+)'
rxsEntry = r'(?P<primaryKey>[^\W]+)(?P<secondaryKeys>(\[' \
+ rxsKey \
+ r'\])*)\W*'
rxKey = re.compile(rxsKey)
rxEntry = re.compile(rxsEntry)
def form2dict( frmDct ):
res = {}
for k, v in frmDct.iteritems():
m = rxEntry.match( k )
if not m: continue
mdct = m.groupdict()
if not 'secondaryKeys' in mdct.keys():
res[mdct['primaryKey']] = v
else:
fullPath = [mdct['primaryKey']]
for sk in re.finditer( rxKey, mdct['secondaryKeys'] ):
k = sk.groupdict()['key']
try:
dpath.util.get(res, fullPath)
except KeyError:
dpath.util.new(res, fullPath, [] if k.isdigit() else {})
fullPath.append(int(k) if k.isdigit() else k)
dpath.util.new(res, fullPath, v)
return res
The practical usage is based on native flask request.form.to_dict() method:
# ... somewhere in a view code
pars = form2dict(request.form.to_dict())
The output structure includes both, dictionary and lists, as one could expect. E.g.:
# A little test:
rs = jQDT_form2dict( {
'columns[2][search][regex]' : False,
'columns[2][search][value]' : None,
'columns[2][search][regex]' : False,
} )
generates:
{
"columns": [
null,
null,
{
"search": {
"regex": false,
"value": null
}
}
]
}
Update: to handle lists as dictionaries (in more efficient way) one may simplify this snippet with following block at else part of if clause:
# ...
else:
fullPathStr = mdct['primaryKey']
for sk in re.finditer( rxKey, mdct['secondaryKeys'] ):
fullPathStr += '/' + sk.groupdict()['key']
dpath.util.new(res, fullPathStr, v)
I decided on a way that is more secure than using exec:
from collections import defaultdict
def get_request_data(form):
'''
return dict list with data from request.form
:param form: MultiDict from `request.form`
:rtype: {id1: {field1:val1, ...}, ...} [fieldn and valn are strings]
'''
# request.form comes in multidict [('data[id][field]',value), ...]
# fill in id field automatically
data = defaultdict(lambda: {})
# fill in data[id][field] = value
for formkey in form.keys():
if formkey == 'action': continue
datapart,idpart,fieldpart = formkey.split('[')
if datapart != 'data': raise ParameterError, "invalid input in request: {}".format(formkey)
idvalue = int(idpart[0:-1])
fieldname = fieldpart[0:-1]
data[idvalue][fieldname] = form[formkey]
# return decoded result
return data
So I'm trying to compare a dict that I have created to a dict response returned by a boto3 call.
The response is a representation of a JSON document and I want to check they are the same.
Boto3 always returned the strings as unicode. Here's the response:
{u'Version': u'2012-10-17', u'Statement': [{u'Action': u'sts:AssumeRole', u'Principal': {u'Service': u'ec2.amazonaws.com'}, u'Effect': u'Allow', u'Sid': u''}]}
I initially created my dict like this:
default_documment = {}
default_documment['Version'] = '2012-10-17'
default_documment['Statement'] = [{}]
default_documment['Statement'][0]['Sid'] = ''
default_documment['Statement'][0]['Effect'] = 'Allow'
default_documment['Statement'][0]['Principal'] = {}
default_documment['Statement'][0]['Principal']['Service'] = 'ec2.amazonaws.com'
default_documment['Statement'][0]['Action'] = 'sts:AssumeRole'
However, when i compare these two dicts with == they are not equal.
So then I tried adding u to all the strings when I create the dict:
# Default document for a new role
default_documment = {}
default_documment[u'Version'] = u'2012-10-17'
default_documment[u'Statement'] = [{}]
default_documment[u'Statement'][0][u'Sid'] = u''
default_documment[u'Statement'][0][u'Effect'] = u'Allow'
default_documment[u'Statement'][0][u'Principal'] = {}
default_documment[u'Statement'][0][u'Principal'][u'Service'] = u'ec2.amazonaws.com'
default_documment[u'Statement'][0][u'Action'] = u'sts:AssumeRole'
This doesn't work either. The dicts are not equally and if i do a print of my dict it doesn't show u'somestring' it just shows 'somestring'.
How can I compare my dict to what boto3 has returned?
Your second attempt works correctly in Python 2.7 and 3.3. Below is just a cut-and-paste of your Boto3 response and your code (with document spelling corrected :)
D = {u'Version': u'2012-10-17', u'Statement': [{u'Action': u'sts:AssumeRole', u'Principal': {u'Service': u'ec2.amazonaws.com'}, u'Effect': u'Allow', u'Sid': u''}]}
default_document = {}
default_document[u'Version'] = u'2012-10-17'
default_document[u'Statement'] = [{}]
default_document[u'Statement'][0][u'Sid'] = u''
default_document[u'Statement'][0][u'Effect'] = u'Allow'
default_document[u'Statement'][0][u'Principal'] = {}
default_document[u'Statement'][0][u'Principal'][u'Service'] = u'ec2.amazonaws.com'
default_document[u'Statement'][0][u'Action'] = u'sts:AssumeRole'
print(D == default_document)
Output:
True