I created a python app to parse a json API.
There is 3 endpoints and 1 of these worries me.
The endpoint is : http://coinmarketcap.northpole.ro/history.json?coin=PCN
My code :
def getHistory(self, coin):
endpoint = "history.json?year=2017&coin=PCN"
data = urllib2.urlopen(self.url + endpoint).read()
data = json.loads(data)['history']
return data
def getOrder(self):
for c in self.getCoinsList():
res = []
symbol = c['symbol']
price = self.getCoinPrice(symbol)
count = 0
count_days = len(self.getHistory(symbol))
for h in self.getHistory(symbol):
if h['price']['usd'] > price:
++count
percent_down = count_days / count * 100
line = {'symbol': symbol, 'price': price, 'percent_down': percent_down}
res.append(line)
return res
When I try to get the h['price']['usd'] I have this :
File "coinmarketcap.py", line 39, in getOrder
if h['price']['usd'] > price:
TypeError: string indices must be integers
When I do print type(h) it return unicode.
getHistory returns a dict, and when you iterate over it like this:
for h in self.getHistory(symbol):
you're iterating over the dict keys, not values.
To iterate over the value instead, use
for h in self.getHistory(symbol).values(): # .itervalues() in python2
#Pixel, I think you are assuming that for h in self.getHistory(symbol): returns the value of the key, which is incorrect, it returns the key.
Try saving the dictionary and fetch by key mapping, like this,
json_data = self.getHistory(symbol)
for h in json_data:
if json_data[h]['price']['usd'] > price:
++count
or retrieve the values from the dictionary values, using
for h in self.getHistory(symbol).values():
if h['price']['usd'] > price:
++count
Related
I'm new in python and sqlalchemy.
I already have a delete method working if I construct the where conditions by hand.
Now, I need to read the columns and values from an enter request in yaml format and create the where conditions.
#enter data as yaml
items:
- item:
table: [MyTable,OtherTable]
filters:
field_id: 1234
#other_id: null
Here is what I try and can't go ahead:
for i in use_case_cfg['items']:
item = i.get('item')
for t in item['table']:
if item['filters']:
filters = item['filters']
where_conditions = ''
count = 0
for column, value in filters.items():
aux = str(getattr(t, column) == bindparam(value))
if count == 0:
where_conditions += aux
else:
where_conditions += ', ' + aux
count += 1
to_delete = inv[t].__table__.delete().where(text(where_conditions))
#to_delete = t.__table__.delete().where(getattr(t, column) == value)
else:
to_delete = inv[t].__table__.delete()
CoreData.session.execute(to_delete)
To me, it looks ok, but when I run, I got the error below:
sqlalchemy.exc.StatementError: (sqlalchemy.exc.InvalidRequestError) A value is required for bind parameter '9876'
[SQL: DELETE FROM MyTable WHERE "MyTable".field_id = %(1234)s]
[parameters: [{}]]
(Background on this error at: http://sqlalche.me/e/cd3x)
Can someone explain to me what is wrong or the proper way to do it?
Thanks.
There are two problems with the code.
Firstly,
str(getattr(t, column) == bindparam(value))
is binding the value as a placeholder, so you end up with
WHERE f2 = :Bob
but it should be the name that maps to the value in filters (so the column name in your case), so you end up with
WHERE f2 = :f2
Secondly, multiple WHERE conditions are being joined with a comma, but you should use AND or OR, depending on what you are trying to do.
Given a model Foo:
class Foo(Base):
__tablename__ = 'foo'
id = sa.Column(sa.Integer, primary_key=True)
f1 = sa.Column(sa.Integer)
f2 = sa.Column(sa.String)
Here's a working version of a segment of your code:
filters = {'f1': 2, 'f2': 'Bob'}
t = Foo
where_conditions = ''
count = 0
for column in filters:
aux = str(getattr(t, column) == sa.bindparam(column))
if count == 0:
where_conditions += aux
else:
where_conditions += ' AND ' + aux
count += 1
to_delete = t.__table__.delete().where(sa.text(where_conditions))
print(to_delete)
session.execute(to_delete, filters)
If you aren't obliged to construct the WHERE conditions as strings, you can do it like this:
where_conditions = [(getattr(t, column) == sa.bindparam(column))
for column in filters]
to_delete = t.__table__.delete().where(sa.and_(*where_conditions))
session.execute(to_delete, filters)
had a question regarding summing the multiple values of duplicate keys into one key with the aggregate total. For example:
1:5
2:4
3:2
1:4
Very basic but I'm looking for an output that looks like:
1:9
2:4
3:2
In the two files I am using, I am dealing with a list of 51 users(column 1 of user_artists.dat) who have the artistID(column 2) and how many times that user has listened to that particular artist given by the weight(column 3).
I am attempting to aggregate the total times that artist has been played, across all users and display it in a format such as:
Britney Spears (289) 2393140. Any help or input would be so appreciated.
import codecs
#from collections import defaultdict
with codecs.open("artists.dat", encoding = "utf-8") as f:
artists = f.readlines()
with codecs.open("user_artists.dat", encoding = "utf-8") as f:
users = f.readlines()
artist_list = [x.strip().split('\t') for x in artists][1:]
user_stats_list = [x.strip().split('\t') for x in users][1:]
artists = {}
for a in artist_list:
artistID, name = a[0], a[1]
artists[artistID] = name
grouped_user_stats = {}
for u in user_stats_list:
userID, artistID, weight = u
grouped_user_stats[artistID] = grouped_user_stats[artistID].astype(int)
grouped_user_stats[weight] = grouped_user_stats[weight].astype(int)
for artistID, weight in u:
grouped_user_stats.groupby('artistID')['weight'].sum()
print(grouped_user_stats.groupby('artistID')['weight'].sum())
#if userID not in grouped_user_stats:
#grouped_user_stats[userID] = { artistID: {'name': artists[artistID], 'plays': 1} }
#else:
#if artistID not in grouped_user_stats[userID]:
#grouped_user_stats[userID][artistID] = {'name': artists[artistID], 'plays': 1}
#else:
#grouped_user_stats[userID][artistID]['plays'] += 1
#print('this never happens')
#print(grouped_user_stats)
how about:
import codecs
from collections import defaultdict
# read stuff
with codecs.open("artists.dat", encoding = "utf-8") as f:
artists = f.readlines()
with codecs.open("user_artists.dat", encoding = "utf-8") as f:
users = f.readlines()
# transform artist data in a dict with "artist id" as key and "artist name" as value
artist_repo = dict(x.strip().split('\t')[:2] for x in artists[1:])
user_stats_list = [x.strip().split('\t') for x in users][1:]
grouped_user_stats = defaultdict(lambda:0)
for u in user_stats_list:
#userID, artistID, weight = u
grouped_user_stats[u[0]] += int(u[2]) # accumulate weights in a dict with artist id as key and sum of wights as values
# extra: "fancying" the data transforming the keys of the dict in "<artist name> (artist id)" format
grouped_user_stats = dict(("%s (%s)" % (artist_repo.get(k,"Unknown artist"), k), v) for k ,v in grouped_user_stats.iteritems() )
# lastly print it
for k, v in grouped_user_stats.iteritems():
print k,v
I am trying to edit this function so the values of the dictionary will not be printed in parentheses and will be iterable:
def traverse_appended(key):
reg_dict = {}
#keypath = r"SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\"
for item in traverse_reg(key):
keypath_str = str(keypath+item)
reg_dict[item] = str(get_reg("Displayversion", keypath_str)), str(get_reg("DisplayName", keypath_str))
#reg_dict[item] = get_reg("DisplayName", keypath_str)
return reg_dict
the expected output is :
{'DXM_Runtime': 'None', 'None'}
The function output:
{'DXM_Runtime': ('None', 'None')}
#Consider traverse_appended returns following dict.
#I think, converting func_dict values which are tuple into string, will help you to get expected output.
func_dict = {"DXM_Runtime":('None','None'),
"TMP_KEY":('A','B')
}
derived_dict = {}
for k,v in func_dict.viewitems():
tmp_str = ",".join(v)
derived_dict[k] = tmp_str
print derived_dict
#Output
E:\tmp_python>python tmp.py
{'DXM_Runtime': 'None,None', 'TMP_KEY': 'A,B'}
#If this doesn't help you, then please post the code for get_reg and traverse_reg function also.
I am taking an Udacity programming course and have been sitting on the same problem for a week. I finally think I am close to getting it right, but I don't get the last objection. Here is my code:
def process_file(f):
# This is example of the datastructure you should return
# Each item in the list should be a dictionary containing all the relevant data
# Note - year, month, and the flight data should be integers
# You should skip the rows that contain the TOTAL data for a year
# data = [{"courier": "FL",
# "airport": "ATL",
# "year": 2012,
# "month": 12,
# "flights": {"domestic": 100,
# "international": 100}
# },
# {"courier": "..."}
# ]
data = []
info = {}
info["courier"], info["airport"] = f[:6].split("-")
with open("{}/{}".format(datadir, f), "r") as html:
soup = BeautifulSoup(html)
car = str(html)[17:19]
airp = str(html)[20:23]
mydict = {}
x = 0
table = soup.find("table", {"class": "dataTDRight"})
rows = table.find_all('tr')
for row in rows:
cells = row.find_all('td')
year = cells[0].get_text()
year = (year.encode('ascii'))
Month = cells[1].get_text()
Month = (Month.encode('ascii'))
domestic = cells[2].get_text()
domestic = (domestic.encode('ascii'))
international = cells[3].get_text()
international = (international.encode('ascii'))
if Month != "Month" and Month != "TOTAL":
Month = int(Month)
year = int(year)
domestic = int(domestic.replace(',', ''))
international = int(international.replace(',', ''))
mydict['courier'] = car
mydict['airport'] = airp
mydict['year'] = year
mydict['month'] = Month
mydict['flights'] = (domestic, international)
data.append(mydict.copy())
#print type(domestic)
#print mydict
print data
return data
def test():
print "Running a simple test..."
open_zip(datadir)
files = process_all(datadir)
data = []
for f in files:
data += process_file(f)
assert len(data) == 399
for entry in data[:3]:
assert type(entry["year"]) == int
assert type(entry["month"]) == int
assert type(entry["flights"]["domestic"]) == int
assert len(entry["airport"]) == 3
assert len(entry["courier"]) == 2
assert data[-1]["airport"] == "ATL"
assert data[-1]["flights"] == {'international': 108289, 'domestic': 701425}
print "... success!"
The error message I get is:
Traceback (most recent call last):
File "vm_main.py", line 33, in <module>
import main
File "/tmp/vmuser_elbzlfkcpw/main.py", line 2, in <module>
import studentMain
File "/tmp/vmuser_elbzlfkcpw/studentMain.py", line 2, in <module>
process.test()
File "/tmp/vmuser_elbzlfkcpw/process.py", line 114, in test
assert type(entry["flights"]["domestic"]) == int
TypeError: tuple indices must be integers, not str
I am a total beginner, I checked both the type of domestic, and international, they are both int.
Can anybody tell me where I can look up or what I did wrong?
You created a tuple here:
mydict['flights'] = (domestic, international)
so mydict['flights'] is a tuple. But you try to treat it as a dictionary here:
assert type(entry["flights"]["domestic"]) == int
That won't work; you'll need to use integer indices here:
assert type(entry["flights"][0]) == int
or better still, use isinstance() to test for types:
assert isinstance(entry["flights"][0], int)
Here you assign your data mydict['flights'] as a tuple.
def process_file(f):
# Omitted code...
mydict['flights'] = (domestic, international)
Your error then comes from an illegal access to that data type. You are attempting to access the first item of that tuple by the name of variable you used in assignment:
assert type(entry["flights"]["domestic"]) == int
You either need to access your data via an integer index:
assert type(entry["flights"][0]) == int
Or you need to change your assignment to:
mydict['flights'] = {"domestic":domestic, "international":international}
tuples are immutable data types which are indexed by integers. The type of access you are attempting is typical of a dictionary, where indexes can be of any type.
I receive data from the Loggly service in dot notation, but to put data back in, it must be in JSON.
Hence, I need to convert:
{'json.message.status.time':50, 'json.message.code.response':80, 'json.time':100}
Into:
{'message': {'code': {'response': 80}, 'status': {'time': 50}}, 'time': 100}
I have put together a function to do so, but I wonder if there is a more direct and simpler way to accomplish the same result.
def dot_to_json(a):
# Create root for JSON tree structure
resp = {}
for k,v in a.items():
# eliminate json. (if metric comes from another type, it will keep its root)
k = re.sub(r'\bjson.\b','',k)
if '.' in k:
# Field has a dot
r = resp
s = ''
k2 = k.split('.')
l = len(k2)
count = 0
t = {}
for f in k2:
count += 1
if f not in resp.keys():
r[f]={}
r = r[f]
if count < l:
s += "['" + f + "']"
else:
s = "resp%s" % s
t = eval(s)
# Assign value to the last branch
t[f] = v
else:
r2 = resp
if k not in resp.keys():
r2[k] = {}
r2[k] = v
return resp
You can turn the path into dictionary access with:
def dot_to_json(a):
output = {}
for key, value in a.iteritems():
path = key.split('.')
if path[0] == 'json':
path = path[1:]
target = reduce(lambda d, k: d.setdefault(k, {}), path[:-1], output)
target[path[-1]] = value
return output
This takes the key as a path, ignoring the first json part. With reduce() you can walk the elements of path (except for the last one) and fetch the nested dictionary with it.
Essentially you start at output and for each element in path fetch the value and use that value as the input for the next iteration. Here dict.setdefault() is used to default to a new empty dictionary each time a key doesn't yet exist. For a path ['foo', 'bar', 'baz'] this comes down to the call output.setdefault('foo', {}).setdefault('bar', {}).setdefault('baz', {}), only more compact and supporting arbitrary length paths.
The innermost dictionary is then used to set the value with the last element of the path as the key.
Demo:
>>> def dot_to_json(a):
... output = {}
... for key, value in a.iteritems():
... path = key.split('.')[1:] # ignore the json. prefix
... target = reduce(lambda d, k: d.setdefault(k, {}), path[:-1], output)
... target[path[-1]] = value
... return output
...
>>> dot_to_json({'json.message.status.time':50, 'json.message.code.response':80, 'json.time':100}))
{'message': {'status': {'time': 50}, 'code': {'response': 80}}, 'time': 100}