I was trying to grab a list of prices. So far my code for such a thing is:
def steamlibrarypull(steamID, key):
#Pulls out a CSV of Steam appids.
steaminfo = {
'key': key,
'steamid': steamID,
'format':'JSON',
'include_appinfo':'1'
}
r = requests.get('http://api.steampowered.com/IPlayerService/GetOwnedGames/v0001/', params=steaminfo)
d = json.loads(r.content)
I = d['response']['games']
B = {}
for games in I:
B[games['name'].encode('utf8')] = games['appid']
with open('games.csv', 'w') as f:
for key, value in B.items():
f.write("%s,%s\r\n" % (key, value))
return B
But I'd like to be able to do a request.get that'll take this dictionary and ouput out a list of prices. https://wiki.teamfortress.com/wiki/User:RJackson/StorefrontAPI Seems to require the need of a CSV list but is that really necessary?
this is a non formal steam api meaning steam modifies as they see fit. currently it does not support multiple appids as noted here.
to use it to get the price of a game you would go
http://store.steampowered.com/api/appdetails/?appids=237110&cc=us&filters=price_overview
working from the code you have above you will need to know how to iterate through the dictionary and update the store price once you get it back.
def steamlibrarypull(steamID, key):
#Pulls out a CSV of Steam appids.
steaminfo = {
'key': key,
'steamid': steamID,
'format':'JSON',
'include_appinfo':'1'
}
r = requests.get('http://api.steampowered.com/IPlayerService/GetOwnedGames/v0001/', params=steaminfo)
d = json.loads(r.content)
response = d['response']['games']
games = {}
for game in response:
getprice = requests.get('http://store.steampowered.com/api/appdetails/?appids=%d&filters=price_overview&cc=us' % game['appid'])
if getprice.status_code == 200:
rjson = json.loads(getprice.text)
# use the appid to fetch the value and convert to decimal
# appid is numeric, cast to string to lookup the price
try:
price = rjson[str(game['appid'])]['data']['price_overview']['initial'] * .01
except:
price = 0
games[game['name']] = {'price': price, 'appid': game['appid']}
this will return the following dictionary:
{u'Half-Life 2: Episode Two': {'price': 7.99, 'appid': 420}
it would be easier to navigate via appid instead of name but as per your request and original structure this is how it should be done. this then gives you the name, appid and price that you can work with further or write to a file.
note that this does not include a sleep timer, if your list of games is long you should sleep your api calls for 2 seconds before making another one or the api will block you and will not return data which will cause an error in python when you parse the price.
Related
I'm trying to create a new list from an API return in python. The purpose of this API call is to pull a list of driver's names, and pair them to a vehicle ID native to the API service. The code currently looks like this:
url = url
headers = {
"accept": "application/json",
"authorization": auth
}
response = requests.get(url, headers=headers)
response = response.json()
for doc in response['data']:
try:
doc['id'],
doc['staticAssignedDriver']['name']
except:
pass
else:
names = {
doc['staticAssignedDriver']['name']: doc['id']
}
names.update(names)
print(type(names))
print(names)
This prints a list of unique names and id's as individual dicts. IE:
{'name1':'id1'}
<class 'dict'>
{'name2':'id2'}
<class 'dict'>
Until I have all of my name:id pairs from my API.
But I'd like to make that a single dict, as such:
{'name1': 'id1', 'name2': 'id2'}
It seems like each new name/id pair ends up being its own var 'names'. Is there a way to make this its own singular dict, instead of individual?
When you do names = {whatever: whatever}, you always create a new dictionary with exactly one key and value. If you want to have only one dictionary that you update over and over, create the dictionary outside of the loop, and just assign a single value into it at a time:
names = {}
for doc in ...:
...
names[doc['staticAssignedDriver']['name']] = doc['id']
x = [{'name1':'id1'},
{'name2':'id2'}]
d = {}
for dct in x:
d.update({key: value for key, value in dct.items()})
print(d)
{'name1': 'id1', 'name2': 'id2'}
So this is a somewhat of a continuation from a previous post of mine except now I have API data to work with. I am trying to get keys Type and Email as columns in a data frame to come up with a final number. My code:
jsp_full=[]
for p in payloads:
payload = {"payload": {"segmentId":p}}
r = requests.post(url,headers = header, json = payload)
#print(r, r.reason)
time.sleep(r.elapsed.total_seconds())
json_data = r.json() if r and r.status_code == 200 else None
json_keys = json_data['payload']['supporters']
json_package = []
jsp_full.append(json_package)
for row in json_keys:
SID = row['supporterId']
Handle = row['contacts']
a_key = 'value'
list_values = [a_list[a_key] for a_list in Handle]
string = str(list_values).split(",")
data = {
'SupporterID' : SID,
'Email' : strip_characters(string[-1]),
'Type' : labels(p)
}
json_package.append(data)
t2 = round(time.perf_counter(),2)
b_key = "Email"
e = len([b_list[b_key] for b_list in json_package])
t = str(labels(p))
#print(json_package)
print(f'There are {e} emails in the {t} segment')
print(f'Finished in {t2 - t1} seconds')
excel = pd.DataFrame(json_package)
excel.to_excel(r'C:\Users\am\Desktop\email parsing\{0} segment {1}.xlsx'.format(t, str(today)), sheet_name=t)
This part works all well and good. Each payload in the API represents a different segment of people so I split them out into different files. However, I am at a point where I need to combine all records into a single data frame hence why I append out to jsp_full. This is a list of a list of dictionaries.
Once I have that I would run the balance of my code which is like this:
S= pd.DataFrame(jsp_full[0], index = {0})
Advocacy_Supporters = S.sort_values("Type").groupby("Type", as_index=False)["Email"].first()
print(Advocacy_Supporters['Email'].count())
print("The number of Unique Advocacy Supporters is :")
Advocacy_Supporters_Group = Advocacy_Supporters.groupby("Type")["Email"].nunique()
print(Advocacy_Supporters_Group)
Some sample data:
[{'SupporterID': '565f6a2f-c7fd-4f1b-bac2-e33976ef4306', 'Email': 'somebody#somewhere.edu', 'Type': 'd_Student Ambassadors'}, {'SupporterID': '7508dc12-7647-4e95-a8b8-bcb067861faf', 'Email': 'someoneelse#email.somewhere.edu', 'Type': 'd_Student Ambassadors'},...`
My desired output is a dataframe that looks like so:
SupporterID Email Type
565f6a2f-c7fd-4f1b-bac2-e33976ef4306 somebody#somewhere.edu d_Student Ambassadors
7508dc12-7647-4e95-a8b8-bcb067861faf someoneelse#email.somewhere.edu d_Student Ambassadors
Any help is greatly appreciated!!
So because this code creates an excel file for each segment, all I did was read back in the excels via a for loop like so:
filesnames = ['e_S Donors', 'b_Contributors', 'c_Activists', 'd_Student Ambassadors', 'a_Volunteers', 'f_Offline Action Takers']
S= pd.DataFrame()
for i in filesnames:
data = pd.read_excel(r'C:\Users\am\Desktop\email parsing\{0} segment {1}.xlsx'.format(i, str(today)),sheet_name= i, engine = 'openpyxl')
S= S.append(data)
This did the trick since it was in a format I already wanted.
I am working with the Twitter Search API which returns a dictionary of dictionaries. My goal is to create a dataframe from a list of keys in the response dictionary.
Example of API response here: Example Response
I have a list of keys within the Statuses dictionary
keys = ["created_at", "text", "in_reply_to_screen_name", "source"]
I would like to loop through each key value returned in the Statuses dictionary and put them in a dataframe with the keys as the columns.
Currently have code to loop through a single key individually and assign to list then append to dataframe but want a way to do more than one key at a time. Current code below:
#w is the word to be queired
w = 'keyword'
#count of tweets to return
count = 1000
#API call
query = twitter.search.tweets(q= w, count = count)
def data_l2 (q, k1, k2):
data = []
for results in q[k1]:
data.append(results[k2])
return(data)
screen_names = data_l3(query, "statuses", "user", "screen_name")
data = {'screen_names':screen_names,
'tweets':tweets}
frame=pd.DataFrame(data)
frame
I will share a more generic solution that I came up with, as I was working with the Twitter API. Let's say you have the ID's of tweets that you want to fetch in a list called my_ids :
# Fetch tweets from the twitter API using the following loop:
list_of_tweets = []
# Tweets that can't be found are saved in the list below:
cant_find_tweets_for_those_ids = []
for each_id in my_ids:
try:
list_of_tweets.append(api.get_status(each_id))
except Exception as e:
cant_find_tweets_for_those_ids.append(each_id)
Then in this code block we isolate the json part of each tweepy status object that we have downloaded and we add them all into a list....
my_list_of_dicts = []
for each_json_tweet in list_of_tweets:
my_list_of_dicts.append(each_json_tweet._json)
...and we write this list into a txt file:
with open('tweet_json.txt', 'w') as file:
file.write(json.dumps(my_list_of_dicts, indent=4))
Now we are going to create a DataFrame from the tweet_json.txt file (I have added some keys that were relevant to my use case that I was working on, but you can add your specific keys instead):
my_demo_list = []
with open('tweet_json.txt', encoding='utf-8') as json_file:
all_data = json.load(json_file)
for each_dictionary in all_data:
tweet_id = each_dictionary['id']
whole_tweet = each_dictionary['text']
only_url = whole_tweet[whole_tweet.find('https'):]
favorite_count = each_dictionary['favorite_count']
retweet_count = each_dictionary['retweet_count']
created_at = each_dictionary['created_at']
whole_source = each_dictionary['source']
only_device = whole_source[whole_source.find('rel="nofollow">') + 15:-4]
source = only_device
retweeted_status = each_dictionary['retweeted_status'] = each_dictionary.get('retweeted_status', 'Original tweet')
if retweeted_status == 'Original tweet':
url = only_url
else:
retweeted_status = 'This is a retweet'
url = 'This is a retweet'
my_demo_list.append({'tweet_id': str(tweet_id),
'favorite_count': int(favorite_count),
'retweet_count': int(retweet_count),
'url': url,
'created_at': created_at,
'source': source,
'retweeted_status': retweeted_status,
})
tweet_json = pd.DataFrame(my_demo_list, columns = ['tweet_id', 'favorite_count',
'retweet_count', 'created_at',
'source', 'retweeted_status', 'url'])
I am getting JIRA data using the following python code,
how do I store the response for more than one key (my example shows only one KEY but in general I get lot of data) and print only the values corresponding to total,key, customfield_12830, summary
import requests
import json
import logging
import datetime
import base64
import urllib
serverURL = 'https://jira-stability-tools.company.com/jira'
user = 'username'
password = 'password'
query = 'project = PROJECTNAME AND "Build Info" ~ BUILDNAME AND assignee=ASSIGNEENAME'
jql = '/rest/api/2/search?jql=%s' % urllib.quote(query)
response = requests.get(serverURL + jql,verify=False,auth=(user, password))
print response.json()
response.json() OUTPUT:-
http://pastebin.com/h8R4QMgB
From the the link you pasted to pastebin and from the json that I saw, its a you issues as list containing key, fields(which holds custom fields), self, id, expand.
You can simply iterate through this response and extract values for keys you want. You can go like.
data = response.json()
issues = data.get('issues', list())
x = list()
for issue in issues:
temp = {
'key': issue['key'],
'customfield': issue['fields']['customfield_12830'],
'total': issue['fields']['progress']['total']
}
x.append(temp)
print(x)
x is list of dictionaries containing the data for fields you mentioned. Let me know if I have been unclear somewhere or what I have given is not what you are looking for.
PS: It is always advisable to use dict.get('keyname', None) to get values as you can always put a default value if key is not found. For this solution I didn't do it as I just wanted to provide approach.
Update: In the comments you(OP) mentioned that it gives attributerror.Try this code
data = response.json()
issues = data.get('issues', list())
x = list()
for issue in issues:
temp = dict()
key = issue.get('key', None)
if key:
temp['key'] = key
fields = issue.get('fields', None)
if fields:
customfield = fields.get('customfield_12830', None)
temp['customfield'] = customfield
progress = fields.get('progress', None)
if progress:
total = progress.get('total', None)
temp['total'] = total
x.append(temp)
print(x)
I want my Echonest response to get song duration, which is supposed to be part of the audio_summary.
params = {
'type':'artist-radio',
'artist':artist,
'results': 3,
'bucket' : ['id:spotify-WW', 'tracks'],
'limit': True
}
response = en.get('playlist/static', **params)
songs = response['songs']
so, to get song duration, which key/value should I use in the example above?
NOTE: the wrapper being used is pyen
durationis an analysis found in song/profile, and not in playlist/static method, so we need a second response from the api.
this is one way of obtaining each song duration (as well as printing artist_name and song_title):
#get 'playlist response'
response_playlist = en.get('playlist/static', **params)
song_playlist = response_playlist['songs']
if len(song_playlist) > 0:
for i, song in enumerate(song_playlist):
#we need to track each song id
song_id = song_playlist[i]['id'] #ok
#in order to get song 'duration', we need to access 'song/profile response'
#and pass the id as an argument to 'audio_summary'
response_profile = en.get('song/profile', id=song_id, bucket="audio_summary")
song_profile = response_profile['songs']
dur = song_profile[0]['audio_summary']['duration']
print dur
#now we access each song 'foreign_id', which is playable by, say, Spotify
for track in song:
track = song['tracks'][i]
track_id = track['foreign_id'].replace('-WW', '')
print '{0} {2} {1}'.format(i, song['artist_name'], song['title'])