I'm using Python 3 and trying to get a list of possible stock symbols from company name using YAHOO.Finance.SymbolSuggest.ssCallback.
With the below code,
import urllib
yahoo_url = 'http://d.yimg.com/autoc.finance.yahoo.com/autoc?query=apple+inc®ion=1&lang=en&callback=YAHOO.Finance.SymbolSuggest.ssCallback'
response = urllib.request.urlopen(yahoo_url)
str_response = response.read().decode('utf-8')
The result is below:
str_response
Out[136]: 'YAHOO.Finance.SymbolSuggest.ssCallback({"ResultSet":{"Query":"AAPL","Result":[{"symbol":"AAPL","name":"Apple Inc.","exch":"NMS","type":"S","exchDisp":"NASDAQ","typeDisp":"Equity"},{"symbol":"^NY2LAAPL","name":"ICE Leveraged 2x AAPL Index","exch":"NYS","type":"I","exchDisp":"NYSE","typeDisp":"Index"},{"symbol":"AAPL.BA","name":"Apple Inc.","exch":"BUE","type":"S","exchDisp":"Buenos Aires","typeDisp":"Equity"},{"symbol":"AAPL34.SA","name":"Apple Inc.","exch":"SAO","type":"S","exchDisp":"Sao Paolo","typeDisp":"Equity"},{"symbol":"AAPL.MX","name":"Apple Inc.","exch":"MEX","type":"S","exchDisp":"Mexico","typeDisp":"Equity"},{"symbol":"AAPL.MI","name":"APPLE","exch":"MIL","type":"S","exchDisp":"Milan","typeDisp":"Equity"},{"symbol":"AAPLD.BA","name":"APPLE INC","exch":"BUE","type":"S","exchDisp":"Buenos Aires","typeDisp":"Equity"},{"symbol":"AAPLC.BA","name":"APPLE INC","exch":"BUE","type":"S","exchDisp":"Buenos Aires","typeDisp":"Equity"},{"symbol":"AAPL.VI","name":"Apple Inc.","exch":"VIE","type":"S","exchDisp":"Vienna","typeDisp":"Equity"}]}});'
How do I extract only the below segment? And then put into dict?
"Result":[{"symbol":"AAPL","name":"Apple Inc.","exch":"NMS","type":"S","exchDisp":"NASDAQ","typeDisp":"Equity"},{"symbol":"^NY2LAAPL","name":"ICE Leveraged 2x AAPL Index","exch":"NYS","type":"I","exchDisp":"NYSE","typeDisp":"Index"},{"symbol":"AAPL.BA","name":"Apple Inc.","exch":"BUE","type":"S","exchDisp":"Buenos Aires","typeDisp":"Equity"},{"symbol":"AAPL34.SA","name":"Apple Inc.","exch":"SAO","type":"S","exchDisp":"Sao Paolo","typeDisp":"Equity"},{"symbol":"AAPL.MX","name":"Apple Inc.","exch":"MEX","type":"S","exchDisp":"Mexico","typeDisp":"Equity"},{"symbol":"AAPL.MI","name":"APPLE","exch":"MIL","type":"S","exchDisp":"Milan","typeDisp":"Equity"},{"symbol":"AAPLD.BA","name":"APPLE INC","exch":"BUE","type":"S","exchDisp":"Buenos Aires","typeDisp":"Equity"},{"symbol":"AAPLC.BA","name":"APPLE INC","exch":"BUE","type":"S","exchDisp":"Buenos Aires","typeDisp":"Equity"},{"symbol":"AAPL.VI","name":"Apple Inc.","exch":"VIE","type":"S","exchDisp":"Vienna","typeDisp":"Equity"}]
Thank you in advance.
convert the string to a dict
use ast.literal_eval
from ast import literal_eval
result = literal_eval(str_response[39:-2])
print(type(result))
>>> dict
# Result key of interest
Result = result['ResultSet']['Result']
print(Result)
[{'symbol': 'AAPL',
'name': 'Apple Inc.',
'exch': 'NMS',
'type': 'S',
'exchDisp': 'NASDAQ',
'typeDisp': 'Equity'},
{'symbol': '^NY2LAAPL',
'name': 'ICE Leveraged 2x AAPL Index',
'exch': 'NYS',
'type': 'I',
'exchDisp': 'NYSE',
'typeDisp': 'Index'},
{'symbol': 'AAPL.BA',
'name': 'Apple Inc.',
'exch': 'BUE',
'type': 'S',
'exchDisp': 'Buenos Aires',
'typeDisp': 'Equity'},
{'symbol': 'AAPL34.SA',
'name': 'Apple Inc.',
'exch': 'SAO',
'type': 'S',
'exchDisp': 'Sao Paolo',
'typeDisp': 'Equity'},
{'symbol': 'AAPL.MX',
'name': 'Apple Inc.',
'exch': 'MEX',
'type': 'S',
'exchDisp': 'Mexico',
'typeDisp': 'Equity'},
{'symbol': 'AAPL.MI',
'name': 'APPLE',
'exch': 'MIL',
'type': 'S',
'exchDisp': 'Milan',
'typeDisp': 'Equity'},
{'symbol': 'AAPLD.BA',
'name': 'APPLE INC',
'exch': 'BUE',
'type': 'S',
'exchDisp': 'Buenos Aires',
'typeDisp': 'Equity'},
{'symbol': 'AAPLC.BA',
'name': 'APPLE INC',
'exch': 'BUE',
'type': 'S',
'exchDisp': 'Buenos Aires',
'typeDisp': 'Equity'},
{'symbol': 'AAPL.VI',
'name': 'Apple Inc.',
'exch': 'VIE',
'type': 'S',
'exchDisp': 'Vienna',
'typeDisp': 'Equity'}]
Related
I have a json file with the following structure:
{'0': {'transaction': [{'transaction_key': '406.l.657872.tr.374',
'transaction_id': '374',
'type': 'add/drop',
'status': 'successful',
'timestamp': '1639593953'},
{'players': {'0': {'player': [[{'player_key': '406.p.100006'},
{'player_id': '100006'},
{'name': {'full': 'Dallas',
'first': 'Dallas',
'last': '',
'ascii_first': 'Dallas',
'ascii_last': ''}},
{'editorial_team_abbr': 'Dal'},
{'display_position': 'DEF'},
{'position_type': 'DT'}],
{'transaction_data': [{'type': 'add',
'source_type': 'freeagents',
'destination_type': 'team',
'destination_team_key': '406.l.657872.t.10',
'destination_team_name': 'Team 1'}]}]},
'1': {'player': [[{'player_key': '406.p.24793'},
{'player_id': '24793'},
{'name': {'full': 'Julio Jones',
'first': 'Julio',
'last': 'Jones',
'ascii_first': 'Julio',
'ascii_last': 'Jones'}},
{'editorial_team_abbr': 'Ten'},
{'display_position': 'WR'},
{'position_type': 'O'}],
{'transaction_data': {'type': 'drop',
'source_type': 'team',
'source_team_key': '406.l.657872.t.10',
'source_team_name': 'Team 1',
'destination_type': 'waivers'}}]},
'count': 2}}]},
'1': {'transaction': [{'transaction_key': '406.l.657872.tr.373',
'transaction_id': '373',
'type': 'add/drop',
'status': 'successful',
'timestamp': '1639575496'},
{'players': {'0': {'player': [[{'player_key': '406.p.32722'},
{'player_id': '32722'},
{'name': {'full': 'Cam Akers',
'first': 'Cam',
'last': 'Akers',
'ascii_first': 'Cam',
'ascii_last': 'Akers'}},
{'editorial_team_abbr': 'LAR'},
{'display_position': 'RB'},
{'position_type': 'O'}],
{'transaction_data': [{'type': 'add',
'source_type': 'freeagents',
'destination_type': 'team',
'destination_team_key': '406.l.657872.t.5',
'destination_team_name': 'Team 2'}]}]},
'1': {'player': [[{'player_key': '406.p.100007'},
{'player_id': '100007'},
{'name': {'full': 'Denver',
'first': 'Denver',
'last': '',
'ascii_first': 'Denver',
'ascii_last': ''}},
{'editorial_team_abbr': 'Den'},
{'display_position': 'DEF'},
{'position_type': 'DT'}],
{'transaction_data': {'type': 'drop',
'source_type': 'team',
'source_team_key': '406.l.657872.t.5',
'source_team_name': 'Team 2',
'destination_type': 'waivers'}}]},
'count': 2}}]},
'2': {'transaction': [{'transaction_key': '406.l.657872.tr.372',
'transaction_id': '372',
'type': 'add/drop',
'status': 'successful',
'timestamp': '1639575448'},
{'players': {'0': {'player': [[{'player_key': '406.p.33413'},
{'player_id': '33413'},
{'name': {'full': 'Travis Etienne',
'first': 'Travis',
'last': 'Etienne',
'ascii_first': 'Travis',
'ascii_last': 'Etienne'}},
{'editorial_team_abbr': 'Jax'},
{'display_position': 'RB'},
{'position_type': 'O'}],
{'transaction_data': [{'type': 'add',
'source_type': 'freeagents',
'destination_type': 'team',
'destination_team_key': '406.l.657872.t.5',
'destination_team_name': 'Team 2'}]}]},
'1': {'player': [[{'player_key': '406.p.24815'},
{'player_id': '24815'},
{'name': {'full': 'Mark Ingram II',
'first': 'Mark',
'last': 'Ingram II',
'ascii_first': 'Mark',
'ascii_last': 'Ingram II'}},
{'editorial_team_abbr': 'NO'},
{'display_position': 'RB'},
{'position_type': 'O'}],
{'transaction_data': {'type': 'drop',
'source_type': 'team',
'source_team_key': '406.l.657872.t.5',
'source_team_name': 'Team 2',
'destination_type': 'waivers'}}]},
'count': 2}}]}
These are transactions for a fantasy football league and I'd like to organize each transaction into a dataframe, however I'm running into issues normalizing the data. I figure I'd need to begin a loop, but am slightly stuck in the mud and would appreciate if anyone has any suggestions. Thank You.
Ideally, I'm looking to summarize each transaction with the following dataframe structure:
transaction_id type added pos_1 dropped pos_2 timestamp
374 add/drop Dallas DEF Julio Jones WR 1639593953
373 add/drop Cam Akers RB Denver DEF 1639575496
372 add/drop Travis Etienne RB Mark Ingram II RB 1639575448
I got the following list of dicts
list_of_dicts = [
{'product': 'car', 'city': 'new york', 'quantity': 13},
{'product': 'car', 'city': 'new york', 'quantity': 25},
{'product': 'bus', 'city': 'miami', 'quantity': 5},
{'product': 'container', 'city': 'atlanta', 'quantity' 5},
{'product': 'container', 'city': 'atlanta', 'quantity' 8}
]
My target is, when values of 'product' and 'city' are the same, sum up the values of 'quantity'.
The result should look like this:
result_list_of_dicts = [
{'product': 'car', 'city': 'new york', 'quantity': 38},
{'product': 'bus', 'city': 'miami', 'quantity': 5},
{'product': 'container', 'city': 'atlanta', 'quantity' 13},
]
Is there a pythonic way? I tried a couple of things but I better not show them because they are really ugly.
Thank you in advance!
You can do the following, using only standard library utils:
from operator import itemgetter
from functools import reduce
from itertools import groupby
pc = itemgetter("product", "city") # sorting and grouping key
q = itemgetter("quantity")
combine = lambda d1, d2: {**d1, "quantity": q(d1) + q(d2)}
[reduce(combine, g) for _, g in groupby(sorted(list_of_dicts, key=pc), key=pc)]
# [{'product': 'bus', 'city': 'miami', 'quantity': 5},
# {'product': 'car', 'city': 'new york', 'quantity': 38},
# {'product': 'container', 'city': 'atlanta', 'quantity': 13}]
Or, maybe even simpler and linear:
from collections import Counter
pc = itemgetter("product", "city")
q = itemgetter("quantity")
totals = Counter()
for dct in list_of_dicts:
totals[pc(dct)] += q(dct)
result_list_of_dicts = [
{"product": p, "city": c, "quantity": q} for (p, c), q in totals.items()
]
One approach using collections.Counter
from collections import Counter
list_of_dicts = [
{'product': 'car', 'city': 'new york', 'quantity': 13},
{'product': 'car', 'city': 'new york', 'quantity': 25},
{'product': 'bus', 'city': 'miami', 'quantity': 5},
{'product': 'container', 'city': 'atlanta', 'quantity': 5},
{'product': 'container', 'city': 'atlanta', 'quantity': 8}
]
counts = sum((Counter({(d["product"], d["city"]): d["quantity"]}) for d in list_of_dicts), Counter())
result = [{"product": product, "city": city, "quantity": quantity} for (product, city), quantity in counts.items()]
print(result)
A pandas implementation
Group by "product" and "city", sum over the groups and reset index to get original columns.
import pandas as pd
list_of_dicts = [
{'product': 'car', 'city': 'new york', 'quantity': 13},
{'product': 'car', 'city': 'new york', 'quantity': 25},
{'product': 'bus', 'city': 'miami', 'quantity': 5},
{'product': 'container', 'city': 'atlanta', 'quantity': 5},
{'product': 'container', 'city': 'atlanta', 'quantity': 8}
]
df = pd.DataFrame(list_of_dicts)
print(df)
df = df.groupby(["product", "city"]).sum().reset_index()
print(df)
summed_dict = df.to_dict("records")
print(summed_dict)
You could do it with a loop, initializing it the first time you encounter the product.
list_of_dicts = [
{'product': 'car', 'city': 'new york', 'quantity': 13},
{'product': 'car', 'city': 'new york', 'quantity': 25},
{'product': 'bus', 'city': 'miami', 'quantity': 5},
{'product': 'container', 'city': 'atlanta', 'quantity': 5},
{'product': 'container', 'city': 'atlanta', 'quantity': 8}
]
new_dict = {}
for ld in list_of_dicts:
if ld['product'] not in new_dict:
new_dict[ld['product']] = {}
new_dict[ld['product']]['city'] = ld['city']
new_dict[ld['product']]['quantity'] = 0
new_dict[ld['product']]['quantity'] += ld['quantity']
# print(new_dict)
# {'car': {'city': 'new york', 'quantity': 38}, 'bus': {'city': 'miami', 'quantity': 5}, 'container': {'city': 'atlanta', 'quantity': 13}}
result_list_of_dicts = [{'product': nd,
'city': new_dict[nd]['city'],
'quantity': new_dict[nd]['quantity']} for nd in new_dict]
# print(result_list_of_dicts)
# [{'product': 'car', 'city': 'new york', 'quantity': 38}, {'product': 'bus', 'city': 'miami', 'quantity': 5}, {'product': 'container', 'city': 'atlanta', 'quantity': 13}]
if i have a dictionary
Singers = [
{'Singer_ID': 'S00464040',
'City': 'BIRMINGHAM',
'First': ' MARTHA RENEE',
'Last': 'BOZEMAN',
'State': 'AL',
'Zipcode': '35201'},
{'Singer_ID': 'S00460410',
'City': 'BIRMINGHAM',
'First': ' EARL FREDERICK JR',
'Last': 'HILLIARD',
'State': 'AL',
'Zipcode': '35202'},
{'Singer_ID': 'S00461038',
'City': 'BIRMINGHAM',
'First': ' SHEILA',
'Last': 'SMOOT',
'State': 'NY',
'Zipcode': '87201'}]
How could I write a function that matches Singer: ID with State?
For example returns:
{'S00464040': 'AL',
'S00460410': 'AL',
'S00461038': 'NY'}
Try this:
>>> {i['Singer_ID']: i['State'] for i in Singers}
{'S00464040': 'AL', 'S00460410': 'AL', 'S00461038': 'NY'}
Looking for a Pythonic way to iterate over a list of Dicts and group them by a certain key.
E.g. a list like this should be grouped by position
[
{'Name': 'Bradley Greer', 'Position': 'Software Engineer', 'Office': 'London', 'Age': '41', 'Start date': '2012/10/13', 'Salary': '$132,000'},
{'Name': 'Brenden Wagner', 'Position': 'Software Engineer', 'Office': 'San Francisco', 'Age': '28', 'Start date': '2011/06/07', 'Salary': '$206,850'},
{'Name': 'Bruno Nash', 'Position': 'Software Engineer', 'Office': 'London', 'Age': '38', 'Start date': '2011/05/03', 'Salary': '$163,500'},
{'Name': 'Cara Stevens', 'Position': 'Sales Assistant', 'Office': 'New York', 'Age': '46', 'Start date': '2011/12/06', 'Salary': '$145,600'},
{'Name': 'Donna Snider', 'Position': 'Customer Support', 'Office': 'New York', 'Age': '27', 'Start date': '2011/01/25', 'Salary': '$112,000'},
{'Name': 'Doris Wilder', 'Position': 'Sales Assistant', 'Office': 'Sydney', 'Age': '23', 'Start date': '2010/09/20', 'Salary': '$85,600'},
{'Name': 'Gavin Joyce', 'Position': 'Sales Assistant', 'Office': 'Edinburgh', 'Age': '42', 'Start date': '2010/12/22', 'Salary': '$92,575'},
{'Name': 'Herrod Chandler', 'Position': 'Sales Assistant', 'Office': 'San Francisco', 'Age': '59', 'Start date': '2012/08/06', 'Salary': '$137,500'}
]
would result in something like this
[
{
'Position': 'Software Engineer',
'Items': [
{'Name': 'Bradley Greer', 'Position': 'Software Engineer', 'Office': 'London', 'Age': '41', 'Start date': '2012/10/13', 'Salary': '$132,000'},
{'Name': 'Brenden Wagner', 'Position': 'Software Engineer', 'Office': 'San Francisco', 'Age': '28', 'Start date': '2011/06/07', 'Salary': '$206,850'},
{'Name': 'Bruno Nash', 'Position': 'Software Engineer', 'Office': 'London', 'Age': '38', 'Start date': '2011/05/03', 'Salary': '$163,500'},
]
},
{
'Position': 'Sales Assistant',
'Items': [
{'Name': 'Cara Stevens', 'Position': 'Sales Assistant', 'Office': 'New York', 'Age': '46', 'Start date': '2011/12/06', 'Salary': '$145,600'},
{'Name': 'Doris Wilder', 'Position': 'Sales Assistant', 'Office': 'Sydney', 'Age': '23', 'Start date': '2010/09/20', 'Salary': '$85,600'},
{'Name': 'Gavin Joyce', 'Position': 'Sales Assistant', 'Office': 'Edinburgh', 'Age': '42', 'Start date': '2010/12/22', 'Salary': '$92,575'},
{'Name': 'Herrod Chandler', 'Position': 'Sales Assistant', 'Office': 'San Francisco', 'Age': '59', 'Start date': '2012/08/06', 'Salary': '$137,500'}
]
},
{
'Position': 'Customer Support',
'Items': [
{'Name': 'Donna Snider', 'Position': 'Customer Support', 'Office': 'New York', 'Age': '27', 'Start date': '2011/01/25', 'Salary': '$112,000'},
]
}
]
You can use itertools.groupby
items is your input list
import itertools
output = []
for k,v in itertools.groupby(items, key=lambda x:x['Position']):
output += [{
'Position': k,
'Items': list(v)
}]
If you want a one-liner (nearly as fast as the previous solution), then here it is:
people = # Your list of dicts
key = "Position" # The key to group by
output = [
{key: k, "items": [person for person in people if person[key] == k]}
for k in {person[key] for person in people}
]
i have nested dictionary with upto 300 items from TYPE1 TO TYPE300 called mainlookup
mainlookup = {'TYPE1': [{'Song': 'Rock', 'Type': 'Hard', 'Price': '10'}],
'TYPE2': [{'Song': 'Jazz', 'Type': 'Slow', 'Price': '5'}],
'TYPE37': [{'Song': 'Country', 'Type': 'Fast', 'Price': '7'}]}
input list to search in lookup based on string TYPE1, TYPE2 and so one
input_list = ['thissong-fav-user:type1-chan-44-John',
'thissong-fav-user:type1-chan-45-kelly-md',
'thissong-fav-user:type2-rock-45-usa',
'thissong-fav-user:type737-chan-45-patrick-md',
'thissong-fav-user:type37-chan-45-kelly-md']
i want to find the string TYPE IN input_list and then create a dictionary as shown below
Output_Desired = {'thissong-fav-user:type1-chan-44-John': [{'Song': 'Rock', 'Type': 'Hard',
'Price':'10'}],
'thissong-fav-user:type1-chan-45-kelly-md': [{'Song': 'Rock', 'Type': 'Hard', 'Price': '10'}],
'thissong-fav-user:type2-rock-45-usa': [{'Song': 'Jazz', 'Type': 'Slow', 'Price': '5'}],
'thissong-fav-user:type37-chan-45-kelly-md': [{'Song': 'Country', 'Type': 'Fast', 'Price': '7'}]}
Note-thissong-fav-user:type737-chan-45-patrick-md in the list has no match so i want to create a
seperate list if value is not found in main lookup
Notfound_list = ['thissong-fav-user:type737-chan-45-patrick-md', and so on..]
Appreciate your help.
You can try this:
mainlookup = {'TYPE1': [{'Song': 'Rock', 'Type': 'Hard', 'Price': '10'}],
'TYPE2': [{'Song': 'Jazz', 'Type': 'Slow', 'Price': '5'}], 'TYPE37': [{'Song': 'Country', 'Type': 'Fast', 'Price': '7'}]}
input_list = ['thissong-fav-user:type1-chan-44-John',
'thissong-fav-user:type1-chan-45-kelly-md', 'thissong-fav-user:type737-chan-45-kelly-md']
dct={i:mainlookup[i.split(':')[1].split('-')[0].upper()] for i in input_list if i.split(':')[1].split('-')[0].upper() in mainlookup.keys()}
Notfoundlist=[i for i in input_list if i not in dct.keys() ]
print(dct)
print(Notfoundlist)
Output:
{'thissong-fav-user:type1-chan-44-John': [{'Song': 'Rock', 'Type': 'Hard', 'Price': '10'}], 'thissong-fav-user:type1-chan-45-kelly-md': [{'Song': 'Rock', 'Type': 'Hard', 'Price': '10'}]}
['thissong-fav-user:type737-chan-45-kelly-md']
An answer using regular expressions:
import re
from pprint import pprint
input_list = ['thissong-fav-user:type1-chan-44-John', 'thissong-fav-user:type1-chan-45-kelly-md', 'thissong-fav-user:type2-rock-45-usa', 'thissong-fav-user:type737-chan-45-patrick-md', 'thissong-fav-user:type37-chan-45-kelly-md']
mainlookup = {'TYPE2': {'Song': 'Reggaeton', 'Type': 'Hard', 'Price': '30'}, 'TYPE1': {'Song': 'Rock', 'Type': 'Hard', 'Price': '10'}, 'TYPE737': {'Song': 'Jazz', 'Type': 'Hard', 'Price': '99'}, 'TYPE37': {'Song': 'Rock', 'Type': 'Soft', 'Price': '1'}}
pattern = re.compile('type[0-9]+')
matches = [re.search(pattern, x).group(0) for x in input_list]
result = {x: [mainlookup[matches[i].upper()]] for i, x in enumerate(input_list)}
pprint(result)
Output:
{'thissong-fav-user:type1-chan-44-John': [{'Price': '10',
'Song': 'Rock',
'Type': 'Hard'}],
'thissong-fav-user:type1-chan-45-kelly-md': [{'Price': '10',
'Song': 'Rock',
'Type': 'Hard'}],
'thissong-fav-user:type2-rock-45-usa': [{'Price': '30',
'Song': 'Reggaeton',
'Type': 'Hard'}],
'thissong-fav-user:type37-chan-45-kelly-md': [{'Price': '1',
'Song': 'Rock',
'Type': 'Soft'}],
'thissong-fav-user:type737-chan-45-patrick-md': [{'Price': '99',
'Song': 'Jazz',
'Type': 'Hard'}]}