Related
I have a JSON feed i need to collect data from but I am running into some issues with iterating and parsing the API data.
The function below collects what I need but some horses get scratched (dont race) and the format of their JSON is different to horses who's status is "Starter".
def race_data():
for race_key in races_list():
data = requests.get(f'{Base_url}events/{race_key}.json?app_id={AppID}&app_key={AppKey}').text
# parse race data
data = json.loads(data)
for race_odds in data['competitors']:
horse_name = race_odds['name']
jockey_name = race_odds['jockey']
jockey_weight = race_odds['weight']
trainer_name = race_odds['trainer']
fixed_win_odds = race_odds['prices'][2]['price']
fixed_place_odds = race_odds['prices'][0]['price']
race_key = data['eventKey']
race_time = data['eventDateTimeUtc']
horse_status = race_odds['status']
print(f'{race_key} {horse_name} {jockey_name} {jockey_weight} {trainer_name} {fixed_win_odds} {fixed_place_odds} {race_time} {horse_status}')
if __name__ == "__main__":
race_data()
which produces
202210290200.T.AUS.quirindi.1 Ponte Pietra Jai Williams 0 Nikki Pollock 1.1 16.0 2022-10-29T02:55:00Z Starter
202210290200.T.AUS.quirindi.1 Gundawarra Vad Bolozhinskyi 0 G A O'brien 4.5 35.0 2022-10-29T02:55:00Z Starter
202210290200.T.AUS.quirindi.1 Cemented Ms Chelsea Hillier 0 Sally Torrens 2.5 35.0 2022-10-29T02:55:00Z Starter
202210290200.T.AUS.quirindi.1 Jack Duggan Ms Madeline Owen 0 J C Deamer 1.1 2.7 2022-10-29T02:55:00Z Starter
202210290200.T.AUS.quirindi.1 Northern Borders Ms Zara Lewis 0 W T Martyn 9.6 68.1 2022-10-29T02:55:00Z Starter
202210290200.T.AUS.quirindi.1 Iffland Ms Amelia Denby 0 K A Lees 1.9 19.4 2022-10-29T02:55:00Z Starter
202210290200.T.AUS.quirindi.1 La Brea Benjamin Osmond 0 K A Lees 2.1 14.0 2022-10-29T02:55:00Z Starter
202210290200.T.AUS.quirindi.1 My Gem Jacob Golden 0 Ms S Grills 2.8 16.1 2022-10-29T02:55:00Z Starter
202210290200.T.AUS.quirindi.1 Another Super Patrick Scorse 0 M J Dwyer 4.9 87.5 2022-10-29T02:55:00Z Starter
I run into an issue when one of the runners is scratched (not racing)
the problem is with the odds price field. When a horse is scratched (defined in horse_status)
fixed_win_odds = race_odds['prices'][2]['price']
fixed_place_odds = race_odds['prices'][0]['price']
The JSON below is for a horse that is scratched:
{
"status": "Scratched",
"startPos": 3,
"trainer": "Ms L Selby",
"shortForm": "0608247373",
"sequence": 7,
"weight": "0",
"name": "Bingo Banko",
"jockey": "Unknown",
"prices":
[
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 0
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 0,
"productType": "Current"
},
{
"price": 16,
"productType": "Max"
},
{
"price": 16,
"productType": "Open"
},
{
"price": 15,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
}
],
"price": 0
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.bingo_banko",
"competitorKey": "202210290200.T.AUS.quirindi.1.bingo_banko"
}
and this is what the JSON looks like when the horse is a starter (not scratched)
{
"status": "Starter",
"startPos": 9,
"trainer": "M J Dwyer",
"shortForm": "7626280005",
"sequence": 6,
"weight": "0",
"name": "Another Super",
"jockey": "Patrick Scorse",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 85
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 3.9
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 5.7
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 17,
"productType": "Current"
},
{
"price": 20,
"productType": "Max"
},
{
"price": 19,
"productType": "Open"
},
{
"price": 17,
"productType": "Last"
},
{
"price": 18,
"productType": "Last"
},
{
"price": 17,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
},
{
"price": 15,
"productType": "Last"
}
],
"price": 17
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.another_super",
"competitorKey": "202210290200.T.AUS.quirindi.1.another_super"
}
I still need to collect the information on the horses that are scratched, so need some help with formatting the iteration to collect the data without getting key error on horses that are scratched.
you can view the full JSON for a race below for reference:
[
{
"status": "Starter",
"startPos": 5,
"trainer": "Nikki Pollock",
"shortForm": "x686623464",
"sequence": 3,
"weight": "0",
"name": "Ponte Pietra",
"jockey": "Jai Williams",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 15.6
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 2.6
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 1.04
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 10,
"productType": "Current"
},
{
"price": 16,
"productType": "Max"
},
{
"price": 16,
"productType": "Open"
},
{
"price": 10,
"productType": "Last"
},
{
"price": 9,
"productType": "Last"
},
{
"price": 9,
"productType": "Last"
},
{
"price": 8,
"productType": "Last"
},
{
"price": 8,
"productType": "Last"
}
],
"price": 10
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.ponte_pietra",
"competitorKey": "202210290200.T.AUS.quirindi.1.ponte_pietra"
},
{
"status": "Starter",
"startPos": 4,
"trainer": "G A O'brien",
"shortForm": "90x4491",
"sequence": 4,
"weight": "0",
"name": "Gundawarra",
"jockey": "Vad Bolozhinskyi",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 34
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 1.45
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 11
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 3,
"productType": "Current"
},
{
"price": 3,
"productType": "Max"
},
{
"price": 3,
"productType": "Open"
},
{
"price": 3,
"productType": "Last"
},
{
"price": 3,
"productType": "Last"
},
{
"price": 3,
"productType": "Last"
},
{
"price": 3,
"productType": "Last"
},
{
"price": 3,
"productType": "Last"
}
],
"price": 3.7
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.gundawarra",
"competitorKey": "202210290200.T.AUS.quirindi.1.gundawarra"
},
{
"status": "Starter",
"startPos": 6,
"trainer": "Sally Torrens",
"shortForm": "7816x64858",
"sequence": 1,
"weight": "0",
"name": "Cemented",
"jockey": "Ms Chelsea Hillier",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 85
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 3.1
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 3
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 13,
"productType": "Current"
},
{
"price": 19,
"productType": "Max"
},
{
"price": 19,
"productType": "Open"
},
{
"price": 13,
"productType": "Last"
},
{
"price": 14,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
},
{
"price": 15,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
}
],
"price": 13
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.cemented",
"competitorKey": "202210290200.T.AUS.quirindi.1.cemented"
},
{
"status": "Starter",
"startPos": 2,
"trainer": "J C Deamer",
"shortForm": "75x5456x73",
"sequence": 2,
"weight": "0",
"name": "Jack Duggan",
"jockey": "Ms Madeline Owen",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 3
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 1.14
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 1.04
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 1,
"productType": "Current"
},
{
"price": 1,
"productType": "Max"
},
{
"price": 1,
"productType": "Open"
},
{
"price": 1,
"productType": "Last"
},
{
"price": 1,
"productType": "Last"
},
{
"price": 1,
"productType": "Last"
},
{
"price": 1,
"productType": "Last"
},
{
"price": 1,
"productType": "Last"
}
],
"price": 1.8
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.jack_duggan",
"competitorKey": "202210290200.T.AUS.quirindi.1.jack_duggan"
},
{
"status": "Starter",
"startPos": 10,
"trainer": "W T Martyn",
"shortForm": "00409",
"sequence": 8,
"weight": "0",
"name": "Northern Borders",
"jockey": "Ms Zara Lewis",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 116
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 8
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 9
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 41,
"productType": "Current"
},
{
"price": 51,
"productType": "Max"
},
{
"price": 34,
"productType": "Open"
},
{
"price": 41,
"productType": "Last"
},
{
"price": 51,
"productType": "Last"
},
{
"price": 41,
"productType": "Last"
},
{
"price": 51,
"productType": "Last"
},
{
"price": 41,
"productType": "Last"
}
],
"price": 41
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.northern_borders",
"competitorKey": "202210290200.T.AUS.quirindi.1.northern_borders"
},
{
"status": "Starter",
"startPos": 8,
"trainer": "K A Lees",
"shortForm": "6",
"sequence": 9,
"weight": "0",
"name": "Iffland",
"jockey": "Ms Amelia Denby",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 33.5
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 3.3
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 1.7
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 14,
"productType": "Current"
},
{
"price": 20,
"productType": "Max"
},
{
"price": 17,
"productType": "Open"
},
{
"price": 14,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
},
{
"price": 14,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
},
{
"price": 14,
"productType": "Last"
}
],
"price": 14
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.iffland",
"competitorKey": "202210290200.T.AUS.quirindi.1.iffland"
},
{
"status": "Starter",
"startPos": 7,
"trainer": "K A Lees",
"shortForm": "55x570x58",
"sequence": 10,
"weight": "0",
"name": "La Brea",
"jockey": "Benjamin Osmond",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 43.3
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 2.6
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 1.9
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 10,
"productType": "Current"
},
{
"price": 13,
"productType": "Max"
},
{
"price": 11,
"productType": "Open"
},
{
"price": 10,
"productType": "Last"
},
{
"price": 9,
"productType": "Last"
},
{
"price": 9,
"productType": "Last"
},
{
"price": 8,
"productType": "Last"
},
{
"price": 9,
"productType": "Last"
}
],
"price": 10
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.la_brea",
"competitorKey": "202210290200.T.AUS.quirindi.1.la_brea"
},
{
"status": "Starter",
"startPos": 1,
"trainer": "Ms S Grills",
"shortForm": "673410x084",
"sequence": 5,
"weight": "0",
"name": "My Gem",
"jockey": "Jacob Golden",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 14.9
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 3.7
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 3.2
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 16,
"productType": "Current"
},
{
"price": 16,
"productType": "Max"
},
{
"price": 13,
"productType": "Open"
},
{
"price": 16,
"productType": "Last"
},
{
"price": 15,
"productType": "Last"
},
{
"price": 14,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
},
{
"price": 15,
"productType": "Last"
}
],
"price": 16
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.my_gem",
"competitorKey": "202210290200.T.AUS.quirindi.1.my_gem"
},
{
"status": "Starter",
"startPos": 9,
"trainer": "M J Dwyer",
"shortForm": "7626280005",
"sequence": 6,
"weight": "0",
"name": "Another Super",
"jockey": "Patrick Scorse",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 85
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 3.9
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 5.7
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 17,
"productType": "Current"
},
{
"price": 20,
"productType": "Max"
},
{
"price": 19,
"productType": "Open"
},
{
"price": 17,
"productType": "Last"
},
{
"price": 18,
"productType": "Last"
},
{
"price": 17,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
},
{
"price": 15,
"productType": "Last"
}
],
"price": 17
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.another_super",
"competitorKey": "202210290200.T.AUS.quirindi.1.another_super"
},
{
"status": "Scratched",
"startPos": 3,
"trainer": "Ms L Selby",
"shortForm": "0608247373",
"sequence": 7,
"weight": "0",
"name": "Bingo Banko",
"jockey": "Unknown",
"prices":
[
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 0
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 0,
"productType": "Current"
},
{
"price": 16,
"productType": "Max"
},
{
"price": 16,
"productType": "Open"
},
{
"price": 15,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
}
],
"price": 0
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.bingo_banko",
"competitorKey": "202210290200.T.AUS.quirindi.1.bingo_banko"
}
]
thanks for your assistance.
Use a for loop to iterate through the prices, and check the betType key to see if it's either FixedWin or FixedPlace, that way the ordering of the different prices doesn't matter.
Also since the race_key and the race event are always the same you should define them at the top of the function so your implementation doesn't need to keep checking the dictionary for them.
def race_data():
for race_key in races_list():
data = requests.get(f'{Base_url}events/{race_key}.json?app_id={AppID}&app_key={AppKey}').text
# parse race data
data = json.loads(data)
race_key = data['eventKey']
race_time = data['eventDateTimeUtc']
for race_odds in data:
horse_name = race_odds['name']
jockey_name = race_odds['jockey']
jockey_weight = race_odds['weight']
trainer_name = race_odds['trainer']
for price in race_odds['prices']:
if price["betType"] == "FixedWin":
fixed_win_odds = price["price"]
elif price["betType"] == "FixedPlace":
fixed_place_odds = price["price"]
horse_status = race_odds['status']
print(f'{race_key} {horse_name} {jockey_name} {jockey_weight} {trainer_name} {fixed_win_odds} {fixed_place_odds} {race_time} {horse_status}')
if __name__ == "__main__":
race_data()
I have the code below which collects data from an API. I am then parsing and collecting the data points I need. I am having an issue where I am getting a KeyError but the key exists and all the data is returned, so if there is an error, why does it work?
any help would be appreciated.
url = f"{Base_url}/betoffer/event/{','.join(map(str, event_ids))}.json?app_id={AppID}&app_key={AppKey}&local={Country}&site={Site}&type=125"
print(url)
player_td_data = requests.get(url).text
player_td_data = json.loads(player_td_data)
for td_data in player_td_data['betOffers']:
if td_data['criterion']['label'] == 'First Touchdown Scorer - Including Overtime' and 'OFFERED_PREMATCH' in td_data['tags']:
for outcomes in td_data['outcomes']:
if outcomes['type'] == 'OT_PLAYER_PARTICIPANT':
event_id = td_data['eventId']
player_id = outcomes['id']
player_name = outcomes['participant']
player_first_td_odds = outcomes['odds']
print(f'{event_id} {player_id} {player_name} {player_first_td_odds}')
which produces the data i am required below
1018663065 3172232074 Wentz, Carson 23000
1018663065 3172232100 McKissic, JD 17000
1018663065 3172232109 Gibson, Antonio 13000
1018663065 3172232117 Robinson Jr., Brian 8500
1018663065 3172232126 Griffin, Ryan 51000
1018663065 3172232134 Pettis, Dante 31000
1018663065 3172232140 St. Brown, Equanimeous 21000
1018663065 3172232146 Kmet, Cole 16000
1018663065 3172232153 Harry, N’Keal 31000
1018663065 3172232159 Fields, Justin 9000
1018663065 3172232164 Montgomery, David 6750
1018663065 3172232170 Blasingame, Khari 76000
1018663065 3172232176 Wesco, Trevon 76000
1018663065 3172232181 Smith-Marsette, Ihmir 61000
1018663065 3172232186 Ebner, Trestan 56000
1018663065 3172232190 Mooney, Darnell 12000
1018663065 3172232194 Herbert, Khalil 16000
1018663065 3172232197 Tonges, Jake 91000
1018663065 3172232200 Jones Jr., Velus 34000
1018663065 3172232259 Samuel, Curtis 10500
1018663065 3172232265 Sims, Cam 41000
1018663065 3172232268 McLaurin, Terry 9000
but it also gives a KeyError below
Traceback (most recent call last):
File "C:\Users\xx\OneDrive\Desktop\xxx-functions\testing.py", line 87, in <module>
first_touchdowns_odds(event_ids)
File "C:\Users\xxx\OneDrive\Desktop\xxx-functions\testing.py", line 77, in first_touchdowns_odds
player_first_td_odds = outcomes['odds']
KeyError: 'odds'
this is the json
{
"betOffers": [{
"eventId": 1018663065,
"criterion": {
"englishLabel": "First Touchdown Scorer - Including Overtime",
"id": 1001488021,
"label": "First Touchdown Scorer - Including Overtime",
"order": [
1
]
},
"tags": [
"OFFERED_PREMATCH",
"BET_BUILDER"
],
"outcomes": [{
"oddsFractional": "22/1",
"englishLabel": "Wentz, Carson",
"label": "Wentz, Carson",
"betOfferId": 2349921831,
"oddsAmerican": "2200",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Wentz, Carson",
"participantId": 1003128420,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 23000,
"cashOutStatus": "ENABLED",
"id": 3172232074,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "50/1",
"englishLabel": "No Touchdown",
"label": "No Touchdown",
"betOfferId": 2349921831,
"oddsAmerican": "5000",
"type": "OT_NO_GOAL",
"changedDate": "2022-10-12T21:37:28Z",
"odds": 51000,
"cashOutStatus": "ENABLED",
"id": 3172232086,
"status": "OPEN"
},
{
"oddsFractional": "16/1",
"englishLabel": "McKissic, JD",
"label": "McKissic, JD",
"betOfferId": 2349921831,
"oddsAmerican": "1600",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "McKissic, JD",
"participantId": 1003405978,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 17000,
"cashOutStatus": "ENABLED",
"id": 3172232100,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "12/1",
"englishLabel": "Gibson, Antonio",
"label": "Gibson, Antonio",
"betOfferId": 2349921831,
"oddsAmerican": "1200",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Gibson, Antonio",
"participantId": 1006035581,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 13000,
"cashOutStatus": "ENABLED",
"id": 3172232109,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "15/2",
"englishLabel": "Robinson Jr., Brian",
"label": "Robinson Jr., Brian",
"betOfferId": 2349921831,
"oddsAmerican": "750",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Robinson Jr., Brian",
"participantId": 1007667637,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 8500,
"cashOutStatus": "ENABLED",
"id": 3172232117,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "50/1",
"englishLabel": "Griffin, Ryan",
"label": "Griffin, Ryan",
"betOfferId": 2349921831,
"oddsAmerican": "5000",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Griffin, Ryan",
"participantId": 1002198957,
"changedDate": "2022-10-13T01:18:07Z",
"odds": 51000,
"cashOutStatus": "ENABLED",
"id": 3172232126,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "30/1",
"englishLabel": "Pettis, Dante",
"label": "Pettis, Dante",
"betOfferId": 2349921831,
"oddsAmerican": "3000",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Pettis, Dante",
"participantId": 1003741140,
"changedDate": "2022-10-13T01:18:07Z",
"odds": 31000,
"cashOutStatus": "ENABLED",
"id": 3172232134,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "20/1",
"englishLabel": "St. Brown, Equanimeous",
"label": "St. Brown, Equanimeous",
"betOfferId": 2349921831,
"oddsAmerican": "2000",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "St. Brown, Equanimeous",
"participantId": 1004561624,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 21000,
"cashOutStatus": "ENABLED",
"id": 3172232140,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "15/1",
"englishLabel": "Kmet, Cole",
"label": "Kmet, Cole",
"betOfferId": 2349921831,
"oddsAmerican": "1500",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Kmet, Cole",
"participantId": 1005209906,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 16000,
"cashOutStatus": "ENABLED",
"id": 3172232146,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "30/1",
"englishLabel": "Harry, N’Keal",
"label": "Harry, N’Keal",
"betOfferId": 2349921831,
"oddsAmerican": "3000",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Harry, N’Keal",
"participantId": 1005252420,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 31000,
"cashOutStatus": "ENABLED",
"id": 3172232153,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "8/1",
"englishLabel": "Fields, Justin",
"label": "Fields, Justin",
"betOfferId": 2349921831,
"oddsAmerican": "800",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Fields, Justin",
"participantId": 1005289956,
"changedDate": "2022-10-12T21:37:30Z",
"odds": 9000,
"cashOutStatus": "ENABLED",
"id": 3172232159,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "23/4",
"englishLabel": "Montgomery, David",
"label": "Montgomery, David",
"betOfferId": 2349921831,
"oddsAmerican": "575",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Montgomery, David",
"participantId": 1005330489,
"changedDate": "2022-10-13T01:18:07Z",
"odds": 6750,
"cashOutStatus": "ENABLED",
"id": 3172232164,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "75/1",
"englishLabel": "Blasingame, Khari",
"label": "Blasingame, Khari",
"betOfferId": 2349921831,
"oddsAmerican": "7500",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Blasingame, Khari",
"participantId": 1005619153,
"changedDate": "2022-10-12T21:37:30Z",
"odds": 76000,
"cashOutStatus": "ENABLED",
"id": 3172232170,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "75/1",
"englishLabel": "Wesco, Trevon",
"label": "Wesco, Trevon",
"betOfferId": 2349921831,
"oddsAmerican": "7500",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Wesco, Trevon",
"participantId": 1005620092,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 76000,
"cashOutStatus": "ENABLED",
"id": 3172232176,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "60/1",
"englishLabel": "Smith-Marsette, Ihmir",
"label": "Smith-Marsette, Ihmir",
"betOfferId": 2349921831,
"oddsAmerican": "6000",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Smith-Marsette, Ihmir",
"participantId": 1005740936,
"changedDate": "2022-10-13T01:18:07Z",
"odds": 61000,
"cashOutStatus": "ENABLED",
"id": 3172232181,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "55/1",
"englishLabel": "Ebner, Trestan",
"label": "Ebner, Trestan",
"betOfferId": 2349921831,
"oddsAmerican": "5500",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Ebner, Trestan",
"participantId": 1005999128,
"changedDate": "2022-10-12T21:37:30Z",
"odds": 56000,
"cashOutStatus": "ENABLED",
"id": 3172232186,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "11/1",
"englishLabel": "Mooney, Darnell",
"label": "Mooney, Darnell",
"betOfferId": 2349921831,
"oddsAmerican": "1100",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Mooney, Darnell",
"participantId": 1006797242,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 12000,
"cashOutStatus": "ENABLED",
"id": 3172232190,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "15/1",
"englishLabel": "Herbert, Khalil",
"label": "Herbert, Khalil",
"betOfferId": 2349921831,
"oddsAmerican": "1500",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Herbert, Khalil",
"participantId": 1007051724,
"changedDate": "2022-10-12T21:37:30Z",
"odds": 16000,
"cashOutStatus": "ENABLED",
"id": 3172232194,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "90/1",
"englishLabel": "Tonges, Jake",
"label": "Tonges, Jake",
"betOfferId": 2349921831,
"oddsAmerican": "9000",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Tonges, Jake",
"participantId": 1007678491,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 91000,
"cashOutStatus": "ENABLED",
"id": 3172232197,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "34/1",
"englishLabel": "Jones Jr., Velus",
"label": "Jones Jr., Velus",
"betOfferId": 2349921831,
"oddsAmerican": "3400",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Jones Jr., Velus",
"participantId": 1007905969,
"changedDate": "2022-10-13T01:18:07Z",
"odds": 35000,
"cashOutStatus": "ENABLED",
"id": 3172232200,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "17/1",
"englishLabel": "Any other Chicago Bears player",
"label": "Any other Chicago Bears player",
"betOfferId": 2349921831,
"oddsAmerican": "1700",
"type": "OT_ANY_OTHER_PARTICIPANT",
"participant": "Chicago Bears",
"participantId": 1000000178,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 18000,
"cashOutStatus": "ENABLED",
"id": 3172232205,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "19/2",
"englishLabel": "Samuel, Curtis",
"label": "Samuel, Curtis",
"betOfferId": 2349921831,
"oddsAmerican": "950",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Samuel, Curtis",
"participantId": 1002450777,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 10500,
"cashOutStatus": "ENABLED",
"id": 3172232259,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "40/1",
"englishLabel": "Sims, Cam",
"label": "Sims, Cam",
"betOfferId": 2349921831,
"oddsAmerican": "4000",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Sims, Cam",
"participantId": 1003050339,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 41000,
"cashOutStatus": "ENABLED",
"id": 3172232265,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "8/1",
"englishLabel": "McLaurin, Terry",
"label": "McLaurin, Terry",
"betOfferId": 2349921831,
"oddsAmerican": "800",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "McLaurin, Terry",
"participantId": 1003741186,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 9000,
"cashOutStatus": "ENABLED",
"id": 3172232268,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"englishLabel": "Thomas, Logan",
"label": "Thomas, Logan",
"betOfferId": 2349921831,
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Thomas, Logan",
"participantId": 1004231988,
"changedDate": "2022-10-12T21:09:17Z",
"cashOutStatus": "ENABLED",
"id": 3172232271,
"status": "SUSPENDED",
"eventParticipantId": 1000000334
},
{
"englishLabel": "Dotson, Jahan",
"label": "Dotson, Jahan",
"betOfferId": 2349921831,
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Dotson, Jahan",
"participantId": 1005740692,
"changedDate": "2022-10-12T21:09:17Z",
"cashOutStatus": "ENABLED",
"id": 3172232274,
"status": "SUSPENDED",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "80/1",
"englishLabel": "Milne, Dax",
"label": "Milne, Dax",
"betOfferId": 2349921831,
"oddsAmerican": "8000",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Milne, Dax",
"participantId": 1007029029,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 81000,
"cashOutStatus": "ENABLED",
"id": 3172232277,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "19/1",
"englishLabel": "Brown, Dyami",
"label": "Brown, Dyami",
"betOfferId": 2349921831,
"oddsAmerican": "1900",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Brown, Dyami",
"participantId": 1007094475,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 20000,
"cashOutStatus": "ENABLED",
"id": 3172232279,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "45/1",
"englishLabel": "Turner, Cole",
"label": "Turner, Cole",
"betOfferId": 2349921831,
"oddsAmerican": "4500",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Turner, Cole",
"participantId": 1007180703,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 46000,
"cashOutStatus": "ENABLED",
"id": 3172232281,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "66/1",
"englishLabel": "Rogers, Armani",
"label": "Rogers, Armani",
"betOfferId": 2349921831,
"oddsAmerican": "6600",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Rogers, Armani",
"participantId": 1007800362,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 67000,
"cashOutStatus": "ENABLED",
"id": 3172232283,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "18/1",
"englishLabel": "Bates, John",
"label": "Bates, John",
"betOfferId": 2349921831,
"oddsAmerican": "1800",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Bates, John",
"participantId": 1007957637,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 19000,
"cashOutStatus": "ENABLED",
"id": 3172232285,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "16/1",
"englishLabel": "Any other Washington Commanders player",
"label": "Any other Washington Commanders player",
"betOfferId": 2349921831,
"oddsAmerican": "1600",
"type": "OT_ANY_OTHER_PARTICIPANT",
"participant": "Washington Commanders",
"participantId": 1000000334,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 17000,
"cashOutStatus": "ENABLED",
"id": 3172232289,
"status": "OPEN",
"eventParticipantId": 1000000334
}
],
"cashOutStatus": "ENABLED",
"closed": "2022-10-14T00:15:00Z",
"betOfferType": {
"englishName": "Player Occurrence Number",
"name": "Player Occurrence Number",
"id": 125
},
"id": 2349921831
}],
"range": {
"total": 1,
"size": 1,
"start": 0
},
"events": [{
"englishName": "Chicago Bears - Washington Commanders",
"groupId": 1000093656,
"homeName": "Chicago Bears",
"groupSortOrder": 2876820019684212700,
"prematchEnd": "2022-10-14T00:15:00Z",
"path": [{
"englishName": "American Football",
"termKey": "american_football",
"name": "American Football",
"id": 1000093199
},
{
"englishName": "NFL",
"termKey": "nfl",
"name": "NFL",
"id": 1000093656
}
],
"id": 1018663065,
"state": "NOT_STARTED",
"group": "NFL",
"start": "2022-10-14T00:15:00Z",
"nonLiveBoCount": 254,
"tags": [
"OFFERED_LIVE",
"BET_BUILDER",
"MATCH",
"LIVE_OCCURRENCE_FEED"
],
"awayName": "Washington Commanders",
"name": "Chicago Bears - Washington Commanders",
"sport": "AMERICAN_FOOTBALL"
}]
}
One small modification to your code allows you to see which record is causing the error.
for td_data in player_td_data['betOffers']:
if td_data['criterion']['label'] == 'First Touchdown Scorer - Including Overtime' and 'OFFERED_PREMATCH' in td_data['tags']:
for outcomes in td_data['outcomes']:
if outcomes['type'] == 'OT_PLAYER_PARTICIPANT':
event_id = td_data['eventId']
player_id = outcomes['id']
player_name = outcomes['participant']
try:
player_first_td_odds = outcomes['odds']
except KeyError:
print(f'KeyError on {player_name}')
break
print(f'{event_id} {player_id} {player_name} {player_first_td_odds}')
You can see that indeed the JSON record for Thomas, Logan is missing the odds field.
I'm having a problem where i can scrape data from a website by using the java pathing. I'm trying to scrape from Rocket League Tracker.
here's my code:
import requests
import re
import json
import math
def rankGetter():
trackerLink = 'https://rocketleague.tracker.network/rocket-league/profile/epic/DirectPanda/overview'
# now we have the tracker link we're going to scrape the website
# all the HTML of the site is now in result
result = requests.get(trackerLink)
# checker to make sure the user used the correct information
if result.status_code == 400:
print('profile not found')
else:
# Extract everything needed to render the current page. Data is stored as Json in the
# JavaScript variable: window.__INITIAL_STATE__={"route":{"path":"\u0 ... }};
json_string = re.search(r"window.__INITIAL_STATE__\s?=\s?(\{.*?\});", result.text).group(1)
# convert text string to structured json data
rocketleague = json.loads(json_string)
# Save structured json data to a text file that helps you orient yourself and pick
# the parts you are interested in.
with open('rocketleague_json_data.txt', 'w') as outfile:
outfile.write(json.dumps(rocketleague, indent=4, sort_keys=True))
The error is the text doc made doesn't have the ranks I want anymore.
"stats": {
"standardLeaderboardLeaders": {},
"standardLeaderboards": [],
"standardPlayers": {},
"standardTitles": {}
},
**"stats-v2": {
"segments": {},
"standardProfileMatches": {},
"standardProfileSummaries": {},
"standardProfiles": {},
"standardProfilesHistory": {},
"standardSessions": {},
"subscriptions": {}
},**
"titles": {
"currentTitle": {
"name": "Rocket League",
"platforms": [
The Ranks should be under stats-V2 but as you can see its empty now.
whats happening and how do i fix it? I was able to get ranks for a week but all the sudden it stopped working today.
Seems that the data are loaded from external URL:
import json
import requests
url = "https://api.tracker.gg/api/v2/rocket-league/standard/profile/epic/DirectPanda"
headers = {
"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0"
}
data = requests.get(url, headers=headers).json()
print(json.dumps(data, indent=4))
Prints:
{
"data": {
"platformInfo": {
"platformSlug": "epic",
"platformUserId": null,
"platformUserHandle": "DirectPanda",
"platformUserIdentifier": "DirectPanda",
"avatarUrl": null,
"additionalParameters": null
},
"userInfo": {
"userId": null,
"isPremium": false,
"isVerified": false,
"isInfluencer": false,
"isPartner": false,
"countryCode": null,
"customAvatarUrl": null,
"customHeroUrl": null,
"socialAccounts": [],
"pageviews": 592,
"isSuspicious": null
},
"metadata": {
"lastUpdated": {
"value": "2021-04-22T17:39:42.277-04:00",
"displayValue": "2021-04-22T21:39:42.2770000+00:00"
},
"playerId": 16603481,
"currentSeason": 17
},
"segments": [
{
"type": "overview",
"attributes": {},
"metadata": {
"name": "Lifetime"
},
"expiryDate": "0001-01-01T00:00:00+00:00",
"stats": {
"wins": {
"rank": 30357,
"percentile": 98.3,
"displayName": "Wins",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 4985,
"displayValue": "4,985",
"displayType": "Number"
},
"goals": {
"rank": 23698,
"percentile": 98.7,
"displayName": "Goals",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 14363,
"displayValue": "14,363",
"displayType": "Number"
},
"mVPs": {
"rank": 35646,
"percentile": 98.0,
"displayName": "MVPs",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 2093,
"displayValue": "2,093",
"displayType": "Number"
},
"saves": {
"rank": 30864,
"percentile": 98.3,
"displayName": "Saves",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 9231,
"displayValue": "9,231",
"displayType": "Number"
},
"assists": {
"rank": 29228,
"percentile": 98.4,
"displayName": "Assists",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 4763,
"displayValue": "4,763",
"displayType": "Number"
},
"shots": {
"rank": 24596,
"percentile": 98.6,
"displayName": "Shots",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 29139,
"displayValue": "29,139",
"displayType": "Number"
},
"goalShotRatio": {
"rank": 1409320,
"percentile": 15.0,
"displayName": "Goal Shot Ratio",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 49.29132777377398,
"displayValue": "49.3",
"displayType": "NumberPrecision1"
},
"score": {
"rank": 28260,
"percentile": 98.4,
"displayName": "TRN Score",
"displayCategory": "General",
"category": "general",
"metadata": {},
"value": 2398222.83,
"displayValue": "2,398,222.8",
"displayType": "NumberPrecision1"
},
"seasonRewardLevel": {
"rank": null,
"percentile": 85.0,
"displayName": "Season Reward Level",
"displayCategory": "General",
"category": "general",
"metadata": {
"iconUrl": "https://trackercdn.com/cdn/tracker.gg/rocket-league/ranks/s4-13.png",
"rankName": "Diamond"
},
"value": 5,
"displayValue": "5",
"displayType": "Number"
},
"seasonRewardWins": {
"rank": null,
"percentile": 95.8,
"displayName": "Season Reward Wins",
"displayCategory": "General",
"category": "general",
"metadata": {},
"value": 9,
"displayValue": "9",
"displayType": "Number"
}
}
},
{
"type": "playlist",
"attributes": {
"playlistId": 0,
"season": 17
},
"metadata": {
"name": "Un-Ranked"
},
"expiryDate": "0001-01-01T00:00:00+00:00",
"stats": {
"tier": {
"rank": null,
"percentile": null,
"displayName": "Matches",
"displayCategory": "General",
"category": "general",
"metadata": {
"iconUrl": "https://trackercdn.com/cdn/tracker.gg/rocket-league/ranks/s4-0.png",
"name": "Unranked"
},
"value": 0,
"displayValue": "0",
"displayType": "Number"
},
"division": {
"rank": null,
"percentile": null,
"displayName": "Matches",
"displayCategory": "General",
"category": "general",
"metadata": {
"name": "Division I"
},
"value": 0,
"displayValue": "0",
"displayType": "Number"
},
"matchesPlayed": {
"rank": null,
"percentile": null,
"displayName": "Matches",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 0,
"displayValue": "0",
"displayType": "Number"
},
"winStreak": {
"rank": null,
"percentile": null,
"displayName": "WinStreak",
"displayCategory": "Performance",
"category": "performance",
"metadata": {
"type": "win"
},
"value": 0,
"displayValue": "0",
"displayType": "Number"
},
"rating": {
"rank": 215152,
"percentile": 90.0,
"displayName": "Rating",
"displayCategory": "Skill",
"category": "skill",
"metadata": {},
"value": 1597,
"displayValue": "1,597",
"displayType": "Number"
}
}
},
{
"type": "playlist",
"attributes": {
"playlistId": 10,
"season": 17
},
"metadata": {
"name": "Ranked Duel 1v1"
},
"expiryDate": "0001-01-01T00:00:00+00:00",
"stats": {
"tier": {
"rank": null,
"percentile": 98.2,
"displayName": "Matches",
"displayCategory": "General",
"category": "general",
"metadata": {
"iconUrl": "https://trackercdn.com/cdn/tracker.gg/rocket-league/ranks/s4-16.png",
"name": "Champion I"
},
"value": 16,
"displayValue": "16",
"displayType": "Number"
},
"division": {
"rank": null,
"percentile": 88.0,
"displayName": "Matches",
"displayCategory": "General",
"category": "general",
"metadata": {
"deltaDown": 13,
"deltaUp": 6,
"name": "Division III"
},
"value": 2,
"displayValue": "2",
"displayType": "Number"
},
"matchesPlayed": {
"rank": null,
"percentile": 57.0,
"displayName": "Matches",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 2,
"displayValue": "2",
"displayType": "Number"
},
"winStreak": {
"rank": null,
"percentile": 60.0,
"displayName": "WinStreak",
"displayCategory": "Performance",
"category": "performance",
"metadata": {
"type": "win"
},
"value": 1,
"displayValue": "1",
"displayType": "Number"
},
"rating": {
"rank": 101541,
"percentile": 96.1,
"displayName": "Rating",
"displayCategory": "Skill",
"category": "skill",
"metadata": {},
"value": 1031,
"displayValue": "1,031",
"displayType": "Number"
}
}
},
{
"type": "playlist",
"attributes": {
"playlistId": 11,
"season": 17
},
"metadata": {
"name": "Ranked Doubles 2v2"
},
"expiryDate": "0001-01-01T00:00:00+00:00",
"stats": {
"tier": {
"rank": null,
"percentile": 87.0,
"displayName": "Matches",
"displayCategory": "General",
"category": "general",
"metadata": {
"iconUrl": "https://trackercdn.com/cdn/tracker.gg/rocket-league/ranks/s4-16.png",
"name": "Champion I"
},
"value": 16,
"displayValue": "16",
"displayType": "Number"
},
"division": {
"rank": null,
"percentile": 90.0,
"displayName": "Matches",
"displayCategory": "General",
"category": "general",
"metadata": {
"deltaDown": 15,
"deltaUp": 3,
"name": "Division IV"
},
"value": 3,
"displayValue": "3",
"displayType": "Number"
},
"matchesPlayed": {
"rank": null,
"percentile": 80.0,
"displayName": "Matches",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 40,
"displayValue": "40",
"displayType": "Number"
},
"winStreak": {
"rank": null,
"percentile": 34.0,
"displayName": "WinStreak",
"displayCategory": "Performance",
"category": "performance",
"metadata": {
"type": "loss"
},
"value": 1,
"displayValue": "-1",
"displayType": "Number"
},
"rating": {
"rank": 311789,
"percentile": 89.0,
"displayName": "Rating",
"displayCategory": "Skill",
"category": "skill",
"metadata": {},
"value": 1177,
"displayValue": "1,177",
"displayType": "Number"
}
}
},
{
"type": "playlist",
"attributes": {
"playlistId": 13,
"season": 17
},
"metadata": {
"name": "Ranked Standard 3v3"
},
"expiryDate": "0001-01-01T00:00:00+00:00",
"stats": {
"tier": {
"rank": null,
"percentile": 96.0,
"displayName": "Matches",
"displayCategory": "General",
"category": "general",
"metadata": {
"iconUrl": "https://trackercdn.com/cdn/tracker.gg/rocket-league/ranks/s4-17.png",
"name": "Champion II"
},
"value": 17,
"displayValue": "17",
"displayType": "Number"
},
"division": {
"rank": null,
"percentile": 79.0,
"displayName": "Matches",
"displayCategory": "General",
"category": "general",
"metadata": {
"deltaDown": 7,
"deltaUp": 27,
"name": "Division III"
},
"value": 2,
"displayValue": "2",
"displayType": "Number"
},
"matchesPlayed": {
"rank": null,
"percentile": 97.8,
"displayName": "Matches",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 95,
"displayValue": "95",
"displayType": "Number"
},
"winStreak": {
"rank": null,
"percentile": 16.0,
"displayName": "WinStreak",
"displayCategory": "Performance",
"category": "performance",
"metadata": {
"type": "loss"
},
"value": 2,
"displayValue": "-2",
"displayType": "Number"
},
"rating": {
"rank": 122500,
"percentile": 95.8,
"displayName": "Rating",
"displayCategory": "Skill",
"category": "skill",
"metadata": {},
"value": 1255,
"displayValue": "1,255",
"displayType": "Number"
}
}
},
...
I am trying to filter through an API (JSON format). I am pretty new to this and have no idea where to start. For simplicity's sake, I will only give part of the API output I would technically get. I am using Python 3, specifically Python 3.9.2.
{
"success": true,
"currency": "CAD",
"timestamp": 1615516661,
"items_list": {
"AK-47 | Aquamarine Revenge (Minimal Wear)": {
"name": "AK-47 | Aquamarine Revenge (Minimal Wear)",
"marketable": 1,
"tradable": 1,
"classid": "1819967861",
"icon_url": "-9a81dlWLwJ2UUGcVs_nsVtzdOEdtWwKGZZLQHTxDZ7I56KU0Zwwo4NUX4oFJZEHLbXH5ApeO4YmlhxYQknCRvCo04DEVlxkKgpot7HxfDhjxszJemkV09-5gZKKkPLLMrfFqWdY781lxLuW8Njw31Dn8xc_YTqmJ4DDJFM2ZwqE_ATtx-u7g8C5vpjOzHM263E8pSGKJ1XuG9M",
"icon_url_large": "-9a81dlWLwJ2UUGcVs_nsVtzdOEdtWwKGZZLQHTxDZ7I56KU0Zwwo4NUX4oFJZEHLbXH5ApeO4YmlhxYQknCRvCo04DEVlxkKgpot7HxfDhjxszJemkV09-5gZKKkPLLMrfFqWdY781lteXA54vwxgbjrkc5ZD3wLNLGcFVrYl6GrAS2x7y7g8PvupidzyRmuCUl4H6IzkSpwUYbC6zHyEM",
"type": "Weapon",
"weapon_type": "Rifle",
"gun_type": "AK-47",
"exterior": "Minimal Wear",
"rarity": "Covert",
"rarity_color": "eb4b4b",
"price": {
"24_hours": {
"average": 65.24,
"median": 64.41,
"sold": "46",
"standard_deviation": "4.43",
"lowest_price": 61.49,
"highest_price": 75.76
},
"7_days": {
"average": 64.7,
"median": 64.49,
"sold": "240",
"standard_deviation": "4.32",
"lowest_price": 53.83,
"highest_price": 75.76
},
"30_days": {
"average": 59.39,
"median": 59.3,
"sold": "982",
"standard_deviation": "8.84",
"lowest_price": 0.18,
"highest_price": 75.76
},
"all_time": {
"average": 38.1,
"median": 40.48,
"sold": "169635",
"standard_deviation": "38.77",
"lowest_price": 0.18,
"highest_price": 125.91
}
},
"first_sale_date": "1432764000"
},
"AK-47 | Aquamarine Revenge (Well-Worn)": {
"name": "AK-47 | Aquamarine Revenge (Well-Worn)",
"marketable": 1,
"tradable": 1,
"classid": "2095030666",
"icon_url": "-9a81dlWLwJ2UUGcVs_nsVtzdOEdtWwKGZZLQHTxDZ7I56KU0Zwwo4NUX4oFJZEHLbXH5ApeO4YmlhxYQknCRvCo04DEVlxkKgpot7HxfDhjxszJemkV09-5gZKKkPLLMrfFqWZU7Mxkh6eU896n0FXk-RJsNzv3cI-WJAA3YFDTqFa2l-u6jJW4uJqdyCBluyEm-z-DyCua9lLK",
"icon_url_large": "-9a81dlWLwJ2UUGcVs_nsVtzdOEdtWwKGZZLQHTxDZ7I56KU0Zwwo4NUX4oFJZEHLbXH5ApeO4YmlhxYQknCRvCo04DEVlxkKgpot7HxfDhjxszJemkV09-5gZKKkPLLMrfFqWZU7Mxkh9bN9J7yjRrhrUFuazjzJteVJlQ6NVHTrFe3wObs15G06picwHFnvid25C3bnhSzn1gSOQz0szG-",
"type": "Weapon",
"weapon_type": "Rifle",
"gun_type": "AK-47",
"exterior": "Well-Worn",
"rarity": "Covert",
"rarity_color": "eb4b4b",
"price": {
"24_hours": {
"average": 34.31,
"median": 34.14,
"sold": "40",
"standard_deviation": "3.67",
"lowest_price": 31.93,
"highest_price": 37.23
},
"7_days": {
"average": 35.72,
"median": 35.8,
"sold": "382",
"standard_deviation": "5.07",
"lowest_price": 29.7,
"highest_price": 38.93
},
"30_days": {
"average": 33.14,
"median": 33.11,
"sold": "1632",
"standard_deviation": "7",
"lowest_price": 26.77,
"highest_price": 38.93
},
"all_time": {
"average": 21.89,
"median": 23.76,
"sold": "212874",
"standard_deviation": "35.16",
"lowest_price": 11.78,
"highest_price": 71.85
}
},
"first_sale_date": "1432764000"
}
}
}
The output I would like is something like the following:
{
"success": true,
"currency": "CAD",
"timestamp": 1615516661,
"items_list": {
"AK-47 | Aquamarine Revenge (Minimal Wear)": {
"name": "AK-47 | Aquamarine Revenge (Minimal Wear)",
"type": "Weapon",
"exterior": "Minimal Wear",
"rarity": "Covert",
"price": {
"24_hours": {
"average": 65.24,
"median": 64.41,
"sold": "46",
"standard_deviation": "4.43",
"lowest_price": 61.49,
"highest_price": 75.76
},
"7_days": {
"average": 64.7,
"median": 64.49,
"sold": "240",
"standard_deviation": "4.32",
"lowest_price": 53.83,
"highest_price": 75.76
},
"30_days": {
"average": 59.39,
"median": 59.3,
"sold": "982",
"standard_deviation": "8.84",
"lowest_price": 0.18,
"highest_price": 75.76
},
"all_time": {
"average": 38.1,
"median": 40.48,
"sold": "169635",
"standard_deviation": "38.77",
"lowest_price": 0.18,
"highest_price": 125.91
}
},
},
"AK-47 | Aquamarine Revenge (Well-Worn)": {
"name": "AK-47 | Aquamarine Revenge (Well-Worn)",
"type": "Weapon",
"exterior": "Well-Worn",
"rarity": "Covert",
"price": {
"24_hours": {
"average": 34.31,
"median": 34.14,
"sold": "40",
"standard_deviation": "3.67",
"lowest_price": 31.93,
"highest_price": 37.23
},
"7_days": {
"average": 35.72,
"median": 35.8,
"sold": "382",
"standard_deviation": "5.07",
"lowest_price": 29.7,
"highest_price": 38.93
},
"30_days": {
"average": 33.14,
"median": 33.11,
"sold": "1632",
"standard_deviation": "7",
"lowest_price": 26.77,
"highest_price": 38.93
},
"all_time": {
"average": 21.89,
"median": 23.76,
"sold": "212874",
"standard_deviation": "35.16",
"lowest_price": 11.78,
"highest_price": 71.85
}
},
}
}
}
As you can see, I am removing some of the contents. How can I accomplish this? Any pointers would be appreciated. Thanks in advance!
Let's say your file with json data is named api_data.json.
import json
# convert json data to a python dict
with open('api_data.json') as f:
data = json.load(f)
# data is a python dict
# delete unwanted keys and associated data
del data['items_list']['AK-47 | Aquamarine Revenge (Minimal Wear)']['marketable']
del data['items_list']['AK-47 | Aquamarine Revenge (Minimal Wear)']['tradable']
# Now, save back as json file
with open('api_data.json') as f:
json.dump(data, f)
{
"took": 0,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 3,
"max_score": 0.2876821,
"hits": [
{
"_index": "product_index",
"_type": "product",
"_id": "1115",
"_score": 0.2876821,
"_source": {
"isactive": true,
"in_use": false,
"brand_name": "Adidas",
"sku_id": "56456487987987",
"long_description": "this is long description",
"key_feature": [
{
"id": 1148,
"key_feature": "sport wear"
},
{
"id": 1147,
"key_feature": "Cotton shirt"
},
{
"id": 1146,
"key_feature": "White and blue"
}
],
"isdeleted": false,
"created_by": null,
"brand_id": 5,
"search_terms": [
{
"label": "white shirt",
"value": 9
}
]
"color_id": 2,
"specific_keywords": "",
"item_list": [
{
"item_id": 1114,
"product_id": 1115,
"isactive": true,
"id": 9,
"isdeleted": false
},
{
"item_id": 1113,
"product_id": 1115,
"isactive": true,
"id": 10,
"isdeleted": false
}
],
"upc_code": "",
"display_size": "L",
"name": "New White shirt",
"updated_by": null,
"id": 1115,
"updated_date": "2020-03-25T08:24:37.644571+00:00",
"color_name": "blue",
"created_date": "2020-03-25T08:11:14.966673+00:00",
"category": [
{
"parent_category_id": 78,
"sub_sub_category": null,
"sub_category": null,
"sub_category_id": null,
"sub_sub_category_id": null,
"parent_category": "new Shirt Cate",
"id": 1151
}
]
}
},
{
"_index": "product_index",
"_type": "product",
"_id": "1113",
"_score": 0.2876821,
"_source": {
"isactive": true,
"in_use": false,
"sku_id": "1456456488",
"brand_name": "Adidas",
"long_description": "",
"key_feature": [
{
"id": 1142,
"key_feature": "Cotton"
},
{
"id": 1141,
"key_feature": "Office Use"
},
{
"id": 1140,
"key_feature": "Black formal"
}
],
"isdeleted": false,
"created_by": null,
"brand_id": 5,
"search_terms": [
]
"color_id": 1,
"specific_keywords": "",
"item_list": [
],
"display_size": "L",
"upc_code": "",
"name": "New Cotton formal shirt black",
"updated_by": null,
"id": 1113,
"updated_date": "2020-03-25T06:48:30.903041+00:00",
"created_date": "2020-03-25T06:48:29.943043+00:00",
"color_name": "black",
"category": [
{
"sub_sub_category": null,
"parent_category_id": 54,
"sub_category": null,
"sub_category_id": null,
"sub_sub_category_id": null,
"parent_category": "MEN'S CLOTHING",
"id": 1149
}
]
}
},
{
"_index": "product_index",
"_type": "product",
"_id": "1114",
"_score": 0.2876821,
"_source": {
"isactive": true,
"in_use": false,
"sku_id": "145645648811",
"brand_name": "Adidas",
"long_description": "",
"key_feature": [
{
"id": 1145,
"key_feature": "Cotton"
},
{
"id": 1144,
"key_feature": "Office Use"
},
{
"id": 1143,
"key_feature": "Black formal"
}
],
"isdeleted": false,
"created_by": null,
"brand_id": 5,
"search_terms": [
],
"color_id": 1,
"specific_keywords": "",
"item_list": [
],
"display_size": "L",
"upc_code": "",
"updated_by": null,
"name": "New Cotton Casual shirt black",
"id": 1114,
"created_date": "2020-03-25T07:13:26.233675+00:00",
"color_name": "black",
"updated_date": "2020-03-25T07:13:27.229363+00:00",
"category": [
{
"sub_sub_category": null,
"parent_category_id": 54,
"sub_category": null,
"sub_category_id": null,
"sub_sub_category_id": null,
"parent_category": "MEN'S CLOTHING",
"id": 1150
}
]
}
}
]
}
}
my requirement is to attach all related documents with specific key value fields which is specify in item_list based on item_id. In above result doc id 1115 has item_list which contains item_id 1114 and 1113. so the particular fields attach in the doc 1115.
what should be the search query for that in elastic search?
You can't do join in Elasticsearch, to achieve your goal, you can do two things:
duplicate the information of item_id 1114 and 1113 in the item_id
1115 (and for sure in all others documents).
Do join at application level, so after this query you can extract the item_id 1114 and 1113 and run two others query to get the information about this items. Then join all the json at application level.