We Keep Coding

Python - trying to convert time from utc to cst in api response

Below is code I am using to get data from an api. And below that is the response. I am trying to convert datetime from UTC to CST and then present the data with that time zone instead. But I am having trouble isolating datetime
import requests
import json
weather = requests.get('...')
j = json.loads(weather.text)
print (json.dumps(j, indent=2))
Response:
{
"metadata": null,
"data": [
{
"datetime": "2022-12-11T05:00:00Z",
"is_day_time": false,
"icon_code": 5,
"weather_text": "Clear with few low clouds and few cirrus",
"temperature": {
"value": 45.968,
"units": "F"
},
"feels_like_temperature": {
"value": 39.092,
"units": "F"
},
"relative_humidity": 56,
"precipitation": {
"precipitation_probability": 4,
"total_precipitation": {
"value": 0.0,
"units": "in"
}
},
"wind": {
"speed": {
"value": 5.144953471725125,
"units": "mi/h"
},
"direction": 25
},
"wind_gust": {
"value": 9.014853256979242,
"units": "mi/h"
},
"pressure": {
"value": 29.4171829577118,
"units": "inHg"
},
"visibility": {
"value": 6.835083114610673,
"units": "mi"
},
"dew_point": {
"value": 31.01,
"units": "F"
},
"cloud_cover": 31
},
{
"datetime": "2022-12-11T06:00:00Z",
"is_day_time": false,
"icon_code": 4,
"weather_text": "Clear with few low clouds",
"temperature": {
"value": 45.068,
"units": "F"
},
"feels_like_temperature": {
"value": 38.066,
"units": "F"
},
"relative_humidity": 56,
"precipitation": {
"precipitation_probability": 5,
"total_precipitation": {
"value": 0.0,
"units": "in"
}
},
"wind": {
"speed": {
"value": 5.167322834645669,
"units": "mi/h"
},
"direction": 27
},
"wind_gust": {
"value": 8.724051539012168,
"units": "mi/h"
},
"pressure": {
"value": 29.4213171559632,
"units": "inHg"
},
"visibility": {
"value": 5.592340730136005,
"units": "mi"
},
"dew_point": {
"value": 30.2,
"units": "F"
},
"cloud_cover": 13
},
{
"datetime": "2022-12-11T07:00:00Z",
"is_day_time": false,
"icon_code": 4,
"weather_text": "Clear with few low clouds",
"temperature": {
"value": 44.33,
"units": "F"
},
"feels_like_temperature": {
"value": 37.364,
"units": "F"
},
"relative_humidity": 56,
"precipitation": {
"precipitation_probability": 4,
"total_precipitation": {
"value": 0.0,
"units": "in"
}
},
"wind": {
"speed": {
"value": 4.988367931281317,
"units": "mi/h"
},
"direction": 28
},
"wind_gust": {
"value": 8.254294917680744,
"units": "mi/h"
},
"pressure": {
"value": 29.4165923579616,
"units": "inHg"
},
"visibility": {
"value": 7.456454306848007,
"units": "mi"
},
"dew_point": {
"value": 29.714,
"units": "F"
},
"cloud_cover": 22
}
],
"error": null

I am assuming what you mean is that you want to present the data in the current time of the Central Time zone. As of the date this question was asked, that would be CST (Central Standard Time). At another time it will be CDT (Central Daylight Time) based on daylight savings time rules that are followed in the Country/City for the time zone for which you wish to localize the data. The rules are all nicely kept in the IANA Timezone Database.
So the trick is that you pick your Country/City from the Timezone DB that follows the rules as they apply to your current time zone. For Central Time, America/Chicago usually works but YMMV.
There are a lot of ways to do this. This example is inefficiently iterating through the dictionary created by json.loads and replacing the time string with a converted string. The key is using the dateutil library to parse the timestamp string and convert using the proper UTC offset as defined for the time zone in the IANA database.
Hopefully this example has enough pieces you can copy and adapt to your own needs.
from dateutil.parser import parse
from dateutil import tz
import json
j = json.loads(weather)
# Loop through each data entry, reformatting the time
for entry in j["data"]:
if "datetime" in entry.keys():
parsed_dt = parse(entry["datetime"])
converted = parsed_dt.astimezone(tz.gettz("America/Chicago"))
entry["datetime"] = converted.isoformat()
print (json.dumps(j, indent=2))
The resulting JSON has datetime fields that contain an ISO timestamp for the CST time.
{
"metadata": null,
"data": [{
"datetime": "2022-12-10T23:00:00-06:00",
"is_day_time": false,
"icon_code": 5,
"weather_text": "Clear with few low clouds and few cirrus",
"temperature": {
"value": 45.968,
"units": "F"
},
"feels_like_temperature": {
"value": 39.092,
"units": "F"
},
"relative_humidity": 56,
"precipitation": {
"precipitation_probability": 4,
"total_precipitation": {
"value": 0.0,
"units": "in"
}
},
"wind": {
"speed": {
"value": 5.144953471725125,
"units": "mi/h"
},
"direction": 25
},
"wind_gust": {
"value": 9.014853256979242,
"units": "mi/h"
},
"pressure": {
"value": 29.4171829577118,
"units": "inHg"
},
"visibility": {
"value": 6.835083114610673,
"units": "mi"
},
"dew_point": {
"value": 31.01,
"units": "F"
},
"cloud_cover": 31
},
{
"datetime": "2022-12-11T00:00:00-06:00",
"is_day_time": false,
"icon_code": 4,
"weather_text": "Clear with few low clouds",
"temperature": {
"value": 45.068,
"units": "F"
},
"feels_like_temperature": {
"value": 38.066,
"units": "F"
},
"relative_humidity": 56,
"precipitation": {
"precipitation_probability": 5,
"total_precipitation": {
"value": 0.0,
"units": "in"
}
},
"wind": {
"speed": {
"value": 5.167322834645669,
"units": "mi/h"
},
"direction": 27
},
"wind_gust": {
"value": 8.724051539012168,
"units": "mi/h"
},
"pressure": {
"value": 29.4213171559632,
"units": "inHg"
},
"visibility": {
"value": 5.592340730136005,
"units": "mi"
},
"dew_point": {
"value": 30.2,
"units": "F"
},
"cloud_cover": 13
},
{
"datetime": "2022-12-11T01:00:00-06:00",
"is_day_time": false,
"icon_code": 4,
"weather_text": "Clear with few low clouds",
"temperature": {
"value": 44.33,
"units": "F"
},
"feels_like_temperature": {
"value": 37.364,
"units": "F"
},
"relative_humidity": 56,
"precipitation": {
"precipitation_probability": 4,
"total_precipitation": {
"value": 0.0,
"units": "in"
}
},
"wind": {
"speed": {
"value": 4.988367931281317,
"units": "mi/h"
},
"direction": 28
},
"wind_gust": {
"value": 8.254294917680744,
"units": "mi/h"
},
"pressure": {
"value": 29.4165923579616,
"units": "inHg"
},
"visibility": {
"value": 7.456454306848007,
"units": "mi"
},
"dew_point": {
"value": 29.714,
"units": "F"
},
"cloud_cover": 22
}
],
"error": null
}

KeyError when parsing json when key is there

I have the code below which collects data from an API. I am then parsing and collecting the data points I need. I am having an issue where I am getting a KeyError but the key exists and all the data is returned, so if there is an error, why does it work?
any help would be appreciated.
url = f"{Base_url}/betoffer/event/{','.join(map(str, event_ids))}.json?app_id={AppID}&app_key={AppKey}&local={Country}&site={Site}&type=125"
print(url)
player_td_data = requests.get(url).text
player_td_data = json.loads(player_td_data)
for td_data in player_td_data['betOffers']:
if td_data['criterion']['label'] == 'First Touchdown Scorer - Including Overtime' and 'OFFERED_PREMATCH' in td_data['tags']:
for outcomes in td_data['outcomes']:
if outcomes['type'] == 'OT_PLAYER_PARTICIPANT':
event_id = td_data['eventId']
player_id = outcomes['id']
player_name = outcomes['participant']
player_first_td_odds = outcomes['odds']
print(f'{event_id} {player_id} {player_name} {player_first_td_odds}')
which produces the data i am required below
1018663065 3172232074 Wentz, Carson 23000
1018663065 3172232100 McKissic, JD 17000
1018663065 3172232109 Gibson, Antonio 13000
1018663065 3172232117 Robinson Jr., Brian 8500
1018663065 3172232126 Griffin, Ryan 51000
1018663065 3172232134 Pettis, Dante 31000
1018663065 3172232140 St. Brown, Equanimeous 21000
1018663065 3172232146 Kmet, Cole 16000
1018663065 3172232153 Harry, N’Keal 31000
1018663065 3172232159 Fields, Justin 9000
1018663065 3172232164 Montgomery, David 6750
1018663065 3172232170 Blasingame, Khari 76000
1018663065 3172232176 Wesco, Trevon 76000
1018663065 3172232181 Smith-Marsette, Ihmir 61000
1018663065 3172232186 Ebner, Trestan 56000
1018663065 3172232190 Mooney, Darnell 12000
1018663065 3172232194 Herbert, Khalil 16000
1018663065 3172232197 Tonges, Jake 91000
1018663065 3172232200 Jones Jr., Velus 34000
1018663065 3172232259 Samuel, Curtis 10500
1018663065 3172232265 Sims, Cam 41000
1018663065 3172232268 McLaurin, Terry 9000
but it also gives a KeyError below
Traceback (most recent call last):
File "C:\Users\xx\OneDrive\Desktop\xxx-functions\testing.py", line 87, in <module>
first_touchdowns_odds(event_ids)
File "C:\Users\xxx\OneDrive\Desktop\xxx-functions\testing.py", line 77, in first_touchdowns_odds
player_first_td_odds = outcomes['odds']
KeyError: 'odds'
this is the json
{
"betOffers": [{
"eventId": 1018663065,
"criterion": {
"englishLabel": "First Touchdown Scorer - Including Overtime",
"id": 1001488021,
"label": "First Touchdown Scorer - Including Overtime",
"order": [
1
]
},
"tags": [
"OFFERED_PREMATCH",
"BET_BUILDER"
],
"outcomes": [{
"oddsFractional": "22/1",
"englishLabel": "Wentz, Carson",
"label": "Wentz, Carson",
"betOfferId": 2349921831,
"oddsAmerican": "2200",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Wentz, Carson",
"participantId": 1003128420,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 23000,
"cashOutStatus": "ENABLED",
"id": 3172232074,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "50/1",
"englishLabel": "No Touchdown",
"label": "No Touchdown",
"betOfferId": 2349921831,
"oddsAmerican": "5000",
"type": "OT_NO_GOAL",
"changedDate": "2022-10-12T21:37:28Z",
"odds": 51000,
"cashOutStatus": "ENABLED",
"id": 3172232086,
"status": "OPEN"
},
{
"oddsFractional": "16/1",
"englishLabel": "McKissic, JD",
"label": "McKissic, JD",
"betOfferId": 2349921831,
"oddsAmerican": "1600",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "McKissic, JD",
"participantId": 1003405978,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 17000,
"cashOutStatus": "ENABLED",
"id": 3172232100,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "12/1",
"englishLabel": "Gibson, Antonio",
"label": "Gibson, Antonio",
"betOfferId": 2349921831,
"oddsAmerican": "1200",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Gibson, Antonio",
"participantId": 1006035581,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 13000,
"cashOutStatus": "ENABLED",
"id": 3172232109,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "15/2",
"englishLabel": "Robinson Jr., Brian",
"label": "Robinson Jr., Brian",
"betOfferId": 2349921831,
"oddsAmerican": "750",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Robinson Jr., Brian",
"participantId": 1007667637,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 8500,
"cashOutStatus": "ENABLED",
"id": 3172232117,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "50/1",
"englishLabel": "Griffin, Ryan",
"label": "Griffin, Ryan",
"betOfferId": 2349921831,
"oddsAmerican": "5000",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Griffin, Ryan",
"participantId": 1002198957,
"changedDate": "2022-10-13T01:18:07Z",
"odds": 51000,
"cashOutStatus": "ENABLED",
"id": 3172232126,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "30/1",
"englishLabel": "Pettis, Dante",
"label": "Pettis, Dante",
"betOfferId": 2349921831,
"oddsAmerican": "3000",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Pettis, Dante",
"participantId": 1003741140,
"changedDate": "2022-10-13T01:18:07Z",
"odds": 31000,
"cashOutStatus": "ENABLED",
"id": 3172232134,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "20/1",
"englishLabel": "St. Brown, Equanimeous",
"label": "St. Brown, Equanimeous",
"betOfferId": 2349921831,
"oddsAmerican": "2000",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "St. Brown, Equanimeous",
"participantId": 1004561624,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 21000,
"cashOutStatus": "ENABLED",
"id": 3172232140,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "15/1",
"englishLabel": "Kmet, Cole",
"label": "Kmet, Cole",
"betOfferId": 2349921831,
"oddsAmerican": "1500",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Kmet, Cole",
"participantId": 1005209906,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 16000,
"cashOutStatus": "ENABLED",
"id": 3172232146,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "30/1",
"englishLabel": "Harry, N’Keal",
"label": "Harry, N’Keal",
"betOfferId": 2349921831,
"oddsAmerican": "3000",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Harry, N’Keal",
"participantId": 1005252420,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 31000,
"cashOutStatus": "ENABLED",
"id": 3172232153,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "8/1",
"englishLabel": "Fields, Justin",
"label": "Fields, Justin",
"betOfferId": 2349921831,
"oddsAmerican": "800",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Fields, Justin",
"participantId": 1005289956,
"changedDate": "2022-10-12T21:37:30Z",
"odds": 9000,
"cashOutStatus": "ENABLED",
"id": 3172232159,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "23/4",
"englishLabel": "Montgomery, David",
"label": "Montgomery, David",
"betOfferId": 2349921831,
"oddsAmerican": "575",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Montgomery, David",
"participantId": 1005330489,
"changedDate": "2022-10-13T01:18:07Z",
"odds": 6750,
"cashOutStatus": "ENABLED",
"id": 3172232164,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "75/1",
"englishLabel": "Blasingame, Khari",
"label": "Blasingame, Khari",
"betOfferId": 2349921831,
"oddsAmerican": "7500",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Blasingame, Khari",
"participantId": 1005619153,
"changedDate": "2022-10-12T21:37:30Z",
"odds": 76000,
"cashOutStatus": "ENABLED",
"id": 3172232170,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "75/1",
"englishLabel": "Wesco, Trevon",
"label": "Wesco, Trevon",
"betOfferId": 2349921831,
"oddsAmerican": "7500",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Wesco, Trevon",
"participantId": 1005620092,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 76000,
"cashOutStatus": "ENABLED",
"id": 3172232176,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "60/1",
"englishLabel": "Smith-Marsette, Ihmir",
"label": "Smith-Marsette, Ihmir",
"betOfferId": 2349921831,
"oddsAmerican": "6000",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Smith-Marsette, Ihmir",
"participantId": 1005740936,
"changedDate": "2022-10-13T01:18:07Z",
"odds": 61000,
"cashOutStatus": "ENABLED",
"id": 3172232181,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "55/1",
"englishLabel": "Ebner, Trestan",
"label": "Ebner, Trestan",
"betOfferId": 2349921831,
"oddsAmerican": "5500",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Ebner, Trestan",
"participantId": 1005999128,
"changedDate": "2022-10-12T21:37:30Z",
"odds": 56000,
"cashOutStatus": "ENABLED",
"id": 3172232186,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "11/1",
"englishLabel": "Mooney, Darnell",
"label": "Mooney, Darnell",
"betOfferId": 2349921831,
"oddsAmerican": "1100",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Mooney, Darnell",
"participantId": 1006797242,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 12000,
"cashOutStatus": "ENABLED",
"id": 3172232190,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "15/1",
"englishLabel": "Herbert, Khalil",
"label": "Herbert, Khalil",
"betOfferId": 2349921831,
"oddsAmerican": "1500",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Herbert, Khalil",
"participantId": 1007051724,
"changedDate": "2022-10-12T21:37:30Z",
"odds": 16000,
"cashOutStatus": "ENABLED",
"id": 3172232194,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "90/1",
"englishLabel": "Tonges, Jake",
"label": "Tonges, Jake",
"betOfferId": 2349921831,
"oddsAmerican": "9000",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Tonges, Jake",
"participantId": 1007678491,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 91000,
"cashOutStatus": "ENABLED",
"id": 3172232197,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "34/1",
"englishLabel": "Jones Jr., Velus",
"label": "Jones Jr., Velus",
"betOfferId": 2349921831,
"oddsAmerican": "3400",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Jones Jr., Velus",
"participantId": 1007905969,
"changedDate": "2022-10-13T01:18:07Z",
"odds": 35000,
"cashOutStatus": "ENABLED",
"id": 3172232200,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "17/1",
"englishLabel": "Any other Chicago Bears player",
"label": "Any other Chicago Bears player",
"betOfferId": 2349921831,
"oddsAmerican": "1700",
"type": "OT_ANY_OTHER_PARTICIPANT",
"participant": "Chicago Bears",
"participantId": 1000000178,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 18000,
"cashOutStatus": "ENABLED",
"id": 3172232205,
"status": "OPEN",
"eventParticipantId": 1000000178
},
{
"oddsFractional": "19/2",
"englishLabel": "Samuel, Curtis",
"label": "Samuel, Curtis",
"betOfferId": 2349921831,
"oddsAmerican": "950",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Samuel, Curtis",
"participantId": 1002450777,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 10500,
"cashOutStatus": "ENABLED",
"id": 3172232259,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "40/1",
"englishLabel": "Sims, Cam",
"label": "Sims, Cam",
"betOfferId": 2349921831,
"oddsAmerican": "4000",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Sims, Cam",
"participantId": 1003050339,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 41000,
"cashOutStatus": "ENABLED",
"id": 3172232265,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "8/1",
"englishLabel": "McLaurin, Terry",
"label": "McLaurin, Terry",
"betOfferId": 2349921831,
"oddsAmerican": "800",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "McLaurin, Terry",
"participantId": 1003741186,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 9000,
"cashOutStatus": "ENABLED",
"id": 3172232268,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"englishLabel": "Thomas, Logan",
"label": "Thomas, Logan",
"betOfferId": 2349921831,
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Thomas, Logan",
"participantId": 1004231988,
"changedDate": "2022-10-12T21:09:17Z",
"cashOutStatus": "ENABLED",
"id": 3172232271,
"status": "SUSPENDED",
"eventParticipantId": 1000000334
},
{
"englishLabel": "Dotson, Jahan",
"label": "Dotson, Jahan",
"betOfferId": 2349921831,
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Dotson, Jahan",
"participantId": 1005740692,
"changedDate": "2022-10-12T21:09:17Z",
"cashOutStatus": "ENABLED",
"id": 3172232274,
"status": "SUSPENDED",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "80/1",
"englishLabel": "Milne, Dax",
"label": "Milne, Dax",
"betOfferId": 2349921831,
"oddsAmerican": "8000",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Milne, Dax",
"participantId": 1007029029,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 81000,
"cashOutStatus": "ENABLED",
"id": 3172232277,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "19/1",
"englishLabel": "Brown, Dyami",
"label": "Brown, Dyami",
"betOfferId": 2349921831,
"oddsAmerican": "1900",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Brown, Dyami",
"participantId": 1007094475,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 20000,
"cashOutStatus": "ENABLED",
"id": 3172232279,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "45/1",
"englishLabel": "Turner, Cole",
"label": "Turner, Cole",
"betOfferId": 2349921831,
"oddsAmerican": "4500",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Turner, Cole",
"participantId": 1007180703,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 46000,
"cashOutStatus": "ENABLED",
"id": 3172232281,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "66/1",
"englishLabel": "Rogers, Armani",
"label": "Rogers, Armani",
"betOfferId": 2349921831,
"oddsAmerican": "6600",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Rogers, Armani",
"participantId": 1007800362,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 67000,
"cashOutStatus": "ENABLED",
"id": 3172232283,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "18/1",
"englishLabel": "Bates, John",
"label": "Bates, John",
"betOfferId": 2349921831,
"oddsAmerican": "1800",
"type": "OT_PLAYER_PARTICIPANT",
"participant": "Bates, John",
"participantId": 1007957637,
"changedDate": "2022-10-12T22:13:43Z",
"odds": 19000,
"cashOutStatus": "ENABLED",
"id": 3172232285,
"status": "OPEN",
"eventParticipantId": 1000000334
},
{
"oddsFractional": "16/1",
"englishLabel": "Any other Washington Commanders player",
"label": "Any other Washington Commanders player",
"betOfferId": 2349921831,
"oddsAmerican": "1600",
"type": "OT_ANY_OTHER_PARTICIPANT",
"participant": "Washington Commanders",
"participantId": 1000000334,
"changedDate": "2022-10-12T21:37:28Z",
"odds": 17000,
"cashOutStatus": "ENABLED",
"id": 3172232289,
"status": "OPEN",
"eventParticipantId": 1000000334
}
],
"cashOutStatus": "ENABLED",
"closed": "2022-10-14T00:15:00Z",
"betOfferType": {
"englishName": "Player Occurrence Number",
"name": "Player Occurrence Number",
"id": 125
},
"id": 2349921831
}],
"range": {
"total": 1,
"size": 1,
"start": 0
},
"events": [{
"englishName": "Chicago Bears - Washington Commanders",
"groupId": 1000093656,
"homeName": "Chicago Bears",
"groupSortOrder": 2876820019684212700,
"prematchEnd": "2022-10-14T00:15:00Z",
"path": [{
"englishName": "American Football",
"termKey": "american_football",
"name": "American Football",
"id": 1000093199
},
{
"englishName": "NFL",
"termKey": "nfl",
"name": "NFL",
"id": 1000093656
}
],
"id": 1018663065,
"state": "NOT_STARTED",
"group": "NFL",
"start": "2022-10-14T00:15:00Z",
"nonLiveBoCount": 254,
"tags": [
"OFFERED_LIVE",
"BET_BUILDER",
"MATCH",
"LIVE_OCCURRENCE_FEED"
],
"awayName": "Washington Commanders",
"name": "Chicago Bears - Washington Commanders",
"sport": "AMERICAN_FOOTBALL"
}]
}

One small modification to your code allows you to see which record is causing the error.
for td_data in player_td_data['betOffers']:
if td_data['criterion']['label'] == 'First Touchdown Scorer - Including Overtime' and 'OFFERED_PREMATCH' in td_data['tags']:
for outcomes in td_data['outcomes']:
if outcomes['type'] == 'OT_PLAYER_PARTICIPANT':
event_id = td_data['eventId']
player_id = outcomes['id']
player_name = outcomes['participant']
try:
player_first_td_odds = outcomes['odds']
except KeyError:
print(f'KeyError on {player_name}')
break
print(f'{event_id} {player_id} {player_name} {player_first_td_odds}')
You can see that indeed the JSON record for Thomas, Logan is missing the odds field.

(Python) Cant scrape data from my targeted site anymore using re, requests, and json

I'm having a problem where i can scrape data from a website by using the java pathing. I'm trying to scrape from Rocket League Tracker.
here's my code:
import requests
import re
import json
import math
def rankGetter():
trackerLink = 'https://rocketleague.tracker.network/rocket-league/profile/epic/DirectPanda/overview'
# now we have the tracker link we're going to scrape the website
# all the HTML of the site is now in result
result = requests.get(trackerLink)
# checker to make sure the user used the correct information
if result.status_code == 400:
print('profile not found')
else:
# Extract everything needed to render the current page. Data is stored as Json in the
# JavaScript variable: window.__INITIAL_STATE__={"route":{"path":"\u0 ... }};
json_string = re.search(r"window.__INITIAL_STATE__\s?=\s?(\{.*?\});", result.text).group(1)
# convert text string to structured json data
rocketleague = json.loads(json_string)
# Save structured json data to a text file that helps you orient yourself and pick
# the parts you are interested in.
with open('rocketleague_json_data.txt', 'w') as outfile:
outfile.write(json.dumps(rocketleague, indent=4, sort_keys=True))
The error is the text doc made doesn't have the ranks I want anymore.
"stats": {
"standardLeaderboardLeaders": {},
"standardLeaderboards": [],
"standardPlayers": {},
"standardTitles": {}
},
**"stats-v2": {
"segments": {},
"standardProfileMatches": {},
"standardProfileSummaries": {},
"standardProfiles": {},
"standardProfilesHistory": {},
"standardSessions": {},
"subscriptions": {}
},**
"titles": {
"currentTitle": {
"name": "Rocket League",
"platforms": [
The Ranks should be under stats-V2 but as you can see its empty now.
whats happening and how do i fix it? I was able to get ranks for a week but all the sudden it stopped working today.

Seems that the data are loaded from external URL:
import json
import requests
url = "https://api.tracker.gg/api/v2/rocket-league/standard/profile/epic/DirectPanda"
headers = {
"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0"
}
data = requests.get(url, headers=headers).json()
print(json.dumps(data, indent=4))
Prints:
{
"data": {
"platformInfo": {
"platformSlug": "epic",
"platformUserId": null,
"platformUserHandle": "DirectPanda",
"platformUserIdentifier": "DirectPanda",
"avatarUrl": null,
"additionalParameters": null
},
"userInfo": {
"userId": null,
"isPremium": false,
"isVerified": false,
"isInfluencer": false,
"isPartner": false,
"countryCode": null,
"customAvatarUrl": null,
"customHeroUrl": null,
"socialAccounts": [],
"pageviews": 592,
"isSuspicious": null
},
"metadata": {
"lastUpdated": {
"value": "2021-04-22T17:39:42.277-04:00",
"displayValue": "2021-04-22T21:39:42.2770000+00:00"
},
"playerId": 16603481,
"currentSeason": 17
},
"segments": [
{
"type": "overview",
"attributes": {},
"metadata": {
"name": "Lifetime"
},
"expiryDate": "0001-01-01T00:00:00+00:00",
"stats": {
"wins": {
"rank": 30357,
"percentile": 98.3,
"displayName": "Wins",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 4985,
"displayValue": "4,985",
"displayType": "Number"
},
"goals": {
"rank": 23698,
"percentile": 98.7,
"displayName": "Goals",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 14363,
"displayValue": "14,363",
"displayType": "Number"
},
"mVPs": {
"rank": 35646,
"percentile": 98.0,
"displayName": "MVPs",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 2093,
"displayValue": "2,093",
"displayType": "Number"
},
"saves": {
"rank": 30864,
"percentile": 98.3,
"displayName": "Saves",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 9231,
"displayValue": "9,231",
"displayType": "Number"
},
"assists": {
"rank": 29228,
"percentile": 98.4,
"displayName": "Assists",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 4763,
"displayValue": "4,763",
"displayType": "Number"
},
"shots": {
"rank": 24596,
"percentile": 98.6,
"displayName": "Shots",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 29139,
"displayValue": "29,139",
"displayType": "Number"
},
"goalShotRatio": {
"rank": 1409320,
"percentile": 15.0,
"displayName": "Goal Shot Ratio",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 49.29132777377398,
"displayValue": "49.3",
"displayType": "NumberPrecision1"
},
"score": {
"rank": 28260,
"percentile": 98.4,
"displayName": "TRN Score",
"displayCategory": "General",
"category": "general",
"metadata": {},
"value": 2398222.83,
"displayValue": "2,398,222.8",
"displayType": "NumberPrecision1"
},
"seasonRewardLevel": {
"rank": null,
"percentile": 85.0,
"displayName": "Season Reward Level",
"displayCategory": "General",
"category": "general",
"metadata": {
"iconUrl": "https://trackercdn.com/cdn/tracker.gg/rocket-league/ranks/s4-13.png",
"rankName": "Diamond"
},
"value": 5,
"displayValue": "5",
"displayType": "Number"
},
"seasonRewardWins": {
"rank": null,
"percentile": 95.8,
"displayName": "Season Reward Wins",
"displayCategory": "General",
"category": "general",
"metadata": {},
"value": 9,
"displayValue": "9",
"displayType": "Number"
}
}
},
{
"type": "playlist",
"attributes": {
"playlistId": 0,
"season": 17
},
"metadata": {
"name": "Un-Ranked"
},
"expiryDate": "0001-01-01T00:00:00+00:00",
"stats": {
"tier": {
"rank": null,
"percentile": null,
"displayName": "Matches",
"displayCategory": "General",
"category": "general",
"metadata": {
"iconUrl": "https://trackercdn.com/cdn/tracker.gg/rocket-league/ranks/s4-0.png",
"name": "Unranked"
},
"value": 0,
"displayValue": "0",
"displayType": "Number"
},
"division": {
"rank": null,
"percentile": null,
"displayName": "Matches",
"displayCategory": "General",
"category": "general",
"metadata": {
"name": "Division I"
},
"value": 0,
"displayValue": "0",
"displayType": "Number"
},
"matchesPlayed": {
"rank": null,
"percentile": null,
"displayName": "Matches",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 0,
"displayValue": "0",
"displayType": "Number"
},
"winStreak": {
"rank": null,
"percentile": null,
"displayName": "WinStreak",
"displayCategory": "Performance",
"category": "performance",
"metadata": {
"type": "win"
},
"value": 0,
"displayValue": "0",
"displayType": "Number"
},
"rating": {
"rank": 215152,
"percentile": 90.0,
"displayName": "Rating",
"displayCategory": "Skill",
"category": "skill",
"metadata": {},
"value": 1597,
"displayValue": "1,597",
"displayType": "Number"
}
}
},
{
"type": "playlist",
"attributes": {
"playlistId": 10,
"season": 17
},
"metadata": {
"name": "Ranked Duel 1v1"
},
"expiryDate": "0001-01-01T00:00:00+00:00",
"stats": {
"tier": {
"rank": null,
"percentile": 98.2,
"displayName": "Matches",
"displayCategory": "General",
"category": "general",
"metadata": {
"iconUrl": "https://trackercdn.com/cdn/tracker.gg/rocket-league/ranks/s4-16.png",
"name": "Champion I"
},
"value": 16,
"displayValue": "16",
"displayType": "Number"
},
"division": {
"rank": null,
"percentile": 88.0,
"displayName": "Matches",
"displayCategory": "General",
"category": "general",
"metadata": {
"deltaDown": 13,
"deltaUp": 6,
"name": "Division III"
},
"value": 2,
"displayValue": "2",
"displayType": "Number"
},
"matchesPlayed": {
"rank": null,
"percentile": 57.0,
"displayName": "Matches",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 2,
"displayValue": "2",
"displayType": "Number"
},
"winStreak": {
"rank": null,
"percentile": 60.0,
"displayName": "WinStreak",
"displayCategory": "Performance",
"category": "performance",
"metadata": {
"type": "win"
},
"value": 1,
"displayValue": "1",
"displayType": "Number"
},
"rating": {
"rank": 101541,
"percentile": 96.1,
"displayName": "Rating",
"displayCategory": "Skill",
"category": "skill",
"metadata": {},
"value": 1031,
"displayValue": "1,031",
"displayType": "Number"
}
}
},
{
"type": "playlist",
"attributes": {
"playlistId": 11,
"season": 17
},
"metadata": {
"name": "Ranked Doubles 2v2"
},
"expiryDate": "0001-01-01T00:00:00+00:00",
"stats": {
"tier": {
"rank": null,
"percentile": 87.0,
"displayName": "Matches",
"displayCategory": "General",
"category": "general",
"metadata": {
"iconUrl": "https://trackercdn.com/cdn/tracker.gg/rocket-league/ranks/s4-16.png",
"name": "Champion I"
},
"value": 16,
"displayValue": "16",
"displayType": "Number"
},
"division": {
"rank": null,
"percentile": 90.0,
"displayName": "Matches",
"displayCategory": "General",
"category": "general",
"metadata": {
"deltaDown": 15,
"deltaUp": 3,
"name": "Division IV"
},
"value": 3,
"displayValue": "3",
"displayType": "Number"
},
"matchesPlayed": {
"rank": null,
"percentile": 80.0,
"displayName": "Matches",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 40,
"displayValue": "40",
"displayType": "Number"
},
"winStreak": {
"rank": null,
"percentile": 34.0,
"displayName": "WinStreak",
"displayCategory": "Performance",
"category": "performance",
"metadata": {
"type": "loss"
},
"value": 1,
"displayValue": "-1",
"displayType": "Number"
},
"rating": {
"rank": 311789,
"percentile": 89.0,
"displayName": "Rating",
"displayCategory": "Skill",
"category": "skill",
"metadata": {},
"value": 1177,
"displayValue": "1,177",
"displayType": "Number"
}
}
},
{
"type": "playlist",
"attributes": {
"playlistId": 13,
"season": 17
},
"metadata": {
"name": "Ranked Standard 3v3"
},
"expiryDate": "0001-01-01T00:00:00+00:00",
"stats": {
"tier": {
"rank": null,
"percentile": 96.0,
"displayName": "Matches",
"displayCategory": "General",
"category": "general",
"metadata": {
"iconUrl": "https://trackercdn.com/cdn/tracker.gg/rocket-league/ranks/s4-17.png",
"name": "Champion II"
},
"value": 17,
"displayValue": "17",
"displayType": "Number"
},
"division": {
"rank": null,
"percentile": 79.0,
"displayName": "Matches",
"displayCategory": "General",
"category": "general",
"metadata": {
"deltaDown": 7,
"deltaUp": 27,
"name": "Division III"
},
"value": 2,
"displayValue": "2",
"displayType": "Number"
},
"matchesPlayed": {
"rank": null,
"percentile": 97.8,
"displayName": "Matches",
"displayCategory": "Performance",
"category": "performance",
"metadata": {},
"value": 95,
"displayValue": "95",
"displayType": "Number"
},
"winStreak": {
"rank": null,
"percentile": 16.0,
"displayName": "WinStreak",
"displayCategory": "Performance",
"category": "performance",
"metadata": {
"type": "loss"
},
"value": 2,
"displayValue": "-2",
"displayType": "Number"
},
"rating": {
"rank": 122500,
"percentile": 95.8,
"displayName": "Rating",
"displayCategory": "Skill",
"category": "skill",
"metadata": {},
"value": 1255,
"displayValue": "1,255",
"displayType": "Number"
}
}
},
...

Extracting data from JSON File to CSV

I have a big JSON file with a very complex structure
you can look on it here: https://drive.google.com/file/d/1tBVJ2xYSCpTTUGPJegvAz2ZXbeN0bteX/view?usp=sharing
it contains more than 7 millions lines, and I want to extract only the "text" field
I have written a python code, to extra all the values of the "text" key or field in the whole file, and it extracted only 12 values! while when I open the JSON file on the Visualstudio, I have more than 19000 values!!
you can see the code here:
import json
import csv
with open("/Users/zahraa-maher/rasa-init-demo/venv/Tickie/external_data/frames2.json") as file:
data = json.load(file)
fname = "outputText8.csv"
with open(fname, "w") as file:
csv_file = csv.writer(file,lineterminator='\n')
csv_file.writerow(["text"])
for item in data[i]["turns"]:
csv_file.writerow([item['text']])
please take a look on the JSON file as it is very large one and with a complex structure, so I an not paste it here to see because it would be not understandable
also this is a part of the son file:
[
{
"user_id": "U22HTHYNP",
"turns": [
{
"text": "I'd like to book a trip to Atlantis from Caprica on Saturday, August 13, 2016 for 8 adults. I have a tight budget of 1700.",
"labels": {
"acts": [
{
"args": [
{
"val": "book",
"key": "intent"
}
],
"name": "inform"
},
{
"args": [
{
"val": "Atlantis",
"key": "dst_city"
},
{
"val": "Caprica",
"key": "or_city"
},
{
"val": "Saturday, August 13, 2016",
"key": "str_date"
},
{
"val": "8",
"key": "n_adults"
},
{
"val": "1700",
"key": "budget"
}
],
"name": "inform"
}
],
"acts_without_refs": [
{
"args": [
{
"val": "book",
"key": "intent"
}
],
"name": "inform"
},
{
"args": [
{
"val": "Atlantis",
"key": "dst_city"
},
{
"val": "Caprica",
"key": "or_city"
},
{
"val": "Saturday, August 13, 2016",
"key": "str_date"
},
{
"val": "8",
"key": "n_adults"
},
{
"val": "1700",
"key": "budget"
}
],
"name": "inform"
}
],
"active_frame": 1,
"frames": [
{
"info": {
"intent": [
{
"val": "book",
"negated": false
}
],
"budget": [
{
"val": "1700.0",
"negated": false
}
],
"dst_city": [
{
"val": "Atlantis",
"negated": false
}
],
"or_city": [
{
"val": "Caprica",
"negated": false
}
],
"str_date": [
{
"val": "august 13",
"negated": false
}
],
"n_adults": [
{
"val": "8",
"negated": false
}
]
},
"frame_id": 1,
"requests": [],
"frame_parent_id": null,
"binary_questions": [],
"compare_requests": []
}
]
},
"author": "user",
"timestamp": 1471272019730.0
},
{
"db": {
"result": [
[
{
"trip": {
"returning": {
"duration": {
"hours": 0,
"min": 51
},
"arrival": {
"hour": 10,
"year": 2016,
"day": 24,
"min": 51,
"month": 8
},
"departure": {
"hour": 10,
"year": 2016,
"day": 24,
"min": 0,
"month": 8
}
},
"seat": "ECONOMY",
"leaving": {
"duration": {
"hours": 0,
"min": 51
},
"arrival": {
"hour": 0,
"year": 2016,
"day": 16,
"min": 51,
"month": 8
},
"departure": {
"hour": 0,
"year": 2016,
"day": 16,
"min": 0,
"month": 8
}
},
"or_city": "Porto Alegre",
"duration_days": 9
},
"price": 2118.81,
"hotel": {
"gst_rating": 7.15,
"vicinity": [],
"name": "Scarlet Palms Resort",
"country": "Brazil",
"amenities": [
"FREE_BREAKFAST",
"FREE_PARKING",
"FREE_WIFI"
],
"dst_city": "Goiania",
"category": "3.5 star hotel"
}
},
{
"trip": {
"returning": {
"duration": {
"hours": 2,
"min": 37
},
"arrival": {
"hour": 12,
"year": 2016,
"day": 10,
"min": 37,
"month": 8
},
"departure": {
"hour": 10,
"year": 2016,
"day": 10,
"min": 0,
"month": 8
}
},
"seat": "ECONOMY",
"leaving": {
"duration": {
"hours": 2,
"min": 37
},
"arrival": {
"hour": 0,
"year": 2016,
"day": 4,
"min": 37,
"month": 8
},
"departure": {
"hour": 22,
"year": 2016,
"day": 3,
"min": 0,
"month": 8
}
},
"or_city": "Porto Alegre",
"duration_days": 7
},
"price": 2369.83,
"hotel": {
"gst_rating": 0,
"vicinity": [],
"name": "Sunway Hostel",
"country": "Argentina",
"amenities": [
"FREE_BREAKFAST",
"FREE_WIFI"
],
"dst_city": "Rosario",
"category": "2.0 star hotel"
}
},
{
"trip": {
"returning": {
"duration": {
"hours": 0,
"min": 51
},
"arrival": {
"hour": 10,
"year": 2016,
"day": 24,
"min": 51,
"month": 8
},
"departure": {
"hour": 10,
"year": 2016,
"day": 24,
"min": 0,
"month": 8
}
},
"seat": "BUSINESS",
"leaving": {
"duration": {
"hours": 0,
"min": 51
},
"arrival": {
"hour": 0,
"year": 2016,
"day": 16,
"min": 51,
"month": 8
},
"departure": {
"hour": 0,
"year": 2016,
"day": 16,
"min": 0,
"month": 8
}
},
"or_city": "Porto Alegre",
"duration_days": 9
},
"price": 2375.72,
"hotel": {
"gst_rating": 7.15,
"vicinity": [],
"name": "Scarlet Palms Resort",
"country": "Brazil",
"amenities": [
"FREE_BREAKFAST",
"FREE_PARKING",
"FREE_WIFI"
],
"dst_city": "Goiania",
"category": "3.5 star hotel"
}
},
{
"trip": {
"returning": {
"duration": {
"hours": 1,
"min": 30
},
"arrival": {
"hour": 11,
"year": 2016,
"day": 1,
"min": 30,
"month": 9
},
"departure": {
"hour": 10,
"year": 2016,
"day": 1,
"min": 0,
"month": 9
}
},
"seat": "BUSINESS",
"leaving": {
"duration": {
"hours": 1,
"min": 30
},
"arrival": {
"hour": 18,
"year": 2016,
"day": 19,
"min": 30,
"month": 8
},
"departure": {
"hour": 17,
"year": 2016,
"day": 19,
"min": 0,
"month": 8
}
},
"or_city": "Porto Alegre",
"duration_days": 13
},
"price": 2492.95,
"hotel": {
"gst_rating": 0,
"vicinity": [],
"name": "Hotel Mundo",
"country": "Brazil",
"amenities": [
"FREE_BREAKFAST",
"FREE_WIFI",
"FREE_PARKING"
],
"dst_city": "Manaus",
"category": "2.5 star hotel"
}
},
{
"trip": {
"returning": {
"duration": {
"hours": 0,
"min": 51
},
"arrival": {
"hour": 10,
"year": 2016,
"day": 31,
"min": 51,
"month": 8
},
"departure": {
"hour": 10,
"year": 2016,
"day": 31,
"min": 0,
"month": 8
}
},
"seat": "ECONOMY",
"leaving": {
"duration": {
"hours": 0,
"min": 51
},
"arrival": {
"hour": 19,
"year": 2016,
"day": 27,
"min": 51,
"month": 8
},
"departure": {
"hour": 19,
"year": 2016,
"day": 27,
"min": 0,
"month": 8
}
},
"or_city": "Porto Alegre",
"duration_days": 4
},
"price": 2538.0,
"hotel": {
"gst_rating": 8.22,
"vicinity": [],
"name": "The Glee",
"country": "Brazil",
"amenities": [
"FREE_BREAKFAST",
"FREE_WIFI"
],
"dst_city": "Recife",
"category": "4.0 star hotel"
}
}
],
[],
[],
[],
[],
[],
[]
],
"search": [
{
"ORIGIN_CITY": "Porto Alegre",
"PRICE_MIN": "2000",
"NUM_ADULTS": "2",
"timestamp": 1471271949.995,
"PRICE_MAX": "3000",
"ARE_DATES_FLEXIBLE": "true",
"NUM_CHILDREN": "5",
"START_TIME": "1470110400000",
"MAX_DURATION": 2592000000.0,
"DESTINATION_CITY": "Brazil",
"RESULT_LIMIT": "10",
"END_TIME": "1472616000000"
},
{
"ORIGIN_CITY": "Atlantis",
"NUM_ADULTS": "8",
"RESULT_LIMIT": "10",
"timestamp": 1471272148.124,
"PRICE_MAX": "1700",
"NUM_CHILDREN": "",
"ARE_DATES_FLEXIBLE": "true",
"START_TIME": "NaN",
"END_TIME": "NaN"
},
{
"ORIGIN_CITY": "Caprica",
"PRICE_MAX": "1700",
"NUM_ADULTS": "8",
"RESULT_LIMIT": "10",
"timestamp": 1471272189.07,
"DESTINATION_CITY": "Atlantis",
"NUM_CHILDREN": "",
"ARE_DATES_FLEXIBLE": "true",
"START_TIME": "1470715200000",
"END_TIME": "1472011200000"
},
{
"ORIGIN_CITY": "Caprica",
"PRICE_MAX": "1700",
"NUM_ADULTS": "8",
"RESULT_LIMIT": "10",
"timestamp": 1471272205.436,
"DESTINATION_CITY": "Atlantis",
"NUM_CHILDREN": "",
"ARE_DATES_FLEXIBLE": "true",
"START_TIME": "1470715200000",
"END_TIME": "1472011200000"
},
{
"ORIGIN_CITY": "Caprica",
"PRICE_MIN": "1700",
"NUM_ADULTS": "8",
"RESULT_LIMIT": "10",
"timestamp": 1471272278.72,
"DESTINATION_CITY": "Atlantis",
"NUM_CHILDREN": "",
"ARE_DATES_FLEXIBLE": "true",
"START_TIME": "1470715200000",
"END_TIME": "1472011200000"
},
{
"ORIGIN_CITY": "Caprica",
"PRICE_MIN": "1700",
"NUM_ADULTS": "8",
"RESULT_LIMIT": "10",
"timestamp": 1471272454.542,
"DESTINATION_CITY": "Atlantis",
"NUM_CHILDREN": "",
"ARE_DATES_FLEXIBLE": "true",
"START_TIME": "1471060800000",
"END_TIME": "1472011200000"
},
{
"ORIGIN_CITY": "Caprica",
"PRICE_MIN": "1700",
"NUM_ADULTS": "8",
"RESULT_LIMIT": "10",
"timestamp": 1471272466.008,
"DESTINATION_CITY": "Atlantis",
"NUM_CHILDREN": "",
"ARE_DATES_FLEXIBLE": "true",
"START_TIME": "1471060800000",
"END_TIME": "1472011200000"
}
]
},
How it could be modified to extract all the "text" values from the JSON file to a CSV file?

This is a potential solution using pandas:
import pandas as pd
#importing data
dj = pd.read_json("frames2.json")
dtext = dj[["user_id","turns"]]
#Saving text records in a list
list_ = []
for record in dtext["turns"].values:
for r in record:
list_.append(r["text"])
#Exporting the csv
out = pd.Series(list_,name="text")
out.to_csv("text.csv")
It gives the following output.

Try:
import json
import csv
with open("/Users/zahraa-maher/rasa-init-demo/venv/Tickie/external_data/frames2.json") as file:
data = json.load(file)
fname = "outputText8.csv"
with open(fname, "w") as file:
csv_file = csv.writer(file,lineterminator='\n')
csv_file.writerow(["text"])
for keys,values in data.items():
now it up to you which of the fields you want to save, if you user a debugger you can see the values and Keys

How to join document in search query

{
"took": 0,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 3,
"max_score": 0.2876821,
"hits": [
{
"_index": "product_index",
"_type": "product",
"_id": "1115",
"_score": 0.2876821,
"_source": {
"isactive": true,
"in_use": false,
"brand_name": "Adidas",
"sku_id": "56456487987987",
"long_description": "this is long description",
"key_feature": [
{
"id": 1148,
"key_feature": "sport wear"
},
{
"id": 1147,
"key_feature": "Cotton shirt"
},
{
"id": 1146,
"key_feature": "White and blue"
}
],
"isdeleted": false,
"created_by": null,
"brand_id": 5,
"search_terms": [
{
"label": "white shirt",
"value": 9
}
]
"color_id": 2,
"specific_keywords": "",
"item_list": [
{
"item_id": 1114,
"product_id": 1115,
"isactive": true,
"id": 9,
"isdeleted": false
},
{
"item_id": 1113,
"product_id": 1115,
"isactive": true,
"id": 10,
"isdeleted": false
}
],
"upc_code": "",
"display_size": "L",
"name": "New White shirt",
"updated_by": null,
"id": 1115,
"updated_date": "2020-03-25T08:24:37.644571+00:00",
"color_name": "blue",
"created_date": "2020-03-25T08:11:14.966673+00:00",
"category": [
{
"parent_category_id": 78,
"sub_sub_category": null,
"sub_category": null,
"sub_category_id": null,
"sub_sub_category_id": null,
"parent_category": "new Shirt Cate",
"id": 1151
}
]
}
},
{
"_index": "product_index",
"_type": "product",
"_id": "1113",
"_score": 0.2876821,
"_source": {
"isactive": true,
"in_use": false,
"sku_id": "1456456488",
"brand_name": "Adidas",
"long_description": "",
"key_feature": [
{
"id": 1142,
"key_feature": "Cotton"
},
{
"id": 1141,
"key_feature": "Office Use"
},
{
"id": 1140,
"key_feature": "Black formal"
}
],
"isdeleted": false,
"created_by": null,
"brand_id": 5,
"search_terms": [
]
"color_id": 1,
"specific_keywords": "",
"item_list": [
],
"display_size": "L",
"upc_code": "",
"name": "New Cotton formal shirt black",
"updated_by": null,
"id": 1113,
"updated_date": "2020-03-25T06:48:30.903041+00:00",
"created_date": "2020-03-25T06:48:29.943043+00:00",
"color_name": "black",
"category": [
{
"sub_sub_category": null,
"parent_category_id": 54,
"sub_category": null,
"sub_category_id": null,
"sub_sub_category_id": null,
"parent_category": "MEN'S CLOTHING",
"id": 1149
}
]
}
},
{
"_index": "product_index",
"_type": "product",
"_id": "1114",
"_score": 0.2876821,
"_source": {
"isactive": true,
"in_use": false,
"sku_id": "145645648811",
"brand_name": "Adidas",
"long_description": "",
"key_feature": [
{
"id": 1145,
"key_feature": "Cotton"
},
{
"id": 1144,
"key_feature": "Office Use"
},
{
"id": 1143,
"key_feature": "Black formal"
}
],
"isdeleted": false,
"created_by": null,
"brand_id": 5,
"search_terms": [
],
"color_id": 1,
"specific_keywords": "",
"item_list": [
],
"display_size": "L",
"upc_code": "",
"updated_by": null,
"name": "New Cotton Casual shirt black",
"id": 1114,
"created_date": "2020-03-25T07:13:26.233675+00:00",
"color_name": "black",
"updated_date": "2020-03-25T07:13:27.229363+00:00",
"category": [
{
"sub_sub_category": null,
"parent_category_id": 54,
"sub_category": null,
"sub_category_id": null,
"sub_sub_category_id": null,
"parent_category": "MEN'S CLOTHING",
"id": 1150
}
]
}
}
]
}
}
my requirement is to attach all related documents with specific key value fields which is specify in item_list based on item_id. In above result doc id 1115 has item_list which contains item_id 1114 and 1113. so the particular fields attach in the doc 1115.
what should be the search query for that in elastic search?

You can't do join in Elasticsearch, to achieve your goal, you can do two things:
duplicate the information of item_id 1114 and 1113 in the item_id
1115 (and for sure in all others documents).
Do join at application level, so after this query you can extract the item_id 1114 and 1113 and run two others query to get the information about this items. Then join all the json at application level.