I have a JSON feed i need to collect data from but I am running into some issues with iterating and parsing the API data.
The function below collects what I need but some horses get scratched (dont race) and the format of their JSON is different to horses who's status is "Starter".
def race_data():
for race_key in races_list():
data = requests.get(f'{Base_url}events/{race_key}.json?app_id={AppID}&app_key={AppKey}').text
# parse race data
data = json.loads(data)
for race_odds in data['competitors']:
horse_name = race_odds['name']
jockey_name = race_odds['jockey']
jockey_weight = race_odds['weight']
trainer_name = race_odds['trainer']
fixed_win_odds = race_odds['prices'][2]['price']
fixed_place_odds = race_odds['prices'][0]['price']
race_key = data['eventKey']
race_time = data['eventDateTimeUtc']
horse_status = race_odds['status']
print(f'{race_key} {horse_name} {jockey_name} {jockey_weight} {trainer_name} {fixed_win_odds} {fixed_place_odds} {race_time} {horse_status}')
if __name__ == "__main__":
race_data()
which produces
202210290200.T.AUS.quirindi.1 Ponte Pietra Jai Williams 0 Nikki Pollock 1.1 16.0 2022-10-29T02:55:00Z Starter
202210290200.T.AUS.quirindi.1 Gundawarra Vad Bolozhinskyi 0 G A O'brien 4.5 35.0 2022-10-29T02:55:00Z Starter
202210290200.T.AUS.quirindi.1 Cemented Ms Chelsea Hillier 0 Sally Torrens 2.5 35.0 2022-10-29T02:55:00Z Starter
202210290200.T.AUS.quirindi.1 Jack Duggan Ms Madeline Owen 0 J C Deamer 1.1 2.7 2022-10-29T02:55:00Z Starter
202210290200.T.AUS.quirindi.1 Northern Borders Ms Zara Lewis 0 W T Martyn 9.6 68.1 2022-10-29T02:55:00Z Starter
202210290200.T.AUS.quirindi.1 Iffland Ms Amelia Denby 0 K A Lees 1.9 19.4 2022-10-29T02:55:00Z Starter
202210290200.T.AUS.quirindi.1 La Brea Benjamin Osmond 0 K A Lees 2.1 14.0 2022-10-29T02:55:00Z Starter
202210290200.T.AUS.quirindi.1 My Gem Jacob Golden 0 Ms S Grills 2.8 16.1 2022-10-29T02:55:00Z Starter
202210290200.T.AUS.quirindi.1 Another Super Patrick Scorse 0 M J Dwyer 4.9 87.5 2022-10-29T02:55:00Z Starter
I run into an issue when one of the runners is scratched (not racing)
the problem is with the odds price field. When a horse is scratched (defined in horse_status)
fixed_win_odds = race_odds['prices'][2]['price']
fixed_place_odds = race_odds['prices'][0]['price']
The JSON below is for a horse that is scratched:
{
"status": "Scratched",
"startPos": 3,
"trainer": "Ms L Selby",
"shortForm": "0608247373",
"sequence": 7,
"weight": "0",
"name": "Bingo Banko",
"jockey": "Unknown",
"prices":
[
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 0
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 0,
"productType": "Current"
},
{
"price": 16,
"productType": "Max"
},
{
"price": 16,
"productType": "Open"
},
{
"price": 15,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
}
],
"price": 0
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.bingo_banko",
"competitorKey": "202210290200.T.AUS.quirindi.1.bingo_banko"
}
and this is what the JSON looks like when the horse is a starter (not scratched)
{
"status": "Starter",
"startPos": 9,
"trainer": "M J Dwyer",
"shortForm": "7626280005",
"sequence": 6,
"weight": "0",
"name": "Another Super",
"jockey": "Patrick Scorse",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 85
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 3.9
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 5.7
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 17,
"productType": "Current"
},
{
"price": 20,
"productType": "Max"
},
{
"price": 19,
"productType": "Open"
},
{
"price": 17,
"productType": "Last"
},
{
"price": 18,
"productType": "Last"
},
{
"price": 17,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
},
{
"price": 15,
"productType": "Last"
}
],
"price": 17
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.another_super",
"competitorKey": "202210290200.T.AUS.quirindi.1.another_super"
}
I still need to collect the information on the horses that are scratched, so need some help with formatting the iteration to collect the data without getting key error on horses that are scratched.
you can view the full JSON for a race below for reference:
[
{
"status": "Starter",
"startPos": 5,
"trainer": "Nikki Pollock",
"shortForm": "x686623464",
"sequence": 3,
"weight": "0",
"name": "Ponte Pietra",
"jockey": "Jai Williams",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 15.6
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 2.6
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 1.04
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 10,
"productType": "Current"
},
{
"price": 16,
"productType": "Max"
},
{
"price": 16,
"productType": "Open"
},
{
"price": 10,
"productType": "Last"
},
{
"price": 9,
"productType": "Last"
},
{
"price": 9,
"productType": "Last"
},
{
"price": 8,
"productType": "Last"
},
{
"price": 8,
"productType": "Last"
}
],
"price": 10
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.ponte_pietra",
"competitorKey": "202210290200.T.AUS.quirindi.1.ponte_pietra"
},
{
"status": "Starter",
"startPos": 4,
"trainer": "G A O'brien",
"shortForm": "90x4491",
"sequence": 4,
"weight": "0",
"name": "Gundawarra",
"jockey": "Vad Bolozhinskyi",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 34
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 1.45
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 11
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 3,
"productType": "Current"
},
{
"price": 3,
"productType": "Max"
},
{
"price": 3,
"productType": "Open"
},
{
"price": 3,
"productType": "Last"
},
{
"price": 3,
"productType": "Last"
},
{
"price": 3,
"productType": "Last"
},
{
"price": 3,
"productType": "Last"
},
{
"price": 3,
"productType": "Last"
}
],
"price": 3.7
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.gundawarra",
"competitorKey": "202210290200.T.AUS.quirindi.1.gundawarra"
},
{
"status": "Starter",
"startPos": 6,
"trainer": "Sally Torrens",
"shortForm": "7816x64858",
"sequence": 1,
"weight": "0",
"name": "Cemented",
"jockey": "Ms Chelsea Hillier",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 85
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 3.1
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 3
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 13,
"productType": "Current"
},
{
"price": 19,
"productType": "Max"
},
{
"price": 19,
"productType": "Open"
},
{
"price": 13,
"productType": "Last"
},
{
"price": 14,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
},
{
"price": 15,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
}
],
"price": 13
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.cemented",
"competitorKey": "202210290200.T.AUS.quirindi.1.cemented"
},
{
"status": "Starter",
"startPos": 2,
"trainer": "J C Deamer",
"shortForm": "75x5456x73",
"sequence": 2,
"weight": "0",
"name": "Jack Duggan",
"jockey": "Ms Madeline Owen",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 3
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 1.14
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 1.04
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 1,
"productType": "Current"
},
{
"price": 1,
"productType": "Max"
},
{
"price": 1,
"productType": "Open"
},
{
"price": 1,
"productType": "Last"
},
{
"price": 1,
"productType": "Last"
},
{
"price": 1,
"productType": "Last"
},
{
"price": 1,
"productType": "Last"
},
{
"price": 1,
"productType": "Last"
}
],
"price": 1.8
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.jack_duggan",
"competitorKey": "202210290200.T.AUS.quirindi.1.jack_duggan"
},
{
"status": "Starter",
"startPos": 10,
"trainer": "W T Martyn",
"shortForm": "00409",
"sequence": 8,
"weight": "0",
"name": "Northern Borders",
"jockey": "Ms Zara Lewis",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 116
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 8
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 9
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 41,
"productType": "Current"
},
{
"price": 51,
"productType": "Max"
},
{
"price": 34,
"productType": "Open"
},
{
"price": 41,
"productType": "Last"
},
{
"price": 51,
"productType": "Last"
},
{
"price": 41,
"productType": "Last"
},
{
"price": 51,
"productType": "Last"
},
{
"price": 41,
"productType": "Last"
}
],
"price": 41
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.northern_borders",
"competitorKey": "202210290200.T.AUS.quirindi.1.northern_borders"
},
{
"status": "Starter",
"startPos": 8,
"trainer": "K A Lees",
"shortForm": "6",
"sequence": 9,
"weight": "0",
"name": "Iffland",
"jockey": "Ms Amelia Denby",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 33.5
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 3.3
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 1.7
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 14,
"productType": "Current"
},
{
"price": 20,
"productType": "Max"
},
{
"price": 17,
"productType": "Open"
},
{
"price": 14,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
},
{
"price": 14,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
},
{
"price": 14,
"productType": "Last"
}
],
"price": 14
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.iffland",
"competitorKey": "202210290200.T.AUS.quirindi.1.iffland"
},
{
"status": "Starter",
"startPos": 7,
"trainer": "K A Lees",
"shortForm": "55x570x58",
"sequence": 10,
"weight": "0",
"name": "La Brea",
"jockey": "Benjamin Osmond",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 43.3
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 2.6
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 1.9
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 10,
"productType": "Current"
},
{
"price": 13,
"productType": "Max"
},
{
"price": 11,
"productType": "Open"
},
{
"price": 10,
"productType": "Last"
},
{
"price": 9,
"productType": "Last"
},
{
"price": 9,
"productType": "Last"
},
{
"price": 8,
"productType": "Last"
},
{
"price": 9,
"productType": "Last"
}
],
"price": 10
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.la_brea",
"competitorKey": "202210290200.T.AUS.quirindi.1.la_brea"
},
{
"status": "Starter",
"startPos": 1,
"trainer": "Ms S Grills",
"shortForm": "673410x084",
"sequence": 5,
"weight": "0",
"name": "My Gem",
"jockey": "Jacob Golden",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 14.9
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 3.7
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 3.2
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 16,
"productType": "Current"
},
{
"price": 16,
"productType": "Max"
},
{
"price": 13,
"productType": "Open"
},
{
"price": 16,
"productType": "Last"
},
{
"price": 15,
"productType": "Last"
},
{
"price": 14,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
},
{
"price": 15,
"productType": "Last"
}
],
"price": 16
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.my_gem",
"competitorKey": "202210290200.T.AUS.quirindi.1.my_gem"
},
{
"status": "Starter",
"startPos": 9,
"trainer": "M J Dwyer",
"shortForm": "7626280005",
"sequence": 6,
"weight": "0",
"name": "Another Super",
"jockey": "Patrick Scorse",
"prices":
[
{
"betType": "Win",
"code": "BT3",
"flucs": [],
"price": 85
},
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 3.9
},
{
"betType": "Place",
"code": "BT2P",
"flucs": [],
"price": 5.7
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 17,
"productType": "Current"
},
{
"price": 20,
"productType": "Max"
},
{
"price": 19,
"productType": "Open"
},
{
"price": 17,
"productType": "Last"
},
{
"price": 18,
"productType": "Last"
},
{
"price": 17,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
},
{
"price": 15,
"productType": "Last"
}
],
"price": 17
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.another_super",
"competitorKey": "202210290200.T.AUS.quirindi.1.another_super"
},
{
"status": "Scratched",
"startPos": 3,
"trainer": "Ms L Selby",
"shortForm": "0608247373",
"sequence": 7,
"weight": "0",
"name": "Bingo Banko",
"jockey": "Unknown",
"prices":
[
{
"betType": "FixedPlace",
"code": "FXD",
"flucs": [],
"price": 0
},
{
"betType": "FixedWin",
"code": "FXD",
"flucs":
[
{
"price": 0,
"productType": "Current"
},
{
"price": 16,
"productType": "Max"
},
{
"price": 16,
"productType": "Open"
},
{
"price": 15,
"productType": "Last"
},
{
"price": 16,
"productType": "Last"
}
],
"price": 0
}
],
"id": "competitor:202210290200.T.AUS.quirindi.1.bingo_banko",
"competitorKey": "202210290200.T.AUS.quirindi.1.bingo_banko"
}
]
thanks for your assistance.
Use a for loop to iterate through the prices, and check the betType key to see if it's either FixedWin or FixedPlace, that way the ordering of the different prices doesn't matter.
Also since the race_key and the race event are always the same you should define them at the top of the function so your implementation doesn't need to keep checking the dictionary for them.
def race_data():
for race_key in races_list():
data = requests.get(f'{Base_url}events/{race_key}.json?app_id={AppID}&app_key={AppKey}').text
# parse race data
data = json.loads(data)
race_key = data['eventKey']
race_time = data['eventDateTimeUtc']
for race_odds in data:
horse_name = race_odds['name']
jockey_name = race_odds['jockey']
jockey_weight = race_odds['weight']
trainer_name = race_odds['trainer']
for price in race_odds['prices']:
if price["betType"] == "FixedWin":
fixed_win_odds = price["price"]
elif price["betType"] == "FixedPlace":
fixed_place_odds = price["price"]
horse_status = race_odds['status']
print(f'{race_key} {horse_name} {jockey_name} {jockey_weight} {trainer_name} {fixed_win_odds} {fixed_place_odds} {race_time} {horse_status}')
if __name__ == "__main__":
race_data()
Related
Below is code I am using to get data from an api. And below that is the response. I am trying to convert datetime from UTC to CST and then present the data with that time zone instead. But I am having trouble isolating datetime
import requests
import json
weather = requests.get('...')
j = json.loads(weather.text)
print (json.dumps(j, indent=2))
Response:
{
"metadata": null,
"data": [
{
"datetime": "2022-12-11T05:00:00Z",
"is_day_time": false,
"icon_code": 5,
"weather_text": "Clear with few low clouds and few cirrus",
"temperature": {
"value": 45.968,
"units": "F"
},
"feels_like_temperature": {
"value": 39.092,
"units": "F"
},
"relative_humidity": 56,
"precipitation": {
"precipitation_probability": 4,
"total_precipitation": {
"value": 0.0,
"units": "in"
}
},
"wind": {
"speed": {
"value": 5.144953471725125,
"units": "mi/h"
},
"direction": 25
},
"wind_gust": {
"value": 9.014853256979242,
"units": "mi/h"
},
"pressure": {
"value": 29.4171829577118,
"units": "inHg"
},
"visibility": {
"value": 6.835083114610673,
"units": "mi"
},
"dew_point": {
"value": 31.01,
"units": "F"
},
"cloud_cover": 31
},
{
"datetime": "2022-12-11T06:00:00Z",
"is_day_time": false,
"icon_code": 4,
"weather_text": "Clear with few low clouds",
"temperature": {
"value": 45.068,
"units": "F"
},
"feels_like_temperature": {
"value": 38.066,
"units": "F"
},
"relative_humidity": 56,
"precipitation": {
"precipitation_probability": 5,
"total_precipitation": {
"value": 0.0,
"units": "in"
}
},
"wind": {
"speed": {
"value": 5.167322834645669,
"units": "mi/h"
},
"direction": 27
},
"wind_gust": {
"value": 8.724051539012168,
"units": "mi/h"
},
"pressure": {
"value": 29.4213171559632,
"units": "inHg"
},
"visibility": {
"value": 5.592340730136005,
"units": "mi"
},
"dew_point": {
"value": 30.2,
"units": "F"
},
"cloud_cover": 13
},
{
"datetime": "2022-12-11T07:00:00Z",
"is_day_time": false,
"icon_code": 4,
"weather_text": "Clear with few low clouds",
"temperature": {
"value": 44.33,
"units": "F"
},
"feels_like_temperature": {
"value": 37.364,
"units": "F"
},
"relative_humidity": 56,
"precipitation": {
"precipitation_probability": 4,
"total_precipitation": {
"value": 0.0,
"units": "in"
}
},
"wind": {
"speed": {
"value": 4.988367931281317,
"units": "mi/h"
},
"direction": 28
},
"wind_gust": {
"value": 8.254294917680744,
"units": "mi/h"
},
"pressure": {
"value": 29.4165923579616,
"units": "inHg"
},
"visibility": {
"value": 7.456454306848007,
"units": "mi"
},
"dew_point": {
"value": 29.714,
"units": "F"
},
"cloud_cover": 22
}
],
"error": null
I am assuming what you mean is that you want to present the data in the current time of the Central Time zone. As of the date this question was asked, that would be CST (Central Standard Time). At another time it will be CDT (Central Daylight Time) based on daylight savings time rules that are followed in the Country/City for the time zone for which you wish to localize the data. The rules are all nicely kept in the IANA Timezone Database.
So the trick is that you pick your Country/City from the Timezone DB that follows the rules as they apply to your current time zone. For Central Time, America/Chicago usually works but YMMV.
There are a lot of ways to do this. This example is inefficiently iterating through the dictionary created by json.loads and replacing the time string with a converted string. The key is using the dateutil library to parse the timestamp string and convert using the proper UTC offset as defined for the time zone in the IANA database.
Hopefully this example has enough pieces you can copy and adapt to your own needs.
from dateutil.parser import parse
from dateutil import tz
import json
j = json.loads(weather)
# Loop through each data entry, reformatting the time
for entry in j["data"]:
if "datetime" in entry.keys():
parsed_dt = parse(entry["datetime"])
converted = parsed_dt.astimezone(tz.gettz("America/Chicago"))
entry["datetime"] = converted.isoformat()
print (json.dumps(j, indent=2))
The resulting JSON has datetime fields that contain an ISO timestamp for the CST time.
{
"metadata": null,
"data": [{
"datetime": "2022-12-10T23:00:00-06:00",
"is_day_time": false,
"icon_code": 5,
"weather_text": "Clear with few low clouds and few cirrus",
"temperature": {
"value": 45.968,
"units": "F"
},
"feels_like_temperature": {
"value": 39.092,
"units": "F"
},
"relative_humidity": 56,
"precipitation": {
"precipitation_probability": 4,
"total_precipitation": {
"value": 0.0,
"units": "in"
}
},
"wind": {
"speed": {
"value": 5.144953471725125,
"units": "mi/h"
},
"direction": 25
},
"wind_gust": {
"value": 9.014853256979242,
"units": "mi/h"
},
"pressure": {
"value": 29.4171829577118,
"units": "inHg"
},
"visibility": {
"value": 6.835083114610673,
"units": "mi"
},
"dew_point": {
"value": 31.01,
"units": "F"
},
"cloud_cover": 31
},
{
"datetime": "2022-12-11T00:00:00-06:00",
"is_day_time": false,
"icon_code": 4,
"weather_text": "Clear with few low clouds",
"temperature": {
"value": 45.068,
"units": "F"
},
"feels_like_temperature": {
"value": 38.066,
"units": "F"
},
"relative_humidity": 56,
"precipitation": {
"precipitation_probability": 5,
"total_precipitation": {
"value": 0.0,
"units": "in"
}
},
"wind": {
"speed": {
"value": 5.167322834645669,
"units": "mi/h"
},
"direction": 27
},
"wind_gust": {
"value": 8.724051539012168,
"units": "mi/h"
},
"pressure": {
"value": 29.4213171559632,
"units": "inHg"
},
"visibility": {
"value": 5.592340730136005,
"units": "mi"
},
"dew_point": {
"value": 30.2,
"units": "F"
},
"cloud_cover": 13
},
{
"datetime": "2022-12-11T01:00:00-06:00",
"is_day_time": false,
"icon_code": 4,
"weather_text": "Clear with few low clouds",
"temperature": {
"value": 44.33,
"units": "F"
},
"feels_like_temperature": {
"value": 37.364,
"units": "F"
},
"relative_humidity": 56,
"precipitation": {
"precipitation_probability": 4,
"total_precipitation": {
"value": 0.0,
"units": "in"
}
},
"wind": {
"speed": {
"value": 4.988367931281317,
"units": "mi/h"
},
"direction": 28
},
"wind_gust": {
"value": 8.254294917680744,
"units": "mi/h"
},
"pressure": {
"value": 29.4165923579616,
"units": "inHg"
},
"visibility": {
"value": 7.456454306848007,
"units": "mi"
},
"dew_point": {
"value": 29.714,
"units": "F"
},
"cloud_cover": 22
}
],
"error": null
}
I have been working with the feeds API. I have been able to create a feed document, encrypt and upload a payload to the returned URL, create the feed, and then check its status. Once status reaches the DONE state, I was also able to get the processing report: The processing report looks like this:
{
"header": {
"sellerId": "A1HGLY0OQBKE5U",
"version": "2.0",
"feedId": "50042018828"
},
"issues": [
{
"messageId": 1,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 2,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 3,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 4,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 5,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 6,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 7,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 8,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 9,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 10,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 11,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 12,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 13,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 14,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 15,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 16,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 17,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 18,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
},
{
"messageId": 19,
"code": "4000003",
"severity": "ERROR",
"message": "The Amazon product type specified is invalid or not supported."
}
],
"summary": {
"errors": 19,
"warnings": 0,
"messagesProcessed": 19,
"messagesAccepted": 0,
"messagesInvalid": 19
}
}
The Payload was this:
{
"header": {
"sellerId": "A1HGLY0OQBKE5U",
"version": "2.0"
},
"messages": [
{
"messageId": 1,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "Awesome Bronze Car",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "sk77"
},
{
"messageId": 2,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "Sleek Steel Chair",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "sk80"
},
{
"messageId": 3,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "Aerodynamic Iron Gloves",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "sk81"
},
{
"messageId": 4,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "Sleek Copper Clock",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "sk86"
},
{
"messageId": 5,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "Mediocre Plastic Wallet",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "sk88"
},
{
"messageId": 6,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "[Sample] 1 L Le Parfait Jar",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "SLLPJ-6088C959"
},
{
"messageId": 7,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "[Sample] 1 L Le Parfait Jar",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "SLLPJ-20D88EFC"
},
{
"messageId": 8,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "[Sample] 1 L Le Parfait Jar",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "SLLPJ-0BAF36BA"
},
{
"messageId": 9,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "[Sample] 1 L Le Parfait Jar",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "SLLPJ-CD28D4F0"
},
{
"messageId": 10,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "[Sample] 1 L Le Parfait Jar",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "SLLPJ-9A544946"
},
{
"messageId": 11,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "[Sample] 1 L Le Parfait Jar",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "SLLPJ-F2FD045F"
},
{
"messageId": 12,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "[Sample] 1 L Le Parfait Jar",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "SLLPJ-0138EE43"
},
{
"messageId": 13,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "[Sample] 1 L Le Parfait Jar",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "SLLPJ-D334BA28"
},
{
"messageId": 14,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "[Sample] 1 L Le Parfait Jar",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "SLLPJ-8650D0FD"
},
{
"messageId": 15,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "[Sample] 1 L Le Parfait Jar",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "SLLPJ-8D93E1C3"
},
{
"messageId": 16,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "Gorgeous Copper Hat",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "sk94"
},
{
"messageId": 17,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "[Sample] Tiered Wire Basket",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "TWB"
},
{
"messageId": 18,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "Heavy Duty Cotton Gloves",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "sk98"
},
{
"messageId": 19,
"operationType": "UPDATE",
"productType": "SHOES",
"attributes": {
"item_name": [
{
"value": "Ergonomic Steel Lamp",
"language_tag": "en_US",
"marketplace_id": "A21TJRUUN4KGV"
}
],
"fulfillment_availability": [
{
"fulfillment_channel_code": "AMAZON_NA",
"quantity": 10
}
]
},
"sku": "sk103"
}
]
}
As can be observed, I am using the product type SHOES. In a previous attempt, I had used the product type LUGGAGE because that was used in the use-case guides. But Amazon still does not like this. Could someone guide me to a list of valid product types?
I've meet the same issue and solve this one, and I'm just sending https://sellingpartnerapi-na.amazon.com/definitions/2020-09-01/productTypes?marketplaceIds=A2EUQ1WTGCTBG2 to get the productType is PRODUCT.
You can reference with https://github.com/amzn/selling-partner-api-docs/blob/main/references/product-type-definitions-api/definitionsProductTypes_2020-09-01.md#searchdefinitionsproducttypes to find out what is the productType it is in the seller centre without keywords paramters.
Therefore you can try it again using "productType": "PRODUCT".
I have a big JSON file with a very complex structure
you can look on it here: https://drive.google.com/file/d/1tBVJ2xYSCpTTUGPJegvAz2ZXbeN0bteX/view?usp=sharing
it contains more than 7 millions lines, and I want to extract only the "text" field
I have written a python code, to extra all the values of the "text" key or field in the whole file, and it extracted only 12 values! while when I open the JSON file on the Visualstudio, I have more than 19000 values!!
you can see the code here:
import json
import csv
with open("/Users/zahraa-maher/rasa-init-demo/venv/Tickie/external_data/frames2.json") as file:
data = json.load(file)
fname = "outputText8.csv"
with open(fname, "w") as file:
csv_file = csv.writer(file,lineterminator='\n')
csv_file.writerow(["text"])
for item in data[i]["turns"]:
csv_file.writerow([item['text']])
please take a look on the JSON file as it is very large one and with a complex structure, so I an not paste it here to see because it would be not understandable
also this is a part of the son file:
[
{
"user_id": "U22HTHYNP",
"turns": [
{
"text": "I'd like to book a trip to Atlantis from Caprica on Saturday, August 13, 2016 for 8 adults. I have a tight budget of 1700.",
"labels": {
"acts": [
{
"args": [
{
"val": "book",
"key": "intent"
}
],
"name": "inform"
},
{
"args": [
{
"val": "Atlantis",
"key": "dst_city"
},
{
"val": "Caprica",
"key": "or_city"
},
{
"val": "Saturday, August 13, 2016",
"key": "str_date"
},
{
"val": "8",
"key": "n_adults"
},
{
"val": "1700",
"key": "budget"
}
],
"name": "inform"
}
],
"acts_without_refs": [
{
"args": [
{
"val": "book",
"key": "intent"
}
],
"name": "inform"
},
{
"args": [
{
"val": "Atlantis",
"key": "dst_city"
},
{
"val": "Caprica",
"key": "or_city"
},
{
"val": "Saturday, August 13, 2016",
"key": "str_date"
},
{
"val": "8",
"key": "n_adults"
},
{
"val": "1700",
"key": "budget"
}
],
"name": "inform"
}
],
"active_frame": 1,
"frames": [
{
"info": {
"intent": [
{
"val": "book",
"negated": false
}
],
"budget": [
{
"val": "1700.0",
"negated": false
}
],
"dst_city": [
{
"val": "Atlantis",
"negated": false
}
],
"or_city": [
{
"val": "Caprica",
"negated": false
}
],
"str_date": [
{
"val": "august 13",
"negated": false
}
],
"n_adults": [
{
"val": "8",
"negated": false
}
]
},
"frame_id": 1,
"requests": [],
"frame_parent_id": null,
"binary_questions": [],
"compare_requests": []
}
]
},
"author": "user",
"timestamp": 1471272019730.0
},
{
"db": {
"result": [
[
{
"trip": {
"returning": {
"duration": {
"hours": 0,
"min": 51
},
"arrival": {
"hour": 10,
"year": 2016,
"day": 24,
"min": 51,
"month": 8
},
"departure": {
"hour": 10,
"year": 2016,
"day": 24,
"min": 0,
"month": 8
}
},
"seat": "ECONOMY",
"leaving": {
"duration": {
"hours": 0,
"min": 51
},
"arrival": {
"hour": 0,
"year": 2016,
"day": 16,
"min": 51,
"month": 8
},
"departure": {
"hour": 0,
"year": 2016,
"day": 16,
"min": 0,
"month": 8
}
},
"or_city": "Porto Alegre",
"duration_days": 9
},
"price": 2118.81,
"hotel": {
"gst_rating": 7.15,
"vicinity": [],
"name": "Scarlet Palms Resort",
"country": "Brazil",
"amenities": [
"FREE_BREAKFAST",
"FREE_PARKING",
"FREE_WIFI"
],
"dst_city": "Goiania",
"category": "3.5 star hotel"
}
},
{
"trip": {
"returning": {
"duration": {
"hours": 2,
"min": 37
},
"arrival": {
"hour": 12,
"year": 2016,
"day": 10,
"min": 37,
"month": 8
},
"departure": {
"hour": 10,
"year": 2016,
"day": 10,
"min": 0,
"month": 8
}
},
"seat": "ECONOMY",
"leaving": {
"duration": {
"hours": 2,
"min": 37
},
"arrival": {
"hour": 0,
"year": 2016,
"day": 4,
"min": 37,
"month": 8
},
"departure": {
"hour": 22,
"year": 2016,
"day": 3,
"min": 0,
"month": 8
}
},
"or_city": "Porto Alegre",
"duration_days": 7
},
"price": 2369.83,
"hotel": {
"gst_rating": 0,
"vicinity": [],
"name": "Sunway Hostel",
"country": "Argentina",
"amenities": [
"FREE_BREAKFAST",
"FREE_WIFI"
],
"dst_city": "Rosario",
"category": "2.0 star hotel"
}
},
{
"trip": {
"returning": {
"duration": {
"hours": 0,
"min": 51
},
"arrival": {
"hour": 10,
"year": 2016,
"day": 24,
"min": 51,
"month": 8
},
"departure": {
"hour": 10,
"year": 2016,
"day": 24,
"min": 0,
"month": 8
}
},
"seat": "BUSINESS",
"leaving": {
"duration": {
"hours": 0,
"min": 51
},
"arrival": {
"hour": 0,
"year": 2016,
"day": 16,
"min": 51,
"month": 8
},
"departure": {
"hour": 0,
"year": 2016,
"day": 16,
"min": 0,
"month": 8
}
},
"or_city": "Porto Alegre",
"duration_days": 9
},
"price": 2375.72,
"hotel": {
"gst_rating": 7.15,
"vicinity": [],
"name": "Scarlet Palms Resort",
"country": "Brazil",
"amenities": [
"FREE_BREAKFAST",
"FREE_PARKING",
"FREE_WIFI"
],
"dst_city": "Goiania",
"category": "3.5 star hotel"
}
},
{
"trip": {
"returning": {
"duration": {
"hours": 1,
"min": 30
},
"arrival": {
"hour": 11,
"year": 2016,
"day": 1,
"min": 30,
"month": 9
},
"departure": {
"hour": 10,
"year": 2016,
"day": 1,
"min": 0,
"month": 9
}
},
"seat": "BUSINESS",
"leaving": {
"duration": {
"hours": 1,
"min": 30
},
"arrival": {
"hour": 18,
"year": 2016,
"day": 19,
"min": 30,
"month": 8
},
"departure": {
"hour": 17,
"year": 2016,
"day": 19,
"min": 0,
"month": 8
}
},
"or_city": "Porto Alegre",
"duration_days": 13
},
"price": 2492.95,
"hotel": {
"gst_rating": 0,
"vicinity": [],
"name": "Hotel Mundo",
"country": "Brazil",
"amenities": [
"FREE_BREAKFAST",
"FREE_WIFI",
"FREE_PARKING"
],
"dst_city": "Manaus",
"category": "2.5 star hotel"
}
},
{
"trip": {
"returning": {
"duration": {
"hours": 0,
"min": 51
},
"arrival": {
"hour": 10,
"year": 2016,
"day": 31,
"min": 51,
"month": 8
},
"departure": {
"hour": 10,
"year": 2016,
"day": 31,
"min": 0,
"month": 8
}
},
"seat": "ECONOMY",
"leaving": {
"duration": {
"hours": 0,
"min": 51
},
"arrival": {
"hour": 19,
"year": 2016,
"day": 27,
"min": 51,
"month": 8
},
"departure": {
"hour": 19,
"year": 2016,
"day": 27,
"min": 0,
"month": 8
}
},
"or_city": "Porto Alegre",
"duration_days": 4
},
"price": 2538.0,
"hotel": {
"gst_rating": 8.22,
"vicinity": [],
"name": "The Glee",
"country": "Brazil",
"amenities": [
"FREE_BREAKFAST",
"FREE_WIFI"
],
"dst_city": "Recife",
"category": "4.0 star hotel"
}
}
],
[],
[],
[],
[],
[],
[]
],
"search": [
{
"ORIGIN_CITY": "Porto Alegre",
"PRICE_MIN": "2000",
"NUM_ADULTS": "2",
"timestamp": 1471271949.995,
"PRICE_MAX": "3000",
"ARE_DATES_FLEXIBLE": "true",
"NUM_CHILDREN": "5",
"START_TIME": "1470110400000",
"MAX_DURATION": 2592000000.0,
"DESTINATION_CITY": "Brazil",
"RESULT_LIMIT": "10",
"END_TIME": "1472616000000"
},
{
"ORIGIN_CITY": "Atlantis",
"NUM_ADULTS": "8",
"RESULT_LIMIT": "10",
"timestamp": 1471272148.124,
"PRICE_MAX": "1700",
"NUM_CHILDREN": "",
"ARE_DATES_FLEXIBLE": "true",
"START_TIME": "NaN",
"END_TIME": "NaN"
},
{
"ORIGIN_CITY": "Caprica",
"PRICE_MAX": "1700",
"NUM_ADULTS": "8",
"RESULT_LIMIT": "10",
"timestamp": 1471272189.07,
"DESTINATION_CITY": "Atlantis",
"NUM_CHILDREN": "",
"ARE_DATES_FLEXIBLE": "true",
"START_TIME": "1470715200000",
"END_TIME": "1472011200000"
},
{
"ORIGIN_CITY": "Caprica",
"PRICE_MAX": "1700",
"NUM_ADULTS": "8",
"RESULT_LIMIT": "10",
"timestamp": 1471272205.436,
"DESTINATION_CITY": "Atlantis",
"NUM_CHILDREN": "",
"ARE_DATES_FLEXIBLE": "true",
"START_TIME": "1470715200000",
"END_TIME": "1472011200000"
},
{
"ORIGIN_CITY": "Caprica",
"PRICE_MIN": "1700",
"NUM_ADULTS": "8",
"RESULT_LIMIT": "10",
"timestamp": 1471272278.72,
"DESTINATION_CITY": "Atlantis",
"NUM_CHILDREN": "",
"ARE_DATES_FLEXIBLE": "true",
"START_TIME": "1470715200000",
"END_TIME": "1472011200000"
},
{
"ORIGIN_CITY": "Caprica",
"PRICE_MIN": "1700",
"NUM_ADULTS": "8",
"RESULT_LIMIT": "10",
"timestamp": 1471272454.542,
"DESTINATION_CITY": "Atlantis",
"NUM_CHILDREN": "",
"ARE_DATES_FLEXIBLE": "true",
"START_TIME": "1471060800000",
"END_TIME": "1472011200000"
},
{
"ORIGIN_CITY": "Caprica",
"PRICE_MIN": "1700",
"NUM_ADULTS": "8",
"RESULT_LIMIT": "10",
"timestamp": 1471272466.008,
"DESTINATION_CITY": "Atlantis",
"NUM_CHILDREN": "",
"ARE_DATES_FLEXIBLE": "true",
"START_TIME": "1471060800000",
"END_TIME": "1472011200000"
}
]
},
How it could be modified to extract all the "text" values from the JSON file to a CSV file?
This is a potential solution using pandas:
import pandas as pd
#importing data
dj = pd.read_json("frames2.json")
dtext = dj[["user_id","turns"]]
#Saving text records in a list
list_ = []
for record in dtext["turns"].values:
for r in record:
list_.append(r["text"])
#Exporting the csv
out = pd.Series(list_,name="text")
out.to_csv("text.csv")
It gives the following output.
Try:
import json
import csv
with open("/Users/zahraa-maher/rasa-init-demo/venv/Tickie/external_data/frames2.json") as file:
data = json.load(file)
fname = "outputText8.csv"
with open(fname, "w") as file:
csv_file = csv.writer(file,lineterminator='\n')
csv_file.writerow(["text"])
for keys,values in data.items():
now it up to you which of the fields you want to save, if you user a debugger you can see the values and Keys
{
"took": 0,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 3,
"max_score": 0.2876821,
"hits": [
{
"_index": "product_index",
"_type": "product",
"_id": "1115",
"_score": 0.2876821,
"_source": {
"isactive": true,
"in_use": false,
"brand_name": "Adidas",
"sku_id": "56456487987987",
"long_description": "this is long description",
"key_feature": [
{
"id": 1148,
"key_feature": "sport wear"
},
{
"id": 1147,
"key_feature": "Cotton shirt"
},
{
"id": 1146,
"key_feature": "White and blue"
}
],
"isdeleted": false,
"created_by": null,
"brand_id": 5,
"search_terms": [
{
"label": "white shirt",
"value": 9
}
]
"color_id": 2,
"specific_keywords": "",
"item_list": [
{
"item_id": 1114,
"product_id": 1115,
"isactive": true,
"id": 9,
"isdeleted": false
},
{
"item_id": 1113,
"product_id": 1115,
"isactive": true,
"id": 10,
"isdeleted": false
}
],
"upc_code": "",
"display_size": "L",
"name": "New White shirt",
"updated_by": null,
"id": 1115,
"updated_date": "2020-03-25T08:24:37.644571+00:00",
"color_name": "blue",
"created_date": "2020-03-25T08:11:14.966673+00:00",
"category": [
{
"parent_category_id": 78,
"sub_sub_category": null,
"sub_category": null,
"sub_category_id": null,
"sub_sub_category_id": null,
"parent_category": "new Shirt Cate",
"id": 1151
}
]
}
},
{
"_index": "product_index",
"_type": "product",
"_id": "1113",
"_score": 0.2876821,
"_source": {
"isactive": true,
"in_use": false,
"sku_id": "1456456488",
"brand_name": "Adidas",
"long_description": "",
"key_feature": [
{
"id": 1142,
"key_feature": "Cotton"
},
{
"id": 1141,
"key_feature": "Office Use"
},
{
"id": 1140,
"key_feature": "Black formal"
}
],
"isdeleted": false,
"created_by": null,
"brand_id": 5,
"search_terms": [
]
"color_id": 1,
"specific_keywords": "",
"item_list": [
],
"display_size": "L",
"upc_code": "",
"name": "New Cotton formal shirt black",
"updated_by": null,
"id": 1113,
"updated_date": "2020-03-25T06:48:30.903041+00:00",
"created_date": "2020-03-25T06:48:29.943043+00:00",
"color_name": "black",
"category": [
{
"sub_sub_category": null,
"parent_category_id": 54,
"sub_category": null,
"sub_category_id": null,
"sub_sub_category_id": null,
"parent_category": "MEN'S CLOTHING",
"id": 1149
}
]
}
},
{
"_index": "product_index",
"_type": "product",
"_id": "1114",
"_score": 0.2876821,
"_source": {
"isactive": true,
"in_use": false,
"sku_id": "145645648811",
"brand_name": "Adidas",
"long_description": "",
"key_feature": [
{
"id": 1145,
"key_feature": "Cotton"
},
{
"id": 1144,
"key_feature": "Office Use"
},
{
"id": 1143,
"key_feature": "Black formal"
}
],
"isdeleted": false,
"created_by": null,
"brand_id": 5,
"search_terms": [
],
"color_id": 1,
"specific_keywords": "",
"item_list": [
],
"display_size": "L",
"upc_code": "",
"updated_by": null,
"name": "New Cotton Casual shirt black",
"id": 1114,
"created_date": "2020-03-25T07:13:26.233675+00:00",
"color_name": "black",
"updated_date": "2020-03-25T07:13:27.229363+00:00",
"category": [
{
"sub_sub_category": null,
"parent_category_id": 54,
"sub_category": null,
"sub_category_id": null,
"sub_sub_category_id": null,
"parent_category": "MEN'S CLOTHING",
"id": 1150
}
]
}
}
]
}
}
my requirement is to attach all related documents with specific key value fields which is specify in item_list based on item_id. In above result doc id 1115 has item_list which contains item_id 1114 and 1113. so the particular fields attach in the doc 1115.
what should be the search query for that in elastic search?
You can't do join in Elasticsearch, to achieve your goal, you can do two things:
duplicate the information of item_id 1114 and 1113 in the item_id
1115 (and for sure in all others documents).
Do join at application level, so after this query you can extract the item_id 1114 and 1113 and run two others query to get the information about this items. Then join all the json at application level.
I have a scenario as depicted below in python code .
In this I am trying to explicitly define new york and ny as synonyms. But unfortunately it is not working. Can you please guide me as I am new to elastic search.
Also I am using custom analyzer.
I also have the file synonyms.txt having text:
ny,newyork,nyork
from datetime import datetime
from elasticsearch import Elasticsearch
es = Elasticsearch()
keywords = ['thousand eyes', 'facebook', 'superdoc', 'quora', 'your story', 'Surgery', 'lending club', 'ad roll',
'the honest company', 'Draft kings', 'newyork']
count = 1
doc_setting = {
"settings": {
"analysis": {
"analyzer": {
"my_analyzer_keyword": {
"type": "custom",
"tokenizer": "keyword",
"filter": [
"asciifolding",
"lowercase",
"synonym"
]
},
"my_analyzer_shingle": {
"type": "custom",
"tokenizer": "standard",
"filter": [
"asciifolding",
"lowercase",
"synonym"
]
}
},
"filter": {
"synonym": {
"type": "synonym",
"synonyms_path": "synonyms.txt",
"ignore_case": "true"
}
}
}
}, "mappings": {
"your_type": {
"properties": {
"keyword": {
"type": "string",
"index_analyzer": "my_analyzer_keyword",
"search_analyzer": "my_analyzer_shingle"
}
}
}
}
}
validate=es.index(index='test', doc_type='your_type', body=doc_setting)
print(validate)
for keyword in keywords:
doc = {
'id': count,
'keyword': keyword
}
res = es.index(index="test", doc_type='your_type', id=count, body=doc)
print(res['result'])
count = count + 1
#res11 = es.get(index="test", doc_type='your_type', id=1)
#print(res11['_source'])
es.indices.refresh(index="test")
question = "I saw news on ny news channel of lending club on facebook, your story and quora"
print("Question asked: %s" % question)
res = es.search(index="test",`enter code here` doc_type='your_type', body={
"query": {"match": {"keyword": question}}})
print("Got %d Hits:" % res['hits']['total'])
for hit in res['hits']['hits']:
print(hit["_source"])
PUT /test_index
{
"settings": {
"analysis": {
"analyzer": {
"my_analyzer_keyword": {
"type": "custom",
"tokenizer": "keyword",
"filter": [
"asciifolding",
"lowercase",
"synonym"
]
},
"my_analyzer_shingle": {
"type": "custom",
"tokenizer": "standard",
"filter": [
"asciifolding",
"lowercase",
"synonym"
]
}
},
"filter": {
"synonym" : {
"type" : "synonym",
"lenient": true,
"synonyms" : ["ny,newyork,nyork"]
}
}
}
}, "mappings": {
"your_type": {
"properties": {
"keyword": {
"type": "text",
"analyzer": "my_analyzer_keyword",
"search_analyzer": "my_analyzer_shingle"
}
}
}
}
}
Then Analyze using
POST /test_index/_analyze
{
"analyzer" : "my_analyzer_shingle",
"text" : "I saw news on ny news channel of lending club on facebook, your story and quorat"
}
The tokens I get are
{
"tokens": [
{
"token": "i",
"start_offset": 0,
"end_offset": 1,
"type": "<ALPHANUM>",
"position": 0
},
{
"token": "saw",
"start_offset": 2,
"end_offset": 5,
"type": "<ALPHANUM>",
"position": 1
},
{
"token": "news",
"start_offset": 6,
"end_offset": 10,
"type": "<ALPHANUM>",
"position": 2
},
{
"token": "on",
"start_offset": 11,
"end_offset": 13,
"type": "<ALPHANUM>",
"position": 3
},
{
"token": "ny",
"start_offset": 14,
"end_offset": 16,
"type": "<ALPHANUM>",
"position": 4
},
{
"token": "newyork",
"start_offset": 14,
"end_offset": 16,
"type": "SYNONYM",
"position": 4
},
{
"token": "nyork",
"start_offset": 14,
"end_offset": 16,
"type": "SYNONYM",
"position": 4
},
{
"token": "news",
"start_offset": 17,
"end_offset": 21,
"type": "<ALPHANUM>",
"position": 5
},
{
"token": "channel",
"start_offset": 22,
"end_offset": 29,
"type": "<ALPHANUM>",
"position": 6
},
{
"token": "of",
"start_offset": 30,
"end_offset": 32,
"type": "<ALPHANUM>",
"position": 7
},
{
"token": "lending",
"start_offset": 33,
"end_offset": 40,
"type": "<ALPHANUM>",
"position": 8
},
{
"token": "club",
"start_offset": 41,
"end_offset": 45,
"type": "<ALPHANUM>",
"position": 9
},
{
"token": "on",
"start_offset": 46,
"end_offset": 48,
"type": "<ALPHANUM>",
"position": 10
},
{
"token": "facebook",
"start_offset": 49,
"end_offset": 57,
"type": "<ALPHANUM>",
"position": 11
},
{
"token": "your",
"start_offset": 59,
"end_offset": 63,
"type": "<ALPHANUM>",
"position": 12
},
{
"token": "story",
"start_offset": 64,
"end_offset": 69,
"type": "<ALPHANUM>",
"position": 13
},
{
"token": "and",
"start_offset": 70,
"end_offset": 73,
"type": "<ALPHANUM>",
"position": 14
},
{
"token": "quorat",
"start_offset": 74,
"end_offset": 80,
"type": "<ALPHANUM>",
"position": 15
}
]
}
and the search produces
POST /test_index/_search
{
"query" : {
"match" : { "keyword" : "I saw news on ny news channel of lending club on facebook, your story and quora" }
}
}
{
"took": 36,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 3,
"max_score": 1.6858001,
"hits": [
{
"_index": "test_index",
"_type": "your_type",
"_id": "4",
"_score": 1.6858001,
"_source": {
"keyword": "newyork"
}
},
{
"_index": "test_index",
"_type": "your_type",
"_id": "2",
"_score": 1.1727304,
"_source": {
"keyword": "facebook"
}
},
{
"_index": "test_index",
"_type": "your_type",
"_id": "5",
"_score": 0.6931472,
"_source": {
"keyword": "quora"
}
}
]
}
}