We Keep Coding

list with a tuple of dicts instead of list of dicts

I have list of dicts
[{'id': 14786,
'sku': '0663370-ZWA',
'sizes': ['38', '40', '42', '44', '46'],
'color': 'zwart'},
{'id': 14787,
'sku': '0663371-ZWA',
'sizes': ['38', '40', '42', '44', '46'],
'color': 'zwart'}]
want to place it in a datastructure for update attributes with woocommerce api
list_of_update_items = []
for index in lst_of_dcts:
lst_of_attributes_items=[]
attributes = {'id': 1, 'name': 'kleur', 'position': 0,'options': index['color'], 'variations': 'false','visible': 'true'},{'id': 6, 'options': index['sizes'], 'variations': 'true','visible': 'true'}
# make a list of attributes_items to use as value in the parent dict
lst_of_attributes_items.append(attributes)
#make a dict with id as key and atribute_list as key
update = {'id': index['id'], 'attributes': lst_of_attributes_items}
list_of_update_items.append(update)
#clear the list of attributes_items for the next iteration
lst_of_attributes_items=[]
#wrap in the top_dict
data = {'update': list_of_update_items}
data results in
{'update': [{'id': 14786,
'attributes': [({'id': 1,
'name': 'kleur',
'position': 0,
'options': 'zwart',
'variations': 'false',
'visible': 'true'},
{'id': 6,
'options': ['38', '40', '42', '44', '46'],
'variations': 'true',
'visible': 'true'})]},
{'id': 14787,
'attributes': [({'id': 1,
'name': 'kleur',
'position': 0,
'options': 'zwart',
'variations': 'false',
'visible': 'true'},
{'id': 6,
'options': ['38', '40', '42', '44', '46'],
'variations': 'true',
'visible': 'true'})]}]}
makes a tuple of dicts in the list of 'attributes'_values. what I need is a list of dicts. What causes the tupple?

You use an intermediate:
lst_of_attributes_items=[]
Which you append a tuple of dicst to:
attributes = {'id': 1, 'name': 'kleur', 'position': 0,'options': index['color'], 'variations': 'false','visible': 'true'},{'id': 6, 'options': index['sizes'], 'variations': 'true','visible': 'true'}
list_of_update_items.append(update)
Notice,
attributes = {...}, {...}
Creates a tuple.
And then that list, with a single item (which is a tuple with two dicts) is set to the "attribute" key of your update dict:
update = {'id': index['id'], 'attributes': lst_of_attributes_items}
This doesn't make any sense, just make attributes a list instead of a tuple (since you dont want tuples... dont use tuples) then just put that directlyin the update dict. The whole thing can be simplified to:
lst_of_dcts = [{'id': 14786,
'sku': '0663370-ZWA',
'sizes': ['38', '40', '42', '44', '46'],
'color': 'zwart'},
{'id': 14787,
'sku': '0663371-ZWA',
'sizes': ['38', '40', '42', '44', '46'],
'color': 'zwart'}]
list_of_update_items = []
for index in lst_of_dcts:
attributes = [
{'id': 1, 'name': 'kleur', 'position': 0,'options': index['color'], 'variations': 'false','visible': 'true'},
{'id': 6, 'options': index['sizes'], 'variations': 'true','visible': 'true'}
]
list_of_update_items.append({'id': index['id'], 'attributes': attributes})
data = {'update': list_of_update_items}
Indeed, you could do the whole thing with a list comprehension (although, I probably wouldn't, since it isn't as readable to me):
update_items = [
{
"id": index["id"],
"attributes": [
{
"id": 1,
"name": "kleur",
"position": 0,
"options": index["color"],
"variations": "false",
"visible": "true",
},
{
"id": 6,
"options": index["sizes"],
"variations": "true",
"visible": "true",
},
],
}
for index in lst_of_dcts
]
Although, using a helper function makes it pretty nice:
def update_from_index(index):
return {
"id": index["id"],
"attributes": [
{
"id": 1,
"name": "kleur",
"position": 0,
"options": index["color"],
"variations": "false",
"visible": "true",
},
{
"id": 6,
"options": index["sizes"],
"variations": "true",
"visible": "true",
},
],
}
update_items = [update_from_index(index) for index in lst_of_dcts]

Partial word search not working in elasticsearch (elasticsearch-py) using mongo-connector

Currently I've indexed my mongoDB collection into Elasticsearch running in a docker container. I am able to query a document by it's exact name, but Elasticsearch is unable to match the query if it is only part of the name. Here is an example:
>>> es = Elasticsearch('0.0.0.0:9200')
>>> es.indices.get_alias('*')
{'mongodb_meta': {'aliases': {}}, 'sigstore': {'aliases': {}}, 'my-index': {'aliases': {}}}
>>> x = es.search(index='sigstore', body={'query': {'match': {'name': 'KEGG_GLYCOLYSIS_GLUCONEOGENESIS'}}})
>>> x
{'took': 198, 'timed_out': False, '_shards': {'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0}, 'hits': {'total': {'value': 1, 'relation': 'eq'}, 'max_score': 8.062855, 'hits': [{'_index': 'sigstore', '_type': 'sigs', '_id': '5d66c23228144432307c2c49', '_score': 8.062855, '_source': {'id': 1, 'name': 'KEGG_GLYCOLYSIS_GLUCONEOGENESIS', 'description': 'http://www.broadinstitute.org/gsea/msigdb/cards/KEGG_GLYCOLYSIS_GLUCONEOGENESIS', 'members': ['ACSS2', 'GCK', 'PGK2', 'PGK1', 'PDHB', 'PDHA1', 'PDHA2', 'PGM2', 'TPI1', 'ACSS1', 'FBP1', 'ADH1B', 'HK2', 'ADH1C', 'HK1', 'HK3', 'ADH4', 'PGAM2', 'ADH5', 'PGAM1', 'ADH1A', 'ALDOC', 'ALDH7A1', 'LDHAL6B', 'PKLR', 'LDHAL6A', 'ENO1', 'PKM2', 'PFKP', 'BPGM', 'PCK2', 'PCK1', 'ALDH1B1', 'ALDH2', 'ALDH3A1', 'AKR1A1', 'FBP2', 'PFKM', 'PFKL', 'LDHC', 'GAPDH', 'ENO3', 'ENO2', 'PGAM4', 'ADH7', 'ADH6', 'LDHB', 'ALDH1A3', 'ALDH3B1', 'ALDH3B2', 'ALDH9A1', 'ALDH3A2', 'GALM', 'ALDOA', 'DLD', 'DLAT', 'ALDOB', 'G6PC2', 'LDHA', 'G6PC', 'PGM1', 'GPI'], 'user': 'naji.taleb#medimmune.com', 'type': 'public', 'level1': 'test', 'level2': 'test2', 'time': '08-28-2019 14:03:29 EDT-0400', 'source': 'File', 'mapped': [''], 'notmapped': [''], 'organism': 'human'}}]}}
When using the full name of the document, elasticsearch is able to successfully query it. But this is what happens when I attempt to search part of the name or use a wildcard:
>>> x = es.search(index='sigstore', body={'query': {'match': {'name': 'KEGG'}}})
>>> x
{'took': 17, 'timed_out': False, '_shards': {'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0}, 'hits': {'total': {'value': 0, 'relation': 'eq'}, 'max_score': None, 'hits': []}}
>>> x = es.search(index='sigstore', body={'query': {'match': {'name': 'KEGG*'}}})
>>> x
{'took': 3, 'timed_out': False, '_shards': {'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0}, 'hits': {'total': {'value': 0, 'relation': 'eq'}, 'max_score': None, 'hits': []}}
In addition to the default index settings I also tried making an index that allows the use of the nGram tokenizer to enable me to do partial search, but that also didn't work. These are the settings I used for that index:
{
"sigstore": {
"aliases": {},
"mappings": {},
"settings": {
"index": {
"max_ngram_diff": "99",
"number_of_shards": "1",
"provided_name": "sigstore",
"creation_date": "1579200699718",
"analysis": {
"filter": {
"substring": {
"type": "nGram",
"min_gram": "1",
"max_gram": "20"
}
},
"analyzer": {
"str_index_analyzer": {
"filter": [
"lowercase",
"substring"
],
"tokenizer": "keyword"
},
"str_search_analyzer": {
"filter": [
"lowercase"
],
"tokenizer": "keyword"
}
}
},
"number_of_replicas": "1",
"uuid": "3nf915U6T9maLdSiJozvGA",
"version": {
"created": "7050199"
}
}
}
}
}
and this is the corresponding python command that created it:
es.indices.create(index='sigstore',body={"mappings": {},"settings": { 'index': { "analysis": {"analyzer": {"str_search_analyzer": {"tokenizer": "keyword","filter": ["lowercase"]},"str_index_analyzer": {"tokenizer": "keyword","filter": ["lowercase", "substring"]}},"filter": {"substring": {"type": "nGram","min_gram": 1,"max_gram": 20}}}},'max_ngram_diff': '99'}})
I use mongo-connector as the pipeline between my mongoDB collection and elasticsearch. This is the command I use to start it:
mongo-connector -m mongodb://username:password#xx.xx.xxx.xx:27017/?authSource=admin -t elasticsearch:9200 -d elastic2_doc_manager -n sigstore.sigs
I'm unsure as to why my elasticsearch is unable to get a partial match, and wondering if there is some setting I'm missing or if there's some crucial mistake I've made somewhere. Thanks for reading.
Versions
MongoDB 4.0.10
elasticsearch==7.1.0
elastic2-doc-manager[elastic5]

Updated after checked your gist:
You need to apply the mapping to your field as written in the doc, cf the first link I share in the comment.
You need to do it after applying the settings on your index according to the gist it's line 11.
Something like:
PUT /your_index/_mapping
{
"properties": {
"name": {
"type": "keyword",
"ignore_above": 256,
"fields": {
"str_search_analyzer": {
"type": "text",
"analyzer": "str_search_analyzer"
}
}
}
}
}
After you set the mapping need to apply it to your document, using update_by_query
https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-update-by-query.html
So you can continue to search with term search on your field name as it will be indexed with a keyword mapping (exact match) and on the sub_field name.str_search_analyzer with part of the word.
your_keyword = 'KEGG_GLYCOLYSIS_GLUCONEOGENESIS' OR 'KEGG*'
x = es.search(index='sigstore', body={'query': {'bool': {'should':[{'term': {'name': your_keyword}},
{'match': {'name.str_search_analyzer': your_keyword}}
]}}
})

how to make a valid JSON out of a string with non-quoted keys?

I have the followign python problem:
'{acorn: 15,acorn_type: 15,area_name: "London",beds_max: 1,beds_min: 1,branch_id: "21062",branch_name: "Realm Estates",brand_name: "Realm Estates",chain_free: false,company_id: "10832",country_code: "gb",county_area_name: "London",currency_code: "GBP",display_address: "Lancaster Mews, Wandsworth SW18",furnished_state: "unfurnished",group_id: "",has_epc: true,has_floorplan: false,incode: "1BA",is_retirement_home: false,is_shared_ownership: false,listing_condition: "pre-owned",listing_id: 45296714,listing_status: "to_rent",listings_category: "residential",location: "London",member_type: "agent",num_baths: 1,num_beds: 1,num_images: 5,num_recepts: 1,outcode: "SW18",post_town_name: "London",postal_area: "SW",price: 1300,price_actual: 1300,price_max: 1500,price_min: 1250,price_qualifier: "",property_highlight: "",property_type: "flat",region_name: "London",section: "to-rent",size_sq_feet: "",tenure: "",zindex: "686544"}'
I would like to json.load() this string, but clearly all the keys are wrong. it should be 'acorn: 15,... and so on. How could I go bout replacing all words before a : into a string? thanks

According to #daniel-roseman comment, it is better to use yaml to load your string into dict:
import yaml,json
string='{acorn: 15,acorn_type: 15,area_name: "London",beds_max: 1,beds_min: 1,branch_id: "21062",branch_name: "Realm Estates",brand_name: "Realm Estates",chain_free: false,company_id: "10832",country_code: "gb",county_area_name: "London",currency_code: "GBP",display_address: "Lancaster Mews, Wandsworth SW18",furnished_state: "unfurnished",group_id: "",has_epc: true,has_floorplan: false,incode: "1BA",is_retirement_home: false,is_shared_ownership: false,listing_condition: "pre-owned",listing_id: 45296714,listing_status: "to_rent",listings_category: "residential",location: "London",member_type: "agent",num_baths: 1,num_beds: 1,num_images: 5,num_recepts: 1,outcode: "SW18",post_town_name: "London",postal_area: "SW",price: 1300,price_actual: 1300,price_max: 1500,price_min: 1250,price_qualifier: "",property_highlight: "",property_type: "flat",region_name: "London",section: "to-rent",size_sq_feet: "",tenure: "",zindex: "686544"}'
dict_version = yaml.safe_load(string)
print dict_version
#if you need json version
json_version = json.dumps(dict_version)
print json_version
which give you:
{'listing_status': 'to_rent', 'outcode': 'SW18', 'brand_name': 'Realm Estates', 'member_type': 'agent', 'chain_free': False, 'area_name': 'London', 'is_shared_ownership': False, 'country_code': 'gb', 'beds_max': 1, 'property_type': 'flat', 'incode': '1BA', 'furnished_state': 'unfurnished', 'branch_id': '21062', 'listing_id': 45296714, 'has_floorplan': False, 'beds_min': 1, 'section': 'to-rent', 'company_id': '10832', 'price_min': 1250, 'zindex': '686544', 'location': 'London', 'county_area_name': 'London', 'acorn_type': 15, 'is_retirement_home': False, 'price_max': 1500, 'listing_condition': 'pre-owned', 'display_address': 'Lancaster Mews, Wandsworth SW18', 'region_name': 'London', 'price': 1300, 'num_images': 5, 'listings_category': 'residential', 'branch_name': 'Realm Estates', 'post_town_name': 'London', 'acorn': 15, 'num_recepts': 1, 'size_sq_feet': '', 'price_qualifier': '', 'postal_area': 'SW', 'property_highlight': '', 'price_actual': 1300, 'num_baths': 1, 'num_beds': 1, 'has_epc': True, 'tenure': '', 'group_id': '', 'currency_code': 'GBP'}
{"listing_status": "to_rent", "outcode": "SW18", "brand_name": "Realm Estates", "member_type": "agent", "chain_free": false, "area_name": "London", "is_shared_ownership": false, "country_code": "gb", "beds_max": 1, "property_type": "flat", "incode": "1BA", "furnished_state": "unfurnished", "branch_id": "21062", "listing_id": 45296714, "has_floorplan": false, "beds_min": 1, "section": "to-rent", "company_id": "10832", "price_min": 1250, "zindex": "686544", "location": "London", "county_area_name": "London", "acorn_type": 15, "is_retirement_home": false, "price_max": 1500, "listing_condition": "pre-owned", "display_address": "Lancaster Mews, Wandsworth SW18", "region_name": "London", "price": 1300, "num_images": 5, "listings_category": "residential", "branch_name": "Realm Estates", "post_town_name": "London", "acorn": 15, "num_recepts": 1, "size_sq_feet": "", "price_qualifier": "", "postal_area": "SW", "property_highlight": "", "price_actual": 1300, "num_baths": 1, "num_beds": 1, "has_epc": true, "tenure": "", "group_id": "", "currency_code": "GBP"}

Loop through multidimensional JSON (Python)

I have a JSON with following structure:
{
'count': 93,
'apps' : [
{
'last_modified_at': '2016-10-21T12:20:26Z',
'frequency_caps': [],
'ios': {
'enabled': True,
'push_enabled': False,
'app_store_id': 'bbb',
'connection_type': 'certificate',
'sdk_api_secret': '--'
},
'organization_id': '--',
'name': '---',
'app_id': 27,
'control_group_percentage': 0,
'created_by': {
'user_id': 'abc',
'user_name': 'def'
},
'created_at': '2016-09-28T11:41:24Z',
'web': {}
}, {
'last_modified_at': '2016-10-12T08:58:57Z',
'frequency_caps': [],
'ios': {
'enabled': True,
'push_enabled': True,
'app_store_id': '386304604',
'connection_type': 'certificate',
'sdk_api_secret': '---',
'push_expiry': '2018-01-14T08:24:09Z'
},
'organization_id': '---',
'name': '---',
'app_id': 87,
'control_group_percentage': 0,
'created_by': {
'user_id': '----',
'user_name': '---'
},
'created_at': '2016-10-12T08:58:57Z',
'web': {}
}
]
}
It's a JSON with two key-value-pairs. The second pair's value is a List of more JSON's.
For me it is too much information and I want to have a JSON like this:
{
'apps' : [
{
'name': 'Appname',
'app_id' : 1234,
'organization_id' : 'Blablabla'
},
{
'name': 'Appname2',
'app_id' : 5678,
'organization_id' : 'Some other Organization'
}
]
}
I want to have a JSON that only contains one key ("apps") and its value, which would be a List of more JSONs that only have three key-value-pairs..
I am thankful for any advice.
Thank you for your help!

#bishakh-ghosh I don't think you need to use the input json as string. It can be used straight as a dictionary. (thus avoid ast)
One more concise way :
# your original json
input_ = { 'count': 93, ... }
And here are the steps :
Define what keys you want to keep
slice_keys = ['name', 'app_id', 'organization_id']
Define the new dictionary as a slice on the slice_keys
dict(apps=[{key:value for key,value in d.items() if key in slice_keys} for d in input_['apps']])
And that's it.
That should yield the JSON formatted as you want, e.g
{
'apps':
[
{'app_id': 27, 'name': '---', 'organization_id': '--'},
{'app_id': 87, 'name': '---', 'organization_id': '---'}
]
}

This might be what you are looking for:
import ast
import json
json_str = """{
'count': 93,
'apps' : [
{
'last_modified_at': '2016-10-21T12:20:26Z',
'frequency_caps': [],
'ios': {
'enabled': True,
'push_enabled': False,
'app_store_id': 'bbb',
'connection_type': 'certificate',
'sdk_api_secret': '--'
},
'organization_id': '--',
'name': '---',
'app_id': 27,
'control_group_percentage': 0,
'created_by': {
'user_id': 'abc',
'user_name': 'def'
},
'created_at': '2016-09-28T11:41:24Z',
'web': {}
}, {
'last_modified_at': '2016-10-12T08:58:57Z',
'frequency_caps': [],
'ios': {
'enabled': True,
'push_enabled': True,
'app_store_id': '386304604',
'connection_type': 'certificate',
'sdk_api_secret': '---',
'push_expiry': '2018-01-14T08:24:09Z'
},
'organization_id': '---',
'name': '---',
'app_id': 87,
'control_group_percentage': 0,
'created_by': {
'user_id': '----',
'user_name': '---'
},
'created_at': '2016-10-12T08:58:57Z',
'web': {}
}
]
}"""
json_dict = ast.literal_eval(json_str)
new_dict = {}
app_list = []
for appdata in json_dict['apps']:
appdata_dict = {}
appdata_dict['name'] = appdata['name']
appdata_dict['app_id'] = appdata['app_id']
appdata_dict['organization_id'] = appdata['organization_id']
app_list.append(appdata_dict)
new_dict['apps'] = app_list
new_json_str = json.dumps(new_dict)
print(new_json_str) # This is your resulting json string

Can't access dictionary data in boto3

I'm using boto3, and I ran this loop:
for i in x["Instances"]
print(i)
Then I get:
{
'AmiLaunchIndex': 0,
'Hypervisor': 'xen',
'VpcId': 'vpc-a790ac1',
'Architecture': 'x86_64',
'InstanceId': 'i-0bab3fb8314',
'PrivateDnsName': 'ip-10-c2.internal',
'BlockDeviceMappings': [{
'Ebs': {
'DeleteOnTermination': True,
'AttachTime': datetime.datetime(2017, 4, 4, 20, 44, 27, tzinfo = tzutc()),
'VolumeId': 'vol-07fd506f45',
'Status': 'attached'
},
'DeviceName': '/dev/xvda'
}, {
'Ebs': {
'DeleteOnTermination': False,
'AttachTime': datetime.datetime(2017, 4, 6, 1, 12, 45, tzinfo = tzutc()),
'VolumeId': 'vol-01ef36c45',
'Status': 'attached'
},
'DeviceName': '/dev/sdf'
}],
'RootDeviceName': '/dev/xvda',
'InstanceType': 't2.micro',
'EnaSupport': True,
'ClientToken': 'ODrMT1465413',
'EbsOptimized': False,
'SubnetId': 'subnet-fb1a4',
'Monitoring': {
'State': 'disabled'
},
'PublicDnsName': '',
'StateTransitionReason': 'User initiated (2017-04-06 01:15:22 GMT)',
'PrivateIpAddress': '10.10.4.116',
'RootDeviceType': 'ebs',
'Tags': [{
'Value': 'wp2',
'Key': 'Name'
}, {
'Value': 'true',
'Key': 'backup'
}],
'ImageId': 'ami-0976f01f',
'StateReason': {
'Code': 'Client.UserInitiadShutdown',
'Message': 'Client.UserInitiatedShutdown: User initiated shutdown'
},
'KeyName': 'pair2',
'ProductCodes': [],
'State': {
'Name': 'stopped',
'Code': 80
},
'LaunchTime': datetime.datetime(2017, 4, 6, 1, 13, 1, tzinfo = tzutc()),
'Placement': {
'AvailabilityZone': 'us-east-1b',
'GroupName': '',
'Tenancy': 'default'
},
'SourceDestCheck': True,
'NetworkInterfaces': [{
'Description': 'Primary network interface',
'PrivateIpAddress': '10.10.4.116',
'PrivateIpAddresses': [{
'Primary': True,
'PrivateIpAddress': '10.10.4.116'
}],
'Status': 'in-use',
'SubnetId': 'subnet-ffbcba4',
'VpcId': 'vpc-a790a7c1',
'Attachment': {
'DeleteOnTermination': True,
'AttachTime': datetime.datetime(2017, 4, 4, 20, 44, 26, tzinfo = tzutc()),
'DeviceIndex': 0,
'AttachmentId': 'eni-attach-c8398',
'Status': 'attached'
},
'Ipv6Addresses': [],
'OwnerId': '895548',
'MacAddress': '0e:31:4c4:b6',
'Groups': [{
'GroupId': 'sg-26c59',
'GroupName': 'web-dmz'
}],
'NetworkInterfaceId': 'eni-5383',
'SourceDestCheck': True
}],
'SecurityGroups': [{
'GroupId': 'sg-2cab59',
'GroupName': 'web-dmz'
}],
'VirtualizationType': 'hvm'
}
I'm trying to access the 'VolumeId' using something like:
for x in ["BlockDeviceMappings"][0]["Ebs"]["VolumeId"]:
print(x)
I get TypeError: string indices must be integers
It looks like 'BlockDeviceMappings' starts as a list with a dictionary in it, but I can't get to 'VolumeId'.
I've also tried:
for x in ["BlockDeviceMappings"][0]:
for k,v in ["Ebs"]:
print(v)
And I get:
ValueError: too many values to unpack (expected 2)
And I tried:
for x in ["BlockDeviceMappings"][0]:
for v in ["Ebs"]:
print(v)
Which prints 'Ebs' several times.
Could someone please point me in the right direction?

To get VolumeId please use
print x["Instances"][0]["BlockDeviceMappings"][0]["Ebs"]["VolumeId"]
you just missed x or _.
You are getting an error because ["BlockDeviceMappings"][0] evalutates to "B".
So you trying getting "orb" from "B"
To get all volumes:
for i in x["Instances"]:
for b in i["BlockDeviceMappings"]
print b["Ebs"]["VolumeId"]
If you have to extract data from complex sturctures like that often, try some quirky search library like github.com/akesterson/dpath-python , it can extract data just using keywords