I'm trying to get details from PostgreSQL about some cameras and I need to insert them all in one JSON response, but I can't imagine how it should be done, because for row in self.data: processes one line per time, how can I add them all in one JSON dump?
I imagine JSON dump like this:
{
"status": "ok",
"total_cameras": 3,
"cameras":[{
"camera_id" : 1,
"camera_name" : "hikvision 1",
"camera_ip": "42.51.56.0"
},
{
"camera_id" : 2,
"camera_name" : "hikvision 2",
"camera_ip": "42.51.56.5"
},
{
"camera_id" : 3,
"camera_name" : "hikvision 3",
"camera_ip": "2.1.58.5"
}]
}
My code which I use to get information from PostgreSQL :
if not self.Data:
self.RES = {'status': 'nocameras'}
return web.Response(text=json.dumps(self.RES), status=403)
else:
self.rows = self.cursor.rowcount
for row in self.Data:
if self.rows > 1:
# Authorizing objects
print(row)
self.Camera_ID = row[0]
self.Camera_Name = row[1]
self.Camera_LAT = row[3]
self.Camera_LOG = row[4]
self.Camera_IP = row[2]
self.Camera_Last_Updated = row[6]
self.Camera_Street = row[5]
self.Camera_API_key = row[7]
print(self.Camera_ID, self.Camera_Name)
else:
self.RES = {'status': 'row_error'}
return web.Response(text=json.dumps(self.RES), status=500)
I would first use the returned rows to build a list of dictionaries:
cameras = [
dict(
camera_id=c_id,
camera_name=c_name,
camera_ip=c_ip
) for c_id, c_name, _, _, c_ip, *_ in self.Data
]
And then create the final JSON object:
json.dumps({
'status': 'ok',
'total_cameras': len(cameras),
'cameras': cameras
})
Related
I'm trying to convert flat structure csv into nested json structure.
I have some data like :
State SubRegion Postcode Suburb
ACT South Canberra 2620 Oaks Estate
ACT North Canberra 2601 Acton
ACT North Canberra 2602 Ainslie
ACT Gungahlin-Hall 2914 Amaroo
I want desired output like this :
[
{
"name":"ACT",
"regions":[
{
"name":"South Canberra",
"suburbs":[
{
"postcode":"2620",
"name":"Oaks Estate"
}
]
},
{
"name":"North Canberra",
"suburbs":[
{
"postcode":"2601",
"name":"Acton"
},
{
"postcode":"2602",
"name":"Ainslie"
}
]
},
{
"name":"Gungahlin-Hall",
"suburbs":[
{
"postcode":"2914",
"name":"Amaroo"
}
]
}
]
}
]
I'm trying to get this structure using pandas and normal script but didn't get the correct structure yet.
I have solved this problem. Here is the solution :
def getindex(convertedList, value):
ivd = -1
for index, item in enumerate(convertedList):
# print("line 7 : ", item, value)
if item['name'] == value:
ivd = index
break
else:
ivd = -1
return ivd
with open('Regions.csv', 'r') as file:
reader = csv.reader(file)
mainData = []
loopIndex = 0
for row in reader:
if loopIndex > 0:
index = getindex(mainData, row[0])
if index > -1:
subindex = getindex(mainData[index]['regions'], row[1])
if subindex > -1:
suburbObj = {
'postcode' : row[3],
'name' : row[4]
}
mainData[index]['regions'][subindex]['suburbs'].append(suburbObj)
else :
regionObj = {
"name" : row[1],
"suburbs" : [{
"name" : row[4],
"postCode" : row[3]
}]
}
mainData[index]['regions'].append(regionObj)
else :
stateObj = {
'name' : row[0],
'regions' : [{
"name" : row[1],
"suburbs" : [{
"name" : row[4],
"postCode" : row[3]
}]
}]
}
mainData.append(stateObj)
loopIndex = loopIndex + 1
If anyone has any better-optimized code, you can post your solutions.
Thanks
i think this should work
import csv
import json
def add_new_region(name, postcode, name2):
d = {"name" : name,
"suburbs" : [add_suburb(postcode, name2)]
}
return d
def add_suburb(postcode, name):
return {"postcode" : postcode,
"name" : name
}
datalist=[]
region_dict={}
region_dict_counter = 0
with open("data.csv", "r") as f:
data = csv.reader(f)
next(data) # skip headers
for row in data:
if row[0] in region_dict.keys():
for x in (datalist[region_dict[row[0]]])["regions"]:
if x["name"] == row[1]:
(x["suburbs"]).append(add_suburb(row[2], row[3]))
break
else :
datalist[region_dict[row[0]]]["regions"].append(add_new_region(row[1], row[2], row[3]))
else:
d = { "name" : row[0],
"regions" : [ add_new_region(row[1], row[2], row[3])]}
datalist.append(d)
region_dict[row[0]] = region_dict_counter
region_dict_counter+=1
json_data=json.dumps(datalist, indent=4)
print(json_data)
with open("data.json", "w") as j:
j.write(json_data)
here are my data :
{
"data": [
{
"date": 1577836800000,
"#NOTIFICATION_SENT": 62629,
"#NOTIFICATION_OPENED": 404
},
{
"date": 1577923200000,
"#NOTIFICATION_OPENED": 734
}
]
}
How can i add all #NOTIFICATION_OPENED" key to get all the notif opened in a same strings ? If it's not possible, how can I select only the first key "#NOTIFICATION_OPENED" ?
With my code, I print the value of the last duplicate key.
Here is my code :
def create_json(id, notificationSent, notificationOpened):
return {(id):{
'id': id,
'notificationSent': notificationSent,
'notificationOpened': notificationOpened,
}}
statUrl = 'myapiurl'
with urlopen (statUrl) as response: sourcedata = response.read()
statdata = json.loads (sourcedata)
def push_data():
newJsonx = dict()
for item in data["data"]:
for item in statdata["data"]:
try:
notificationOpened = item["#NOTIFICATION_OPENED"]
except:
notificationOpened = '0'
print(notificationOpened)
try:
notificationSent = item["#NOTIFICATION_SENT"]
except:
notificationSent = '0'
# JSON DATA
newJson = create_json(notificationSent, notificationOpened)
newJsonx.update(newJson)
with open('myfile.json', 'w', encoding='utf-8') as json_file:
json.dump(newJsonx, json_file, ensure_ascii=False, sort_keys=True, indent=2)
print('JSON: %s' % json.dumps(newJsonx, sort_keys=True, indent=2))
push_data()
# ...snipped for brevity...
with urlopen(statUrl) as response:
sourcedata = response.read()
statdata = json.loads(sourcedata)
print(statdata)
# should be this format as in your question post
# {
# "data": [{
# "date": 1577836800000,
# "#NOTIFICATION_SENT": 62629,
# "#NOTIFICATION_OPENED": 404
# },
# {
# "date": 1577923200000,
# "#NOTIFICATION_OPENED": 734
# }]
# }
notificationSent = []
notificationOpened = []
for i, d in enumerate(statdata['data']):
notificationOpened.append(d.get('#NOTIFICATION_OPENED', 0))
notificationSent.append(d.get('#NOTIFICATION_SENT', 0))
print(sum(notificationOpened))
1138
Here's a variant of your original code that will create the dictionary you're after.
# create data
statdata = {
"data": [
{
"date": 1577836800000,
"#NOTIFICATION_SENT": 62629,
"#NOTIFICATION_OPENED": 404
},
{
"date": 1577923200000,
"#NOTIFICATION_OPENED": 734
}
]
}
# use your create_json function
def create_json(id, notificationSent, notificationOpened):
return {(id):{
'id': id,
'notificationSent': notificationSent,
'notificationOpened': notificationOpened,
}}
# define an ID value (needed for the create_json function)
id = 0
# initialise output dictionary
newJsonx = dict()
# add items to the dictionary
for elem in statdata["data"]:
for item in elem:
try:
notificationOpened = elem["#NOTIFICATION_OPENED"]
except:
notificationOpened = '0'
print(notificationOpened)
try:
notificationSent = elem["#NOTIFICATION_SENT"]
except:
notificationSent = '0'
newJson = create_json(id, notificationSent, notificationOpened)
newJsonx.update(newJson)
id =+ 1
This gives:
newJsonx
{0: {'id': 0, 'notificationSent': 62629, 'notificationOpened': 404}, 1: {'id': 1, 'notificationSent': '0', 'notificationOpened': 734}}
When I run the following code I get this error.
{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name "album_id": Proto field is not repeating, cannot start list.', 'status': 'INVALID_ARGUMENT', 'details': [{'#type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'description': 'Invalid JSON payload received. Unknown name "album_id": Proto field is not repeating, cannot start list.'}]}]}}
If I remove the "albumId": ["albumid code"] it works fine and returns
10 new items, total 10
def _actually_list_media_items(session):
ret = []
params = {
'fields': 'mediaItems(id,baseUrl,filename,mimeType,productUrl),nextPageToken',
}
search_json = {
"pageSize": 10,
"albumId": ["<albumid code>"],
"filters": {
"includeArchivedMedia": False,
"contentFilter": {
"excludedContentCategories": [
"DOCUMENTS",
"RECEIPTS",
"SCREENSHOTS",
"UTILITY",
"WHITEBOARDS",
]
},
"mediaTypeFilter": {
"mediaTypes": [
"PHOTO",
],
},
},
}
tmp = 0
while tmp < 1:
rsp = session.post(
'https://photoslibrary.googleapis.com/v1/mediaItems:search',
params=params,
json=search_json,
).json()
if 'error' in rsp:
print(rsp)
cur = [m for m in rsp.get('mediaItems', [])]
ret += cur
print(f'{len(cur)} new items, total {len(ret)}')
pageToken = rsp.get('nextPageToken')
if pageToken is None:
break
params['pageToken'] = pageToken
tmp = tmp + 1
return ret
The comment about albumId and filters being exclusive is correct, so you need to pick one or the other. However, assuming you want to use the albumId by itself, you need to remove the square brackets around your albumid code, here's a clip from my code:
searchbody = {
"albumId": album_id,
"pageSize": 10
}
print(searchbody)
mediaresults = gAPIservice.mediaItems().search(body=searchbody).execute()
mediaitems = mediaresults.get('mediaItems', [])
for item in mediaitems:
print(u'{0} ({1})'.format(item['filename'], item['id']))
Edit:
Apparently you can't use albumId and filters together: source
filters: object(Filters)
Filters to apply to the request. Can't be set in conjunction with an albumId.
Aside from that, albumId is a supposed to be a string not an array: source
"albumId": "<albumid code>",
I have some json data loaded in MongoDb. e.g. doc1 = {"id": 1,"name": "x1"},doc2 = {"id": 2,"name": "x2"},doc3 = {"id": 3,"name": "x3"}. Now I want this data to import from MongoDb to Elasticsearch. I wrote this piece of code.
mgclient = MongoClient()
db = mgclient['light-test']
col = db['test']
es1 = Elasticsearch()
print ("Connected", es1.info())
es1.indices.create(index='light-test', ignore=400)
# Pull from mongo and dump into ES using bulk API
actions = []
for data in tqdm(col.find(), total=col.count()):
data.pop('_id')
action = {
"_index": 'light-test',
"_type": 'test',
"_source": data
}
actions.append(action)
print("complete")
# Dump x number of objects at a time
if len(actions) >= 100:
deque(parallel_bulk(es1, actions), maxlen=0)
actions = []
print("done")
a = es1.search(index='light-test', body={
'query': {
'match_all': {
}
}
})
print(a)
The problem is in the query returned. The hits shows blank whereas it should had returned the json files.
results
Help me in importing the data from MongoDb to Elasticsearch.
app = Flask(__name__)
MONGO_URL = '...'
mgclient = MongoClient(MONGO_URL, ssl=True, ssl_cert_reqs=ssl.CERT_NONE)
db = mgclient['light']
col = db['task']
doc1 = {...}
doc2 = {...}
doc3 = {...}
post_id = col.insert_many([doc1, doc2, doc3])
print(col.count())
es1 = Elasticsearch(...)
ESinfo=(es1.info())
# Pull from mongo and dump into ES using bulk API
actions = []
for data in tqdm(col.find(), total=col.count()):
data.pop('_id')
action = {
"index": {
"_index": 'light',
"_type": 'task',
}
}
actions.append(action)
actions.append(data)
#delete = es1.indices.delete(index = 'light')
request_body = {
"settings" : {
"number_of_shards": 1,
"number_of_replicas": 0
}
}
es1.indices.create(index='light', body = request_body, ignore=400)
res = es1.bulk(index = 'light', body = actions, refresh = True)
result = col.find()
names = []
for obj in col.find():
name = obj['name']
names.append(name)
print(names)
#app.route('/query')
def Query():
a = es1.search(index='light', body={
'query': {
'match': {
'name': '...',
}
}
})
return jsonify(query=a)
if __name__ == "__main__":
app.run(host='0.0.0.0', port=1024)
This has helped. thank you :)
I do not understand why I get this error Bytes_Written is in the dataset but why can't python find it? I am getting this information(see dataset below) from a VM, I want to select Bytes_Written and Bytes_Read and then subtract the previous values from current value and print a json object like this
{'Bytes_Written': previousValue-currentValue, 'Bytes_Read': previousValue-currentValue}
here is what the data looks like:
{
"Number of Devices": 2,
"Block Devices": {
"bdev0": {
"Backend_Device_Path": "/dev/disk/by-path/ip-192.168.26.1:3260-iscsi-iqn.2010-10.org.openstack:volume-d1c8e7c6-8c77-444c-9a93-8b56fa1e37f2-lun-010.0.0.142",
"Capacity": "2147483648",
"Guest_Device_Name": "vdb",
"IO_Operations": "97069",
"Bytes_Written": "34410496",
"Bytes_Read": "363172864"
},
"bdev1": {
"Backend_Device_Path": "/dev/disk/by-path/ip-192.168.26.1:3260-iscsi-iqn.2010-10.org.openstack:volume-b27110f9-41ba-4bc6-b97c-b5dde23af1f9-lun-010.0.0.146",
"Capacity": "2147483648",
"Guest_Device_Name": "vdb",
"IO_Operations": "93",
"Bytes_Written": "0",
"Bytes_Read": "380928"
}
}
}
This is the complete code that I am running.
FIELDS = ("Bytes_Written", "Bytes_Read", "IO_Operation")
def counterVolume_one(state):
url = 'http://url'
r = requests.get(url)
data = r.json()
for field in FIELDS:
state[field] += data[field]
return state
state = {"Bytes_Written": 0, "Bytes_Read": 0, "IO_Operation": 0}
while True:
counterVolume_one(state)
time.sleep(1)
for field in FIELDS:
print("{field:s}: {count:d}".format(field=field, count=state[field]))
counterVolume_one(state)
Your returned JSON structure does not have any of these FIELDS = ("Bytes_Written", "Bytes_Read", "IO_Operation") keys directly.
You'll need to modify your code slightly.
data = r.json()
for block_device in data['Block Devices'].iterkeys():
for field in FIELDS:
state[field] += int(data['Block Devices'][block_device][field])