I have files with the following structure:
{
"function": "ComAl_Set_nad_crtl_xcall_state",
"timeStamp": 1488500329974,
"Param1": "SIG_NAD_XCALL_ATTEMPTS_COUNT",
"Value1": "2"
}
These JSON files are created by some functions which I have in my program. But I have an issue getting the last value from these files (Value1). Currently this is the code I am using to get data from the file:
def get_json_from_stub(self, file_name):
def jsonize_stub(raw_data):
end = raw_data.rfind(",")
parsed_data = "[" + raw_data[:end] + "]"
return json.loads(parsed_data.replace("\00", ""))
command = "'cat " + self.stub_path + file_name + "'"
content = self.send_ssh_command(command)
json_stub = jsonize_stub(content)
return json_stub
and this is the code for getting Value1:
#app.route('/stub/comal/getSignal/ComAl_Set_nad_crtl_xcall_requests', methods=['GET'])
def get_nad_crtl_xcall_requests():
file_name = "ComAl_Set_nad_crtl_xcall_requests.out"
json_stub = self.stubManager.get_json_from_stub(file_name)
return MapEcallRequests().tech_to_business(json_stub[-1]["Value1"])
more specifically I want to replace json_stub[-1]["Value1"] with another way of getting Value1. The problem is that sometimes these files don´t get written so I would like to get Value1 in a different way and to raise an error message in case Value1 isn´t there, just to avoid my application crashing in case the value is not there. Is there are way to do it? Thanks.
You can check if the key exists (you can also check if the length is correct):
if len(json_stub) > 0 and json_stub[-1].get('Value1') is not None:
value1_node = json_stub[-1]('Value1')
else:
# 'Value1' key does not exist
Related
Here's the code I'm trying to run. Works when executed from PyCharm. I set up a cronjob and it worked wonders for weeks. It's now giving a KeyError out of the bloom. Can't find what's wrong since I haven't touched anything in the cronjob.
import csv
import json
import os
import random
import time
from urllib.parse import urlencode
import requests
API_URL = "https://community.koodomobile.com/widget/pointsLeaderboard?"
LEADERBOARD_FILE = "leaderboard_data.json"
def get_leaderboard(period: str = "allTime", max_results: int = 20) -> list:
payload = {"period": period, "maxResults": max_results}
return requests.get(f"{API_URL}{urlencode(payload)}").json()
def dump_leaderboard_data(leaderboard_data: dict) -> None:
with open("leaderboard_data.json", "w") as jf:
json.dump(leaderboard_data, jf, indent=4, sort_keys=True)
def read_leaderboard_data(data_file: str) -> dict:
with open(data_file) as f:
return json.load(f)
def parse_leaderboard(leaderboard: list) -> dict:
return {
item["name"]: {
"id": item["id"],
"score_data": {
time.strftime("%Y-%m-%d %H:%M"): item["points"],
},
"rank": item["leaderboardPosition"],
} for item in leaderboard
}
def update_leaderboard_data(target: dict, new_data: dict) -> dict:
for player, stats in new_data.items():
target[player]["rank"] = stats["rank"]
target[player]["score_data"].update(stats["score_data"])
return target
def leaderboard_to_csv(leaderboard: dict) -> None:
data_rows = [
[player] + list(stats["score_data"].values())
for player, stats in leaderboard.items()
]
random_player = random.choice(list(leaderboard.keys()))
dates = list(leaderboard[random_player]["score_data"])
with open("the_data.csv", "w") as output:
w = csv.writer(output)
w.writerow([""] + dates)
w.writerows(data_rows)
def script_runner():
if os.path.isfile(LEADERBOARD_FILE):
fresh_data = update_leaderboard_data(
target=read_leaderboard_data(LEADERBOARD_FILE),
new_data=parse_leaderboard(get_leaderboard()),
)
leaderboard_to_csv(fresh_data)
dump_leaderboard_data(fresh_data)
else:
dump_leaderboard_data(parse_leaderboard(get_leaderboard()))
if __name__ == "__main__":
script_runner()
Here's the error taht it gives me. Seems like there's a problem with dictionary hence the KeyError.
File "/Users/Rob/PycharmProjects/Koodo/TEST.Json.py", line 75, in <module>
script_runner()
File "/Users/Rob/PycharmProjects/Koodo/TEST.Json.py", line 64, in script_runner
fresh_data = update_leaderboard_data(
File "/Users/Rob/PycharmProjects/Koodo/TEST.Json.py", line 44, in update_leaderboard_data
target[player]["rank"] = stats["rank"]
KeyError: 'triggered123'
Here's the data in JSON file : https://pastebin.com/HQyL4Kyx
It looks to me like the first time your code is run, it caches the leaderboard.
On subsequent runs, it will update the existing leaderboard with the new values, by looping over each of the new entries, finding their username, and looking up that key in the older dictionary. However, if there's a new user, that key will not exist in the old dictionary.
You're getting the error because the player triggered123 is a new user to the leaderboard, and was listed after you first ran the script.
You need to update update_leaderboard_data to handle this case (by checking if the key exists in the dictionary before attempting to access it).
That's because the key named triggered123 is NOT present in the dictionary target.
Source: https://wiki.python.org/moin/KeyError
The problem is that in update_leaderboard_data you iterate over the new data and use them to lookup in the old data. If a new key does not already exists in the old data, you'll get the KeyError.
Try to print the dict target to see if the key triggered123 is in it.
Or use the get method of the dict with a default value: so no error wil be raised, and you'll be able to search this default value in your output.
I try to pass a variable to nested JSON in Python script.
Script as below,
import requests, request
group = request.form['grp']
zon = request.form['zone']
load = { "extra_vars": {
"g_name": "' +str(group)+ '",
"z_name": "' +str(zon)+ '"
}
}
----
--
-
However when i post the value to the API, it seem i post word '+str(group)+' and '+str(zon)+' instead the actual value that assign under declared variable.
Since i'm very new in Python programming, does passing value to nested JSON is allow in Python?
Try the following:
group = request.form['grp']
zon = request.form['zone']
load = { "extra_vars": {
"g_name": f"{group}",
"z_name": f"{zon}"
}
}
You can pass variables into a string using f-strings and brackets around your variable (note {group}):
>>> group = "my_group"
>>> {"g_name": f"'{group}'"}
{'g_name': "'my_group'"}
Or doing simple string concatenation also, which is what you almost done in your code (but just did not properly close the ' character using "'":
>>> "'" + str(group) + "'"
"'my_group'"
All in all here's your code adapted:
load = { "extra_vars": {
"g_name": f"'{group}'",
"z_name": f"'{zon}'"
}
}
Yes, yet another. I can't figure out what the issue is. I'm trying to iterate over a list that is a subsection of JSON output from an API call.
This is the section of JSON that I'm working with:
[
{
"created_at": "2017-02-22 17:20:29 UTC",
"description": "",
"id": 1,
"label": "FOO",
"name": "FOO",
"title": "FOO",
"updated_at": "2018-12-04 16:37:09 UTC"
}
]
The code that I'm running that retrieves this and displays it:
#!/usr/bin/python
import json
import sys
try:
import requests
except ImportError:
print "Please install the python-requests module."
sys.exit(-1)
SAT_API = 'https://satellite6.example.com/api/v2/'
USERNAME = "admin"
PASSWORD = "password"
SSL_VERIFY = False # Ignore SSL for now
def get_json(url):
# Performs a GET using the passed URL location
r = requests.get(url, auth=(USERNAME, PASSWORD), verify=SSL_VERIFY)
return r.json()
def get_results(url):
jsn = get_json(url)
if jsn.get('error'):
print "Error: " + jsn['error']['message']
else:
if jsn.get('results'):
return jsn['results']
elif 'results' not in jsn:
return jsn
else:
print "No results found"
return None
def display_all_results(url):
results = get_results(url)
if results:
return json.dumps(results, indent=4, sort_keys=True)
def main():
orgs = display_all_results(KATELLO_API + "organizations/")
for org in orgs:
print org
if __name__ == "__main__":
main()
I appear to be missing a concept because when I print org I get each character per line such as
[
{
"
c
r
e
a
t
e
d
_
a
t
"
It does this through to the final ]
I've also tried to print org['name'] which throws the TypeError: list indices must be integers, not str Python error. This makes me think that org is being seen as a list rather than a dictionary which I thought it would be due to the [{...}] format.
What concept am I missing?
EDIT: An explanation for why I'm not getting this: I'm working with a script in the Red Hat Satellite API Guide which I'm using to base another script on. I'm basically learning as I go.
display_all_results is returning a string since you are doing json.dumps in json.dumps(results, indent=4, sort_keys=True), which converts the dictionary to a string (you are getting that dictionary from r.json() in get_json function)
You then end up iterating over the characters of that string in main, and you see one character per line
Instead just return results from display_all_results and the code will work as intended
def display_all_results(url):
#results is already a dictionary, just return it
results = get_results(url)
if results:
return results
Orgs is a result of json.dump which produces a string. So instead of this code:
for org in orgs:
print(org)
replace it with simply:
#for org in orgs:
print(orgs)
I am trying to do reverse geocoding and extract pincodes for lot-long. The .csv file has around 1 million records..
Below is my problem
1. Google API failing to give address for large records, and taking huge amount of time. I will later move it to Batch-Process though.
2. I tried to split the file into chunks and ran few files manually one by one (1000 records in each file after splitting), then i surprisingly get 100% result.
3. Later, I ran in loop one by one, again, Google API fails to give the result
Note: Right now we are looking for free API's only
**Below is my code**
def reverse_geocode(latlng):
result = {}
url = 'https://maps.googleapis.com/maps/api/geocode/json?latlng={}'
request = url.format(latlng)
key= '&key=' + api_key
request = request + key
data = requests.get(request).json()
if len(data['results']) > 0:
result = data['results'][0]
return result
def parse_postal_code(geocode_data):
if (not geocode_data is None) and ('formatted_address' in geocode_data):
for component in geocode_data['address_components']:
if 'postal_code' in component['types']:
return component['short_name']
return None
dfinal = pd.DataFrame(columns=colnames)
dmiss = pd.DataFrame(columns=colnames)
for fl in files:
df = pd.read_csv(fl)
print ('Processing file : ' + fl[36:])
df['geocode_data'] = ''
df['Pincode'] = ''
df['geocode_data']=df['latlng'].map(reverse_geocode)
df['Pincode'] = df['geocode_data'].map(parse_postal_code)
if (len(df[df['Pincode'].isnull()]) > 0):
d0=df[df['Pincode'].isnull()]
print("Missing Picodes : " + str(len(df[df['Pincode'].isnull()])) + " / " + str(len(df)))
dmiss.append(d0)
d0=df[~df['Pincode'].isnull()]
dfinal.append(d0)
else:
dfinal.append(df)
Can anybody help me out, what is the problem in my code? or if any additional info required please let me know....
You've run into Google API usage limits.
Basically what I am trying to do is generate a json list of SSH keys (public and private) on a server using Python. I am using nested dictionaries and while it does work to an extent, the issue lies with it displaying every other user's keys; I need it to list only the keys that belong to the user for each user.
Below is my code:
def ssh_key_info(key_files):
for f in key_files:
c_time = os.path.getctime(f) # gets the creation time of file (f)
username_list = f.split('/') # splits on the / character
user = username_list[2] # assigns the 2nd field frome the above spilt to the user variable
key_length_cmd = check_output(['ssh-keygen','-l','-f', f]) # Run the ssh-keygen command on the file (f)
attr_dict = {}
attr_dict['Date Created'] = str(datetime.datetime.fromtimestamp(c_time)) # converts file create time to string
attr_dict['Key_Length]'] = key_length_cmd[0:5] # assigns the first 5 characters of the key_length_cmd variable
ssh_user_key_dict[f] = attr_dict
user_dict['SSH_Keys'] = ssh_user_key_dict
main_dict[user] = user_dict
A list containing the absolute path of the keys (/home/user/.ssh/id_rsa for example) is passed to the function. Below is an example of what I receive:
{
"user1": {
"SSH_Keys": {
"/home/user1/.ssh/id_rsa": {
"Date Created": "2017-03-09 01:03:20.995862",
"Key_Length]": "2048 "
},
"/home/user2/.ssh/id_rsa": {
"Date Created": "2017-03-09 01:03:21.457867",
"Key_Length]": "2048 "
},
"/home/user2/.ssh/id_rsa.pub": {
"Date Created": "2017-03-09 01:03:21.423867",
"Key_Length]": "2048 "
},
"/home/user1/.ssh/id_rsa.pub": {
"Date Created": "2017-03-09 01:03:20.956862",
"Key_Length]": "2048 "
}
}
},
As can be seen, user2's key files are included in user1's output. I may be going about this completely wrong, so any pointers are welcomed.
Thanks for the replies, I read up on nested dictionaries and found that the best answer on this post, helped me solve the issue: What is the best way to implement nested dictionaries?
Instead of all the dictionaries, I simplfied the code and just have one dictionary now. This is the working code:
class Vividict(dict):
def __missing__(self, key): # Sets and return a new instance
value = self[key] = type(self)() # retain local pointer to value
return value # faster to return than dict lookup
main_dict = Vividict()
def ssh_key_info(key_files):
for f in key_files:
c_time = os.path.getctime(f)
username_list = f.split('/')
user = username_list[2]
key_bit_cmd = check_output(['ssh-keygen','-l','-f', f])
date_created = str(datetime.datetime.fromtimestamp(c_time))
key_type = key_bit_cmd[-5:-2]
key_bits = key_bit_cmd[0:5]
main_dict[user]['SSH Keys'][f]['Date Created'] = date_created
main_dict[user]['SSH Keys'][f]['Key Type'] = key_type
main_dict[user]['SSH Keys'][f]['Bits'] = key_bits