I have a django form that checks if a postcode is valid and if valid it will perform a check against postcodes that are permitted for delivery. I cant get the second condition working in the nested if structure (works fine independently). This is the code:
from django import forms
import requests
class PostCodeForm (forms.Form):
pcode = forms.CharField()
def clean_pcode(self):
pcode = self.cleaned_data['pcode'].lower()
permitted = {'gu15','GF34','FG34','BT25'}
url = 'https://api.postcodes.io/postcodes/{}/validate'.format(pcode)
r = requests.get(url)
is_correct = r.json()['result']
if not is_correct:
raise forms.ValidationError("Your postcode is invalid. Please re-enter a valid entry.")
if not pcode[:4] in (permitted):
raise forms.ValidationError("Apologies, but does not currently deliver to you postcode.")
return pcode
Fix your indentation, so that the second if statment and return statement are outside of the first if statement.
def clean_pcode(self):
pcode = self.cleaned_data['pcode'].lower()
permitted = {'gu15','GF34','FG34','BT25'}
url = 'https://api.postcodes.io/postcodes/{}/validate'.format(pcode)
r = requests.get(url)
is_correct = r.json()['result']
if not is_correct:
raise forms.ValidationError("Your postcode is invalid. Please re-enter a valid entry.")
if not pcode[:4] in (permitted):
raise forms.ValidationError("Apologies, but does not currently deliver to you postcode.")
return pcode
As an aside, you should change the slicing from pcode[:4] to pcode[:-3]. This means that your code will still work if you add 3-character post districts (e.g. GU1) to your permitted set.
Also, since you call lower(), you need to make sure all your entries in permitted are lowercase. Personally, I would use upper() for normalising postcodes.
Related
I try to get data from a webpage. This page contains several release information, but allow values not to be set. I.e. the date for testing from/to might be an empty string.
Now I try to deserialize all my data sucked from the page to insert it to a database and face problems handling empty dates.
from marshmallow import fields, Schema, ValidationError
class TestSchema(Schema):
training_necessary = fields.Function(deserialize=lambda x: True if x == 'Yes' else False)
test_from = fields.Date()
test_to = fields.Date()
data = dict(training_necessary='Yes', test_from='', test_to='')
try:
validated = TestSchema().load(data)
except ValidationError as err:
print(f"{err}")
Result:
{'test_to': ['Not a valid date.'], 'test_from': ['Not a valid date.']}
I already tried several combinations of allow_none=True or default='' but none of them helped my to get through. So, how to manage to allow empty dates? Setting a default to somewhat like 1970-01-01 won't help in that case.
Any hints?
Regards, Thomas
+++ EDIT: SOLUTION +++
Here's the working code I ended up after Jérômes helpful tipp:
from marshmallow import fields, Schema, ValidationError, pre_load
class TestSchema(Schema):
training_necessary = fields.Function(deserialize=lambda x: True if x == 'Yes' else False)
test_from = fields.Date(allow_none=True)
test_to = fields.Date(allow_none=True)
#pre_load(pass_many=False)
def string_to_none(self, data, many, **kwargs):
turn_to_none = lambda x: None if x == '' else x
for k, v in data.items():
data[k] = turn_to_none(v)
return data
data = dict(training_necessary='Yes', test_from='', test_to='')
try:
validated = TestSchema().load(data)
except ValidationError as err:
print(f"{err}")
I would pass no value at all.
data = dict(training_necessary='Yes')
Or I'd make the date fields allow_none and I'd pass None, not an empty string.
data = dict(training_necessary='Yes', test_from=None, test_to=None)
If the issue is that your input contains empty strings, I'd say this is a client issue, but you can add a pre_load method to delete empty strings from the input before deserializing. This is more or less equivalent to modifying the values you scrape from the page before feeding them to marshmallow.
I am trying to use two Google API calls to get a restaurant's price_level and phone number.
First, looping through
for restaurant in name:
find_place_url = "https://maps.googleapis.com/maps/api/place/findplacefromtext/json?"
# use separate parameter dictionary b.c. findplace and findplacedetail have diff field.
find_place_param ={}
find_place_param["input"] = restaurant
find_place_param["inputtype"] = "textquery"
find_place_param["key"] = google_key
# get place_id then use it to get phone number
a = requests.get(find_place_url, parameters).json()
this is first findplace api used to grab place_id for given restaurant. It will look like:
{'candidates': [{'place_id': 'ChIJdTDCTdT4cUgRqxush2XhgnQ'}], 'status': 'OK'}
if given restaurant has proper place_id or else it will give:
{'candidates': [], 'status': 'ZERO_RESULTS'}
now this is all of my code: from here I grab place_id however put it in try and except because as stated above status is either zero or ok. But even if I go pass except it will run find_place_detail api call which requires place_id thus it fails. How can I skip last block of code if I do not receive place_id?
price_level2 = []
phone_number = []
for restaurant in name:
find_place_url = "https://maps.googleapis.com/maps/api/place/findplacefromtext/json?"
# use separate parameter dictionary b.c. findplace and findplacedetail have diff field.
find_place_param ={}
find_place_param["input"] = restaurant
find_place_param["inputtype"] = "textquery"
find_place_param["key"] = google_key
# get place_id then use it to get phone number
a = requests.get(find_place_url, parameters).json()
print(a)
# adding it to original parameter. since only this and findplace parameter has to be different.
try:
parameters["place_id"] = a["candidates"][0]["place_id"]
except:
print("Phone number not available")
phone_number.append(None)
# passing in fields of our interest
parameters["fields"] = "name,price_level,formatted_phone_number"
find_place_detail_url ="https://maps.googleapis.com/maps/api/place/details/json?"
b = requests.get(find_place_detail_url, parameters).json()
phone_number.append(b["result"]["formatted_phone_number"])
price_level2.append(b["result"]['price_level'])
You can use an else clause:
try:
parameters["place_id"] = a["candidates"][0]["place_id"]
except KeyError:
print("Phone number not available")
phone_number.append(None)
else:
parameters["fields"] = "name,price_level,formatted_phone_number"
find_place_detail_url ="https://maps.googleapis.com/maps/api/place/details/json?"
b = requests.get(find_place_detail_url, parameters).json()
...
Also, your except clause should be more specific (I guess the case you're trying to catch is a KeyError). For more information on exception handling in Python, see the documentation.
I am trying to check if I have an entry in my database using this code:
def device_update(request):
json_data = json.loads(request.body)
email = json_data['email']
imei = json_data['imei']
sdk_version = json_data['sdk_version']
date = json_data['updateDate']
rule = json_data['ruleName']
group_name = json_data['group']
if Group.objects.filter(group=group_name).exists():
print("group does exists")
else:
print("group doesn't exists")
return HttpResponse("Successful")
However, when the code reaches the if statement to check if the group exists, it returns error 500.
I tried to check with two groups one that exists and another one that doesn't, in both cases I got error 500.
How can I fix this and why is this happening?
The logic for checking if a Group exists, i.e. the line:
if Group.objects.filter(group=group_name).exists()
is not throwing the error here. It is likely that json_data is missing one of the keys you expect it to have, for example, 'group'.
I'd recommend using the get method that dictionaries have. This provides default values when the specified key is not present in the dictionary. You should also have error handling for when the request body is not in valid JSON format.
Here's an example:
def device_update(request):
try:
json_data = json.loads(request.body)
except json.JSONDecodeError:
return HttpResponse('Request body must be in valid JSON format')
email = json_data.get('email', '')
imei = json_data.get('imei', '')
sdk_version = json_data.get('sdk_version', '')
date = json_data.get('updateDate', '')
rule = json_data.get('ruleName', '')
group_name = json_data.get('group', '')
if Group.objects.filter(group=group_name).exists():
print("group does exists")
else:
print("group doesn't exists")
return HttpResponse("Successful")
I set the defaults to the empty string '', but you may want to change that.
Your view doesn't have any error handling. Looking at it quickly, at least two things could go wrong. The request body might not be valid json, and if it is valid json, it might not contain the required keys.
def device_update(request):
try:
json_data = json.loads(request.body)
except ValueError:
return HttpResponse("Invalid json")
try:
email = json_data['email']
imei = json_data['imei']
sdk_version = json_data['sdk_version']
date = json_data['updateDate']
rule = json_data['ruleName']
group_name = json_data['group']
except KeyError as e:
return HttpResponse("Missing Key %s" % e[0])
...
Writing your own validation for a single view like this is ok. As it gets more complicated, you might want to look at django rest framework. It has serializers which will help you manage validation.
Alasdair/Keselme, looks that your view is correct.
Try to put the ipdb into your code in order to debug your code, and than you can print the request.data and see what is comming in the request.
Im working on a small project of retrieving information about books from the Google Books API using Python 3. For this i make a call to the API, read out the variables and store those in a list. For a search like "linkedin" this works perfectly. However when i enter "Google", it reads the second title from the JSON input. How can this happen?
Please find my code below (Google_Results is the class I use to initialize the variables):
import requests
def Book_Search(search_term):
parms = {"q": search_term, "maxResults": 3}
r = requests.get(url="https://www.googleapis.com/books/v1/volumes", params=parms)
print(r.url)
results = r.json()
i = 0
for result in results["items"]:
try:
isbn13 = str(result["volumeInfo"]["industryIdentifiers"][0]["identifier"])
isbn10 = str(result["volumeInfo"]["industryIdentifiers"][1]["identifier"])
title = str(result["volumeInfo"]["title"])
author = str(result["volumeInfo"]["authors"])[2:-2]
publisher = str(result["volumeInfo"]["publisher"])
published_date = str(result["volumeInfo"]["publishedDate"])
description = str(result["volumeInfo"]["description"])
pages = str(result["volumeInfo"]["pageCount"])
genre = str(result["volumeInfo"]["categories"])[2:-2]
language = str(result["volumeInfo"]["language"])
image_link = str(result["volumeInfo"]["imageLinks"]["thumbnail"])
dict = Google_Results(isbn13, isbn10, title, author, publisher, published_date, description, pages, genre,
language, image_link)
gr.append(dict)
print(gr[i].title)
i += 1
except:
pass
return
gr = []
Book_Search("Linkedin")
I am a beginner to Python, so any help would be appreciated!
It does so because there is no publisher entry in volumeInfo of the first entry, thus it raises a KeyError and your except captures it. If you're going to work with fuzzy data you have to account for the fact that it will not always have the expected structure. For simple cases you can rely on dict.get() and its default argument to return a 'valid' default entry if an entry is missing.
Also, there are a few conceptual problems with your function - it relies on a global gr which is bad design, it shadows the built-in dict type and it captures all exceptions guaranteeing that you cannot exit your code even with a SIGINT... I'd suggest you to convert it to something a bit more sane:
def book_search(search_term, max_results=3):
results = [] # a list to store the results
parms = {"q": search_term, "maxResults": max_results}
r = requests.get(url="https://www.googleapis.com/books/v1/volumes", params=parms)
try: # just in case the server doesn't return valid JSON
for result in r.json().get("items", []):
if "volumeInfo" not in result: # invalid entry - missing volumeInfo
continue
result_dict = {} # a dictionary to store our discovered fields
result = result["volumeInfo"] # all the data we're interested is in volumeInfo
isbns = result.get("industryIdentifiers", None) # capture ISBNs
if isinstance(isbns, list) and isbns:
for i, t in enumerate(("isbn10", "isbn13")):
if len(isbns) > i and isinstance(isbns[i], dict):
result_dict[t] = isbns[i].get("identifier", None)
result_dict["title"] = result.get("title", None)
authors = result.get("authors", None) # capture authors
if isinstance(authors, list) and len(authors) > 2: # you're slicing from 2
result_dict["author"] = str(authors[2:-2])
result_dict["publisher"] = result.get("publisher", None)
result_dict["published_date"] = result.get("publishedDate", None)
result_dict["description"] = result.get("description", None)
result_dict["pages"] = result.get("pageCount", None)
genres = result.get("authors", None) # capture genres
if isinstance(genres, list) and len(genres) > 2: # since you're slicing from 2
result_dict["genre"] = str(genres[2:-2])
result_dict["language"] = result.get("language", None)
result_dict["image_link"] = result.get("imageLinks", {}).get("thumbnail", None)
# make sure Google_Results accepts keyword arguments like title, author...
# and make them optional as they might not be in the returned result
gr = Google_Results(**result_dict)
results.append(gr) # add it to the results list
except ValueError:
return None # invalid response returned, you may raise an error instead
return results # return the results
Then you can easily retrieve as much info as possible for a term:
gr = book_search("Google")
And it will be far more tolerant of data omissions, provided that your Google_Results type makes most of the entries optional.
Following #Coldspeed's recommendation it became clear that missing information in the JSON file caused the exception to run. Since I only had a "pass" statement there it skipped the entire result. Therefore I will have to adapt the "Try and Except" statements so errors do get handled properly.
Thanks for the help guys!
After running my script I notice that my "parse_doc" function throws error when it find's any url None. Turn out that, my "process_doc" function were supposed to produce 25 links but it produces only 19 because few pages doesn't have any link to lead to another page. However, when my second function receives that link with None value, it produces that error indicating "MissingSchema". How to get around this so that when it finds any link with None value it will go for another. Here is the partial portion of my script which will give you an idea what I meant:
def process_doc(medium_link):
page = requests.get(medium_link).text
tree = html.fromstring(page)
try:
name = tree.xpath('//span[#id="titletextonly"]/text()')[0]
except IndexError:
name = ""
try:
link = base + tree.xpath('//section[#id="postingbody"]//a[#class="showcontact"]/#href')[0]
except IndexError:
link = ""
parse_doc(name, link) "All links get to this function whereas some links are with None value
def parse_doc(title, target_link):
page = requests.get(target_link).text # Error thrown here when it finds any link with None value
tel = re.findall(r'\d{10}', page)[0] if re.findall(r'\d{10}', page) else ""
print(title, tel)
The error what I'm getting:
raise MissingSchema(error)
requests.exceptions.MissingSchema: Invalid URL '': No schema supplied. Perhaps you meant http://?
Btw, in my first function there is a variable named "base" which is for concatenating with the produced result to make a full-fledged link.
If you want to avoid cases when your target_link == None then try
def parse_doc(title, target_link):
if target_link:
page = requests.get(target_link).text
tel = re.findall(r'\d{10}', page)[0] if re.findall(r'\d{10}', page) else ""
print(tel)
print(title)
This should allow you to handle only non-empty links or do nothing otherwise
First of all make sure that your schema, meaning url, is correct. Sometimes you are just missing a character or have one too much in https://.
If you have to raise an exception though you can do it like this:
import requests
from requests.exceptions import MissingSchema
...
try:
res = requests.get(linkUrl)
print(res)
except MissingSchema:
print('URL is not complete')