Following code will query the table and result in a JSON as output
import boto3
import json
import re
def lambda_handler(event, context):
dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table('master')
response = table.scan()
data = response['Items']
while 'LastEvaluatedKey' in response:
response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
data.extend(response['Items'])
return {
'statusCode': 200,
'headers': {
'Access-Control-Allow-Origin' : '*',
},
'body': json.dumps(data)
}
My JSON response contains a \r.
[
{
"r_dt": "29-Oct-18",
"de_dt": "31-Dec-99\r",
"v_status": "R",
"v_num": "M13020"
},
{
"r_dt": "29-Oct-18",
"de_dt": "31-Dec-99\r",
"v_status": "R",
"v_num": "O03873"
}
}
How do I remove \r from from my JSON response? I tried JSON.loads/RegEx, but it didn't work
How to remove \r from from my json reposnse. I tried json.loads/regex
, didnt work
Your input data contains \r so it ends up in JSON response. So, instead of "fixing" JSON output, you have to fix input data. If that's not possible, you have to sanitize data as early as possible, before serializing it to JSON:
def sanitize(item):
item['de_dt'] = item['de_dt'].rstrip('\r')
# OR
item['de_dt'] = item['de_dt'].replace('\r', '')
return item
# ...
while 'LastEvaluatedKey' in response:
response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
data.extend(sanitize(item) for item in response['Items'])
import boto3
import json
import re
def lambda_handler(event, context):
dynamodb = boto3.resource('dynamodb', region_name='ap-southeast-1')
table = dynamodb.Table('dsl_vehicle_master')
response = table.scan()
data =[sanitize(item) for item in response['Items']]
while 'LastEvaluatedKey' in response:
response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
data.extend(sanitize(item) for item in response['Items'])
return {
'statusCode': 200,
'headers': {
'Access-Control-Allow-Origin' : '*',
},
'body': json.dumps(data)
}
def sanitize(item):
item['dereg_dt'] = item['dereg_dt'].rstrip('\r\n')
# OR
#item['dereg_dt'] = item['dereg_dt'].replace('\r', '')
return item
Related
I am setting up a lambda function to put data into a db. I am using API gateway to control the endpoint.
The request I am making sends the data as a string:
fetch(
url,
{
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({ name: "hello", contact: "Tim", phone: "555" }),
}
);
My lambda function keeps erroring out because it doesn't know how to handle the string.
My function is:
import re
import psycopg2
import os
import json
def lambda_handler(event, context):
password = os.environ['DB_SECRET']
host = os.environ['HOST']
connection = psycopg2.connect(user="postgres", password=password,
host=host, port="5432",
database="postgres")
cursor = connection.cursor()
postgres_insert_query = "INSERT INTO clients (name, phone, contact) VALUES ('{0}','{1}','{2}')".format(event.get('name'), event.get('phone'), event.get('contact'))
cursor.execute(postgres_insert_query)
print(cursor.rowcount)
print(event['body'])
connection.commit()
return {
'statusCode': 200,
'headers': {
"Access-Control-Allow-Origin" : "*"
},
}
Now my print statement print(event['body']) returns '{"name":"hello","contact":"Tim","phone":"555"}' which is the exact data I need. However for the life of me I can't figure out how to get the data out of the string.
import json
data = json.loads(event['body'])
print(data['name'])
I am trying to send email from lambda by passing variable in the html body inside payload(in send_email()).I tried using +str(latesturgency)+ and also using {latestimpact} .This doesn.t work.I am new to lambda and python.
How can I solve this?
import json
import logging
import re
import http.client
import mimetypes
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
def send_email(email,latestdescription,latestservice,latestimpact,latesturgency):
conn = http.client.HTTPSConnection("mail.us-east-1.aws.cloud.bmw")
payload = {
"from":"xxx#yyy.com",
"to": "xxx#yyy.com",
"subject": "Test mail","textbody": "body",
"htmlbody": "<h3>Test body!Please create a +str(latesturgency)+ priority ticket to {latestservice} , with below message:{latestdescription}</h3>"
}
print(payload)
headers = {
'Content-Type': 'application/json'
}
data=json.dumps(payload)
print(data)
conn.request("POST", "", data, headers)
res = conn.getresponse()
data = res.read()
print(data.decode("utf-8"))
print("done")
def lambda_handler(event, context):
empidemployee= event['currentIntent']["slots"]["empidemployee"]
latestdescription= event['currentIntent']["slots"]["latestdescription"]
latestservice= event['currentIntent']["slots"]["latestservice"]
latestimpact= event['currentIntent']["slots"]["latestimpact"]
latesturgency= event['currentIntent']["slots"]["latesturgency"]
email=event['currentIntent']["slots"]["email"]
send_email(email,latestdescription,latestservice,latestimpact,latesturgency)
You can do this using the format function fro your string, I have added this below.
import json
import logging
import re
import http.client
import mimetypes
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
def send_email(email,latestdescription,latestservice,latestimpact,latesturgency):
conn = http.client.HTTPSConnection("mail.us-east-1.aws.cloud.bmw")
payload = {
"from":"xxx#yyy.com",
"to": "xxx#yyy.com",
"subject": "Test mail","textbody": "body",
"htmlbody": "<h3>Test body!Please create a {} priority ticket to {}, with below message:{}</h3>".format(latesturgency, latestservice, latestdescription)
}
print(payload)
headers = {
'Content-Type': 'application/json'
}
data=json.dumps(payload)
print(data)
conn.request("POST", "", data, headers)
res = conn.getresponse()
data = res.read()
print(data.decode("utf-8"))
print("done")
def lambda_handler(event, context):
empidemployee= event['currentIntent']["slots"]["empidemployee"]
latestdescription= event['currentIntent']["slots"]["latestdescription"]
latestservice= event['currentIntent']["slots"]["latestservice"]
latestimpact= event['currentIntent']["slots"]["latestimpact"]
latesturgency= event['currentIntent']["slots"]["latesturgency"]
email=event['currentIntent']["slots"]["email"]
send_email(email,latestdescription,latestservice,latestimpact,latesturgency)
Use the {} notation to indicate where a variable should be replace then use .format() after your string passing in the variables in the ordering you will use them in.
In your case it is now
"<h3>Test body!Please create a {} priority ticket to {}, with below message:{}</h3>".format(latesturgency, latestservice, latestdescription)
For more information about the format function take a read in the documentation.
I'm trying to parse a website with the requests module:
import requests
some_data = {'a':'',
'b':''}
with requests.Session() as s:
result = s.post('http://website.com',data=some_data)
print(result.text)
The page is responding as below:
{
"arrangetype":"U",
"list": [
{
"product_no":43,
"display_order":4,
"is_selling":"T",
"product_empty":"F",
"fix_position":null,
"is_auto_sort":false
},
{
"product_no":44,
"display_order":6,
"is_selling":"T",
"product_empty":"F",
"fix_position":null,
"is_auto_sort":false
}
],
"length":2
}
I found that instead of parsing full HTML, it would be better to deal with the response as all the data I want is in that response.
What I want to get is a list of the values of product_no, so the expected result is:
[43,44]
How do I do this?
Convert your JSON response to a dictionary with json.loads(), and collect your results in a list comprehension.
Demo:
from json import loads
data = """{
"arrangetype":"U",
"list": [
{
"product_no":43,
"display_order":4,
"is_selling":"T",
"product_empty":"F",
"fix_position":null,
"is_auto_sort":false
},
{
"product_no":44,
"display_order":6,
"is_selling":"T",
"product_empty":"F",
"fix_position":null,
"is_auto_sort":false
}
],
"length":2
}"""
json_dict = loads(data)
print([x['product_no'] for x in json_dict['list']])
# [43, 44]
Full Code:
import requests
from json import loads
some_data = {'a':'',
'b':''}
with requests.Session() as s:
result = s.post('http://website.com',data=some_data)
json_dict = loads(result.text)
print([x["product_no"] for x in json_dict["list"]])
Am working with an API that asks to specify a file (Excel in my case) to upload to the API. Documentation specifies: JSON Property = file, Data Type = FileUpload. My question: What is FileUpload. I tried simply specifying the name of the file (e.g. c:\test\Data.xls) but obviously that does not work.
I am working with an API for FastField Mobile Forms (www.fastfield.com). Documentation shown in screen shot, as is code and result. Somehow, I am not posting the file data to the API correctly.
So this is the Python code I am attempting to run
import requests
import json
from requests.auth import HTTPBasicAuth
import base64
# Get session token, this must be specified in header of subsequent request and returns a JSON object ready for insertion into header
rqstResponse = requests.post('https://manage.fastfieldforms.com/api/authenticate', auth=HTTPBasicAuth('***', '***'))
jsonObj = json.loads(rqstResponse.content)
sessionToken = jsonObj['data']['sessionToken']
headers = {'X-Gatekeeper-SessionToken': sessionToken}
# run this code to get listIds - which are hard coded further down
rqstResponse = requests.get("https://manage.fastfieldforms.com/api/globallists", headers=headers)
print (rqstResponse.content)
del rqstResponse
# Read file and convert to binary string
filePath = r"J:\Properties\PropGIS\proj\20150820140457_TelecoMapping\data\Survey_Feb17\FastField_Test01.xlsx"
with open(filePath, 'r') as f:
filecontents = f.read()
fileDataEncoded = base64.b64encode(filecontents)
# create JSON
payloadDictObj = {}
payloadDictObj['file'] = fileDataEncoded
payloadDictObj['id'] = "03c804cb-b983-4e4c-956b-96ac23da16b2"
#payloadDictObj['listname'] = "Test02"
serializedJsonStr = json.dumps(payloadDictObj)
print serializedJsonStr
# Update Global List
rqstResponse = requests.post("https://manage.fastfieldforms.com/api//globallist", data=serializedJsonStr, headers=headers)
print (rqstResponse.content)
del rqstResponse
# --------------------
# Response
{
"code": 200,
"data": {
"searchResults": [
{
"id": 7793,
"accountId": 43600,
"name": "Test01",
"active": true,
"createdAt": "2017-05-24T06:37:28.49Z",
"updatedAt": "2017-05-24T06:37:28.49Z",
"version": 1,
"listId": "03c804cb-b983-4e4c-956b-96ac23da16b2",
"path": "{ bucket:'fastfield-globallists', key:'43600/ca4b89df75db4ef8b513d15d59f654d8.csv'}"
}
]
}
}
{"id": "03c804cb-b983-4e4c-956b-96ac23da16b2", "file": "UEsDB...qaJXQ=="}
{
"code": 403,
"error": "listname name is required",
"data": {}
}
OK. For what it's worth. This is how I finally got this to work. I now realise that this question was more about my understanding, or lack thereof, of the Python request module.
import requests
import json
from requests.auth import HTTPBasicAuth
import base64
# Get session token, this must be specified in header of subsequent request and returns a JSON object ready for insertion into header
rqstResponse = requests.post('https://manage.fastfieldforms.com/api/authenticate', auth=HTTPBasicAuth('XXX', 'XXX'))
jsonObj = json.loads(rqstResponse.content)
sessionToken = jsonObj['data']['sessionToken']
headers = {'X-Gatekeeper-SessionToken': sessionToken}
def getGloballistsDetails():
# run this code to get listIds - which are hard coded further down
rqstResponse = requests.get("https://manage.fastfieldforms.com/api/globallists", headers=headers)
print (rqstResponse.content)
del rqstResponse
def createGloballist(listname, filePath):
# usage example createGloballist("test01", r"c:\temp\test.xlsx")
files = {'file': open(filePath, 'rb')}
data = {'listname': listname}
rqstResponse = requests.post("https://manage.fastfieldforms.com/api//globallist", files=files, data=data, headers=headers)
print (rqstResponse.content)
del rqstResponse
def updateGloballist(id, filePath):
# usage example createGloballist("f03c7db1-cfea-4486-8350-53381ac048b4", r"c:\temp\test.xlsx")
files = files = {'file': open(filePath, 'rb')}
data = {'id': id}
rqstResponse = requests.post("https://manage.fastfieldforms.com/api//globallist", files=files, data=data, headers=headers)
print (rqstResponse.content)
del rqstResponse
filePath = r"J:\Properties\PropGIS\proj\20150820140457_TelecoMapping\data\Survey_Feb17\FastField_Test01.xlsx"
getGloballistsDetails()
#createGloballist("Test02", filePath)
updateGloballist('f03c7db1-cfea-4486-8350-53381ac048b4', filePath)
I am trying to send a message via Firebase to a certain client. This is my current (test) code:
import json
import requests
import urllib
def send_message():
server = "https://fcm.googleapis.com/fcm/send"
api_key = "xxx"
user_token = "xxx"
headers = {'Content-Type': 'application/json', 'Authorization': 'key=' + api_key}
data = {"type": "dataUpdate"}
payload = {"data": data, "to": user_token}
payload = json.dumps(payload)
res = requests.post(server, headers=headers, json=payload)
return res
which produces the following error, returned by Firebase:
JSON_PARSING_ERROR: Unexpected token END OF FILE at position 0.
The following JSON sent to Firebase seems correct to me:
{
"data": {
"type":"dataUpdate"
},
"to":"xxx"
}
which is in the format described by the Firebase documentation. Any idea why Firebase doesn't accept the given data?
When you use json=payload as a parameter to requests.post() you don't need to specify 'Content-Type': 'application/json' in the header. In addition, you are passing a string when the parameter should be payload as a dict (ie. no need for json.dumps())
Try this:
def send_message():
server = "https://fcm.googleapis.com/fcm/send"
api_key = "xxx"
user_token = "xxx"
headers = {'Authorization': 'key=' + api_key}
data = {"type": "dataUpdate"}
payload = {"data": data, "to": user_token}
res = requests.post(server, headers=headers, json=payload)
return res