When I try to connect to AWS RDS (MySQL), most of the time I receive an InterfaceError. When I edit the Lambda code and re-run, it will work fine the first time, but then the same error occurs.
My code:
import sys
import logging
import pymysql
import json
import traceback
rds_host = "*****.rds.amazonaws.com"
name = "*****"
password = "****"
db_name = "myDB"
logger = logging.getLogger()
logger.setLevel(logging.INFO)
try:
conn = pymysql.connect(rds_host, user=name, passwd=password, db=db_name, connect_timeout=5)
except:
logger.error("ERROR: Unexpected error: Could not connect to MySql instance.")
sys.exit()
logger.info("SUCCESS: Connection to RDS mysql instance succeeded")
def handler(event, context):
sub = event['sub']
username = event['username']
givenname = event['givenname']
isAdmin = event['isAdmin']
print (sub)
print (username)
print (givenname)
print (isAdmin)
data = {}
cur = conn.cursor()
try:
cmd = "SELECT AuthState FROM UserInfo WHERE UserName=" + "\'" + username + "\'"
rowCnt = cur.execute(cmd)
print (cmd)
except:
print("ERROR: DB Query Execution failed.")
traceback.print_exc()
data['errorMessage'] = 'Internal server error'
response = {}
response['statusCode'] = 500
response['body'] = data
return response
if rowCnt <= 0:
print (username)
data['errorMessage'] = 'No User Name Found'
response = {}
response['statusCode'] = 400
response['body'] = data
conn.close()
return response
for row in cur:
print row[0]
if int(row[0]) == 0:#NOT_AUTHORIZED
ret = "NOT_AUTHORIZED"
elif int(row[0]) == 1:#PENDING
ret = "PENDING"
elif int(row[0]) == 2:#AUTHORIZED
ret = "AUTHORIZED"
else:#BLOCKED
ret = "BLOCKED"
data['state'] = ret
response = {}
response['statusCode'] = 200
response['body'] = data
conn.close()
return response
The stacktrace:
Traceback (most recent call last):
File "/var/task/app.py", line 37, in handler
File "/var/task/pymysql/connections.py", line 851, in query
self._execute_command(COMMAND.COM_QUERY, sql)
File "/var/task/pymysql/connections.py", line 1067, in _execute_command
raise err.InterfaceError("(0, '')")
InterfaceError: (0, '')
Read Understanding Container Reuse in Lambda.
It was written about Node but is just as accurate for Python.
Your code doesn't run from the top with each invocation. Sometimes it starts with the handler.
Why? It's faster.
How do you know when this will happen? You don't... except for each time you redeploy the function, of course, you'll always get a fresh container on the first invocation, because the old containers would have been abandoned by the redeploy.
If you're going to do your DB connection outside the handler, don't call conn.close(), because on the next invocation of the function, you might find your container is still alive, and the handler is invoked with an already-closed database handle.
You have to write Lambda functions so that they neither fail if a container is reused, nor fail if a container is not reused.
The simpler solution is to open the DB connection inside the handler. The more complex but also more optimal solution (in terms of runtime) is to never close it, so that it can potentially be reused.
Related
I have the lambda function code below that transfers objects from s3 buckets to AWS RDS database.
import json
import boto3
import pymysql
s3_client = boto3.client('s3')
def lambda_handler(event, context):
bucket_name = event["bucket"]
s3_file_name = event["object"]
resp = s3_client.get_object(Bucket=bucket_name, Key=s3_file_name)
data = resp['Body']
rds_endpoint = ""
username = #username for RDS Mysql
password = # RDS Mysql password
db_name = # RDS MySQL DB name
conn = None
try:
conn = pymysql.connect(host=rds_endpoint, user=username, password=password, database=db_name)
except pymysql.MySQLError as e:
print("ERROR: Unexpected error: Could not connect to MySQL instance.")
try:
cur = conn.cursor()
cur.execute(#db stuff)
conn.commit()
except Exception as e:
print(e)
return 'Table not created!'
with conn.cursor() as cur:
try:
cur.execute(#db stuff)
conn.commit()
output = cur.execute()
except:
output = ("Entry not inputted! Error!")
print("Deleting the csv file from s3 bucket")
return {
'statusCode': 200,
'body': 'Successfully uploaded!'
}
The code above works fine with this given test ev:
{
"bucket": "python-bucket",
"object": "bobmarley.mp3"
}
However, when I try to adapt it to the s3 bucket by changing the lines of code to below as seen in this tutorial: https://www.data-stats.com/s3-data-ingestion-to-rds-through-lambda/
bucket_name = event["Records"][0]["s3"]["bucket"]["name"]
s3_file_name = event["Records"][0]["s3"]["object"]["key"]
I get this error:
[ERROR] TypeError: list indices must be integers or slices, not str
Traceback (most recent call last):
File "/var/task/lambda_function.py", line 7, in lambda_handler
bucket_name = event["Records"]["s3"]["bucket"]["name"]
I have an Azure Function that sends queries to snowflake using the Python snowflake-connector. It opens a connection, creates a cursor, sends the query and does not check to ensure the query was successful using _no_results=True. When I run it it works fine. However when I use it to run multiple queries at once, some queries are randomly failing with status code 604: Query Execution was canceled. Is there some sort of concurrent limit that I'm hitting? I cannot find any information in the documentation. The queries being sent are very simple (truncate table x) and are not timing out.
My code is attached below.
import logging
import json
import time
import gc
from flatten_json import flatten
import os
import snowflake.connector
import azure.functions as func
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
# Deserialize request body
req_body = req.get_json()
logging.info('Deserialized input successfully. Request body: ' + json.dumps(req_body))
#Create result JSON to be returned to caller
result = {
"TaskName":req_body['TaskName'],
"Status":"Complete",
"TaskKey": req_body['TaskKey'],
"Query_ID":"",
"Session_ID":""
}
#Create the Snowflake parameters for connection
USER = <sfusername>
PASSWD = <sfpw>
ACCOUNT = <sfAcc>
WAREHOUSE = <sfwh>
DATABASE = <sfdb>
logging.info('Connection string created')
copy_sql_statement = create_sql_statement(req_body)
logging.info('Insert SQL Statement: ' + copy_sql_statement)
logging.info('Attempting to Connect to Snowflake...')
try:
# Try to connect to Snowflake
connection = snowflake.connector.connect(user=USER, password=PASSWD, account=ACCOUNT, warehouse=WAREHOUSE, database=DATABASE)
logging.info('Connection Successful')
except Exception as e:
raise e
logging.info('Try block for query_snowflake started.')
try:
# Call function to execute copy into
output_list = query_snowflake(req_body, connection, copy_sql_statement) #return queryid and sessionid from Snowflake
queryid = output_list[0]
sessionid = output_list[1]
result['Query_ID']= queryid
result['Session_ID']= sessionid
logging.info('Query sent to Snowflake Successfully.')
return func.HttpResponse(json.dumps(result), status_code = 200)
except Exception as e:
result['Status'] = 'Failed'
result['Error_Message'] = str(e)
logging.info('Copy Into function failed. Error: ' + str(e))
return func.HttpResponse(json.dumps(result),
status_code=400)
def create_sql_statement(req_body):
# Replace TaskKey and CDCMin
copy_sql_statement = req_body['InsertSQL'].replace('#TaskKey', req_body['TaskKey']).replace('#CDCMinDate', req_body['CDCMinDate']).replace('#CDCMaxDate', req_body['CDCMaxDate'])
return copy_sql_statement
def query_snowflake(req_body, connection, copy_sql_statement):
try:
# Execute copy into statement
cur = connection.cursor()
sessionid = cur.execute("select current_session()").fetchone()
cur.execute(copy_sql_statement, _no_results=True)
#connection.execute('COMMIT;')
return [cur.sfqid, sessionid[0]] #return queryid and sessionid as list for result body
except Exception as e:
raise e
#finally:
# Close and dispose connection
cur.close()
connection.close()
NEW CODE:
import logging
import json
import time
import gc
from flatten_json import flatten
import os
import snowflake.connector
import azure.functions as func
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
# Deserialize request body
req_body = req.get_json()
logging.info('Deserialized input successfully. Request body: ' + json.dumps(req_body))
#Create result JSON to be returned to caller
result = {
"TaskName":req_body['TaskName'],
"Status":"Complete",
"TaskKey": req_body['TaskKey'],
"Query_ID":"",
"Session_ID":"",
"Error_Message":""
}
#Create the Snowflake parameters for connection
USER = <sfusername>
PASSWD = <sfpw>
ACCOUNT = <sfAcc>
WAREHOUSE = <sfwh>
DATABASE = <sfdb>
logging.info('Connection string created')
copy_sql_statement = create_sql_statement(req_body)
logging.info('SQL Statement: ' + copy_sql_statement)
logging.info('Attempting to Connect to Snowflake...')
try:
# Try to connect to Snowflake
connection = snowflake.connector.connect(user=USER, password=PASSWD, account=ACCOUNT, warehouse=WAREHOUSE, database=DATABASE)
logging.info('Connection Successful')
except Exception as e:
raise e
logging.info('Try block for send query started.')
try:
# Call function to execute copy into
logging.info('Sending Query to Snowflake...')
output_list = query_snowflake(req_body, connection, copy_sql_statement) #return queryid and sessionid from Snowflake
queryid = output_list[0]
sessionid = output_list[1]
result['Query_ID']= queryid
result['Session_ID']= sessionid
logging.info('Ensuring Query was Sent...')
status_stmt = create_status_statement(queryid, sessionid)
for x in range(1,14): #it will try for 3.5min in case query is pending
time.sleep(5)
returnValues = get_query_status(status_stmt, connection)
# check result, if error code 604 we know the query canceled.
if returnValues[1] == '604':
result['Status'] = 'Failed'
result['Error_Message'] = 'SQL Execution Canceled'
return func.HttpResponse(json.dumps(result), status_code = 400)
# if its anything but pending, we know the query was sent to snowflake
# 2nd Function worries about the result
elif returnValues[0] != 'PENDING':
result['Status'] = returnValues[0]
logging.info('Query sent to Snowflake Successfully.')
return func.HttpResponse(json.dumps(result), status_code = 200)
else:
logging.info('Loop ' + str(x) + ' completed, trying again...')
time.sleep(10)
#if it exits for loop, mark success, let 2nd function surface any failures.
result['Status'] = 'Success'
return func.HttpResponse(json.dumps(result), status_code = 200)
except Exception as e:
result['Status'] = 'Failed'
result['Error_Message'] = str(e)
logging.info('Copy Into function failed. Error: ' + str(e))
return func.HttpResponse(json.dumps(result),
status_code=400)
def create_sql_statement(req_body):
# Replace TaskKey and CDCMin
copy_sql_statement = req_body['InsertSQL'].replace('#TaskKey', req_body['TaskKey']).replace('#CDCMinDate', req_body['CDCMinDate']).replace('#CDCMaxDate', req_body['CDCMaxDate'])
return copy_sql_statement
def query_snowflake(req_body, connection, copy_sql_statement):
try:
# Execute copy into statement
cur = connection.cursor()
sessionid = cur.execute("select current_session()").fetchone()
cur.execute(copy_sql_statement, _no_results=True)
# return queryid and sessionid as list for result body
return [cur.sfqid, sessionid[0]]
except Exception as e:
raise e
def create_status_statement(queryid, sessionid):
sql_statement = "SELECT execution_status, error_code, query_id \
FROM TABLE(INFORMATION_SCHEMA.QUERY_HISTORY_BY_SESSION(SESSION_ID => " + sessionid + ")) \
WHERE QUERY_ID = '" + queryid + "'"
return sql_statement
def get_query_status(etl_sql_statement, conn):
QueryStatus = ''
ErrorCode = ''
QueryID = ''
try:
#Execute sql statement.
cur = conn.cursor()
Result = cur.execute(etl_sql_statement)
row = Result.fetchone()
if row is None:
ErrorCode = 'PENDING'
else:
QueryStatus = str(row[0])
ErrorCode = str(row[1])
QueryID = str(row[2])
except Exception as e:
logging.info('Failed to get query status. Error: ' + str(e))
raise e
finally:
#Close and dispose cursor
cur.close()
return (QueryStatus, ErrorCode, QueryID)
I am currently working with a basic client/server application and implementing a simple RSA / public-key authentication system. I have ran into this error and can not, for the life of me, figure it out.
I am using the latest version of python.
server.py
def getUserData(username):
global privateKeysFilename
filename = privateKeysFilename
with open(filename, "r") as keysFile:
for line in keysFile:
line = [token.rstrip("\n") for token in line.split(",")]
if(username == line[0]):
if DEBUG:
print("\n=== DEBUG\nUser data = %s\n===\n" %(line))
return line
return False
def running(self):
global BUFFER, DEBUG, start, final
while 1:
print('Waiting for a connection')
connection, client_address = self.server_socket.accept()
connection.send("Successful connection!".encode())
x = randint(start, final)
self.fx = function(x)
connection.send(str(x).encode())
try:
# Output that a client has connected
print('connection from', client_address)
write_connection()
# Set the time that the client connected
start_time = datetime.datetime.now()
# Loop until the client disconnects from the server
while 1:
# Receive information from the client
userData = connection.recv(BUFFER)
#data = connection.recv(1024).decode()
if(userData != "0"):
#define split character
ch = ","
userData = userData.split(ch.encode())
username = userData[0]
r = int(userData[1])
userData = getUserData(username)
e, n = int(userData[1]), int(userData[2])
y = modularPower(r, e, n)
if DEBUG:
print("=== DEBUG\ne = %d\nn = %d\nr = %d\ny = %d\n===\n" %(e, n, r, y))
if(self.fx == y):
#if authentication passed
connection.send("Welcome!!!".encode())
else:
connection.send("Failure!!!".encode())
if (userData != 'quit') and (userData != 'close'):
print('received "%s" ' % userData)
connection.send('Your request was successfully received!'.encode())
write_data(userData)
# Check the dictionary for the requested artist name
# If it exists, get all their songs and return them to the user
if userData in self.song_dictionary:
songs = ''
for i in range(len(self.song_dictionary.get(userData))):
songs += self.song_dictionary.get(userData)[i] + ', '
songs = songs[:-2]
print('sending data back to the client')
connection.send(songs.encode())
print("Sent", songs)
# If it doesn't exist return 'error' which tells the client that the artist does not exist
else:
print('sending data back to the client')
connection.send('error'.encode())
else:
# Exit the while loop
break
# Write how long the client was connected for
write_disconnection(start_time)
except socket.error:
# Catch any errors and safely close the connection with the client
print("There was an error with the connection, and it was forcibly closed.")
write_disconnection(start_time)
connection.close()
data = ''
finally:
if data == 'close':
print('Closing the connection and the server')
# Close the connection
connection.close()
# Exit the main While loop, so the server does not listen for a new client
break
else:
print('Closing the connection')
# Close the connection
connection.close()
# The server continues to listen for a new client due to the While loop
and here is the output with error:
Traceback <most recent call last>:
File "server.py", line 165, in running
e, n = int(userData[1]), int(userData[2])
TypeError: 'bool' object is not subscriptable
Any help would be greatly appreciated! :)
By using userData[n] you are trying to access the nth element in a subscriptable object.
This can be a list, dict, tuple or even a string.
The error you see means that your object userData is neither of the previous mentioned types, and it's a bool ( True or False )
Since it's the result of calling the function getUserData(), I recommend you check the return type of this function and make sure it's of the mentioned types and revise your code logic.
[Update]
By checking the function getUserData() it looks it only returns line if the username is included, if not it returns False which is not handled in the main code.
I suggest this edit to the code to inlclude success status to the return value as follows.
def getUserData(username):
global privateKeysFilename
filename = privateKeysFilename
with open(filename, "r") as keysFile:
for line in keysFile:
line = [token.rstrip("\n") for token in line.split(",")]
if(username == line[0]):
if DEBUG:
print("\n=== DEBUG\nUser data = %s\n===\n" %(line))
return True, line
return False, None
And then in your code when you call getUserData() you check for the success first before parsing data like this
userData = getUserData(username)
if userData [0]:
e, n = int(userData[1]), int(userData[2])
y = modularPower(r, e, n)
else:
# Your failure condition
I'm trying to connect and to read data in a MongoDB database, to learn Python. I'm using Pymongo and I have this error :
Traceback (most recent call last):
File "secondtest.py", line 107, in <module>
user_info = dbco.find({})
TypeError: expected a character buffer object
This is my database.ini :
[postgresql]
host=monhostname
database=monpass
port=15000
user=monuser
password=monpass
[mongodb]
hostname=127.0.0.1
database=Mydatabase
username=Myname
password=Myn#me!
collection=measure
port=27017
And my code using it :
# -*- coding: utf-8 -*-
# secondtest.py
import psycopg2
import sys
import pymongo
from urllib import quote_plus
from pymongo import MongoClient
from configparser import ConfigParser
# Connection information in database.ini
params = ''
mongollection = ''
# Variables to connect to a database, to use a cursor object and to fetch all results from a query
mongoClient = ''
pgsqlClient = ''
pgsqlCursor = ''
pgsqlRecords = ''
mongoRecords = ''
dbco = ''
# Retrieve connection information from ini file
def dbConfig(section, filename='database.ini'):
# Create a parser
parser = ConfigParser()
# Read config file
parser.read(filename)
# Get section, depending on the database engine
db = {}
if parser.has_section(section):
params = parser.items(section)
for param in params:
db[param[0]] = param[1]
else:
raise Exception('Section {0} not found in the {1} file'.format(section, filename))
# Return data or directly as a string
if section == 'postgresql':
return db
elif section == 'mongodb':
host = ''
username = ''
passwd = ''
port = ''
dbmongo = ''
connectstring = ''
# Make a string to connect to MongoDB
for key, value in db.iteritems():
if key == 'hostname':
host = value.encode("utf-8")
elif key == 'username':
username = value
elif key == 'password':
passwd = value
elif key == 'database':
dbmongo = value
elif key == 'collection':
mongollection = value
elif key == 'port':
port = value
connectstring = "mongodb://" + username + ":" + quote_plus(passwd) + "#" + host + ":" + port
print("Internal test = " + connectstring)
return connectstring.encode('iso-8859-1')
# Connection to MongoDB
def connectToMongoDb():
# The f-string is only available in Python >= 3.6
params = dbConfig('mongodb')
print("Parameters : " + params)
mongoClient = MongoClient(params)
try:
# print("Connection to database")
dbco = mongoClient.mongollection
print("Here")
print("Test dbco : " + dbco)
print("Connected to MongoDB !")
return dbco
except:
return "Error : can't connect to MongoDB !"
# Close MongoDB connection
def closeMongoDbConnection():
# try:
mongoClient.close()
return 'Connection closed'
# except:
# return "Can't close the connection. See if you already had one or if you didn't mispell its name."
# Make a query in MongoDB
def mongoDbQuery():
#mongocursor = mongoClient.mongollection.find()
#for document in cursor:
#print(document)
mongoClient.database_names()
if __name__ == '__main__':
dataconnect = connectToMongoDb()
print("Connection\n")
#mongoDbQuery()
#collec = mongoClient.measure
user_info = dbco.find({})
print(user_info)
print(closeMongoDbConnection())
Could you help me with this problem ? I think quote_plus() or even dbco = mongoClient.mongollection is what makes this error occurs. But I'm not 100% sure and I don't see, even with the documentation, how could I resolve this.
Thank you.
I'm reading your code. I see in the beginning of the program, you create the dbco variable to point to an empty string:
dbco = ''
following after that, I see you define a couple functions, then you call find() on that string:
user_info = dbco.find({})
You're passing {} (empty dict) as parameter to the method... but as you can see in the documentation here, this method needs another string as first parameter. That causes the error you see.
Now I'm not entirely sure how to fix it because I don't know what you mean. Maybe you mean to use the dataconnect variable, since that is the one that gets the results of the connectToMongoDb function:
dataconnect = connectToMongoDb()
I made it again and changed some little things. Now, it works. Here is the code, for people who'd need it in the future.
import sys
import pymongo
from urllib import quote_plus
from pymongo import MongoClient
from configparser import ConfigParser
client = MongoClient()
connected = ''
# Retrieve connection information from ini file
def dbConfig(section, filename='database.ini'):
# Keep result in global variable when the function is finished
global client
# Create a parser
parser = ConfigParser()
# Read config file
parser.read(filename)
# Get section, depending on the database engine
db = {}
if parser.has_section(section):
params = parser.items(section)
for param in params:
db[param[0]] = param[1]
else:
raise Exception('Section {0} not found in the {1} file'.format(section, filename))
# Return data or directly as a string
if section == 'postgresql':
return db
elif section == 'mongodb':
# Variables for the connection
host = ''
username = ''
passwd = ''
port = ''
connectstring = ''
# Make a string to connect to MongoDB
for key, value in db.iteritems():
if key == 'hostname':
host = value.encode("utf-8")
elif key == 'username':
username = value
elif key == 'password':
passwd = value
elif key == 'database':
dbmongo = value
elif key == 'collection':
mongollection = value
elif key == 'port':
port = value
# Make the URI needed for the connection to Mongo DB
passwing = "mongodb://" + username + ":" + quote_plus(passwd) + "#" + host + ":" + port
client = MongoClient(passwing)
return client
# Close MongoDB connection
def closeMongoDbConnection():
# Try to close the connection to Mongo DB
try:
client.close()
return 'Connection closed'
except:
return "Can't close the connection. See if you already had one or if you didn't mispell its name."
# Connection to MongoDB
def connectToMongoDb(mydb):
db = client.get_database(mydb)
return db.measure
# Make a query in MongoDB
def mongoDbQuery():
docs = connected.find().count()
#for document in docs:
#print(document)
print(docs)
if __name__ == '__main__':
connected = connectToMongoDb('neocampus')
#docs = connected.find()
# print(test)
#for document in docs:
#print(document)
mongoDbQuery()
# Show if the connection to Mongo DB is a success or not
print(closeMongoDbConnection())
The problems were :
- about global variables in and out of functions
- the database variable empty (because of that)
- a first call of MongoClient()
I'm using an open source piece of python code that basically pulls in a location of an entity and saves these details to a DB in real time. lets call it scanner the scanner program. DB file it saves it to is a sqlite file: db.sqlite.
As this is happening my piece of code in question is searching the db file every 45 seconds performing a select statement to find a certain value. This will work a couple of times but after running for a couple of minutes concurrently with the scanner program they run into a DB lock error:
sqlite3.OperationalError: database is locked
So what can I do to my code to ensure this lock does not happen. I cannot change how the scanner program accesses the DB. Only my program.
Any help here would be great. I've seen timeouts mentioned along with threading but I am not sure on either.
from datetime import datetime
import sqlite3
import time
import json
import tweepy
def get_api(cfg):
auth = tweepy.OAuthHandler(cfg['consumer_key'], cfg['consumer_secret'])
auth.set_access_token(cfg['access_token'], cfg['access_token_secret'])
return tweepy.API(auth)
# Fill in the values noted in previous step here
cfg = {
"consumer_key" : "X",
"consumer_secret" : "X",
"access_token" : "X",
"access_token_secret" : "X"
}
with open('locales/pokemon.en.json') as f:
pokemon_names = json.load(f)
currentid = 1
pokemonid = 96 #test
while 1==1:
conn = sqlite3.connect('db.sqlite')
print "Opened database successfully";
print "Scanning DB....";
time.sleep(1)
cur = conn.execute("SELECT * FROM sightings WHERE pokemon_id = ? and id > ?", (pokemonid, currentid))
row = cur.fetchone()
if row is None:
print "No Pokemon Found \n "
time.sleep(1)
while row is not None:
#get pokemon name
name = pokemon_names[str(pokemonid)]
#create expiry time
datestr = datetime.fromtimestamp(row[3])
dateoutput = datestr.strftime("%H:%M:%S")
#create location
location = "https://www.google.com/maps/place/%s,%s" % (row[5], row[6])
#inform user
print "%s found! - Building tweet! \n" % (name)
time.sleep(1)
#create tweet
buildtweet = "a wild %s spawned in #Dublin - It will expire at %s. %s #PokemonGo \n "%(name, dateoutput, location)
#print tweet
#log
print buildtweet
currentid = row[0]
time.sleep(1)
#send tweet
api = get_api(cfg)
tweet = buildtweet
try:
status = api.update_status(status=tweet)
print "sent!"
except:
pass
print "this tweet failed \n"
time.sleep(30)
row = cur.fetchone()
cur.close()
conn.close()
print "Waiting..... \n "
time.sleep(45)
conn.close()