I'm using an open source piece of python code that basically pulls in a location of an entity and saves these details to a DB in real time. lets call it scanner the scanner program. DB file it saves it to is a sqlite file: db.sqlite.
As this is happening my piece of code in question is searching the db file every 45 seconds performing a select statement to find a certain value. This will work a couple of times but after running for a couple of minutes concurrently with the scanner program they run into a DB lock error:
sqlite3.OperationalError: database is locked
So what can I do to my code to ensure this lock does not happen. I cannot change how the scanner program accesses the DB. Only my program.
Any help here would be great. I've seen timeouts mentioned along with threading but I am not sure on either.
from datetime import datetime
import sqlite3
import time
import json
import tweepy
def get_api(cfg):
auth = tweepy.OAuthHandler(cfg['consumer_key'], cfg['consumer_secret'])
auth.set_access_token(cfg['access_token'], cfg['access_token_secret'])
return tweepy.API(auth)
# Fill in the values noted in previous step here
cfg = {
"consumer_key" : "X",
"consumer_secret" : "X",
"access_token" : "X",
"access_token_secret" : "X"
}
with open('locales/pokemon.en.json') as f:
pokemon_names = json.load(f)
currentid = 1
pokemonid = 96 #test
while 1==1:
conn = sqlite3.connect('db.sqlite')
print "Opened database successfully";
print "Scanning DB....";
time.sleep(1)
cur = conn.execute("SELECT * FROM sightings WHERE pokemon_id = ? and id > ?", (pokemonid, currentid))
row = cur.fetchone()
if row is None:
print "No Pokemon Found \n "
time.sleep(1)
while row is not None:
#get pokemon name
name = pokemon_names[str(pokemonid)]
#create expiry time
datestr = datetime.fromtimestamp(row[3])
dateoutput = datestr.strftime("%H:%M:%S")
#create location
location = "https://www.google.com/maps/place/%s,%s" % (row[5], row[6])
#inform user
print "%s found! - Building tweet! \n" % (name)
time.sleep(1)
#create tweet
buildtweet = "a wild %s spawned in #Dublin - It will expire at %s. %s #PokemonGo \n "%(name, dateoutput, location)
#print tweet
#log
print buildtweet
currentid = row[0]
time.sleep(1)
#send tweet
api = get_api(cfg)
tweet = buildtweet
try:
status = api.update_status(status=tweet)
print "sent!"
except:
pass
print "this tweet failed \n"
time.sleep(30)
row = cur.fetchone()
cur.close()
conn.close()
print "Waiting..... \n "
time.sleep(45)
conn.close()
Related
I'm trying to put the data file of schools.data which is just a file listing many universities. It says 'type' object is not subscriptable in terminal. Here is the code
import urllib
import sqlite3
import json
import time
import ssl
conn = sqlite3.connect('universityrawdata.sqlite')
cur = conn.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS Universitylocations (address TEXT, geodata TEXT)''')
fh = open("schools.data")
count = 0
for line in fh:
if count > 200:
print ('Retrieved 200 locations, restart to retrieve more')
break
address = line.strip()
print('')
cur.execute("SELECT geodata FROM Universitylocations WHERE address= ?",(bytes[address]))
print("Resolving", data)
url = fh + urllib.urlencode({"sensor":"false", "address": address})
print("Retrieving", url)
uh = urllib.urlopen(url, context=scontext)
data = uh.read()
print('Retrieved',len(data),'characters',data[:20].replace('\n',''))
count = count + 1
try:
js = json.loads(str(data))
except:
continue
if 'status' not in js or (js['status'] != 'OK' and js['status'] != 'ZERO_RESULTS') :
print('==== Failed to Retrieve ====')
print (data)
continue
cur.execute('''INSERT INTO Universitylocations (address, geodata) VALUES (?, ?)''', (bytes[address],bytes[data]))
conn.commit()
if count % 10 == 0 :
print('Pausing for a bit...')
time.sleep(5)
print("Run process.py to read the data on a database")
Can anyone help? I've been having this issue for a while.
This line is culprit:
cur.execute("SELECT geodata FROM Universitylocations WHERE address= ?",(bytes[address]))
change bytes[address] with (address,). Means:
cur.execute('''SELECT geodata FROM Universitylocations WHERE address= ?''',(address,))
Check what your data type is the database.
I am exploring pytest to automate test activities. I'd like to pass var values store in one .py file, which is outside 'TestCases' folder. JSON body is:
req_json= "{ \r\n\"ReqParamList\": { \r\n \"StdValue\": \"age#name\" \r\n } \r\n}"
resp = requests.request("POST", cURL, data=req_json, headers=headers, params=eid)
Age and name are fetched from DB and stored in as var in a separate py file
import mysql.connector
from mysql.connector import errors
try:
cnx = mysql.connector.connect(
host="hostname",
port=port,
database="dbname",
user="uname",
connection_timeout=5,
password="pw")
# Fname
FnameCur = cnx.cursor()
RandomFname = ("select fname from firstnames limit 1")
FnameCur .execute(RandomFname)
for row in FnameCur :
fname = (row[0])
print("fname is:", fname)
# Lname
LnameCur = cnx.cursor()
RandomLname = ("select Lname from lasttnames where Lname in (select "
"Lname from (select Lname from lasttnames ORDER BY RAND("
") limit 1)t)")
LnameCur.execute(RandomLname )
for row in LnameCur:
Lname = (row[0])
print ("Lname is:",Lname)
except Error as e:
print("Error", e)
finally:
if (cnx.is_connected()):
fnameCur .close()
LnameCur.close()
Could anyone please help me with this?
I have created a database and needs to check that one for corresponding user name and password... if it is there it should print "success" else not...
I tried many ways, it will print only if the there is a correct username and password... else it shows nothing.
It does not show can't log in or error. It only exits with 0 error without any display.
#!/usr/bin/python
import pymysql
# Open database connection
db = pymysql.connect("localhost","root","","text" )
# prepare a cursor object using cursor() method
cursor = db.cursor()
eg = input("What's your user name? ")
password = input("What's your user password? ")
sql = ("SELECT * FROM login WHERE name = '%s' AND password= '%s' " % (eg,password))
try:
# Execute the SQL command
cursor.execute(sql)
results = None
# Fetch all the rows in a list of lists.
results = cursor.fetchall()
try:
if results is not None:
for row in results:
id = row[0]
name = row[1]
password = row[2]
print("login success")
except:
print("Error you arenot in the planet dude sign up first")
except:
print("Error you arenot in the planet dude sign up first")
# disconnect from server
db.close()
Something seems off with the indentation, try the following segment:
try:
# Execute the SQL command
cursor.execute(sql)
results = None
# Fetch all the rows in a list of lists.
results = cursor.fetchall()
try:
if results is not None :
for row in results:
id = row[0]
name = row[1]
password= row[2]
print ("login success")
except:
print("Error you arenot in the planet dude sign up first")
except:
print ("Error you are not in the planet dude sign up first")
for row in results:
id = row[0]
name = row[1]
password= row[2]
print ("login success")
Should be
success = False
for row in results:
db_id = row[0]
db_name = row[1]
db_password= row[2]
if db_name == eg and db_password == password:
success = True
print ("login success")
The variables created from row[1] and row[2] need to be different from the eq and password variables created above. Once that is done, you just need to compare them to see if they match up.
You can check the success variable later on to perform an action only if the login process was successful or not successful.
The following script is an extract from
https://github.com/RittmanMead/obi-metrics-agent/blob/master/obi-metrics-agent.py
The script is written in jython & it hits the weblogic admin console to extract metrics
The problem is it runs only once and does not loop infinitely
Here's the script that I've extracted from the original for my purpose:
import calendar, time
import sys
import getopt
print '---------------------------------------'
# Check the arguments to this script are as expected.
# argv[0] is script name.
argLen = len(sys.argv)
if argLen -1 < 2:
print "ERROR: got ", argLen -1, " args, must be at least two."
print '$FMW_HOME/oracle_common/common/bin/wlst.sh obi-metrics-agent.py <AdminUserName> <AdminPassword> [<AdminServer_t3_url>] [<Carbon|InfluxDB>] [<target host>] [<target port>] [targetDB influx db>'
exit()
outputFormat='CSV'
url='t3://localhost:7001'
targetHost='localhost'
targetDB='obi'
targetPort='8086'
try:
wls_user = sys.argv[1]
wls_pw = sys.argv[2]
url = sys.argv[3]
outputFormat=sys.argv[4]
targetHost=sys.argv[5]
targetPort=sys.argv[6]
targetDB=sys.argv[7]
except:
print ''
print wls_user, wls_pw,url, outputFormat,targetHost,targetPort,targetDB
now_epoch = calendar.timegm(time.gmtime())*1000
if outputFormat=='InfluxDB':
import httplib
influx_msgs=''
connect(wls_user,wls_pw,url)
results = displayMetricTables('Oracle_BI*','dms_cProcessInfo')
while True:
for table in results:
tableName = table.get('Table')
rows = table.get('Rows')
rowCollection = rows.values()
iter = rowCollection.iterator()
while iter.hasNext():
row = iter.next()
rowType = row.getCompositeType()
keys = rowType.keySet()
keyIter = keys.iterator()
inst_name= row.get('Name').replace(' ','-')
try:
server= row.get('Servername').replace(' ','-').replace('/','_')
except:
try:
server= row.get('ServerName').replace(' ','-').replace('/','_')
except:
server='unknown'
try:
host= row.get('Host').replace(' ','-')
except:
host=''
while keyIter.hasNext():
columnName = keyIter.next()
value = row.get(columnName )
if columnName.find('.value')>0:
metric_name=columnName.replace('.value','')
if value is not None:
if outputFormat=='InfluxDB':
influx_msg= ('%s,server=%s,host=%s,metric_group=%s,metric_instance=%s value=%s %s') % (metric_name,server,host,tableName,inst_name, value,now_epoch*1000000)
influx_msgs+='\n%s' % influx_msg
conn = httplib.HTTPConnection('%s:%s' % (targetHost,targetPort))
## TODO pretty sure should be urlencoding this ...
a=conn.request("POST", ("/write?db=%s" % targetDB), influx_msg)
r=conn.getresponse()
if r.status != 204:
print 'Failed to send to InfluxDB! Error %s Reason %s' % (r.status,r.reason)
print influx_msg
#sys.exit(2)
else:
print 'Skipping None value %s,server=%s,host=%s,metric_group=%s,metric_instance=%s value=%s %s' % (metric_name,server,host,tableName,inst_name, value,now_epoch*1000000)
I've tried to use the While loop, but that just stopped the code from exiting and not re-looping
What I want to achieve is to loop it infinitely post connection to weblogic
i.e. after this line
connect(wls_user,wls_pw,url)
and perhaps sleep for 5 seconds before re-running
Any and all help will be appreciated
Thanks
P
You can use this kind of condition for the loop :
mainLoop = 'true'
while mainLoop == 'true' :
and this for the pause between iterations :
java.lang.Thread.sleep(3 * 1000)
I am trying to bypass "Cannot delete or update a parent row: a foreign key constraint fails" inside my python script.
So I am planning to drop all tables but this error throws up due to inter relationship.
My query is I need to get this automated and i know I am gonna come with the same error, but I know how to bypass it by calling SET FOREIGN_KEY_CHECKS=0; and then once deleted enable the feature again SET FOREIGN_KEY_CHECKS=1;.
Need to know how to automate this inside python
import MySQLdb
import sys
if len(sys.argv) != 4:
print "please enter the Hostname to connect followed by:"
print "mysql username;"
print "mysql db to connect;"
else:
_host = sys.argv[1]
_user = sys.argv[2]
# _pass = sys.argv[3]
_db = sys.argv[3]
cham = raw_input("please enter the command to be executed:- ")
_pass = raw_input("please enter password:- ")
if cham == "drop table":
db = MySQLdb.connect(host = _host, user = _user,db = _db, passwd = _pass )
cursor = db.cursor()
cursor.execute("show tables")
for i in cursor.fetchall():
cursor.execute("drop table" + " " + (i[0]))
print cursor.fetchall()
print "all the tables has been deleted"
db.close()
else:
db = MySQLdb.connect(host = _host, user = _user,db = _db, passwd = _pass )
cursor = db.cursor()
cursor.execute(cham)
print cursor.fetchall()
db.close()
I tried the following snip and it worked, thanks anyways.
if cham == "drop table":
db = MySQLdb.connect(host = _host, user = _user,db = _db, passwd = _pass )
cursor = db.cursor()
cursor.execute("show tables")
for i in cursor.fetchall():
try:
cursor.execute("drop table" + " " + (i[0]))
#print cursor.fetchall()
except:
cursor.execute("SET FOREIGN_KEY_CHECKS=0")
cursor.execute("drop table" + " " + (i[0]))
cursor.execute("SET FOREIGN_KEY_CHECKS=1")
# print "all the tables has been deleted"
db.close()