I have a python loop, which uses selenium to get some data from a website and store that in a SQL Database. At the beginning every loop takes about one second but after some time it is slowing down more and more... I think the problem is a memory problem but I don't know how to solve it.
This is my code:
count = 0
driver = webdriver.PhantomJS()
driver.set_window_size(1120, 550)
con = sql.connect(user="user", passwd="passwd", db="db", host="localhost")
cur = con.cursor()
def create():
if random.random() < 0.5:
driver.get('http://www.example.com/w')
else:
driver.get('http://www.example.com/p')
name = driver.find_element_by_xpath("//div[#class='address']/h3").text
name1 = name.split(" ")[0]
name2 = name.split(" ")[1]
test = driver.find_element_by_xpath("//div[#class='adr']").text
test2 = test.replace("\n", " ")
dd = driver.find_element_by_xpath("(//dl[#class='dl-horizontal')[1]/dd").text
dd2 = driver.find_element_by_xpath("(//dl[#class='dl-horizontal'])[2]/dd/a").text
day = driver.find_element_by_xpath("(//dl[#class='dl-horizontal'])[5]/dd").text
i = "','"
try:
values = unidecode("'" + name1 + i + name2 + i + dd + i + dd2 + i + day + i + test2 + "'")
cur.execute("INSERT INTO accounts (name1,name2,dd,dd2,day,test2) VALUES (" + values + ")")
con.commit()
global anzahl
anzahl += 1
sys.stdout.write('.')
sys.stdout.flush()
gc.collect()
except sql.Error as e:
print("Error %d: %s" % (e.args[0], e.args[1]))
gc.collect()
start = time.time()
for _ in range(200):
create()
cur.close()
con.close()
end = time.time()
I don't see anything what yould slow down the loop. I tried gc.collect() but it doesn't change anything.
What can i do to that my loop does not slow down after a some time?
Things that can slow down you code:
The web server, which can reduce bandwidth to prevent DoS,
You driver object?
Network (Database) can be slow,
I/O access (with sys.stdout.write and print), depending on the real stream. Is it a console?
Related
i'm trying to get files name's in a ftp directory.acctualy my problem is that i get eatch time an empty string when i use a loop.in case that i run my program without a lopp i get the right files name's.
This is my program
class Watch:
def __init__(self):
self.m=""
def goh(self):
while True:
j = 0
ftp = FTP('')
ftp.connect('127.0.0.1', 1026)
ftp.login(user='user', passwd='12345')
ftp.cwd("/FTM/Simulateur/1.MPTC_ACK")
files = ftp.nlst()
while j < len(files):
timestamp = ftp.voidcmd("MDTM " + files[j])[4:].strip()
time = parser.parse(timestamp)
self.time_dic = str(time)
self.tab_file = files[j]
os.chdir("/Users/ouhejjouyou/Desktop/eleclink/Fichier_in/1.MPTC_ACK")
fhandle = open(files[j], 'wb')
ftp.retrbinary("RETR " + str(self.tab_file), fhandle.write)
fhandle.close()
ftp.delete(self.tab_file)
self.m = self.time_dic + " Reception du fichier " + self.tab_file + " réussi\n"
j = j + 1
a=Watch()
t = Thread(target=a.goh)
t.start()
print(a.m)
t.join()
I have the below Python script and it works very well, but I would like to introduce some fail safe options .. That fail safe options being ..
1) if I cannot find (in this example) Michael I would like to write to the file ERROR ..
2) If the database does not allow me to connect for whatever reason I would like to write to another file CONNECTION_ERROR
Here is my script:
#! /usr/bin/python
import pymssql
import sys
sys.path.insert(0, '/opt/mount/safe')
from secrets import password
conn = pymssql.connect(
server="server",
port=port,
user="user",
password=password,
database="database")
conn
cursor = conn.cursor()
cursor.execute("SELECT name, address FROM database WHERE name = 'michael'")
with open('safe/file.txt', 'w') as f:
for row in cursor.fetchall():
print ( "Person " + (row[0])),
print ( "has this address " + (row[1]))
f.write(str( "Person " + (row[0])))
f.write("%s\n" % str( " has this address " + (row[1])))
conn.close()
Took me a while but the below works really really well
import sys, pymssql, string, os, calendar, datetime, traceback, socket, platform
try:
d = datetime.datetime.now()
log = open("LogFile.txt","a")
log.write("----------------------------" + "\n")
log.write("----------------------------" + "\n")
log.write("Log: " + str(d) + "\n")
log.write("\n")
# Start process...
starttime = datetime.datetime.now()
log.write("Begin process:\n")
log.write(" Process started at "
+ str(starttime) + "\n")
log.write("\n")
xxxxxx
CODE HERE
XXXXXX
endtime = datetime.datetime.now()
# Process Completed...
log.write(" Completed successfully in "
+ str(endtime - starttime) + "\n")
log.write("\n")
log.close()
except:
tb = sys.exc_info()[2]
tbinfo = traceback.format_tb(tb)[0]
# Concatenate information together concerning
# the error into a message string
pymsg = "PYTHON ERRORS:\nTraceback info:\n" + tbinfo + "\nError Info:\n" + str(sys.exc_info())
# Return python error messages for use in
# script tool or Python Window
log.write("" + pymsg + "\n")
log.close()
I've got a problem with a LCD screen HD44780. The screen is connected to RPi3 via I2C. I've prepared a code which shows data on it.
from thread import *
import socket
import sys
import string
import select
import lcddriver
import MySQLdb
import time
import mysql.connector
BUFFER_SIZE = 1024
display = lcddriver.lcd()
database = 'data'
username = 'admin'
password = 'mypasss'
date_now = time.strftime("%Y-%m-%d %H:%M:%S")
def clientthread(conn):
received = []
while True:
data = conn.recv(BUFFER_SIZE)
if data == ";" :
conn.close()
if received[0] == '1':
esp_id = int(received[0])
temperature = float (received[1] + received[2] + received[3] + received[4])
humidity = float (received[5] + received[6] + received[7] + received[8])
esp1 = received[0] + ":T=" + received[1] + received[2] + received[3] + received[4] + "H=" + received[5] + received[6] + received[7] + received[8]
display.lcd_display_string(str(esp1), 1)
print esp1
datehour = time.strftime("%Y-%m-%d %H:%M:%S")
db = MySQLdb.connect("localhost","root","raspberry","projekt_grupowy")
cursor = db.cursor()
sql = "INSERT INTO pomiary(esp_id, temperature, humidity, datehour) VALUES('%d', '%.2f', '%.2f', '%s')" % (esp_id, temperature, humidity, datehour)
try:
cursor.execute(sql)
db.commit()
except:
print "Nieudana proba wpisu"
db.rollback()
else:
print "Data sent"
received = []
break
elif received[0] == '2':
esp_id = int (received[0])
temperature = float(received[1] + received[2] + received[3] + received[4])
humidity = float(received[5] + received[6] + received[7] + received[8])
esp2 = received[0] + ":T=" + received[1] + received[2] + received[3] + received[4] + "H=" + received[5] + received[6] + received[7] + received[8]
print esp2
display.lcd_display_string(str(esp2), 2)
datehour = time.strftime("%Y-%m-%d %H:%M:%S")
db = MySQLdb.connect("localhost","root","raspberry","projekt_grupowy")
cursor = db.cursor()
sql = "INSERT INTO pomiary(esp_id, temperature, humidity, datehour) VALUES('%d', '%.2f', '%.2f', '%s')" % (esp_id, temperature, humidity, datehour)
try:
cursor.execute(sql)
db.commit()
except:
print "Nieudana proba wpisu"
db.rollback()
else:
print "Data sent"
received = []
break
else:
print "error sensor"
received = []
else:
received.append(data)
conn.close()
def main():
try:
host = '192.168.42.1'
port = 8888
tot_socket = 1
list_sock = []
for i in range(tot_socket):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
s.bind((host, port+i))
s.listen(10)
list_sock.append(s)
print "[*] Server listening on %s %d" %(host, (port+i))
while 1:
for j in range(len(list_sock)):
conn, addr = list_sock[j].accept()
print '[*] Connected with ' + addr[0] + ':' + str(addr[1])
start_new_thread(clientthread ,(conn,))
s.close()
except KeyboardInterrupt as msg:
sys.exit(0)
if __name__ == "__main__":
main()
I receive data from two ESP modules and show them on the screen. 3
display.lcd_display_string(str(esp1), 1)
display.lcd_display_string(str(esp2), 2)
Sometimes there's a moment when two modules are sending data at the same time. In result the screen shows errors even though I've implemented multithreading. What can I do to avoid that?
Running Firebird on my Raspberry Pi at the moment and using FlameRobin to control it. With another Raspberry Pi I want to some statements to it, I don't get any error codes but the data just doesn't get there.
import RPi.GPIO as GPIO
import time
import fdb
con = fdb.connect(dsn='10.100.2.197/3050:/home/trainee2/Desktop/ice', user='sysdba', password='trainee')
text_file = open("namen1.txt", "r")
lines = text_file.read().split(',')
namen = lines
text_file.close()
status = [0] * 12
indexSpatie = 0
pinnen = [18,23,24,25,20,21,17,27,6,13,19,26]
controlepin = [1] * 12
GPIO.setmode(GPIO.BCM)
for p in range(0,12):
GPIO.setup(pinnen[p],GPIO.IN)
print pinnen[p]
cur = con.cursor()
while True:
for e in range(0,12):
status[e] = GPIO.input(pinnen[e])
if (status[e] != controlepin[e]):
n = e
naam = str(namen[n])
indexSpatie = naam.index(' ')
voornaam = naam[:indexSpatie]
achternaam = naam[indexSpatie:]
stat = str(status[n])
datum = time.strftime("%d/%m/%Y")
print( voornaam + achternaam + " met pinnummer: " + str(pinnen[n]) + " heeft status van " + stat + " op vandaag: " + datum)
cur.execute("insert into ICEDATA (PRENAME, NAME, DATUM) values(?,?,?)",(voornaam,achternaam,datum))
controlepin[e] = status[e]
time.sleep(1)
I post the whole code cause I don't see anything wrong with the statement and connection itself.
Like #ain said, forgot the commit statement on the connection
con.commit()
I'm trying to run a function that has an infinite loop (to check data after few seconds of delay) using multithreading. Since I read data from a csv file, I'm also using Queues.
My current function fine when I do not use multithreading/queues but when I use them, the function only loops once and then stops.
Here's my function that has a infinite loop. Please note that the first while True loop is for threads (in case I use less number of threads than rows in csv) , the function only requires the second while True loop.
def doWork(q):
while True:
#logging.info('Thread Started')
row=q.get()
url = row[0]
target_price = row[1]
#logging.info('line 79')
while True:
delay=randint(5,10)
headers = {'User-Agent': generate_user_agent()}
print datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S')+': '+'Sleeping for ' + str(delay) + ' seconds'
#logging.info('line 81')
eventlet.sleep(delay)
try:
#logging.info('line 85')
with requests.Session() as s:
#logging.info('line 87')
with eventlet.Timeout(10, False):
page = s.get(url,headers=headers,proxies=proxyDict,verify=False)
#logging.info('line 89')
tree = html.fromstring(page.content)
#logging.info('line 91')
price = tree.xpath('//div[#class="a-row a-spacing-mini olpOffer"]/div[#class="a-column a-span2 olpPriceColumn"]/span[#class="a-size-large a-color-price olpOfferPrice a-text-bold"]/text()')[0]
title = tree.xpath('//h1/text()')[0]
#logging.info('line 93')
new_price = re.findall("[-+]?\d+[\.]?\d+[eE]?[-+]?\d*", price)[0]
#logging.info('line 95')
old_price = new_price
#logging.info('line 97')
#print price
print new_price
print title + 'Current price:' + new_price
if float(new_price)<float(target_price):
print 'Lower price found!'
mydriver = webdriver.Chrome()
send_simple_message()
login(mydriver)
print 'Old Price: ' + old_price
print 'New Price: ' + new_price
else:
print 'Trying again'
q.task_done()
except Exception as e:
print e
print 'Error!'
q.task_done()
And here is my thread driver function;
q = Queue(concurrent * 2)
if __name__ == "__main__":
for i in range(concurrent):
t = Thread(target=doWork,args=(q,))
t.daemon = True
t.start()
try:
with open('products.csv','r') as f:
reader = csv.reader(f.read().splitlines())
for row in reader:
q.put((row[0],row[1]))
q.join()
except KeyboardInterrupt:
sys.exit(1)
For anyone that is facing the same issue, here's how I solved it.
I removed q.task_done() from the while loop and put it outside the while loop. This is working as intended but I'm not sure if this is the right approach.