We Keep Coding

created this python script to add 100 members but adds only 3 to 5 and stops automatically

created this python script to add 100 members but adds only 3 to 5 and sometime only 1 and after that it stops automatically dont know what is the problem but maybe i think the problem is in time stamp
created this python script to add 100 members but adds only 3 to 5 and sometime only 1 and after that it stops automatically dont know what is the problem but maybe i think the problem is in time stamp
from telethon.sync import TelegramClient
from telethon.tl.functions.messages import GetDialogsRequest
from telethon.tl.types import InputPeerEmpty, InputPeerChannel, InputPeerUser, PeerUser
from telethon.errors.rpcerrorlist import PeerFloodError, UserPrivacyRestrictedError, ChatWriteForbiddenError, UserAlreadyParticipantError
from telethon.tl.functions.channels import InviteToChannelRequest
from telethon.tl.functions.channels import GetFullChannelRequest, JoinChannelRequest
from telethon import types, utils, errors
import configparser
import sys
import csv
from csv import reader
import traceback
import time
import random
from telethon.sessions import StringSession
print(" ")
print(" " )
print(" ")
print(" ")
delta = 1
with open('../phone.csv', 'r') as read_obj:
csv_reader = reader(read_obj)
list_of_rows = list(csv_reader)
row_number = delta
col_number = 1
value = list_of_rows[row_number - 1][col_number - 1]
with open('../api.csv', 'r') as api_obj_id:
csv_reader = reader(api_obj_id)
list_of_rows = list(csv_reader)
row_number = delta
col_number = 1
deltaop = list_of_rows[row_number - 1][col_number - 1]
with open('../api.csv', 'r') as hash_obj:
csv_reader = reader(hash_obj)
list_of_rows = list(csv_reader)
row_number = delta
col_number = 2
deltaxd = list_of_rows[row_number - 1][col_number - 1]
api_id = int(deltaop)
api_hash = str(deltaxd)
pphone = value
config = configparser.ConfigParser()
config.read("../config.ini")
to_group = config['Telegram']['to_channel']
def autos():
channel_username = to_group
phone = utils.parse_phone(pphone)
client = TelegramClient(f"../sessions/{phone}", api_id, api_hash)
client.connect()
if not client.is_user_authorized():
print('some thing has changed')
client.send_code_request(phone)
client.sign_in(phone, input ('Enter the code: '))
input_file = '../data.csv'
users = []
with open(input_file, encoding='UTF-8') as f:
rows = csv.reader(f, delimiter=",", lineterminator="\n")
next(rows, None)
for row in rows:
user = {}
user['srno'] = row[0]
user['username'] = row[1]
user['id'] = int(row[2])
#user['access_hash'] = int(row[2])
user['name'] = row[3]
users.append(user)
startfrom = int(input("Start From = "))
endto = int(input("End To = "))
for user in users:
if (int(startfrom) <= int (user['srno'])) and (int(user['srno']) <= int(endto)):
try:
status = 'delta'
if user['username'] == "":
print("no username, moving to next")
continue
client(InviteToChannelRequest(channel_username,[user['username']]))
status = 'DONE'
#print("Waiting for 60-180 Seconds...")
time.sleep(random.randrange(0, 5))
except UserPrivacyRestrictedError:
status = 'PrivacyRestrictedError'
except UserAlreadyParticipantError:
status = 'ALREADY'
except PeerFloodError as g:
status = 'PeerFloodError :('
print('Script Is Stopping Now')
time.sleep(86400)
except ChatWriteForbiddenError as cwfe:
client(JoinChannelRequest(channel_username))
continue
except errors.RPCError as e:
status = e.__class__.__name__
except Exception as d:
status = d
except:
traceback.print_exc()
print("Unexpected Error")
continue
channel_connect = client.get_entity(channel_username)
channel_full_info = client(GetFullChannelRequest(channel=channel_connect))
countt = int(channel_full_info.full_chat.participants_count)
print(f"ADDING {user['name']} TO {channel_username} TOTAL: {countt} - {status}")
elif int(user['srno']) > int(endto):
print("Members Added Successfully!")
stat = input('Done!\nChoose From Below:\n\n1 - Repeat The Script\nOR Just Hit Enter To Quit\n\nEnter: ')
if stat == '1':
autos()
else:
quit()
autos()

Python limiting array

I'm trying to insert some data from a text file (fields terminated by ";") to a table in MySQL using Python. Inserting one by one takes too long, so I decided to write a loop to insert a large block of rows at once, but not the whole file (no memory available for that). The table has 21 columns.
import datetime
import mysql.connector
print(datetime.datetime.now())
with open("/backup/backup/backupDB/csv/file.txt", "r", encoding = "latin-1") as data:
dbconn = mysql.connector.connect(
host = "server", user = "user", password = "password", port = 3306
)
cur = dbconn.cursor(prepared = True)
cur.execute("SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema='schema' AND table_name='table'")
iterColumn = cur.fetchall()
columns = str(iterColumn).replace("(","").replace(",)","").replace("[","(").replace("]",")").replace("'","")
next(data)
cur = dbconn.cursor()
block = 5000
y = []
try:
while True:
for x in data:
x = x.split(";")
y.append(tuple(x))
if len(y) == block:
break
cur.executemany("insert ignore into schema.table " + columns + " values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", y)
dbconn.commit()
y=[]
except:
print(datetime.datetime.now())
The code above works, but it caps the insert to 60000 rows, exactly, although I have 210000+ in my file.
Where am I doing wrong?

Errors with LCD screen

I've got a problem with a LCD screen HD44780. The screen is connected to RPi3 via I2C. I've prepared a code which shows data on it.
from thread import *
import socket
import sys
import string
import select
import lcddriver
import MySQLdb
import time
import mysql.connector
BUFFER_SIZE = 1024
display = lcddriver.lcd()
database = 'data'
username = 'admin'
password = 'mypasss'
date_now = time.strftime("%Y-%m-%d %H:%M:%S")
def clientthread(conn):
received = []
while True:
data = conn.recv(BUFFER_SIZE)
if data == ";" :
conn.close()
if received[0] == '1':
esp_id = int(received[0])
temperature = float (received[1] + received[2] + received[3] + received[4])
humidity = float (received[5] + received[6] + received[7] + received[8])
esp1 = received[0] + ":T=" + received[1] + received[2] + received[3] + received[4] + "H=" + received[5] + received[6] + received[7] + received[8]
display.lcd_display_string(str(esp1), 1)
print esp1
datehour = time.strftime("%Y-%m-%d %H:%M:%S")
db = MySQLdb.connect("localhost","root","raspberry","projekt_grupowy")
cursor = db.cursor()
sql = "INSERT INTO pomiary(esp_id, temperature, humidity, datehour) VALUES('%d', '%.2f', '%.2f', '%s')" % (esp_id, temperature, humidity, datehour)
try:
cursor.execute(sql)
db.commit()
except:
print "Nieudana proba wpisu"
db.rollback()
else:
print "Data sent"
received = []
break
elif received[0] == '2':
esp_id = int (received[0])
temperature = float(received[1] + received[2] + received[3] + received[4])
humidity = float(received[5] + received[6] + received[7] + received[8])
esp2 = received[0] + ":T=" + received[1] + received[2] + received[3] + received[4] + "H=" + received[5] + received[6] + received[7] + received[8]
print esp2
display.lcd_display_string(str(esp2), 2)
datehour = time.strftime("%Y-%m-%d %H:%M:%S")
db = MySQLdb.connect("localhost","root","raspberry","projekt_grupowy")
cursor = db.cursor()
sql = "INSERT INTO pomiary(esp_id, temperature, humidity, datehour) VALUES('%d', '%.2f', '%.2f', '%s')" % (esp_id, temperature, humidity, datehour)
try:
cursor.execute(sql)
db.commit()
except:
print "Nieudana proba wpisu"
db.rollback()
else:
print "Data sent"
received = []
break
else:
print "error sensor"
received = []
else:
received.append(data)
conn.close()
def main():
try:
host = '192.168.42.1'
port = 8888
tot_socket = 1
list_sock = []
for i in range(tot_socket):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
s.bind((host, port+i))
s.listen(10)
list_sock.append(s)
print "[*] Server listening on %s %d" %(host, (port+i))
while 1:
for j in range(len(list_sock)):
conn, addr = list_sock[j].accept()
print '[*] Connected with ' + addr[0] + ':' + str(addr[1])
start_new_thread(clientthread ,(conn,))
s.close()
except KeyboardInterrupt as msg:
sys.exit(0)
if __name__ == "__main__":
main()
I receive data from two ESP modules and show them on the screen. 3
display.lcd_display_string(str(esp1), 1)
display.lcd_display_string(str(esp2), 2)
Sometimes there's a moment when two modules are sending data at the same time. In result the screen shows errors even though I've implemented multithreading. What can I do to avoid that?

Firebird statements no error but no result

Running Firebird on my Raspberry Pi at the moment and using FlameRobin to control it. With another Raspberry Pi I want to some statements to it, I don't get any error codes but the data just doesn't get there.
import RPi.GPIO as GPIO
import time
import fdb
con = fdb.connect(dsn='10.100.2.197/3050:/home/trainee2/Desktop/ice', user='sysdba', password='trainee')
text_file = open("namen1.txt", "r")
lines = text_file.read().split(',')
namen = lines
text_file.close()
status = [0] * 12
indexSpatie = 0
pinnen = [18,23,24,25,20,21,17,27,6,13,19,26]
controlepin = [1] * 12
GPIO.setmode(GPIO.BCM)
for p in range(0,12):
GPIO.setup(pinnen[p],GPIO.IN)
print pinnen[p]
cur = con.cursor()
while True:
for e in range(0,12):
status[e] = GPIO.input(pinnen[e])
if (status[e] != controlepin[e]):
n = e
naam = str(namen[n])
indexSpatie = naam.index(' ')
voornaam = naam[:indexSpatie]
achternaam = naam[indexSpatie:]
stat = str(status[n])
datum = time.strftime("%d/%m/%Y")
print( voornaam + achternaam + " met pinnummer: " + str(pinnen[n]) + " heeft status van " + stat + " op vandaag: " + datum)
cur.execute("insert into ICEDATA (PRENAME, NAME, DATUM) values(?,?,?)",(voornaam,achternaam,datum))
controlepin[e] = status[e]
time.sleep(1)
I post the whole code cause I don't see anything wrong with the statement and connection itself.

Like #ain said, forgot the commit statement on the connection
con.commit()

Python Multi-threaded socket listener error with threads not releasing

I have 500+ units in the world that connects to my server and dump their data. Up to now i have been using a PHP script to act as a socket listener, but I need to go multi-threaded as the load is increasing and PHP can not keep up. I am quite new to Python and decided to use it for my new platform, over the past few days i have struggled and tried many examples to no avail. Through my search i came across some questions trying to answer this problem, but none did. I will attach my code.
The problem : as the units connect to the server and the server accepts it, the server creates a new thread to handle the connection, the problem comes when the unit drops the connection the thread stays open and active and the total thread count grows, and this is linked to the system limit : "number of open files", i can increase this limit but this only make it a time bomb , it does not solve this.
Please help.
#! /usr/bin/python
import multiprocessing
import socket
import sys
import pprint
import datetime
import MySQLdb
import time
import datetime
import re
import select
import resource
import threading
max_connections = 1024
max_connections_set = max_connections
resource.setrlimit(resource.RLIMIT_NOFILE, (max_connections_set, max_connections_set))
#the incomming port
the_global_port = xxxx #(any port)
#display id
the_global_id = "UNIT TYPE"
class ConnectionObject(object):
the_id = None
the_socket = None
the_socket_address = None
the_socket_address_ip = None
the_socket_address_port = None
the_server = None
the_process = None
the_process_id = None
the_process_name = None
the_imei = None
identifier = ""
# The class "constructor" - It's actually an initializer
def __init__(self, in_process_nr, in_process , in_socket , in_socket_address, in_server):
self.the_id = in_process_nr
self.the_process = in_process
self.the_process_id = self.the_process.exitcode
self.the_process_name = self.the_process.name
self.the_socket = in_socket
self.the_socket_address = in_socket_address
self.the_socket_address_ip = self.the_socket_address[0]
self.the_socket_address_port = self.the_socket_address[1]
self.the_server = in_server
self.identifier = str(self.the_id) + " " + str(self.the_process_name) + " " + str(self.the_socket_address_ip) + " " + str(self.the_socket_address_port) + " "
#end def init
#end def class
def processData(the_connection_object , the_data):
def mysql_open_connection_inside():
try:
the_conn = MySQLdb.connect(host= "127.0.0.1",
user="user",
passwd="password",
db="mydb")
except MySQLdb.Error, e:
print "Error %d: %s" % (e.args[0], e.args[1])
time.sleep(30)
try:
the_conn = MySQLdb.connect(host= "127.0.0.1",
user="user",
passwd="password",
db="mydb")
except MySQLdb.Error, e:
print "Error %d: %s" % (e.args[0], e.args[1])
print "Unexpected error:", sys.exc_info()[0]
raise
sys.exit(0)
#end time 2
#end try except
return the_conn
#end def mysql_open_connection
conn = mysql_open_connection_inside()
x = conn.cursor()
add_rawdata = ("INSERT INTO RawData"
"(ID,RawData,Type) "
"VALUES (%s, %s, %s)")
data_raw = ('123456', 'ABCD','')
records_inserted = 0
the_connection_object.the_imei = ""
#global clients
imei = ""
try:
thedata = ""
thedata = " ".join("{:02x}".format(ord(c)) for c in the_data)
record_to_save = ' '.join(thedata)
seqtoreply = ""
seqtoreply = "OK"
#reply part
if (seqtoreply != ""): #reply to unit
try:
the_connection_object.the_socket.sendall(seqtoreply)
#echoout(the_connection_object.identifier+"We Replyed With : " + seqtoreply)
except:
echoout(the_connection_object.identifier+"Send Reply Error : " + str(sys.exc_info()[1]))
#end except
#end of if
the_id = "some generated id"
data_raw = (the_id, werk_data, 'UNIT')
try:
x.execute(add_rawdata, data_raw)
conn.commit()
echoout(the_connection_object.identifier+"Raw Data Saved.")
except:
conn.rollback()
echoout(the_connection_object.identifier+" Raw Data NOT Saved : " + str(sys.exc_info()[1]))
#end of data save insert
#echoout("=============================")
endme = 1
echoout("")
conn.close()
#end try
except:
conn.close()
echoout(the_connection_object.identifier+"Error : " + str(sys.exc_info()[1]))
#end try except
#end def handel function
def handle_p(processnr, server, connection, address):
this_connection = ConnectionObject(processnr,multiprocessing.current_process(), connection, address, server)
thisprocess = multiprocessing.current_process()
this_connection.the_id = ""
the_address = this_connection.the_socket_address_ip
the_port = this_connection.the_socket_address_port
try:
echoout("New connection from : "+str(the_address)+" on port "+str(the_port))
close_the_socket = False
while True:
#--------------------- recive part -------------------------------------------------
data = connection.recv(512)
thedata = ""
thedata = " ".join("{:02x}".format(ord(c)) for c in data)
if ((thedata == "") or (thedata == " ") or (data == False)):
echoout("Socket Closed Remotely : No Data")
close_the_socket = True
break
#end - if data blank
else :
processData(this_connection, data)
#end there is data
echoout("=============================")
#end if while true
#end try
except:
print "handling request, Error : " + str(sys.exc_info()[1])
close_the_socket = True
connection.close()
finally:
close_the_socket = True
echoout("Closing socket")
connection.close()
#end try finally
#end def handel function
def mysql_update(update_statement, update_values):
conn_update = MySQLdb.connect(host= "127.0.0.1",
user="user",
passwd="password",
db="mydb")
x_update = conn_update.cursor(MySQLdb.cursors.DictCursor)
rawdata_data = (update_statement)
data_rawdata = (update_values)
allupdateok = False
#end if there is more
try:
x_update.execute(rawdata_data, data_rawdata)
conn_update.commit()
allupdateok = True
conn_update.close()
except:
conn_update.rollback()
allupdateok = False
conn_update.close()
print "Unexpected error:", sys.exc_info()[0]
raise
#end of data save insert
if (allupdateok == False):
echoout("Update Raw Data Table Error")
#end if update
return allupdateok
#end def mysqlupdate
def echoout(thestring):
datestring = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if (thestring != ""):
outstring = datestring + " " + thestring
print outstring
else :
outstring = thestring
print outstring
#end - def echoout
class Server(object):
threads = []
all_threads = []
high_proc = ""
def __init__(self, hostname, port):
self.hostname = hostname
self.port = port
def start(self):
echoout("Listening for conncetions")
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind((self.hostname, self.port))
self.socket.listen(10)
process_number = 1
inputs = [self.socket]
while True:
inready, outready, excready = select.select(inputs, [], [], 30);
for s in inready:
if s == self.socket:
conn, address = self.socket.accept()
high_processname = ""
echoout("Got a connection...")
process = threading.Thread(target=handle_p, args=(process_number,self, conn, address))
high_processname = process.name
self.high_proc = high_processname
process.daemon = True
process.start()
self.all_threads.append((process,conn))
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
self.threads.append((process_number,conn,st,0,process))
process_number = process_number + 1
print "ACTIVE Threads = " + str(threading.activeCount())
the_total_count = 0
dead_count = 0
alive_count = 0
for the_thread in self.all_threads :
if (the_thread[0].is_alive() == True):
alive_count = alive_count + 1
else :
dead_count = dead_count + 1
the_thread[1].close()
the_thread[0].join(0.3)
self.all_threads.pop(the_total_count)
#end if alive else
the_total_count = the_total_count + 1
#end for threads
print "LIVE Threads = " + str(alive_count)
print "DEAD Threads = " + str(dead_count)
print "TOTAL Threads = " + str(the_total_count)
print ""
#end if s = socke, new connection
#end for loop
#end while truw
self.socket.close()
#end def start
#main process part
if __name__ == "__main__":
start_ip = "0.0.0.0"
start_port = the_global_port
#start server
server = Server(start_ip, start_port)
try:
print "Listening on " , start_port
server.start()
except:
print "unexpected, Error : " + str(sys.exc_info()[1])
finally:
print "shutting down"
active_clients = 0
for process in multiprocessing.active_children():
try:
active_clients = active_clients + 1
process.terminate()
#process.join()
except:
print "Process not killed = " + str(sys.exc_info()[1])
#end try except
#close mysql connection
print "Active clients = " + str(active_clients)
#end try finally
server.socket.close()
server.threads = []
server = None
print "All done."
#end def main

First of all, it is silly to use threads when you can have 500+ connected clients, you should go asynchronous - look at gevent for example for
a very good library, or at least use select (see Python documentation).
Then, your code to close the socket in handle_p looks good, indeed when
the recv() call comes back with an empty string it means the remote
end is disconnected so you break the while, fine.
However, it looks like the remote closed the connection but it is not
detected on your side (recv() doesn't return). The best would be
then to have a kind of heartbeat to know when you can close the
connection.