We Keep Coding

Python Multithreading with requests

i have one scraper which initiate the "requestes" session and fetch some data, using a IPV6, i have now 10000 ip list, I have prepared it using threading, but its giving error.
Need support to find out the issue.
import requests, queue,threading, urllib3,jso,pandas as pd, os, time, datetime,inspect
num_threads = 2
root = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
with open (root+ "/ip_list.txt") as ips:
device_ip = list(ips)
class Writer_Worker(threading.Thread):
def __init__(self, queue, df, *args, **kwargs):
if not queue:
print("Device Queue not specified")
exit(1)
self.out_q = queue
self.df = df
super().__init__(*args, **kwargs)
def run(self):
while True:
try:
device_details = self.out_q.get(timeout=3)
except queue.Empty:
return
self.df[device_details[0]] = device_details
self.out_q.task_done()
class Worker(threading.Thread):
def __init__(self, queue, out_queue, device_password, *args, **kwargs):
if not queue:
print("Device Queue not specified")
exit(1)
self.queue = queue
self.pas = device_password
self.out_q = out_queue
super().__init__(*args, **kwargs)
def run(self):
while True:
try:
device_ip = self.queue.get(timeout=3)
except queue.Empty:
return
self.connect_to_device_and_process(device_ip)
self.queue.task_done()
def connect_to_device_and_process(self, device_ip):
st = str("Online")
try:
r = requests.post("https://["+device_ip+"]/?q=index.login&mimosa_ajax=1", {"username":"configure", "password":self.pas}, verify=False)
except requests.exceptions.ConnectionError:
st = str("Offline")
self.out_q.put([device_ip,st,"","","","","","","","","","","","","","","","","",""])
return
finally:
if 'Online' in st:
r = requests.get("https://["+device_ip+"]/cgi/dashboard.php", cookies=r.cookies, verify=False)
if "Response [401]" in str(r):
st2 = str("Password Error")
self.out_q.put([device_ip,st2,"","","","","","","","","","","","","","","","","",""])
else:
data = json.loads(r.content.decode())
output5 = data ['config'] ['Spectrum_Power']
self.out_q.put([device_ip,st,output5['Auto_Power'].replace('2', 'Max Power').replace('1', 'Min Power').replace('0', 'off'),output5['AutoConfig']])
def main():
start = time.time()
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
pas = input("Enter Device Password:")
df =pd.DataFrame(columns = ["IP","Status","Auto_Power","AutoChannel"])
q = queue.Queue(len(device_ip))
for ip in device_ip:
q.put_nowait(ip)
out_q = queue.Queue(len(device_ip))
Writer_Worker(out_q, df).start()
for _ in range(num_threads):
Worker(q, out_q, pas).start()
q.join()
print(df)
df.to_excel('iBridge_C5x_Audit_Report.xlsx', sheet_name='Detail', index = False)
if __name__ == "__main__":
main()
below is the error while running the script, seeps I am unable to login to this device.
Any help is appreciable.

You should use a thread pool that distributes the work between a fixed number of threads. This is a core feature of Python since version 3.2.
from concurrent.futures import ThreadPoolExecutor
Define a function perform(ip) that performs the request for one ip
Set variable numThreads to the number of desired threads
Run the thread-pool executor:
print(f'Using {numThreads} threads')
with ThreadPoolExecutor(max_workers=numThreads) as pool:
success = all(pool.map(perform, ips))
Source: https://docs.python.org/3/library/concurrent.futures.html
On that page you find an example even better tailored to your application: https://docs.python.org/3/library/concurrent.futures.html#threadpoolexecutor-example

from threading import Thread
th = Thread(target=self.fill_imdb, args=(movies_info_part, "thread " + str(count)))
th.start()
fill_imdb is my method

Why is my connection being refused half of the time

I'm trying to find a way to forward stdin input from my main process to a child process, and what I've found that works is basically to open a socket on the main process and then send text via the socket to the children processes. But what I'm finding is that half of the time my socket gets refused, and I have no idea what's going on.
I've followed the instructions on this question 16130786 but to no avail, I can connect via telnet, but the software still fails.
Here is the minimally reproducable example I've made
from multiprocessing import Process, Queue
from queue import Full, Empty
from io import TextIOBase
import socket
import selectors
class SocketConsoleClient(TextIOBase):
def __init__(self, port: int):
self.port = port
self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.conn.connect(('', self.port))
self.selector = selectors.DefaultSelector()
self.conn.setblocking(False)
self.selector.register(self.conn, selectors.EVENT_WRITE, data='hello')
def readline(self, size: int = ...) -> str:
while True:
for k, _ in self.selector.select(timeout=None):
if k.data == 'hello':
try:
return str(self.conn.recv(1024).decode('latin1'))
except Exception as e:
# print(e)
continue
class SocketConsoleWriter(Process):
def __init__(self):
super().__init__()
self.writes = Queue()
self.connections = []
self.listener = None
self.selector = None
self.port = 10000
def run(self) -> None:
while True:
try:
self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.listener.bind(('', self.port))
self.listener.listen()
print('listening on', ('', self.port))
self.listener.setblocking(False)
break
except Exception as _:
self.port += 1 # if errno is 98, then port is not available.
self.selector = selectors.DefaultSelector()
self.selector.register(self.listener, selectors.EVENT_READ, data='test')
while True:
try:
w = self.writes.get_nowait()
if w == '$$$EXIT!!!':
break
else:
for c in self.connections:
c.send(w.encode('latin1'))
except Empty:
pass
try:
d = self.selector.select(1)
for k, _ in d:
if k.data == 'test':
conn, addr = self.listener.accept()
print('{} connected'.format(addr))
self.connections.append(conn)
except Exception as e:
# print(e)
pass
class SocketConsoleServer:
server = None
def __init__(self):
if SocketConsoleServer.server is None:
SocketConsoleServer.server = SocketConsoleWriter()
SocketConsoleServer.server.start()
#staticmethod
def port() -> int:
if SocketConsoleServer.server is None:
SocketConsoleServer.server = SocketConsoleWriter()
SocketConsoleServer.server.start()
return SocketConsoleServer.server.port
#staticmethod
def write(msg: str):
if SocketConsoleServer.server is None:
SocketConsoleServer.server = SocketConsoleWriter()
SocketConsoleServer.server.start()
SocketConsoleServer.server.writes.put(msg)
if __name__ == '__main__':
import sys, time
serv = SocketConsoleServer()
time.sleep(1)
class TestProcessSocket(Process):
def run(self):
sys.stdin = SocketConsoleClient(serv.port())
time.sleep(1)
print(input())
client = TestProcessSocket()
client.start()
serv.write(input('Type something: '))
client.join()
Why is my socket connection getting refused, I'm using ubuntu?

why is queue empty for the file to be downloaded

Below is the code that I have that downloads various URLS into each separate thread, I was in attempt to make some changes before I implement the thread pool but with this change the queue is coming to be empty and download is not beginning.
import Queue
import urllib2
import os
import utils as _fdUtils
import signal
import sys
import time
import threading
class ThreadedFetch(threading.Thread):
""" docstring for ThreadedFetch
"""
def __init__(self, queue, out_queue):
super(ThreadedFetch, self).__init__()
self.queueItems = queue.get()
self.__url = self.queueItems[0]
self.__saveTo = self.queueItems[1]
self.outQueue = out_queue
def run(self):
fileName = self.__url.split('/')[-1]
path = os.path.join(DESKTOP_PATH, fileName)
file_size = int(_fdUtils.getUrlSizeInBytes(self.__url))
while not STOP_REQUEST.isSet():
urlFh = urllib2.urlopen(self.__url)
_log.info("Download: %s" , fileName)
with open(path, 'wb') as fh:
file_size_dl = 0
block_sz = 8192
while True:
buffer = urlFh.read(block_sz)
if not buffer:
break
file_size_dl += len(buffer)
fh.write(buffer)
status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size)
status = status + chr(8)*(len(status)+1)
sys.stdout.write('%s\r' % status)
time.sleep(.05)
sys.stdout.flush()
if file_size_dl == file_size:
_log.info("Download Completed %s%% for file %s, saved to %s",
file_size_dl * 100. / file_size, fileName, DESKTOP_PATH)
below is the main function that does the call and initiation.
def main(appName):
args = _fdUtils.getParser()
urls_saveTo = {}
# spawn a pool of threads, and pass them queue instance
# each url will be downloaded concurrently
for i in range(len(args.urls)):
t = ThreadedFetch(queue, out_queue)
t.daemon = True
t.start()
try:
for url in args.urls:
urls_saveTo[url] = args.saveTo
# urls_saveTo = {urls[0]: args.saveTo, urls[1]: args.saveTo, urls[2]: args.saveTo}
# populate queue with data
for item, value in urls_saveTo.iteritems():
queue.put([item, value])
# wait on the queue until everything has been processed
queue.join()
print '*** Done'
except (KeyboardInterrupt, SystemExit):
lgr.critical('! Received keyboard interrupt, quitting threads.')

You create the queue and then the first thread which immediately tries to fetch an item from the still empty queue. The ThreadedFetch.__init__() method isn't run asynchronously, just the run() method when you call start() on a thread object.
Store the queue in the __init__() and move the get() into the run() method. That way you can create all the threads and they are blocking in their own thread, giving you the chance to put items into the queue in the main thread.
class ThreadedFetch(threading.Thread):
def __init__(self, queue, out_queue):
super(ThreadedFetch, self).__init__()
self.queue = queue
self.outQueue = out_queue
def run(self):
url, save_to = self.queue.get()
# ...
For this example the queue is unnecessary by the way as every thread gets exactly one item from the queue. You could pass that item directly to the thread when creating the thread object:
class ThreadedFetch(threading.Thread):
def __init__(self, url, save_to, out_queue):
super(ThreadedFetch, self).__init__()
self.url = url
self.save_to = save_to
self.outQueue = out_queue
def run(self):
# ...
And when the ThreadedFetch class really just consists of the __init__() and run() method you may consider moving the run() method into a function and start that asynchronously.
def fetch(url, save_to, out_queue):
# ...
# ...
def main():
# ...
thread = Thread(target=fetch, args=(url, save_to, out_queue))
thread.daemon = True
thread.start()

Python - Asyncore (client) socket - Can not determaine connection status

For some reason, self.connected of the asyncore.dispatcher class doesn't consider my socket to be connected on the client side. The server side sees the socket as connected and treats it as such, but the client doesn't know if it's connected or not and handle_connect doesn't "proc", so i can't use a overridden version of it to check if the socket is connected either.
Any thoughts on this code why it ain't working:
#!/usr/bin/python
# -*- coding: iso-8859-15 -*-
import asyncore
from threading import *
from socket import *
from time import sleep
from os import _exit
from logger import *
from config import *
class logDispatcher(asyncore.dispatcher):
def __init__(self, config=None):
self.inbuffer = ''
self.buffer = ''
self.lockedbuffer = False
self.is_writable = False
asyncore.dispatcher.__init__(self)
#Thread.__init__(self)
self.create_socket(AF_INET, SOCK_STREAM)
#self.is_writable = True
#self.start()
def compare(self, obj, otherobj):
return (str(obj).lower() == str(otherobj).lower()[:len(str(obj))])
def _in(self, obj, otherobj):
return (str(obj).lower() in str(otherobj).lower())
def parse(self):
if self.inbuffer[-2:] != '\r\n':
return False
self.lockedbuffer = True
self.inbuffer = ''
self.lockedbuffer = False
def readable(self):
return True
def handle_connect(self):
log('Connected to ' + str(server), 'SOCK_CORE')
def handle_close(self):
self.close()
def handle_read(self):
data = self.recv(8192)
while self.lockedbuffer:
sleep(0.01)
self.inbuffer += data
def writable(self):
return (len(self.buffer) > 0)
def handle_write(self):
while self.is_writable:
sent = self.send(self.buffer)
sleep(1)
self.buffer = self.buffer[sent:]
if len(self.buffer) <= 0:
self.is_writable = False
sleep(0.01)
def _send(self, what):
self.buffer += what + '\r\n'
self.is_writable = True
def handle_error(self):
log('Error, closing socket!', 'SOCK_CORE')
self.close()
def run(self):
log('Log socket engine initating', 'SOCK_CORE')
self.connect((server, server_port))
print self.connected
sleep(3)
print self.connected
class start(Thread):
def __init__(self):
Thread.__init__(self)
self.start()
def run(self):
asyncore.loop(0.1)
start()
logDisp = logDispatcher()
logDisp.run()

def handle_connect_event(self):
self.is_connected = True
Adding that to your dispatcher will give you a way to check if the socket is connected or not, thanks to some stack trace (python -m trace -t script.py) in Python I managed to figure out that the asyncore class automaticly created that function for whatever reason, and it was called continiously as long as the socket was connected or in a connected state.
After that, i also replaced the threaded asyncore.loop() and replaced it with a "static" placement locking your main thread, one of these two combinations (or both) solved the issue for now.. the logic isn't the same as in my problem which i don't like but i assume that i'll be needing to create my own dispach_event just like if i were to do a OpenGL GUI class where i would call dispatch_event() manually every loop some how in the thread to "keep things alive".. it's just a thought..
Anyway, here's a working example:
#!/usr/bin/python
# -*- coding: iso-8859-15 -*-
import asyncore, socket
from threading import *
from time import sleep
from os import _exit
from logger import *
from config import *
def _map():
return {}
def _array():
return []
class logDispatcher(Thread, asyncore.dispatcher):
def __init__(self, config=None):
self.inbuffer = ''
self.buffer = ''
self.lockedbuffer = False
self.is_writable = False
self.is_connected = False
self.exit = False
self.initated = False
asyncore.dispatcher.__init__(self)
Thread.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.connect((server, server_port))
except:
log('Could not connect to ' + server, 'LOG_SOCK')
return None
self.start()
def handle_connect_event(self):
self.is_connected = True
def handle_connect(self):
self.is_connected = True
log('Connected to ' + str(server), 'LOG_SOCK')
def handle_close(self):
self.is_connected = False
self.close()
def handle_read(self):
data = self.recv(8192)
while self.lockedbuffer:
sleep(0.01)
self.inbuffer += data
def handle_write(self):
while self.is_writable:
sent = self.send(self.buffer)
sleep(1)
self.buffer = self.buffer[sent:]
if len(self.buffer) <= 0:
self.is_writable = False
sleep(0.01)
def _send(self, what):
self.buffer += what + '\r\n'
self.is_writable = True
def run(self):
sleep(1)
log('Log engine initating (hooking on to main)', 'LOG_CORE')
main = None
for t in enumerate():
if t.getName() == 'MainThread':
main = t
log('Log engine attached to main', 'LOG_CORE')
while (main and main.isAlive()) and (self.connected or self.is_connected):
print 'WHAM', self.connected, self.is_connected
sleep(1)
while 1:
logDisp = logDispatcher()
asyncore.loop(0.1)
log('Logserver disconnected, trying to reconnect!', 'CORE')
sleep(10)

Socket in python will only send data it receives

I must be missing something in the code. I've rewritten an 'echo server' example to do a bit more when it receives something.
This is how it currently looks:
#!/usr/bin/env python
import select
import socket
import sys
import threading
import time
import Queue
globuser = {}
queue = Queue.Queue()
class Server:
def __init__(self):
self.host = ''
self.port = 2000
self.backlog = 5
self.size = 1024
self.server = None
self.threads = []
def open_socket(self):
try:
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server.bind((self.host,self.port))
self.server.listen(5)
except socket.error, (value,message):
if self.server:
self.server.close()
print "Could not open socket: " + message
sys.exit(1)
def run(self):
self.open_socket()
input = [self.server,sys.stdin]
running = 1
while running:
inputready,outputready,exceptready = select.select(input,[],[])
for s in inputready:
if s == self.server:
# handle the server socket
c = Client(self.server.accept(), queue)
c.start()
self.threads.append(c)
elif s == sys.stdin:
# handle standard input
junk = sys.stdin.readline()
running = 0
# close all threads
self.server.close()
for c in self.threads:
c.join()
class Client(threading.Thread):
initialized=0
def __init__(self,(client,address), queue):
threading.Thread.__init__(self)
self.client = client
self.address = address
self.size = 1024
self.queue = queue
self.threads = []
global globuser
print 'Client thread created!'
def run(self):
running = 1
while running:
print 'While running client'
data = self.client.recv(self.size)
print 'Dit we receive data?'
if data:
print 'Data received!'
print 'Fetching data from socket: ',
if data[0] == 'I':
print 'Initializing user: ' + data
user = {'uid': data[1:6] ,'x': data[6:9], 'y': data[9:12]}
globuser[user['uid']] = user
print globuser
initialized=1
m=updateClient(user['uid'], queue)
m.start()
self.threads.append(m)
self.client.send('Beginning - Initialized'+';')
elif data[0] == 'A':
print 'Client has received previous data: ' + data
#On deactivation, nothing works.
self.client.send(data+';')
#print 'Client Data sent: ' + data
else:
print 'Closing'
self.client.close()
running = 0
if self.queue.empty():
print 'Queue is empty'
else:
print 'Queue has information: ',
data2 = self.queue.get(1, 1)
isdata2 = 1
if data2 == 'Exit':
running = 0
print 'Client is being closed'
self.client.close()
if isdata2 == 1:
print 'Sending data to client: ' + data2,
self.client.send(data2)
self.queue.task_done()
isdata2 = 0
time.sleep(1)
class updateClient(threading.Thread):
def __init__(self,uid, queue):
threading.Thread.__init__(self)
self.uid = uid
self.queue = queue
global globuser
print 'updateClient thread started!'
def run(self):
running = 20
test=0
while running > 0:
test = test + 1
self.queue.put('Test Queue Data #' + str(test))
running = running - 1
time.sleep(1)
print 'Updateclient has stopped'
if __name__ == "__main__":
s = Server()
s.run()
This runs fine, although it's kind of silly to keep sending the same data back again along with other data.
In the middle of the code you'll see
#On deactivation, nothing works.
self.client.send(data+';')
#print 'Client Data sent: ' + data
When I DO deactive the self.client.send(data+';') or change it into self.client.send('something else;') it does not work! And the client receives nothing.
Is there something special about the "data" variable? Do I need to format the string in some way?

Here's a cleaned-up, functional version of your code! I tested it myself, though I didn't write unit tests.
There were some syntax errors and other miscellaneous problems with the
original code, so I took some liberties. I'm assuming that the protocol is
framed by using ; as a delimiter, since a ; is sent at the end of every
message to the client, though no framing was being done in the original code.
from twisted.internet import reactor, protocol, task
from twisted.protocols import basic
from twisted.python import log
import sys
class ServerProtocol(basic.LineOnlyReceiver):
delimiter = ';'
def lineReceived(self, line):
if line.startswith('I'):
user = dict(uid=line[1:6], x=line[6:9], y=line[9:12])
self.factory.users[user['uid']] = user
log.msg(repr(self.factory.users))
self.startUpdateClient()
self.sendLine('Beginning - Initialized')
elif line.startswith('A'):
self.sendLine(line)
else:
self.transport.loseConnection()
def _updateClient(self):
if self._running == 0:
self._looper.stop()
return
self._running -= 1
self._test += 1
self.sendLine('Test Queue Data #%d' % (self._test,))
def startUpdateClient(self):
self._running, self._test = 20, 0
self._looper = task.LoopingCall(self._updateClient)
self._looper.start(1, now=False)
class Server(protocol.ServerFactory):
protocol = ServerProtocol
def __init__(self):
self.users = {}
if __name__ == '__main__':
log.startLogging(sys.stderr)
reactor.listenTCP(2000, Server())
reactor.run()