We Keep Coding

Python Multithreading with requests

i have one scraper which initiate the "requestes" session and fetch some data, using a IPV6, i have now 10000 ip list, I have prepared it using threading, but its giving error.
Need support to find out the issue.
import requests, queue,threading, urllib3,jso,pandas as pd, os, time, datetime,inspect
num_threads = 2
root = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
with open (root+ "/ip_list.txt") as ips:
device_ip = list(ips)
class Writer_Worker(threading.Thread):
def __init__(self, queue, df, *args, **kwargs):
if not queue:
print("Device Queue not specified")
exit(1)
self.out_q = queue
self.df = df
super().__init__(*args, **kwargs)
def run(self):
while True:
try:
device_details = self.out_q.get(timeout=3)
except queue.Empty:
return
self.df[device_details[0]] = device_details
self.out_q.task_done()
class Worker(threading.Thread):
def __init__(self, queue, out_queue, device_password, *args, **kwargs):
if not queue:
print("Device Queue not specified")
exit(1)
self.queue = queue
self.pas = device_password
self.out_q = out_queue
super().__init__(*args, **kwargs)
def run(self):
while True:
try:
device_ip = self.queue.get(timeout=3)
except queue.Empty:
return
self.connect_to_device_and_process(device_ip)
self.queue.task_done()
def connect_to_device_and_process(self, device_ip):
st = str("Online")
try:
r = requests.post("https://["+device_ip+"]/?q=index.login&mimosa_ajax=1", {"username":"configure", "password":self.pas}, verify=False)
except requests.exceptions.ConnectionError:
st = str("Offline")
self.out_q.put([device_ip,st,"","","","","","","","","","","","","","","","","",""])
return
finally:
if 'Online' in st:
r = requests.get("https://["+device_ip+"]/cgi/dashboard.php", cookies=r.cookies, verify=False)
if "Response [401]" in str(r):
st2 = str("Password Error")
self.out_q.put([device_ip,st2,"","","","","","","","","","","","","","","","","",""])
else:
data = json.loads(r.content.decode())
output5 = data ['config'] ['Spectrum_Power']
self.out_q.put([device_ip,st,output5['Auto_Power'].replace('2', 'Max Power').replace('1', 'Min Power').replace('0', 'off'),output5['AutoConfig']])
def main():
start = time.time()
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
pas = input("Enter Device Password:")
df =pd.DataFrame(columns = ["IP","Status","Auto_Power","AutoChannel"])
q = queue.Queue(len(device_ip))
for ip in device_ip:
q.put_nowait(ip)
out_q = queue.Queue(len(device_ip))
Writer_Worker(out_q, df).start()
for _ in range(num_threads):
Worker(q, out_q, pas).start()
q.join()
print(df)
df.to_excel('iBridge_C5x_Audit_Report.xlsx', sheet_name='Detail', index = False)
if __name__ == "__main__":
main()
below is the error while running the script, seeps I am unable to login to this device.
Any help is appreciable.

You should use a thread pool that distributes the work between a fixed number of threads. This is a core feature of Python since version 3.2.
from concurrent.futures import ThreadPoolExecutor
Define a function perform(ip) that performs the request for one ip
Set variable numThreads to the number of desired threads
Run the thread-pool executor:
print(f'Using {numThreads} threads')
with ThreadPoolExecutor(max_workers=numThreads) as pool:
success = all(pool.map(perform, ips))
Source: https://docs.python.org/3/library/concurrent.futures.html
On that page you find an example even better tailored to your application: https://docs.python.org/3/library/concurrent.futures.html#threadpoolexecutor-example

from threading import Thread
th = Thread(target=self.fill_imdb, args=(movies_info_part, "thread " + str(count)))
th.start()
fill_imdb is my method

Callback function does not see correct values in instance

I'm having a strange phenomena in Python with callback functions and handlers.
I use ZMQ to handle my communication and use a stream for the socket. I have the base class:
import multiprocessing
import zmq
from concurrent.futures import ThreadPoolExecutor
from zmq.eventloop import ioloop, zmqstream
from zmq.utils import jsonapi as json
# Types of messages
TYPE_A = 'type_a'
TYPE_B = 'type_b'
class ZmqProcess(multiprocessing.Process):
def __init__(self):
super(ZmqProcess, self).__init__()
self.context = None
self.loop = None
self.handle_stream = None
def setup(self):
self.context = zmq.Context()
self.loop = ioloop.IOLoop.instance()
def send(self, msg_type, msg, host, port):
sock = zmq.Context().socket(zmq.PAIR)
sock.connect('tcp://%s:%s' % (host, port))
sock.send_json([msg_type, msg])
def stream(self, sock_type, addr):
sock = self.context.socket(sock_type)
if isinstance(addr, str):
addr = addr.split(':')
host, port = addr if len(addr) == 2 else (addr[0], None)
if port:
sock.bind('tcp://%s:%s' % (host, port))
else:
port = sock.bind_to_random_port('tcp://%s' % host)
stream = zmqstream.ZMQStream(sock, self.loop)
return stream, int(port)
class MessageHandler(object):
def __init__(self, json_load=-1):
self._json_load = json_load
self.pool = ThreadPoolExecutor(max_workers=10)
def __call__(self, msg):
i = self._json_load
msg_type, data = json.loads(msg[i])
msg[i] = data
if msg_type.startswith('_'):
raise AttributeError('%s starts with an "_"' % msg_type)
getattr(self, msg_type)(*msg)
And I have a class that inherits from it:
import zmq
import zmq_base
class ZmqServerMeta(zmq_base.ZmqProcess):
def __init__(self, bind_addr, handlers):
super(ZmqServerMeta, self).__init__()
self.bind_addr = bind_addr
self.handlers = handlers
def setup(self):
super(ZmqServerMeta, self).setup()
self.handle_stream, _ = self.stream(zmq.PAIR, self.bind_addr)
self.handle_stream.on_recv(StreamHandler(self.handle_stream, self.stop,
self.handlers))
def run(self):
self.setup()
self.loop.start()
def stop(self):
self.loop.stop()
class StreamHandler(zmq_base.MessageHandler):
def __init__(self, handle_stream, stop, handlers):
super(StreamHandler, self).__init__()
self._handle_stream = handle_stream
self._stop = stop
self._handlers = handlers
def type_a(self, data):
if zmq_base.TYPE_A in self._handlers:
if self._handlers[zmq_base.TYPE_A]:
for handle in self._handlers[zmq_base.TYPE_A]:
self.pool.submit(handle, data)
else:
pass
else:
pass
def type_b(self, data):
if zmq_base.TYPE_B in self._handlers:
if self._handlers[zmq_base.TYPE_B]:
for handle in self._handlers[zmq_base.TYPE_B]:
self.pool.submit(handle, data)
else:
pass
else:
pass
def endit(self):
self._stop()
Additionally, I have a class that I want to use as storage. And here is where the trouble starts:
import threading
import zmq_server_meta as server
import zmq_base as base
class Storage:
def __init__(self):
self.list = []
self.list_lock = threading.RLock()
self.zmq_server = None
self.host = '127.0.0.1'
self.port = 5432
self.bind_addr = (self.host, self.port)
def setup(self):
handlers = {base.TYPE_A: [self. remove]}
self.zmq_server = server.ZmqServerMeta(handlers=handlers, bind_addr=self.bind_addr)
self.zmq_server.start()
def add(self, data):
with self.list_lock:
try:
self.list.append(data)
except:
print "Didn't work"
def remove(self, msg):
with self.list_lock:
try:
self.list.remove(msg)
except:
print "Didn't work"
The idea is that that class stores some global information that it receives.
It is all started in a file to test:
import sys
import time
import storage
import zmq_base as base
import zmq_server_meta as server
def printMsg(msg):
print msg
store = storage.Storage()
store.setup()
handlers = {base.TYPE_B: [printMsg]}
client = server.ZmqServerMeta(handlers=handlers, bind_addr=('127.0.0.1', 5431))
client.start()
message = "Test"
store.add(message)
client.send(base.TYPE_A, message, '127.0.0.1', 5432)
I simplified it to reduce clutter. Instead of just adding it, it is usually send and then a response comes back. The response, the client sending, should be processed by the correct callback, remove(), and it should remove something out of the list. The problem that occurs is, that the remove() function sees an empty list, although there should be an element in the list. If I check from the testing file, I can see the element after it was added, and if I call remove() from there, I see a non-empty list and can remove it. My question is, why does the callback sees an empty list and how can I make sure it does see the correct elements in the list?
Kind regards
Patrick

I believe the problem lays in the fact that the ZmqProcess class inherits from multiprocessing.Process. Multiprocessing does not allow to share objects among different processes, except by using a shared memory map using Value or Array ( as can be seen in the documentation: https://docs.python.org/3/library/multiprocessing.html#sharing-state-between-processes )
If you want to use your custom object, you can use a Server process / proxy object, which can be found in on the same page of the documentation.
So you can, for instance, define a manager in the init function of the Storage class like: self.manager = Manager() Afterwards you put self.list = self.manager.list(). This should do the trick.

Can pyzmq publishers be operated from class instances?

I have following code for a Publisher, which instantiates a few class instances and publishes some messages.
However, I don't receive anything at Subscriber side.
Publisher
import zmq
import time
from multiprocessing import Process
class SendData:
def __init__(self, msg, port):
self.msg = msg
self.port = port
ctx = zmq.Context()
self.sock = ctx.socket(zmq.PUB)
self.sock.bind('tcp://127.0.0.1:'+str(self.port))
time.sleep(1)
def sender(self):
self.sock.send_json(self.msg)
def main():
for device, port in zip(['2.2.2.2', '5.5.5.5'],[5001, 5002]):
msg = {device:'Some random message'}
instance = SendData(device, port)
Process(target=instance.sender).start()
if __name__ == "__main__":
main()
Subscriber
import zmq
ctx = zmq.Context()
recv_sock1 = ctx.socket(zmq.SUB)
recv_sock1.connect('tcp://127.0.0.1:5001')
recv_sock1.setsockopt(zmq.SUBSCRIBE, '')
recv_sock2 = ctx.socket(zmq.SUB)
recv_sock2.connect('tcp://127.0.0.1:5002')
recv_sock2.setsockopt(zmq.SUBSCRIBE, '')
while True:
if recv_sock1.poll(10):
msg = recv_sock1.recv_json()
print msg
if recv_sock2.poll(10):
msg = recv_sock2.recv_json()
print msg
I had subscribers started before publishers could publish anything. Also, I can see TCP connections are in Established so connections are made.
pyzmq version 16.0.0
python version 2.7
Q1: Are 0mq publishers supported from class instances?
Q2: What am I missing?

As was said before, trying to share the ZeroMQ context between processes is the problem here and the solution by user3666197 will work.
However, I would suggest subclassing multiprocessing.Process in this case. That way, it is much clearer what part of the code is executed in which process. It also makes your code more readable and reusable.
The following code creates one sender process per device. The sender processes can be reused during the runtime of your program to send more data:
import multiprocessing
import queue
import zmq
import time
class Sender(multiprocessing.Process):
def __init__(self, port):
super(Sender, self).__init__()
self._port = port
self._messages = multiprocessing.Queue()
self._do_stop = multiprocesing.Event()
def run(self):
"""
This code is executed in a new process.
"""
ctx = zmq.Context()
sock = ctx.socket(zmq.PUB)
sock.bind("tcp://127.0.0.1:" + str(self._port))
while not self._do_stop.is_set():
try:
msg = self._message.get_nowait()
sock.send_json(msg)
except queue.Empty:
time.sleep(0.01)
def stop(self):
self._do_stop.set()
def send_message(self, msg):
self._messages.put(msg)
def main():
data = zip(['2.2.2.2', '5.5.5.5'], [5001, 5002])
# create senders
senders = {device: Sender(port) for device, port in data}
# start senders
for device in senders:
senders[device].start()
# send messages
for device, port in zip(['2.2.2.2', '5.5.5.5'],[5001, 5002]):
msg = {device: 'Some random message'}
senders[device].send_message(msg)
# do more stuff here....
# ... e.g. send more messages
# ...
# once you are finished, stop the subprocesses
for device in senders:
senders[device].stop()
I hope this helps solving your problem.

A1: Yes, they are.
A2: Conflicts of scope-of-use v/s Zero-sharing, one of ZeroMQ maxims
Once your original Publisher code is being executed in main(), the class instantiation process creates ( i.e. inside the main()-process scope-of-use ), via the .__init__() constructor-method, it's own Context() -instance, which thus belongs ( incl. all of it's derived child-objects ( sockets et al ) ) to this main()-process.
Next, the call to the Process(...) initiates another few processes, that receive the class-instances ( the pity is that these have already created ZeroMQ non-shareable toys ) from the main()-scope-of-use.
Solution?
A possible dirty quick hack could be to defer the ZeroMQ Context() instantiation -- yes, just move it from .__init__() to some .aDeferredSETUP() that will be executed specifically under different scope-of-use inside each of the spinned-of Process()-process, different from main()-process and you ought be done, as Zero-sharing is safely obeyed.
class SendData:
def __init__(self, msg, port):
self.msg = msg
self.port = port
self.NotSETUP = True
self.ctx = None
self.sock = None
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ L8R
# ctx = zmq.Context()
# self.sock = ctx.socket( zmq.PUB )
# self.sock.bind( 'tcp://127.0.0.1:' + str( self.port ) )
# time.sleep( 1 )
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ L8R
def sender( self ):
if self.NotSETUP:
self.aDeferredSETUP()
self.sock.send_json( self.msg )
def aDeferredSETUP( self ): # create I/O-threads in Process(), not main()
self.ctx = zmq.Context()
self.sock = self.ctx.socket( zmq.PUB )
self.sock.bind( 'tcp://127.0.0.1:' + str( self.port ) )
time.sleep( 1 )

zmq PUBhandler could not be used in __init__() method

I have a class MyLogger for sending messages to log server by using PUBhandler.
An exception gets raised when MyLogger is instanced in LogWorker.init() method (like version 1), however, it is ok if MyLogger is instanced in LogWorker.log_worker() method (version 2).
Any suggestions would be appreciated.
import logging
from multiprocessing import Process
import os
import random
import sys
import time
import zmq
from zmq.log.handlers import PUBHandler
class MyLogger(object):
''''''
def __init__(self, port, handler=None):
self.port = port
self.handler = handler or self._construct_sock_handler()
self.logger = logging.getLogger()
self.logger.setLevel(logging.INFO)
if not self.logger.handlers:
self.logger.addHandler(self.handler)
def _construct_sock_handler(self):
context = zmq.Context()
log_sock = context.socket(zmq.PUB)
log_sock.connect("tcp://127.0.0.1:%i" % self.port)
time.sleep(0.1)
handler = PUBHandler(log_sock)
return handler
def get_logger(self):
return self.logger
def sub_logger(port, level=logging.DEBUG):
ctx = zmq.Context()
sub = ctx.socket(zmq.SUB)
sub.bind('tcp://127.0.0.1:%i' % port)
sub.setsockopt(zmq.SUBSCRIBE, "")
logging.basicConfig(level=level)
while True:
level, message = sub.recv_multipart()
if message.endswith('\n'):
# trim trailing newline, which will get appended again
message = message[:-1]
log = getattr(logging, level.lower())
log(message)
class LogWorker(object):
def __init__(self):
- pass # version 1
+ self.logger = MyLogger(port).get_logger() # version 2
def log_worker(self, port):
- self.logger = MyLogger(port).get_logger() # version 1
print "starting logger at %i with level=%s" % (os.getpid(), logging.DEBUG)
while True:
level = logging.INFO
self.logger.log(level, "Hello from %i!" % os.getpid())
time.sleep(1)
if __name__ == '__main__':
if len(sys.argv) > 1:
n = int(sys.argv[1])
else:
n = 2
port = 5555
workers = [Process(target=LogWorker().log_worker, args=(port,)) for _ in range(n)]
[w.start() for w in workers]
try:
sub_logger(port)
except KeyboardInterrupt:
pass
finally:
[ w.terminate() for w in workers ]

answer from pyzmq owner minrk:
You cannot pass zmq contexts or sockets across the fork boundary that happens when you instantiate a subprocess with multiprocessing. You have to make sure that you create your Context after you are in the subprocess.
solution:
def work():
worker = LogWorker(port)
worker.log_worker()
workers = [ Process(target=work) for _ in range(n) ]

Using threads in the right way

I'm working on server written in python. When the client sends a cmd the server will call a function with unknown running time. So to avoid blocking I used threading. But when looking at the child process it seems that they're not terminating, causing a lot of memory usage.
EDIT : Here is the tree of the directory : http://pastebin.com/WZDxLquC
Following answers I found on stackoverflow I implemented a custom Thread class:
sThreads.py :
import threading
class Thread(threading.Thread):
def __init__(self, aFun, args = ()):
super(Thread, self).__init__(None, aFun, None, args)
self.stopped = threading.Event()
def stop(self):
self.stopped.set()
def isStopped(self):
return self.stopped.isSet()
Then here is the server's loop:
some where in mainServer.py:
def serve_forever(self, aCustomClass, aSize = 1024):
while True:
self.conn, self.addr = self.sock.accept()
msg = self.recvMSG(4096)
if(msg):
self.handShake(msg)
print 'Accepted !'
while True:
msg = self.recvMSG(aSize)
if(msg):
t = sThreads.Thread(self.handle, (aCustomClass,))
t.start()
self.currentThreads.append(t)
if(self.workers > 0):
tt = sThreads.Thread(self.respond)
tt.start()
if(self.workers == 0 and len(self.currentThreads) > 0):
for th in self.currentThreads:
th.stop()
Using a custom Thread class will not solve the issue and it still does not stop the terminated threads!
EDIT : added the handle() and respond() methods :
def handle(self, aClass):
self.workers += 1
self.queue.put(aClass._onRecieve(self.decodeStream()))
def respond(self):
while self.workers > 0:
msgToSend, wantToSend = self.queue.get()
self.workers -= 1
if(wantToSend):
print 'I want to send :', msgToSend
continue #Send is not yet implemented !

It seems that self.queue.get() was causing all the issue ...