I have a script that checks a gmail account using the imap IDLE protocol. To do this I use imaplib2, hosted here. Every so often it throws an unhandled exception:
Traceback (most recent call last):
File "C:\Python27\lib\site-packages\imaplib2\imaplib2.py", line 1830, in _reader
raise IOError("Too many read 0")
IOError: Too many read 0
(line 1839 from the posted link)
Here is the offending section (halfway down):
def _reader(self):
threading.currentThread().setName(self.identifier + 'reader')
if __debug__: self._log(1, 'starting using select')
line_part = ''
rxzero = 0
terminate = False
while not (terminate or self.Terminate):
if self.state == LOGOUT:
timeout = 1
else:
timeout = self.read_poll_timeout
try:
r,w,e = select.select([self.read_fd], [], [], timeout)
if __debug__: self._log(5, 'select => %s, %s, %s' % (r,w,e))
if not r: # Timeout
continue
data = self.read(self.read_size) # Drain ssl buffer if present
start = 0
dlen = len(data)
if __debug__: self._log(5, 'rcvd %s' % dlen)
if dlen == 0:
rxzero += 1
if rxzero > 5:
raise IOError("Too many read 0") # <- This is the error I'm
time.sleep(0.1) # getting
else:
rxzero = 0
while True:
stop = data.find('\n', start)
if stop < 0:
line_part += data[start:]
break
stop += 1
line_part, start, line = \
'', stop, line_part + data[start:stop]
if __debug__: self._log(4, '< %s' % line)
self.inq.put(line)
if self.TerminateReader:
terminate = True
except:
reason = 'socket error: %s - %s' % sys.exc_info()[:2]
if __debug__:
if not self.Terminate:
self._print_log()
if self.debug: self.debug += 4 # Output all
self._log(1, reason)
self.inq.put((self.abort, reason))
break
I can't catch this error from my script because imaplib2 creates separate threads for its _reader and _writer functions. I don't really understand the error, so my question is should I modify the imaplib2 source code to ignore this error or change the conditions of it or what?
Thanks
I've been getting a variety of errors from imaplib2 including errno 10054 connection forciblely closed and too many read 0. These errors would cause my program to hang for about a half hour. To work around these issues I used multiprocessing to do the work in a separate process and implemented an activity check. If there is no activity for a period of time the main process terminates (I know, not ideal) the child process and spawns another. Here is some of the relevant code.
def MasterRun():
from multiprocessing import Value
counter = Value("I", 0)
last = counter.value
elapsed = 0
interval = 1
TIMEOUT = 90
proc = _make_process(counter)
# value < 0 signals process quit naturally
while counter.value >= 0:
if counter.value != last:
elapsed = 0
last = counter.value
if elapsed >= TIMEOUT or not proc.is_alive():
print "terminating process reason: %s\n" % \
("no activity time was exceeded" if proc.is_alive() else "process committed suicide")
proc.terminate()
proc.join(25)
proc = _make_process(counter)
proc.start()
elapsed = 0
sleep(interval)
elapsed += interval
proc.join(25)
def _make_process(counter):
from multiprocessing import Process
print "spawning child process"
proc = Process(target=_make_instance, args=(counter, ))
proc.daemon = True
proc.start()
return proc
Related
I have been trying a lot lately to come up with a good approach to wait on a producer of tasks for its completion by celery worker. I came up with one approach but that doesn't seem to work fast enough, Here it is:
celery producer:
leafs = []
def chain_tasks():
for i in range(1, 10):
p1 = ping1.si(i)
p2 = ping2.si(i)
p3 = ping3.si(i)
p4 = ping4.si(i)
mychain = chain(p1, p2, p3, p4)
leaf_id = mychain.apply_async()
leafs.append(leaf_id)
print('[INFO] Total leafs ->', leafs)
def _cancel_tasks(msg):
print("[ERROR] Dummy Task canceller->", msg)
def parent_succeeds(t):
if t.parent == None:
return True
else:
parent_succeeded = True
parent = t.parent
if parent.state == 'PENDING':
parent_succeeded = parent_succeeds(parent)
if not parent_succeeded:
return False
print('[INFO] Waiting on parentTask({0})...at {1} - {2}'.format(parent, datetime.now().strftime("%H:%M:%S"), parent.state), end='')
parent.wait(propagate=True)
print('Done.')
return parent.state != 'FAILURE'
def wait_for_comp():
print("[INFO] Waiting for celery to finish...")
max_fail = round(len(leafs) / 2)
fail_count = 0
for t in leafs:
if fail_count <= max_fail:
print('[INFO] Waiting on Task({0})...at {1}'.format(t, datetime.now().strftime("%H:%M:%S")))
try:
if parent_succeeds(t):
t.wait(propagate=True)
else:
print('[ERROR] One of the parent failed -> ', t.parent)
except Exception as e:
fail_count += 1
print('[ERROR] Exception Occurred [' + datetime.now().strftime("%H:%M:%S") + '] ->', str(e), fail_count)
print('[ERROR] Traceback [' + datetime.now().strftime("%H:%M:%S") + '] ->', traceback.format_exc())
else:
print("[ERROR] Failed!")
_cancel_tasks('failure of more than half tasks({0}/{1})'.format(fail_count, max_fail))
break
print("[INFO] Done.")
if __name__ == '__main__':
time_start = time.time()
chain_tasks()
wait_for_comp()
print('Finish time %s', time.time() - time_start)
One things that's wrong with this approach is that it WAITS on a sequence(for-loop) of tasks that doesn't necessarily need to be maintained at worker side as worker execution is based on rabbit-mq entry. So it involves lot of waiting.
Is there an alternative way to make the wait more efficient?
If I am not missing something important, then the simple solution is to change the leaf_id = mychain.apply_async() to something like:
result_as = mychain.apply_async()
result = result_as.get() # will block until the task is done
Note: do not call get() in your tasks.
Using the script below, I cannot seem to exit the threads. The script runs smoothly without issues but never exits when done. I can still see the thread alive, I have to use htop to kill them or completely exit the command line.
How can I get this script to exit and the threads to die?
def async_dns():
s = adns.init()
while True:
dname = q.get()
response = s.synchronous(dname,adns.rr.NS)[0]
if response == 0:
dot_net.append("Y")
print(dname + ", is Y")
elif response == 300 or response == 30 or response == 60:
dot_net.append("N")
print(dname + ", is N")
elif q.empty() == True:
q.task_done()
q = queue.Queue()
threads = []
for i in range(20):
t = threading.Thread(target=async_dns)
threads.append(t)
t.start()
for name in names:
q.put_nowait(name)
Remove and return an item from the queue. If optional args block is true and timeout is None (the default), block if necessary until an item is available. If timeout is a positive number, it blocks at most timeout seconds and raises the Empty exception if no item was available within that time. Otherwise (block is false), return an item if one is immediately available, else raise the Empty exception (timeout is ignored in that case).
Remember to check your queue.
See the document of queue.
Your threads are stuck in dname = q.get()
If you reaching empty queue, q.get() will wait forever for value to arrive.
You can replace get with get_nowait() but get ready to catch Queue.Empty execption
Background
A small server which waits for different types of jobs which are represented
as Python functions (async_func and async_func2 in the sample code below).
Each job gets submitted to a Pool with apply_async and takes a different amount of time, i.e. I cannot be sure that a job which was submitted first, also finishes first
I can check whether the job was finished with .get(timeout=0.1)
Question
How I can check whether the job is still waiting in the queue or is already running?
Is using a Queue the correct way or is there a more simple way?
Code
import multiprocessing
import random
import time
def async_func(x):
iterations = 0
x = (x + 0.1) % 1
while (x / 10.0) - random.random() < 0:
iterations += 1
time.sleep(0.01)
return iterations
def async_func2(x):
return(async_func(x + 0.5))
if __name__ == "__main__":
results = dict()
status = dict()
finished_processes = 0
worker_pool = multiprocessing.Pool(4)
jobs = 10
for i in range(jobs):
if i % 2 == 0:
results[i] = worker_pool.apply_async(async_func, (i,))
else:
results[i] = worker_pool.apply_async(async_func2, (i,))
status[i] = 'submitted'
while finished_processes < jobs:
for i in range(jobs):
if status[i] != 'finished':
try:
print('{0}: iterations needed = {1}'.format(i, results[i].get(timeout=0.1)))
status[i] = 'finished'
finished_processes += 1
except:
# how to distinguish between "running but no result yet" and "waiting to run"
status[i] = 'unknown'
Just send the status dict, to the function, since dicts are mutable all you need to do is change a bit your functions:
def async_func2(status, x):
status[x] = 'Started'
return(async_func(x + 0.5))
Of course you can change the status to pending just before calling your apply_async
for i in range(0, 650):
s = ticket[i]
try:
response = resource.get(path='ticket/%s' % s[0]) # Get ticket data from RT server
except urllib2.URLError, e: # If connection fails
resource = RTResource(url, user, pwd, CookieAuthenticator) # Reconnect to RT server
count -= 1 # Count re-connection attempts
if count < 0:
print "Connection failed at ticket %s" % s[0]
print "Got %s tickets out of %s" % {i + 1, len(ticket) + 1}
wb.save(fname)
sys.exit(1)
print 'Trying again...'
i -= 1
continue
count = 10
...more code here...
The above code executes well but will skip an iteration when exception is thrown. I am trying to decreament the value of i and then continuing the loop so that when exception is thrown, the loop will repeat for same value of i. When a value of i is skipped I lose one ticket from RT server. How do I fix it?
You ... can't do that in python. You can't affect the value of the iterator - it's using it's own internal value for each step in the loop, not paying attention to your attempts to override. If you have to succeed for each iteration I use something like this:
while True:
# code here
if success:
break
And place that inside your for loop. Or better, extract a method to simplify readability, but that's another post.
(Besides the correct point raised by g.d.d.c. about the inability to decrement the loop counter the specific way you've gone, )this sort of stuff is exactly the motivation for finally. You should probably organize your code as follows:
try - the part that's supposed to run but might not
except - the part to do only if there was a problem
else (optional) - the part to do only if there wasn't a problem
finally - stuff to do in any case
An alternative to embedding a while loop in your for loop, as suggested by g.d.d.c, is to simply use a while loop instead of a for loop, like so:
i = 0
while i < 650:
s = ticket[i]
try:
response = resource.get(path='ticket/%s' % s[0]) # Get ticket data from RT server
except urllib2.URLError, e: # If connection fails
resource = RTResource(url, user, pwd, CookieAuthenticator) # Reconnect to RT server
count -= 1 # Count re-connection attempts
if count < 0:
print "Connection failed at ticket %s" % s[0]
print "Got %s tickets out of %s" % {i + 1, len(ticket) + 1}
wb.save(fname)
sys.exit(1)
print 'Trying again...'
continue
count = 10
i += 1
...more code here...
I have the following sample code that I am trying to use the multiprocessing module on. The following statement had been working previously under other applications, but one process (which receives a very small amount of data just due to the breakup) finishes first and causes the program to finish. Could someone help me understand why this is not waiting for the others?
def mpProcessor(basePath, jsonData, num_procs = mp.cpu_count()):
manager = mp.Manager()
map = manager.dict()
procs = mp.Pool(processes = num_procs, maxtasksperchild = 1)
chunkSize = len(jsonData) / (num_procs)
dataChunk = [(i, i + chunkSize) for i in range(0, len(jsonData), chunkSize)]
count = 1
for i in dataChunk:
print 'test'
s, e = i
procs.apply_async(processJSON, args = (count, basePath, jsonData[s:e]))
count += 1
procs.close()
procs.join()
return map
def processJSON(proc, basePath, records):
print 'Spawning new process: %d' %os.getpid()
outDict = dict()
print len(records)
for i in range(len(records)):
valid = False
idx = 0
while valid == False:
jsonObject = json.loads(records[i][1])['results'][idx]
if jsonObject['kind'] == 'song':
valid = True
break
else:
idx += 1
tunesTrack = Track()
tunesTrack.setTrackId(jsonObject['trackId'])
print 'Finished processing %d records with process %d' %(len(records), os.getpid())
You seem to be reinventing the wheel.
What you are trying to do could be much more easily achieved by using an initializer with the pool and using map rather than apply_async. As it stands your code snippet is not runnable so I can't be sure what the actual problem is. However, the following should simplify your code and make it easier to debug.
import math
import multiprocessing as mp
def pool_init(basePath_):
global basePath, job_count
basePath = basePath_
job_count = 0
print 'Spawning new process: %d' %os.getpid()
def mpProcessor(basePath, jsonData, num_procs=mp.cpu_count()):
pool = mp.Pool(processes=num_procs, initializer=pool_init, initargs=(basePath,))
# could specify a chunksize, but multiprocessing works out the optimal chunksize
return pool.map(processJSON, jsonData)
# change processJSON to work with single records and
# remove proc and basePath args (as not needed)
def processJSON(record):
global job_count
print 'Starting job %d in process: %d' % (job_count, os.getpid())
valid = False
idx = 0
while valid == False:
jsonObject = json.loads(record[1])['results'][idx]
if jsonObject['kind'] == 'song':
valid = True
break
else:
idx += 1
tunesTrack = Track()
tunesTrack.setTrackId(jsonObject['trackId'])
print 'Finished processing job %d with process %d' % (job_count, os.getpid())
job_count += 1