#!/usr/bin/env python
#coding=utf-8
import sys,os,threading
import Queue
keyword = sys.argv[1]
path = sys.argv[2]
class keywordMatch(threading.Thread):
def __init__(self,queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
while True:
line = self.queue.get()
if keyword in line:
print line
queue.task_done()
def main():
concurrent = 100 # Number of threads
queue = Queue.Queue()
for i in range(concurrent):
t = keywordMatch(True)
t.setDaemon(True)
t.start()
allfiles = os.listdir(path)
for files in allfiles:
pathfile = os.path.join(path,files)
fp = open(pathfile)
lines = fp.readlines()
for line in lines:
queue.put(line.strip())
queue.join()
if __name__ == '__main__':
main()
This program is for searching the keyword in a directory,
but there occurs an error:
Exception in thread Thread-100:
Traceback (most recent call last):
File "/usr/local/Cellar/python/2.7.3/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "du.py", line 17, in run
line = self.queue.get()
AttributeError: 'bool' object has no attribute 'get'
How can I get rid of the error?
You're instantiating the thread with t = keywordMatch(True), and then in __init__ you're taking this argument and saving it as self.queue - so naturally self.queue is going to be a bool. If you want there to be a Queue instance there, you should pass it in.
In main() you wrote:
t = keywordMatch(True)
The keywordMatch class's __init__ does this:
def __init__(self,queue):
self.queue = queue
So now self.queue is True! Later, trying to do self.queue.get fails because it isn't a queue at all.
Related
When i run the code below, i got an exception
# System
import time
import logging
import sys
import os
import threading
# cv2 and helper:
import cv2
class inic_thread(threading.Thread):
def __init__(self, threadID, name, counter):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
self.counter = counter
def run(self):
print "Starting " + self.name
if self.counter == 1: capture_continuos()
elif self.counter == 2: face_search()
def capture_continuos():
#os.system('python capture_continuos.py')
while(1):
print 'a'
def face_search():
# atributes
pool = []
path_pool = './pool/'
while(1):
pool_get = os.listdir(path_pool)
if len(pool_get) > 0:
#print(str(len(pool_get))+' images in the pool')
for image in pool_get:
print(image)
os.system('python face_search.py -i '+str(image))
else:
print('Empty Pool')
try:
capture_continuos = inic_thread(1, "capture_continuos_1", 1)
face_search_2 = inic_thread(2, "face_search_2", 2)
capture_continuos.start()
face_search_2.start()
except:
print("Error: unable to start thread")
But it don't make sense to me, because one of the threads run normal, (face_search) but the other one give this exception.
Starting capture_continuos_1
Exception in thread capture_continuos_1:
Traceback (most recent call last):
File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run()
File "main.py", line 44, in run
if self.counter == 1: capture_continuos()
TypeError: 'inic_thread' object is not callable
What i'm doing wrong?
I run in a Raspberry Pi 3 model B with Ubuntu MATE 14.04; Python 2.7.12
At the bottom of your script you redefine variable capture_continuos assigning thread object to it.
Also as was mentioned to terminate thread it's better to call os._exit() instead of sys.exit().
I am trying to loop an alarm (in the file beep.wav) and show an alert.
Once the alert is closed, I want to instantly stop the alarm.
I am attempting a solution which uses threading to control the alarm's playback.
However, it throws an error:
Traceback (most recent call last):
File "test.py", line 28, in <module>
thread.stop()
File "test.py", line 21, in stop
self.process.kill()
AttributeError: 'AlarmThread' object has no attribute 'process'
I don't really know why this error would get thrown, but it looks like self.process is, for some reason, not assigned when AlarmThread.stop is called.
This makes no sense to me, as from my code it looks like thread.stop is only called after thread.start:
import subprocess
import threading
class AlarmThread(threading.Thread):
def __init__(self, file_name="beep.wav"):
super(AlarmThread, self).__init__()
self.file_name = file_name
self.ongoing = None
def run(self):
self.ongoing = True
while self.ongoing:
self.process = subprocess.Popen(["afplay", self.file_name])
self.process.wait()
def stop(self):
if self.ongoing is not None:
self.ongoing = False
self.process.kill()
thread = AlarmThread()
thread.start()
# show_alert is synchronous, an alert must be closed before the script continues
show_alert("1 second timer")
thread.stop()
thread.join()
You have a race condition. The thread hasn't had time to start, create the process and assign self.process by the time you call thread.stop(). You could initialize self.process in __init__ and use that to see if the process is really there
import subprocess
import threading
class AlarmThread(threading.Thread):
def __init__(self, file_name="beep.wav"):
super(AlarmThread, self).__init__()
self.lock = threading.Lock()
self.file_name = file_name
self.ongoing = False
self.process = None
def run(self):
self.ongoing = True
while True:
with self.lock:
if not self.ongoing:
break
self.process = subprocess.Popen(["afplayer", self.file_name])
self.process.wait()
def stop(self):
with self.lock:
if self.ongoing:
self.ongoing = False
if self.process:
self.process.kill()
thread = AlarmThread()
thread.start()
# show_alert is synchronous, an alert must be closed before the script continues
show_alert("1 second timer")
thread.stop()
thread.join()
Yes, it was caused by a race condition.
The thread hasn't had time to start, create the process and assign self.process by the time you call thread.stop()
However, I found a fix that relied upon simply waiting until thread.process was assigned:
thread = AlarmThread()
thread.start()
while not thread.process:
time.sleep(0.1)
show_alert(message)
thread.stop()
thread.join()
My class also changed slightly to ensure thread.process is always assigned:
class AlarmThread(threading.Thread):
def __init__(self, file_name="beep.wav"):
super(AlarmThread, self).__init__()
self.file_name = file_name
self.ongoing = None
self.process = None
def run(self):
self.ongoing = True
while self.ongoing:
self.process = subprocess.Popen(["afplay", self.file_name])
self.process.wait()
self.process = None
def stop(self):
if self.ongoing is not None:
self.ongoing = False
self.process.kill()
I have pretty simple code of worker, might be doing some silly mistake.
class BingWorker(object):
def __init__(self):
self.gm_worker = gearman.GearmanWorker(['localhost:4730'])
completed_job_request = self.gm_worker.register_task('bingmedia', callBing)
def callBing(self, gearman_worker, gearman_job):
print "In worker ", gearman_job.data
return "Kools"
def run(self):
self.gm_worker.work()
if __name__ == '__main__':
BingWorker().run()
gives
Traceback (most recent call last):
File "worker.py", line 16, in <module>
BingWorker().run()
File "worker.py", line 6, in __init__
completed_job_request = self.gm_worker.register_task('bingmedia', callBing)
NameError: global name 'callBing' is not defined
any hints? example is very similar to python example at http://gearman.org/examples/reverse/. just put in class structure
Improved code:
class BingWorker(object):
def __init__(self):
self.gm_worker = gearman.GearmanWorker(['localhost:4730'])
def callBing(self, gearman_worker, gearman_job):
print "In worker ", gearman_job.data
return "Kools"
def run(self):
completed_job_request = self.gm_worker.register_task('bingmedia', self.callBing)
self.gm_worker.work()
if __name__ == '__main__':
BingWorker().run()
You need to change callBing to self.callBing and it might be better to move the registration to the first line of the run method rather than the __init__ method.
The fist is because callBing is a reference to a missing global while self.callBing is a reference to the classes method. The second is because potentially you could get a call to self.callBing before init has finished which would be bad news.
I'm using thread pool while using Tornado to do some work. This is the code:
common/thread_pool.py
import tornado.ioloop
class Worker(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self._queue = queue
def run(self):
logging.info('Worker start')
while True:
content = self._queue.get()
if isinstance(content, str) and content == 'quit':
break
#content: (func, args, on_complete)
func = content[0]
args = content[1]
on_complete = content[2]
resp = func(args)
tornado.ioloop.IOLoop.instance().add_callback(lambda: on_complete(resp))
#i dont know is correct to call this
#self._queue.task_done()
logging.info('Worker stop')
class WorkerPool(object):
_workers = []
def __init__(self, num):
self._queue = Queue.Queue()
self._size = num
def start(self):
logging.info('WorkerPool start %d' % self._size)
for _ in range(self._size):
worker = Worker(self._queue)
worker.start()
self._workers.append(worker)
def stop(self):
for worker in self._workers:
self._queue.put('quit')
for worker in self._workers:
worker.join()
logging.info('WorkerPool stopd')
def append(self, content):
self._queue.put(content)
gateway.py
import tornado.ioloop
import tornado.web
from common import thread_pool
workers = None
class MainServerHandler(tornado.web.RequestHandler):
#tornado.web.asynchronous
def get(self):
start_time = time.time()
method = 'get'
content = (self.handle, (method, self.request, start_time), self.on_complete)
workers.append(content)
#tornado.web.asynchronous
def post(self):
start_time = time.time()
method = 'post'
content = (self.handle, (method, self.request, start_time), self.on_complete)
workers.append(content)
def handle(self, args):
method, request, start_time = args
#for test, just return
return 'test test'
def on_complete(self, res):
logging.debug('on_complete')
self.write(res)
self.finish()
return
def main(argv):
global workers
workers = thread_pool.WorkerPool(conf_mgr.thread_num)
workers.start()
application = tornado.web.Application([(r"/", MainServerHandler)])
application.listen(8888)
tornado.ioloop.IOLoop.instance().start()
if __name__ == "__main__":
main(sys.argv[1:])
When I make many concurrent requests, I get this error:
ERROR: 2014-09-15 18:04:03: ioloop.py:435 * 140500107065056 Exception in callback <tornado.stack_context._StackContextWrapper object at 0x7fc8b4d6b9f0>
Traceback (most recent call last):
File "/home/work/nlp_arch/project/ps/se/nlp-arch/gateway/gateway/../third-party/tornado-2.4.1/tornado/ioloop.py", line 421, in _run_callback
callback()
File "/home/work/nlp_arch/project/ps/se/nlp-arch/gateway/gateway/../common/thread_pool.py", line 39, in <lambda>
tornado.ioloop.IOLoop.instance().add_callback(lambda: on_complete(resp))
File "/home/work/nlp_arch/project/ps/se/nlp-arch/gateway/gateway/gateway.py", line 92, in on_complete
self.write(res)
File "/home/work/nlp_arch/project/ps/se/nlp-arch/gateway/gateway/../third-party/tornado-2.4.1/tornado/web.py", line 489, in write
raise RuntimeError("Cannot write() after finish(). May be caused "
RuntimeError: Cannot write() after finish(). May be caused by using async operations without the #asynchronous decorator.
But I didn't call write after finish. I'm also using the #asynchronous decorator. At the same time, in the logs I see that write/finish is called by same thread.
The issue is with the way you're adding the callback to the I/O loop. Add it like this:
tornado.ioloop.IOLoop.instance().add_callback(on_complete, resp)
And the errors will go away.
You're seeing this strange behavior because when you use a lambda function, you're creating a closure in the local scope of the function, and the variables used in that closure get bound at the point the lambda is executed, not when its created. Consider this example:
funcs = []
def func(a):
print a
for i in range(5):
funcs.append(lambda: func(i))
for f in funcs:
f()
Output:
4
4
4
4
4
Because your worker method is running in a while loop, on_complete ends up getting redefined several times, which also changes the value of on_complete inside the lambda. That means if one worker thread sets on_complete for a handler A, but then gets another task and sets on_complete for handler B prior to the callback set for handler A running, both callbacks end up up running handler B's on_complete.
If you really wanted to use a lambda, you could also avoid this by binding on_complete in the local scope of the lambda:
tornado.ioloop.IOLoop.instance().add_callback(lambda on_complete=on_complete: on_complete(resp))
But just adding the function and its argument directly is much nicer.
I am trying to write a wrapper like thing on vlc player on windows 7 so that it can load next and previous files in folder with a keystroke. What I do in it is, I take a filepath as argument make an instance of the vlc class and using pyHook, scan for keys and when specific keystrokes are detected call the play_next and play_prev methods on the instance. The methods work by killing the last process and spawning new vlc with next file found by get_new_file method. It works the first couple of times and then gives the peculiar error.
None
None
Traceback (most recent call last):
File "C:\Python27\pyHook\HookManager.py", line 351, in KeyboardSwitch
return func(event)
File "filestuff.py", line 64, in kbwrap
kbeventhandler(event,instance)
File "filestuff.py", line 11, in kbeventhandler
instance.play_prev()
File "filestuff.py", line 34, in play_prev
f=self.get_new_file(-1)
File "filestuff.py", line 40, in get_new_file
dirname= os.path.dirname(self.fn)
File "C:\Python27\lib\ntpath.py", line 205, in dirname
return split(p)[0]
File "C:\Python27\lib\ntpath.py", line 178, in split
while head2 and head2[-1] in '/\\':
TypeError: an integer is required
here is the code:
import os
import sys
import pythoncom, pyHook
import win32api
import subprocess
import ctypes
def kbeventhandler(event,instance):
if event.Key=='Home':
instance.play_prev()
if event.Key=='End':
instance.play_next()
return True
class vlc(object):
def __init__(self,filepath,vlcp):
self.fn=filepath
self.vlcpath=vlcp
self.process = subprocess.Popen([self.vlcpath, self.fn])
def kill(self):
PROCESS_TERMINATE = 1
handle = ctypes.windll.kernel32.OpenProcess(PROCESS_TERMINATE, False, self.process.pid)
ctypes.windll.kernel32.TerminateProcess(handle, -1)
ctypes.windll.kernel32.CloseHandle(handle)
print self.process.poll()
def play_next(self):
self.kill()
f=self.get_new_file(1)
self.process = subprocess.Popen([self.vlcpath, f])
self.fn=f
def play_prev(self):
self.kill()
f=self.get_new_file(-1)
self.process = subprocess.Popen([self.vlcpath, f])
self.fn=f
def get_new_file(self,switch):
dirname= os.path.dirname(self.fn)
supplist=['.mkv','.flv','.avi','.mpg','.wmv']
files = [os.path.join(dirname,f) for f in os.listdir(dirname) if (os.path.isfile(os.path.join(dirname,f)) and os.path.splitext(f)[-1]in supplist)]
files.sort()
try: currentindex=files.index(self.fn)
except: currentindex=0
i=0
if switch==1:
if currentindex<(len(files)-1):i=currentindex+1
else:
if currentindex>0:i=currentindex-1
return files[i]
def main():
vlcpath='vlc'
if os.name=='nt': vlcpath='C:/Program Files (x86)/VideoLAN/VLC/vlc.exe'
fn='H:\\Anime\\needless\\Needless_[E-D]\\[Exiled-Destiny]_Needless_Ep11v2_(04B16479).mkv'
if len(sys.argv)>1:
fn=sys.argv[1] #use argument if available or else use default file
instance=vlc(fn,vlcpath)
hm = pyHook.HookManager()
def kbwrap(event):
kbeventhandler(event,instance)
hm.KeyDown = kbwrap
hm.HookKeyboard()
pythoncom.PumpMessages()
if __name__ == '__main__':
main()
here too: http://pastebin.com/rh82XGzd
The problem was that
in main I set hm.KeyDown = kbwrap and then from function kbwrap called the actual event handler kbeventhandler but didn't return any value from kbwrap
def kbwrap(event):
return kbeventhandler(event,flag)
hm.KeyDown = kbwrap
and I also offloaded the vlc work to a different thread as pyHook wasn't playing nice with subprocess.
final working code:
import os
import sys
import pythoncom, pyHook
import win32api
import subprocess
import ctypes
import threading
from multiprocessing import *
class vlcThread(threading.Thread):
def __init__(self,filepath,vlcp,fl):
threading.Thread.__init__(self)
self.fn,self.vlcpath,self.flag=filepath,vlcp,fl
self.daemon=True
self.start() # invoke the run method
def run(self):
vlcinstance=vlc(self.fn,self.vlcpath)
while True:
if(self.flag.value==1):
vlcinstance.play_next()
self.flag.value=0
if(self.flag.value==-1):
vlcinstance.play_prev()
self.flag.value=0
def kbeventhandler(event,flag):
if event.Key=='Home':
flag.value =-1
return False
if event.Key=='End':
flag.value =1
return False
return True
class vlc(object):
def __init__(self,filepath,vlcp):
self.fn=filepath
self.vlcpath=vlcp
self.process = subprocess.Popen([self.vlcpath,self.fn],close_fds=True)
def kill(self):
p, self.process = self.process, None
if p is not None and p.poll() is None:
p.kill()
p.wait()
def play_next(self):
self.kill()
f=self.get_new_file(1)
self.process = subprocess.Popen([self.vlcpath,f],close_fds=True)
self.fn=f
def play_prev(self):
self.kill()
f=self.get_new_file(-1)
self.process = subprocess.Popen([self.vlcpath, f],close_fds=True)
self.fn=f
def get_new_file(self,switch):
dirname= os.path.dirname(self.fn)
supplist=['.mkv','.flv','.avi','.mpg','.wmv','ogm','mp4']
files = [os.path.join(dirname,f) for f in os.listdir(dirname) if (os.path.isfile(os.path.join(dirname,f)) and os.path.splitext(f)[-1]in supplist)]
files.sort()
try: currentindex=files.index(self.fn)
except: currentindex=0
i=0
if switch==1:
if currentindex<(len(files)-1):i=currentindex+1
else:
if currentindex>0:i=currentindex-1
return files[i]
def main():
vlcpath='vlc'
flag=Value('i')
flag.value=0
if os.name=='nt': vlcpath='C:/Program Files (x86)/VideoLAN/VLC/vlc.exe'
fn='H:\\Anime\\needless\\Needless_[E-D]\\[Exiled-Destiny]_Needless_Ep11v2_(04B16479).mkv'
if len(sys.argv)>1:
fn=sys.argv[1] #use argument if available or else use default file
t=vlcThread(fn,vlcpath,flag)
hm = pyHook.HookManager()
def kbwrap(event):
return kbeventhandler(event,flag)
hm.KeyDown = kbwrap
hm.HookKeyboard()
pythoncom.PumpMessages()
if __name__ == '__main__':
main()