We Keep Coding

Logging to a single file from multiple processes in multiple modules

In Logging Cookbook I found Logging to a single file from multiple processes, I would like to use it with multiple modules, each module is a process, do you have any idea?
I used first code in the Logging to a single file from multiple processes.
Lets say the main in module, and worker_process in another, how to do that?
main.py:
def listener_configurer():
root = logging.getLogger()
h = logging.handlers.RotatingFileHandler('mptest.log', 'a', 300, 10)
f = logging.Formatter('%(asctime)s %(processName)-10s %(name)s %(levelname)-8s %(message)s')
h.setFormatter(f)
root.addHandler(h)
def listener_process(queue, configurer):
configurer()
while True:
try:
record = queue.get()
if record is None: # We send this as a sentinel to tell the listener to quit.
break
logger = logging.getLogger(record.name)
logger.handle(record) # No level or filter logic applied - just do it!
except Exception:
import sys, traceback
print('Whoops! Problem:', file=sys.stderr)
traceback.print_exc(file=sys.stderr)
def main():
queue = multiprocessing.Queue(-1)
listener = multiprocessing.Process(target=listener_process,
args=(queue, listener_configurer))
listener.start()
workers = []
for i in range(10):
worker = worker_process.TEST(queue)
workers.append(worker)
worker.start()
for w in workers:
w.join()
queue.put_nowait(None)
listener.join()
if __name__ == '__main__':
main()
worker_process.py
def worker_configurer(queue):
h = logging.handlers.QueueHandler(queue)
root = logging.getLogger()
root.addHandler(h)
root.setLevel(logging.DEBUG)
class TEST(multiprocessing.Process):
def __init__(self, queue, func=worker_configurer):
super(TEST, self).__init__()
self.queue = queue
self.func = func
self.LEVELS = [logging.DEBUG, logging.INFO, logging.WARNING,
logging.ERROR, logging.CRITICAL]
self.LOGGERS = ['a.b.c', 'd.e.f']
self.MESSAGES = [
'Random message #1',
'Random message #2',
'Random message #3',
]
def run(self):
self.func(self.queue)
name = multiprocessing.current_process().name
print('Worker started: %s' % name)
for i in range(10):
time.sleep(random())
logger = logging.getLogger(choice(self.LOGGERS))
level = choice(self.LEVELS)
message = choice(self.MESSAGES)
logger.log(level, message)
print('Worker finished: %s' % name)
This is not work correctly, I would like to use the code (first code) in the doc with multiple modules, as I mentioned before.

Python3: Thread still alive after joined

This my code
def timeout(seconds_before_timeout):
def deco(func):
#functools.wraps(func)
def wrapper(*args, **kwargs):
res = [
Exception("function [%s] timeout [%s seconds] exceeded!"
% (func.__name__, seconds_before_timeout))
]
def new_func():
try:
res[0] = func(*args, **kwargs)
except Exception as ex:
res[0] = ex
thread = Thread(target=new_func)
thread.daemon = True
try:
thread.start()
thread.join(seconds_before_timeout)
except Exception as ex:
print("error starting thread")
raise ex
ret = res[0]
if isinstance(ret, BaseException):
raise ret
return ret
return wrapper
return deco
And timeout function i used for:
#timeout(2)
def listen_for_a_new_campaign(self):
"""
Start listening for new campaign in list_campaign queue
"""
while True:
try:
for method_frame, properties, body \
in self.obj_requester_channel.consume(LIST_CAMPAIGN_QUEUE):
body_dict = literal_eval(body.decode("utf-8"))
message_number = body_dict["Msg_Count"]
n_message = min(message_number, BATCH_SIZE)
identify(n_message)
a_request = {
"campaign_request": body_dict,
"campaign_ack" : method_frame.delivery_tag,
"n_message" : n_message
}
identify(a_request)
return a_request
# Acknowledge the message
n_requeued_messages = self.obj_requester_channel.cancel()
print("Requeued %i messages" % n_requeued_messages)
break
except pika.exceptions.ConnectionWrongStateError:
print("Create connection ...")
self.create_connection()
continue
except pika.exceptions.ChannelWrongStateError:
print("Create connection ...")
self.create_connection()
self.obj_requester_channel = self.obj_connection.channel()
self.obj_requester_channel.queue_declare(queue=LIST_CAMPAIGN_QUEUE)
self.obj_campaign_channel = self.obj_connection.channel()
continue
When I run my program, I checked all process by htop and below is result, all thread is alive:
I don't know what's wrong with that.
I run this code on my laptop everything was OK, but when I deploy them to EC2 instance I found that problems.
Help me!!

mock and side_effect substitution - keeping access to the original class and its attributes

I want to mock method _subprocess on a particular instance of a class.
Specifically when the task fires off pip freeze as a command (in that case its taskname is freeze).
class Command(object):
def __init__(self, mgr, taskname, config):
self.mgr = mgr
self.taskname = taskname
self.config = config
self.append = self.config.get("append", False)
self.stderr = ""
def _subprocess(self, cmd, fnp_o, self_=None):
try:
mode = "a" if self.append else "w"
fnp_stderr = self.mgr._get_fnp("log")
with open(fnp_stderr, "a") as ferr:
ferr.write("cmd: %s\nstderr begin:\n" % (cmd))
with open(fnp_o, mode) as fo:
proc = subprocess.check_call(
cmd.split(),
stdout=fo,
stderr=ferr,
cwd=self.mgr.workdir,
encoding="utf-8",
)
ferr.write("stderr end\n\n")
except (Exception,) as e:
if cpdb(): pdb.set_trace()
raise
This is the test method:
def fake_subprocess(self, cmd, fnp_o, self_):
try:
raise NotImplementedError("fake_subprocess(%s)" % (locals()))
except (Exception,) as e:
pdb.set_trace()
raise
def test_001_scan(self):
try:
with patch.object(Command, '_subprocess', side_effect = self.fake_subprocess) as mock_method:
options = self.get_options()
self.mgr = Main(options)
self.mgr.process()
except (Exception,) as e:
pdb.set_trace()
raise
My problem is two-fold.
First, the self in fake_subprocess refers to the UnitTest object, not the Command object. My use of the self_ parameter gets around that.
Second, in most cases, except for pip freeze I want to run the original subprocess, not the fake one.
Now, I can probably power through this by keeping an extra reference to Command._subprocess and using self_
But is there a more elegant way? Very naive when it comes to unittest.Mock.

This is what ended up working for me:
test-side
def fake_subprocess(self, cmd, fnp_o, self_):
try:
if self_.taskname != "freeze":
return self_._subprocess_actual(cmd, fnp_o, self_)
with open(fnp_o, self_.mode) as fo:
fo.write(self.fake_subprocess_payload["freeze"])
except (Exception,) as e:
raise
def test_001_scan(self):
try:
with patch.object(
Command, "_subprocess", side_effect=self.fake_subprocess
) as mock_method:
options = self.get_options()
self.mgr = Main(options)
self.mgr.process()
except (Exception,) as e:
raise
actual code-side
class Command(object):
def _subprocess(self, cmd, fnp_o, self_=None):
try:
fnp_stderr = self.mgr._get_fnp("log")
with open(fnp_stderr, "a") as ferr:
ferr.write("cmd: %s\nstderr begin:\n" % (cmd))
with open(fnp_o, self.mode) as fo:
proc = subprocess.check_call(
cmd.split(), stdout=fo, stderr=ferr, cwd=self.mgr.workdir
)
ferr.write("stderr end\n\n")
except (Exception,) as e:
if cpdb():
pdb.set_trace()
raise
_subprocess_actual = _subprocess
def run(self):
try:
t_cmd = self.config["cmdline"] # .replace(r"\\","\\")
t_fnp = os.path.join(self.mgr.workdir, self.config["filename"])
fnp_log = "subprocess.log"
cmd = sub_template(t_cmd, self, self.mgr.vars)
fnp_o = sub_template(t_fnp, self, self.mgr.vars)
self._subprocess(cmd=cmd, fnp_o=fnp_o, self_=self)
except (Exception,) as e:
if cpdb():
pdb.set_trace()
raise

Exception in thread StompReceiverThread-1

I'm having trouble with this error:
Exception in thread StompReceiverThread-1 (most likely raised during
interpreter shutdown):
That is no traceback at all.. just that.
Usualy everything works fine but rarely it happens and then the action does not conclude.
Any tips?
My code:
class Listener(stomp.ConnectionListener):
def __init__(self, conn, request):
self.conn = conn
self.request = request
def on_error(self, headers, message):
global WAITING_RESPONSE
print('received an error: ' + message)
WAITING_RESPONSE = False
def on_message(self, headers, message):
global WAITING_RESPONSE
try:
msg = json.loads(message)
if str(msg.get('transaction_id','')) == str(CURRENT_ID):
printDebugLine('Queue response:'+str(message))
manageQueueResponse(message,self.request)
WAITING_RESPONSE = False
self.conn.ack(headers['message-id'], '11')
except stomp.exception.ConnectFailedException:
print('Stomp error on message')
sys.exit(3)
except Exception as e:
print('ERROR: %s' % str(e))
sys.exit(3)
class Queue(object):
def __init__(self):
self.host = xx
self.port = xx
self.login = xx
self.passwd = xx
self.request = {}
self.start()
def start(self):
try:
self.conn = stomp.Connection(host_and_ports=[(self.host, self.port)])
self.conn.start()
self.conn.connect(self.login, self.passwd, wait=True)
self.conn.set_listener('xx', Listener(self.conn, self.request))
self.conn.subscribe(destination='xx', id='xx', ack='xx')
except stomp.exception.ConnectFailedException:
print('ERROR: unable to connect')
sys.exit(3)
except Exception as e:
print('ERROR: %s' % str(e))
sys.exit(3)
def send(self, data):
global CURRENT_ID
while WAITING_RESPONSE:
time.time(0.1)
try:
CURRENT_ID = str(uuid.uuid4())
data.update({'transaction_id': CURRENT_ID})
b = json.dumps(data)
self.request.update(data)
printDebugLine('Queue request:'+str(data))
self.conn.send(body=b, destination='xx')
timeout(data,self.request,29)
except stomp.exception.ConnectFailedException:
print('ERROR: unable to connect')
except Exception as e:
print('ERROR: %s' % str(e))

It looks like your main program is exiting, the interpreter is cleaning up things, but the stomp receiver thread was not shutdown first. The receiver thread goes to do something but basic modules are no longer available, so it gives an exception message, but cannot print a Traceback because that fuctionality is no longer available due to the program exiting.
Look at why the main program would be exiting.

Where do I change this Python code snippet to save the temp file in the tmp-folder?

this is a code example implementing a file lock, so the application can only open one instance. It currently works, but saves the the lock file in the Home-folder (Ubuntu). If the application crashes, the lock file does not get removed which is not good....
I can not easily see where I should change the code to save it in the c:/tmp-folder instead?
#!/usr/bin/python
# -*- coding: utf-8 -*-
#implements a lockfile if program already is open
import os
import socket
from fcntl import flock
class flock(object):
'''Class to handle creating and removing (pid) lockfiles'''
# custom exceptions
class FileLockAcquisitionError(Exception): pass
class FileLockReleaseError(Exception): pass
# convenience callables for formatting
addr = lambda self: '%d#%s' % (self.pid, self.host)
fddr = lambda self: '<%s %s>' % (self.path, self.addr())
pddr = lambda self, lock: '<%s %s#%s>' %\
(self.path, lock['pid'], lock['host'])
def __init__(self, path, debug=None):
self.pid = os.getpid()
self.host = socket.gethostname()
self.path = path
self.debug = debug # set this to get status messages
def acquire(self):
'''Acquire a lock, returning self if successful, False otherwise'''
if self.islocked():
if self.debug:
lock = self._readlock()
print 'Previous lock detected: %s' % self.pddr(lock)
return False
try:
fh = open(self.path, 'w')
fh.write(self.addr())
fh.close()
if self.debug:
print 'Acquired lock: %s' % self.fddr()
except:
if os.path.isfile(self.path):
try:
os.unlink(self.path)
except:
pass
raise (self.FileLockAcquisitionError,
'Error acquiring lock: %s' % self.fddr())
return self
def release(self):
'''Release lock, returning self'''
if self.ownlock():
try:
os.unlink(self.path)
if self.debug:
print 'Released lock: %s' % self.fddr()
except:
raise (self.FileLockReleaseError,
'Error releasing lock: %s' % self.fddr())
return self
def _readlock(self):
'''Internal method to read lock info'''
try:
lock = {}
fh = open(self.path)
data = fh.read().rstrip().split('#')
fh.close()
lock['pid'], lock['host'] = data
return lock
except:
return {'pid': 8**10, 'host': ''}
def islocked(self):
'''Check if we already have a lock'''
try:
lock = self._readlock()
os.kill(int(lock['pid']), 0)
return (lock['host'] == self.host)
except:
return False
def ownlock(self):
'''Check if we own the lock'''
lock = self._readlock()
return (self.fddr() == self.pddr(lock))
def __del__(self):
'''Magic method to clean up lock when program exits'''
self.release()
#now testing to see if file is locked = other instance of this program is running already
lock = flock('tmp.lock', True).acquire()
if lock:
print 'doing stuff'
else:
print 'locked!'
exit()
#end of lockfile

Use tempfile

At the end of the script:
lock = flock('tmp.lock', True).acquire()
The 'tmp.lock' is the path to the file in the current directory. Change it to the path you need, i.e. 'c:/tmp-folder/tmp.lock'.
lock = flock('c:/tmp-folder/tmp.lock', True).acquire()
However, as #g19fanatic notes: are you on a Windows ('c:/...') or Linux (Ubuntu) system?