We Keep Coding

Python3: Thread still alive after joined

This my code
def timeout(seconds_before_timeout):
def deco(func):
#functools.wraps(func)
def wrapper(*args, **kwargs):
res = [
Exception("function [%s] timeout [%s seconds] exceeded!"
% (func.__name__, seconds_before_timeout))
]
def new_func():
try:
res[0] = func(*args, **kwargs)
except Exception as ex:
res[0] = ex
thread = Thread(target=new_func)
thread.daemon = True
try:
thread.start()
thread.join(seconds_before_timeout)
except Exception as ex:
print("error starting thread")
raise ex
ret = res[0]
if isinstance(ret, BaseException):
raise ret
return ret
return wrapper
return deco
And timeout function i used for:
#timeout(2)
def listen_for_a_new_campaign(self):
"""
Start listening for new campaign in list_campaign queue
"""
while True:
try:
for method_frame, properties, body \
in self.obj_requester_channel.consume(LIST_CAMPAIGN_QUEUE):
body_dict = literal_eval(body.decode("utf-8"))
message_number = body_dict["Msg_Count"]
n_message = min(message_number, BATCH_SIZE)
identify(n_message)
a_request = {
"campaign_request": body_dict,
"campaign_ack" : method_frame.delivery_tag,
"n_message" : n_message
}
identify(a_request)
return a_request
# Acknowledge the message
n_requeued_messages = self.obj_requester_channel.cancel()
print("Requeued %i messages" % n_requeued_messages)
break
except pika.exceptions.ConnectionWrongStateError:
print("Create connection ...")
self.create_connection()
continue
except pika.exceptions.ChannelWrongStateError:
print("Create connection ...")
self.create_connection()
self.obj_requester_channel = self.obj_connection.channel()
self.obj_requester_channel.queue_declare(queue=LIST_CAMPAIGN_QUEUE)
self.obj_campaign_channel = self.obj_connection.channel()
continue
When I run my program, I checked all process by htop and below is result, all thread is alive:
I don't know what's wrong with that.
I run this code on my laptop everything was OK, but when I deploy them to EC2 instance I found that problems.
Help me!!

Exception in thread StompReceiverThread-1

I'm having trouble with this error:
Exception in thread StompReceiverThread-1 (most likely raised during
interpreter shutdown):
That is no traceback at all.. just that.
Usualy everything works fine but rarely it happens and then the action does not conclude.
Any tips?
My code:
class Listener(stomp.ConnectionListener):
def __init__(self, conn, request):
self.conn = conn
self.request = request
def on_error(self, headers, message):
global WAITING_RESPONSE
print('received an error: ' + message)
WAITING_RESPONSE = False
def on_message(self, headers, message):
global WAITING_RESPONSE
try:
msg = json.loads(message)
if str(msg.get('transaction_id','')) == str(CURRENT_ID):
printDebugLine('Queue response:'+str(message))
manageQueueResponse(message,self.request)
WAITING_RESPONSE = False
self.conn.ack(headers['message-id'], '11')
except stomp.exception.ConnectFailedException:
print('Stomp error on message')
sys.exit(3)
except Exception as e:
print('ERROR: %s' % str(e))
sys.exit(3)
class Queue(object):
def __init__(self):
self.host = xx
self.port = xx
self.login = xx
self.passwd = xx
self.request = {}
self.start()
def start(self):
try:
self.conn = stomp.Connection(host_and_ports=[(self.host, self.port)])
self.conn.start()
self.conn.connect(self.login, self.passwd, wait=True)
self.conn.set_listener('xx', Listener(self.conn, self.request))
self.conn.subscribe(destination='xx', id='xx', ack='xx')
except stomp.exception.ConnectFailedException:
print('ERROR: unable to connect')
sys.exit(3)
except Exception as e:
print('ERROR: %s' % str(e))
sys.exit(3)
def send(self, data):
global CURRENT_ID
while WAITING_RESPONSE:
time.time(0.1)
try:
CURRENT_ID = str(uuid.uuid4())
data.update({'transaction_id': CURRENT_ID})
b = json.dumps(data)
self.request.update(data)
printDebugLine('Queue request:'+str(data))
self.conn.send(body=b, destination='xx')
timeout(data,self.request,29)
except stomp.exception.ConnectFailedException:
print('ERROR: unable to connect')
except Exception as e:
print('ERROR: %s' % str(e))

It looks like your main program is exiting, the interpreter is cleaning up things, but the stomp receiver thread was not shutdown first. The receiver thread goes to do something but basic modules are no longer available, so it gives an exception message, but cannot print a Traceback because that fuctionality is no longer available due to the program exiting.
Look at why the main program would be exiting.

Python: Aioimaplib catch exceptions

im trying to check multiple imap login informations asynchronously with aioimaplib.
This code works as long as the imap servers are reachable and / or the clients don't time out.
What is the correct way to catch the exceptions?
Example exception:
ERROR:asyncio:Task exception was never retrieved future: <Task finished coro=<BaseEventLoop.create_connection() done, defined at G:\WinPython-3.5.4\python-3.5.4.amd64\lib\asyncio\base_events.py:679> exception=TimeoutError(10060, "Connect call failed ('74.117.114.100', 993)")>
Code:
account_infos = [
# User Password Server
('user1#web.com', 'password1', 'imap.google.com'),
('user2#web.com', 'password2', 'imap.yandex.com'),
('user3#web.com', 'password3', 'imap.server3.com'),
]
class MailLogin:
def __init__(self):
self.loop = asyncio.get_event_loop()
self.queue = asyncio.Queue(loop=self.loop)
self.max_workers = 2
async def produce_work(self):
for i in account_infos:
await self.queue.put(i)
for _ in range(max_workers):
await self.queue.put((None, None, None))
async def worker(self):
while True:
(username, password, server) = await self.queue.get()
if username is None:
break
while True:
try:
s = IMAP4_SSL(server)
await s.wait_hello_from_server()
r = await s.login(username, password)
await s.logout()
if r.result != 'NO':
print('Information works')
except Exception as e:
# DOES NOT CATCH
print(str(e))
else:
break
def start(self):
try:
self.loop.run_until_complete(
asyncio.gather(self.produce_work(), *[self.worker() for _ in range(self.max_workers)],
loop=self.loop, return_exceptions=True)
)
finally:
print('Done')
if __name__ == '__main__':
MailLogin().start()

There are several ways to do this but the TimeoutError is probably caught in your except. You don't see it because str(e) is an empty string.
You can see the stacks enabling debug mode of asyncio.
First, you can catch the exception as you did:
async def fail_fun():
try:
imap_client = aioimaplib.IMAP4_SSL(host='foo', timeout=1)
await imap_client.wait_hello_from_server()
except Exception as e:
print('Exception : ' + str(e))
if __name__ == '__main__':
get_event_loop().run_until_complete(fail_fun())
Second, you can catch the exception at run_until_complete
async def fail_fun():
imap_client = aioimaplib.IMAP4_SSL(host='foo', timeout=1)
await imap_client.wait_hello_from_server()
if __name__ == '__main__':
try:
get_event_loop().run_until_complete(fail_fun())
except Exception as e:
print('Exception : ' + str(e))
The connection is established wrapping the loop.create_connection coroutine with create_task : we wanted to establish the connection in the IMAP4 constructor and __init__ should return None.
So if your host has a wrong value, you could test it before, or wait for the timeout :
socket.gaierror: [Errno -5] No address associated with hostname
if a host is not responding before the timeout, you can raise the timeout. And if the connection is lost during the connection, you can add a connection lost callback in the IMAP4 constructor.

TCP client not able to write to socket but receive works fine

I have client program written in python that talks to some server.
[Client]
import asyncore
import logging
import socket
import sys, threading, traceback
from cStringIO import StringIO
class Client(threading.Thread, asyncore.dispatcher):
def __init__(self, host, port):
self.logger = logging.getLogger()
threading.Thread.__init__(self)
self._thread_sockets = dict()
asyncore.dispatcher.__init__(self, map=self._thread_sockets)
# data members for the module
self.host = host
self.port = port
self.write_buffer = ""
self.is_connected = False
self.read_buffer = StringIO()
# Ok now to run the thread !!
self.start()
def run(self) :
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
address = (self.host, self.port)
self.logger.debug('connecting to %s', address)
# wait until server is up
while not self.is_connected :
try :
self.connect(address)
except Exception as ex :
pass #do nothing, proceed forward !!
asyncore.loop(map=self._thread_sockets)
def handle_connect(self):
self.is_connected = True
self.logger.debug('handle_connect()')
def handle_close(self):
self.logger.debug('handle_close()')
self.close()
def handle_error(self):
traceback.print_exc(sys.stderr)
self.close()
def writable(self):
self.logger.debug('writable() : len is %d bytes', len(self.write_buffer))
is_writable = (len(self.write_buffer) > 0)
if is_writable:
self.logger.debug('writable() -> %s', is_writable)
return is_writable
def readable(self):
self.logger.debug('readable() -> True')
return True
def handle_write(self):
sent = self.send(self.write_buffer)
self.logger.debug('data len written to socket -> %s', sent)
self.logger.debug('handle_write() -> "%s"', self.write_buffer[:sent])
#self.write_buffer = self.write_buffer[sent:]
def handle_read(self):
data = self.recv(8192)
self.logger.debug('handle_read() -> %d bytes', len(data))
self.read_buffer.write(data)
self.logger.debug('data received from socket -> %s', self.read_buffer.getvalue())
self.read_buffer.truncate(0)
def send(self, data) :
self.write_buffer = data
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(name)s: %(message)s',
)
try :
client = Client("127.0.0.1", 8182)
client.send('sending data from client')
except Exception as ex :
logging.exception(ex)
sys.exit(1)
I am able to receive data from server correctly but send call to the server always fails. As from the log the send always return 'None'.
Am i missing anything ?

You override the send method of asyncore.dispatcher with code that does not send any data and returns no value:
def send(self, data) :
self.write_buffer = data
At the least, you need to change your code to look similar to this:
def send_data(self, data):
self.write_buffer = data
and this:
client.send_data('sending data from client')
The asyncore.dispatcher class already has a send method which is a wrapper around the socket.send method. From asyncore.py:
def send(self, data):
try:
result = self.socket.send(data)
return result
except socket.error, why:
if why.args[0] == EWOULDBLOCK:
return 0
elif why.args[0] in _DISCONNECTED:
self.handle_close()
return 0
else:
raise
Because you override this method, your send method gets called in your handle_write method, and no data is sent to the server.

Python asyncio - heartbeat() method not writing to stream

I trying to create a proof of concept with Python 3 asyncio, implementing a client that sends heartbeats periodically to a server in order to keep the connection alive.
Note that the server is simply an echo server and doesn't close the connection. But it is important that the client is able to send a heartbeat periodically.
Here is the current implementation:
stream_client.py
import asyncio
class StreamClient:
def __init__(self, heartbeat_int, loop=None):
self.heartbeat_int = heartbeat_int
if loop is not None:
self.loop = loop
else:
self.loop = asyncio.get_event_loop()
def start(self):
""" start manages the event loop, but it is not a coroutine """
self.loop.run_until_complete(self.get_client())
self.loop.create_task(self.start_timed_session())
msg = self.loop.run_until_complete(self.logon('hello'))
if msg == 'hello':
try:
self.loop.run_forever()
except KeyboardInterrupt:
print('Closing connection with server...')
self.writer.close()
self.loop.close()
else:
print('Logon unsuccessful, closing connection with server...')
self.writer.close()
self.loop.close()
#asyncio.coroutine
def get_client(self):
self.reader, self.writer = yield from asyncio.open_connection(
'127.0.0.1',
9871,
loop=self.loop
)
print('Connection established at "localhost:9871"')
#asyncio.coroutine
def timed_session(self):
yield from asyncio.sleep(self.heartbeat_int)
self.loop.create_task(self.start_timed_session())
#asyncio.coroutine
def start_timed_session(self):
heartbeat_task = self.loop.create_task(self.timed_session())
heartbeat_task.add_done_callback(self.heartbeat)
#asyncio.coroutine
def logon(self, msg):
print('Sending message:', msg)
self.writer.write(msg.encode())
data = yield from self.reader.read(15)
resp = data.decode()
print('Data received:', resp)
return resp
def heartbeat(self, fut):
"""
This is future's callback:
1) Can't be a coroutine
2) Takes a future as an argument
"""
print('> Sending heartbeat...')
self.writer.write('heartbeat'.encode())
# Start the client
client = StreamClient(5)
client.start()
stream_server.py
import asyncio
class StreamServer:
def __init__(self, loop=None):
if loop is not None:
self.loop = loop
else:
self.loop = asyncio.get_event_loop()
#asyncio.coroutine
def server_handler(self, reader, writer):
data = yield from reader.read(15)
msg = data.decode()
if data is not 'bye':
print('Received data: "{}" from {}'.format(msg, writer.get_extra_info('peername')))
print('Echoing the message...')
writer.write(data)
yield from writer.drain()
else:
print('Received data: "{}" from {}'.format(
data,
writer.get_extra_info('peername')
)
)
print('Closing the connection...')
writer.close()
loop = asyncio.get_event_loop()
stream_server = StreamServer(loop)
coro_server = asyncio.start_server(
stream_server.server_handler,
'127.0.0.1',
9871,
loop=stream_server.loop
)
server = loop.run_until_complete(coro_server)
print('Listening on:', server.sockets[0].getsockname())
try:
loop.run_forever()
except KeyboardInterrupt:
pass
print('Closing server')
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
Question:
On the heartbeat() method the line self.writer.write('heartbeat'.encode()) seems to never be executed.
How can I get it to work?

The code is being executed on the client side, you just don't have any code on the server-side to receive the message. server_handler is only called once per connection, not once per message. So, if you want it to be able to receive an infinite number of heartbeats from a given client, you need to set up a loop inside of server_handler to receive them:
#asyncio.coroutine
def server_handler(self, reader, writer):
while True: # Keep receiving data from client.
data = yield from reader.read(15)
msg = data.decode()
if data is not 'bye':
print('Received data: "{}" from {}'.format(msg, writer.get_extra_info('peername')))
print('Echoing the message...')
writer.write(data)
yield from writer.drain()
else:
print('Received data: "{}" from {}'.format(
data,
writer.get_extra_info('peername')
)
)
print('Closing the connection...')
writer.close()