We Keep Coding

Python3: Thread still alive after joined

This my code
def timeout(seconds_before_timeout):
def deco(func):
#functools.wraps(func)
def wrapper(*args, **kwargs):
res = [
Exception("function [%s] timeout [%s seconds] exceeded!"
% (func.__name__, seconds_before_timeout))
]
def new_func():
try:
res[0] = func(*args, **kwargs)
except Exception as ex:
res[0] = ex
thread = Thread(target=new_func)
thread.daemon = True
try:
thread.start()
thread.join(seconds_before_timeout)
except Exception as ex:
print("error starting thread")
raise ex
ret = res[0]
if isinstance(ret, BaseException):
raise ret
return ret
return wrapper
return deco
And timeout function i used for:
#timeout(2)
def listen_for_a_new_campaign(self):
"""
Start listening for new campaign in list_campaign queue
"""
while True:
try:
for method_frame, properties, body \
in self.obj_requester_channel.consume(LIST_CAMPAIGN_QUEUE):
body_dict = literal_eval(body.decode("utf-8"))
message_number = body_dict["Msg_Count"]
n_message = min(message_number, BATCH_SIZE)
identify(n_message)
a_request = {
"campaign_request": body_dict,
"campaign_ack" : method_frame.delivery_tag,
"n_message" : n_message
}
identify(a_request)
return a_request
# Acknowledge the message
n_requeued_messages = self.obj_requester_channel.cancel()
print("Requeued %i messages" % n_requeued_messages)
break
except pika.exceptions.ConnectionWrongStateError:
print("Create connection ...")
self.create_connection()
continue
except pika.exceptions.ChannelWrongStateError:
print("Create connection ...")
self.create_connection()
self.obj_requester_channel = self.obj_connection.channel()
self.obj_requester_channel.queue_declare(queue=LIST_CAMPAIGN_QUEUE)
self.obj_campaign_channel = self.obj_connection.channel()
continue
When I run my program, I checked all process by htop and below is result, all thread is alive:
I don't know what's wrong with that.
I run this code on my laptop everything was OK, but when I deploy them to EC2 instance I found that problems.
Help me!!

AsyncSSH saving output to list in Python

I'm trying to use AsyncSSH module to execute a command and capture the output for further processing. I was following the examples on the official documentation but they all print() the result, I want to save it as a list. How would I do that in Python 3.5? I tried to modify the class by adding getdata() that would return 'out' list but it doesn't seem to be working
import asyncio, asyncssh, sys
class MySSHClientSession(asyncssh.SSHClientSession):
out = []
def data_received(self, data, datatype):
if datatype == asyncssh.EXTENDED_DATA_STDERR:
print(data, end='', file=sys.stderr)
else:
#print(data, end='')
self.out.append(data)
def connection_lost(self, exc):
if exc:
print('SSH session error: ' + str(exc), file=sys.stderr)
def getdata(self):
return self.out
async def run_client(zone, cmd):
async with asyncssh.connect(zone, password='xxx', username='xxx', known_hosts=None) as conn:
chan, session = await conn.create_session(MySSHClientSession, cmd)
await chan.wait_closed()
try:
x = asyncio.get_event_loop().run_until_complete(run_client('fetish', 'sudo fl-service status'))
# the code to process x would go here
print(x.getdata())
except (OSError, asyncssh.Error) as exc:
sys.exit('SSH connection failed: ' + str(exc))

I have corrected your code.
your out[] wasn't initiated, you tried to append to None
self.out can be used in its lifecycle only.
import asyncio, asyncssh, sys
class MySSHClientSession(asyncssh.SSHClientSession):
def __init__(self):
self.out = []
def data_received(self, data, datatype):
if datatype == asyncssh.EXTENDED_DATA_STDERR:
print(data, end='', file=sys.stderr)
else:
#print(data, end='')
self.out.append(data)
def connection_lost(self, exc):
if exc:
print('SSH session error: ' + str(exc), file=sys.stderr)
else:
print(self.out)
async def run_client(zone, cmd):
async with asyncssh.connect(zone, password='xxx', username='xxx', known_hosts=None) as conn:
chan, session = await conn.create_session(MySSHClientSession, cmd)
await chan.wait_closed()
try:
x = asyncio.get_event_loop().run_until_complete(run_client('fetish', 'whoami'))
# the code to process x would go here
#print(x.getdata())
except (OSError, asyncssh.Error) as exc:
sys.exit('SSH connection failed: ' + str(exc))

Python: Aioimaplib catch exceptions

im trying to check multiple imap login informations asynchronously with aioimaplib.
This code works as long as the imap servers are reachable and / or the clients don't time out.
What is the correct way to catch the exceptions?
Example exception:
ERROR:asyncio:Task exception was never retrieved future: <Task finished coro=<BaseEventLoop.create_connection() done, defined at G:\WinPython-3.5.4\python-3.5.4.amd64\lib\asyncio\base_events.py:679> exception=TimeoutError(10060, "Connect call failed ('74.117.114.100', 993)")>
Code:
account_infos = [
# User Password Server
('user1#web.com', 'password1', 'imap.google.com'),
('user2#web.com', 'password2', 'imap.yandex.com'),
('user3#web.com', 'password3', 'imap.server3.com'),
]
class MailLogin:
def __init__(self):
self.loop = asyncio.get_event_loop()
self.queue = asyncio.Queue(loop=self.loop)
self.max_workers = 2
async def produce_work(self):
for i in account_infos:
await self.queue.put(i)
for _ in range(max_workers):
await self.queue.put((None, None, None))
async def worker(self):
while True:
(username, password, server) = await self.queue.get()
if username is None:
break
while True:
try:
s = IMAP4_SSL(server)
await s.wait_hello_from_server()
r = await s.login(username, password)
await s.logout()
if r.result != 'NO':
print('Information works')
except Exception as e:
# DOES NOT CATCH
print(str(e))
else:
break
def start(self):
try:
self.loop.run_until_complete(
asyncio.gather(self.produce_work(), *[self.worker() for _ in range(self.max_workers)],
loop=self.loop, return_exceptions=True)
)
finally:
print('Done')
if __name__ == '__main__':
MailLogin().start()

There are several ways to do this but the TimeoutError is probably caught in your except. You don't see it because str(e) is an empty string.
You can see the stacks enabling debug mode of asyncio.
First, you can catch the exception as you did:
async def fail_fun():
try:
imap_client = aioimaplib.IMAP4_SSL(host='foo', timeout=1)
await imap_client.wait_hello_from_server()
except Exception as e:
print('Exception : ' + str(e))
if __name__ == '__main__':
get_event_loop().run_until_complete(fail_fun())
Second, you can catch the exception at run_until_complete
async def fail_fun():
imap_client = aioimaplib.IMAP4_SSL(host='foo', timeout=1)
await imap_client.wait_hello_from_server()
if __name__ == '__main__':
try:
get_event_loop().run_until_complete(fail_fun())
except Exception as e:
print('Exception : ' + str(e))
The connection is established wrapping the loop.create_connection coroutine with create_task : we wanted to establish the connection in the IMAP4 constructor and __init__ should return None.
So if your host has a wrong value, you could test it before, or wait for the timeout :
socket.gaierror: [Errno -5] No address associated with hostname
if a host is not responding before the timeout, you can raise the timeout. And if the connection is lost during the connection, you can add a connection lost callback in the IMAP4 constructor.

asyncores's handle_close keeps firing continuously

self.handlers.append(ConnHandler(sock, self.handlers))I'm new to python and I tried to write a simple socket server to test stuff out and get to know the language better.
import asyncore
import socket
import json
class ConnHandler(asyncore.dispatcher_with_send):
def __init__(self, conn, handlerlist):
asyncore.dispatcher_with_send.__init__(self, conn)
self.handlers = handlerlist
def handle_close(self):
self.close()
print 'Socket closed'
if(self.handlers.count(self) > 0):
self.handlers.remove(self);
def handle_read(self):
data = ''
more = True
while more:
try:
data += self.recv(1024)
except socket.error, e:
more = False
if data == '':
return
try:
message = json.loads(data)
except ValueError:
self.send('Invalid JSON\n')
return
print message
class TestServer(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
self.handlers = []
def handle_accept(self):
pair = self.accept()
if pair is not None:
sock, addr = pair
print 'Incoming connection from %s' % repr(addr)
self.handlers.append(ConnHandler(sock, self.handlers))
def sendToAll(self, string):
for h in self.handlers:
h.send(string + '\n')
server = TestServer('localhost', 55555)
asyncore.loop()
My problem is the following.
If I connect to the server with telnet and then quit telnet to close the connection, python just spams 'Socket closed' in the terminal. How can I fix this? Are there obvious beginner mistakes in the above code?

handle_close is not meant to detect if peer is disconnected, you get this information in handle_read if .recv call returns an empty string. Then you can close the socket, and then handle_close is called.
Here is a modified version of your code:
def handle_close(self):
print 'Socket closed'
if(self.handlers.count(self) > 0):
self.handlers.remove(self);
def handle_read(self):
data = ''
more = True
while more:
try:
new_data = self.recv(1024)
if not new_data:
self.close()
return
else:
data += new_data
except socket.error, e:
more = False
if data == '':
return
try:
message = json.loads(data)
except ValueError:
self.send('Invalid JSON\n')
return
print message

Paramiko exec_command, timeout not working

Setup: paramiko 1.15.1
According to the documentation a timeout was created for exec_command since 1.10.0 but for some reason it's not working for me. Do I have an error in my code that I've missed or is it in fact a bug?
I have this code
class CustomSSH(object):
def __init__(self, node):
self.node = node
self.ssh = paramiko.SSHClient()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
self.privkey = paramiko.RSAKey.from_private_key_file('./secret.key')
except:
self.use_password = True
def connect_ssh(self, timeout=60):
try:
if self.use_password:
self.ssh.connect(self.node,
timeout=60,
username='xxx',
password='xxx',
look_for_keys=False)
else:
"""Using SSH Key"""
self.ssh.connect(self.node,
timeout=60,
username='xxx',
pkey=self.privkey,
look_for_keys=False)
self.using_ssh = True
return True
except paramiko.AuthenticationException, e:
if self.use_password:
raise paramiko.SSHException
print "Private key {}".format(e)
print "Trying using username/password instead"
self.use_password = True
self.connect_ssh()
except paramiko.SSHException, e:
print "some other error ", e
self.close()
raise
except Exception:
print "exception"
raise
def close(self):
print "Closing connection"
try:
self.ssh.close()
except:
print "Error closing connection"
def send_cmd(self, command, regexmatch='', timeout=60):
"""Issue Commands using SSH
returns a Tuple of (True/False, results)"""
try:
stdin, stdout, stderr = self.ssh.exec_command(command, timeout)
xresults = stdout.readlines()
results = ''.join(xresults)
re_status = re.compile(regexmatch, re.IGNORECASE)
if re_status.search(results):
return (True, xresults)
else:
return (False, xresults)
except Exception as e:
raise KeyError, e
That I'm executing as follows:
ssh = CustomSSH(node)
ssh.connect_ssh()
ssh.send_cmd(abort_cmd)
What is wrong here?

It was of course a simple typo.
When calling exec_command I sent the value of timeout instead of timeout=value.
self.ssh.exec_command(command, timeout)
should have been
self.ssh.exec_command(command, timeout=60)