I got this sample code for python websocket client from here and I am running into a problem below. I edited this code a bit from the original one in couple of places since I was getting into a problem:
Removed the connect method and merged it into constructor. I was
getting 'self._ws_connection' not found issue.
Changed the WebSocketClient __init__()method to accept 'url'
and removed the '' argument. I am not sure what passing of ''
means.
Problem1:
Traceback (most recent call last):
File "websocketcli.py", line 138, in <module>
main()
File "websocketcli.py", line 129, in main
client.send('Hello world!')
File "websocketcli.py", line 49, in send
self._ws_connection.write_message(escape.utf8(json.dumps(data)))
File "/users/anjangam/pyvenv/venv/lib/python2.7/site-packages/tornado/websocket.py", line 970, in write_message
return self.protocol.write_message(message, binary)
AttributeError: 'NoneType' object has no attribute 'write_message'
Problem 2:
When I remove the '*' from the WbSocketClient constructor, I get the following error, almost same as the one encountered in problem 1. Note: I am using python version 2.7.8.
Traceback (most recent call last):
File "websocketcli.py", line 138, in <module>
main()
File "websocketcli.py", line 129, in main
client.send('Hello world!')
File "websocketcli.py", line 52, in send
self._ws_connection.write_message(escape.utf8(json.dumps(data)))
File "/users/anjangam/pyvenv/venv/lib/python2.7/site-packages/tornado/websocket.py", line 970, in write_message
return self.protocol.write_message(message, binary)
AttributeError: 'NoneType' object has no attribute 'write_message'
Client code:
from tornado import escape
from tornado import gen
from tornado import httpclient
from tornado import httputil
from tornado import ioloop
from tornado import websocket
import functools
import json
import time
APPLICATION_JSON = 'application/json'
DEFAULT_CONNECT_TIMEOUT = 60
DEFAULT_REQUEST_TIMEOUT = 60
class WebSocketClient():
"""Base for web socket clients.
"""
def __init__(self, connect_timeout=DEFAULT_CONNECT_TIMEOUT,
request_timeout=DEFAULT_REQUEST_TIMEOUT):
self.connect_timeout = connect_timeout
self.request_timeout = request_timeout
def connect(self, url):
"""Connect to the server.
:param str url: server URL.
"""
headers = httputil.HTTPHeaders({'Content-Type': APPLICATION_JSON})
request = httpclient.HTTPRequest(url=url,
connect_timeout=self.connect_timeout,
request_timeout=self.request_timeout,
headers=headers)
self._ws_connection = websocket.WebSocketClientConnection(ioloop.IOLoop.current(),
request)
self._ws_connection.connect_future.add_done_callback(self._connect_callback)
print 'Connection: ', self._ws_connection
def send(self, data):
"""Send message to the server
:param str data: message.
"""
if not self._ws_connection:
raise RuntimeError('Web socket connection is closed.')
self._ws_connection.write_message(escape.utf8(json.dumps(data)))
def close(self):
"""Close connection.
"""
if not self._ws_connection:
raise RuntimeError('Web socket connection is already closed.')
self._ws_connection.close()
def _connect_callback(self, future):
if future.exception() is None:
self._ws_connection = future.result()
self._on_connection_success()
self._read_messages()
else:
self._on_connection_error(future.exception())
#gen.coroutine
def _read_messages(self):
while True:
msg = yield self._ws_connection.read_message()
if msg is None:
self._on_connection_close()
break
self._on_message(msg)
def _on_message(self, msg):
"""This is called when new message is available from the server.
:param str msg: server message.
"""
pass
def _on_connection_success(self):
"""This is called on successful connection ot the server.
"""
pass
def _on_connection_close(self):
"""This is called when server closed the connection.
"""
pass
def _on_connection_error(self, exception):
"""This is called in case if connection to the server could
not established.
"""
pass
class TestWebSocketClient(WebSocketClient):
def __init__(self, url):
WebSocketClient.__init__(self, url)
def _on_message(self, msg):
print(msg)
deadline = time.time() + 1
ioloop.IOLoop().instance().add_timeout(
deadline, functools.partial(self.send, str(int(time.time()))))
def _on_connection_success(self):
print('Connected!')
self.send(str(int(time.time())))
def _on_connection_close(self):
print('Connection closed!')
def _on_connection_error(self, exception):
print('Connection error: %s', exception)
def main():
client = TestWebSocketClient()
client.connect('ws://localhost:8888/ws')
client.send('Hello world!')
try:
ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
client.close()
if __name__ == '__main__':
main()
The original code you referenced works just fine on python 3.5, however I suspect you are using some version of python 2. The reason it would break would be due to the asterisk you mentioned, which is not a feature of python 2. PEP 3102 talks about this, but in short that asterisk will just force function calls to pass in keyword arguments as opposed to relying on the order that the parameters were supplied in.
So take for example the following functions:
# we can use positional arguments on this one
def test(a, b):
print(a, b)
# this will require keyword arguments to be supplied on function calls
def test_keywords_args_required(*, a, b):
print(a, b)
test(1, 2) # test can be called using positional arguments
test_keywords_args_required(1, 2) # this will error out. Needs keywords args!
test_keywords_args_required(a=1, b=2) # this will work!
If you use the code originally supplied from that link, and remove the asterisk, the code should work (confirmed with python 2.7).
Related
To start, I'm working with aiosmtpd, and am trying to write a class that wraps around it to start up the SMTP server programmatically with StartTLS. Now, up until very recently, this code worked as expected with any handler you might pass into it, such as a basic Message handler that I wrote to adjust parameters of the message, etc. and passing that in as part of the message headers.
import asyncio
import aiosmtpd
import aiosmtpd.controller
import aiosmtpd.handlers
import aiosmtpd.smtp
import email
import regex
import logging
import ssl
EMPTYBYTES = b''
COMMASPACE = ', '
CRLF = b'\r\n'
NLCRE = regex.compile(br'\r\n|\r|\n')
class StartTLSServer(aiosmtpd.controller.Controller):
def __init__(self, handler, ssl_cert_file, ssl_key_file, loop=None, hostname=None,
port=8025, *, ready_timeout=1.0, enable_SMTPUTF8=True, decode_data=False,
require_starttls=True, smtp_ident=None, data_size_limit=10485760,
smtp_timeout=300):
context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
context.load_cert_chain(ssl_cert_file, ssl_key_file)
self.tls_context = context
self.require_starttls = require_starttls
self.enable_SMTPUTF8 = enable_SMTPUTF8
self.decode_data = decode_data
self.smtp_ident = smtp_ident
self.data_size_limit = data_size_limit
self.smtp_timeout = smtp_timeout
super().__init__(handler, loop=loop, hostname=hostname, port=port,
ready_timeout=ready_timeout, enable_SMTPUTF8=enable_SMTPUTF8)
def factory(self):
return aiosmtpd.smtp.SMTP(self.handler, data_size_limit=self.data_size_limit,
enable_SMTPUTF8=self.enable_SMTPUTF8,
decode_data=self.decode_data,
require_starttls=self.require_starttls,
hostname=self.smtp_ident,
ident=self.smtp_ident,
tls_context=self.tls_context,
timeout=self.smtp_timeout)
class MessageHandler(aiosmtpd.handlers.Message):
def __init__(self, message_class=None, *, loop=None):
super().__init__(message_class)
self.loop = loop or asyncio.get_event_loop()
async def handle_DATA(self, server, session, envelope):
message = self.prepare_message(session, envelope)
await self.handle_message(message)
return '250 OK'
def prepare_message(self, session, envelope):
# If the server was created with decode_data True, then data will be a
# str, otherwise it will be bytes.
data = envelope.content
if isinstance(data, bytes):
message = email.message_from_bytes(data, self.message_class)
else:
assert isinstance(data, str), (
'Expected str or bytes, got {}'.format(type(data)))
message = email.message_from_string(data, self.message_class)
message['X-Peer'] = str(session.peer)
message['X-Envelope-MailFrom'] = envelope.mail_from
message['X-Envelope-RcptTo'] = COMMASPACE.join(envelope.rcpt_tos)
return message # This is handed off to handle_message directly.
async def handle_message(self, message):
print(message.as_string())
return
This resides in custom_handlers.py which is then subsequently called in testing via the Python console as follows:
>>> from custom_handlers import StartTLSServer, MessageHandler
>>> server = StartTLSServer(MessageHandler, ssl_cert_file="valid cert path", ssl_key_file="valid key path", hostname="0.0.0.0", port=25, require_starttls=True, smtp_ident="StartTLSSMTPServer01")
>>> server.start()
When I want to stop the test server, I'll simply do a server.stop() however during processing of any message, we get hard-stopped by this evil error:
Traceback (most recent call last):
File "/home/sysadmin/.local/lib/python3.8/site-packages/aiosmtpd/smtp.py", line 728, in _handle_client
await method(arg)
File "/home/sysadmin/.local/lib/python3.8/site-packages/aiosmtpd/smtp.py", line 1438, in smtp_DATA
status = await self._call_handler_hook('DATA')
File "/home/sysadmin/.local/lib/python3.8/site-packages/aiosmtpd/smtp.py", line 465, in _call_handler_hook
status = await hook(self, self.session, self.envelope, *args)
TypeError: handle_DATA() missing 1 required positional argument: 'envelope'
Now, I can replicate this with ANY handler passed into the SMTP factory.
However, I can't replicate this with a plain aiosmtpd with a Debugging handler, like defined in the docs:
aiosmtpd -c aiosmtpd.handlers.Debugging stdout -l 0.0.0.0:8025
... which works fine. Passing the Debugging handler into the StartTLSServer causes the same error as the custom MessageHandler class, even with the Debugging handler.
Am I missing something obvious here about my class that's exploding here in a way that is different to the programmatic usage as expected by aiosmtpd?
You are missing () in order to instantiate an object of your MessageHandler class:
>>> server = StartTLSServer(MessageHandler(), ...)
When you simply pass MessageHandler without the (), aiosmtpd will try to invoke the regular function MessageHandler.handle_DATA(...) (as opposed to the bound method function MessageHandler().handle_DATA(...)).
This regular function takes four arguments: an instance of MessageHandler as its first argument followed by the usual server, session and envelope arguments. This explains why the error message complains about a missing positional argument.
PS, do note that your handle_DATA implementation is superfluous since it is identical to the implementation in the base class aiosmtpd.handlers.Message - so you can just delete it, and it should still work just fine.
I'm following this Route_Guide sample.
The sample in question fires off and reads messages without replying to a specific message. The latter is what i'm trying to achieve.
Here's what i have so far:
import grpc
...
channel = grpc.insecure_channel(conn_str)
try:
grpc.channel_ready_future(channel).result(timeout=5)
except grpc.FutureTimeoutError:
sys.exit('Error connecting to server')
else:
stub = MyService_pb2_grpc.MyServiceStub(channel)
print('Connected to gRPC server.')
this_is_just_read_maybe(stub)
def this_is_just_read_maybe(stub):
responses = stub.MyEventStream(stream())
for response in responses:
print(f'Received message: {response}')
if response.something:
# okay, now what? how do i send a message here?
def stream():
yield my_start_stream_msg
# this is fine, i receive this server-side
# but i can't check for incoming messages here
I don't seem to have a read() or write() on the stub, everything seems to be implemented with iterators.
How do i send a message from this_is_just_read_maybe(stub)?
Is that even the right approach?
My Proto is a bidirectional stream:
service MyService {
rpc MyEventStream (stream StreamingMessage) returns (stream StreamingMessage) {}
}
What you're trying to do is perfectly possible and will probably involve writing your own request iterator object that can be given responses as they arrive rather than using a simple generator as your request iterator. Perhaps something like
class MySmarterRequestIterator(object):
def __init__(self):
self._lock = threading.Lock()
self._responses_so_far = []
def __iter__(self):
return self
def _next(self):
# some logic that depends upon what responses have been seen
# before returning the next request message
return <your message value>
def __next__(self): # Python 3
return self._next()
def next(self): # Python 2
return self._next()
def add_response(self, response):
with self._lock:
self._responses.append(response)
that you then use like
my_smarter_request_iterator = MySmarterRequestIterator()
responses = stub.MyEventStream(my_smarter_request_iterator)
for response in responses:
my_smarter_request_iterator.add_response(response)
. There will probably be locking and blocking in your _next implementation to handle the situation of gRPC Python asking your object for the next request that it wants to send and your responding (in effect) "wait, hold on, I don't know what request I want to send until after I've seen how the next response turned out".
Instead of writing a custom iterator, you can also use a blocking queue to implement send and receive like behaviour for client stub:
import queue
...
send_queue = queue.SimpleQueue() # or Queue if using Python before 3.7
my_event_stream = stub.MyEventStream(iter(send_queue.get, None))
# send
send_queue.push(StreamingMessage())
# receive
response = next(my_event_stream) # type: StreamingMessage
This makes use of the sentinel form of iter, which converts a regular function into an iterator that stops when it reaches a sentinel value (in this case None).
I need to call a celery task for each GRPC request, and return the result.
In default GRPC implementation, each request is processed in a separate thread from a threadpool.
In my case, the server is supposed to process ~400 requests in batch mode per second. So one request may have to wait 1 second for the result due to the batch processing, which means the size of the threadpool must be larger than 400 to avoid blocking.
Can this be done asynchronously?
Thanks a lot.
class EventReporting(ss_pb2.BetaEventReportingServicer, ss_pb2.BetaDeviceMgtServicer):
def ReportEvent(self, request, context):
res = tasks.add.delay(1,2)
result = res.get() ->here i have to block
return ss_pb2.GeneralReply(message='Hello, %s!' % result.message)
As noted by #Michael in a comment, as of version 1.32, gRPC now supports asyncio in its Python API. If you're using an earlier version, you can still use the asyncio API via the experimental API: from grpc.experimental import aio. An asyncio hello world example has also been added to the gRPC repo. The following code is a copy of the example server:
import logging
import asyncio
from grpc import aio
import helloworld_pb2
import helloworld_pb2_grpc
class Greeter(helloworld_pb2_grpc.GreeterServicer):
async def SayHello(self, request, context):
return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name)
async def serve():
server = aio.server()
helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server)
listen_addr = '[::]:50051'
server.add_insecure_port(listen_addr)
logging.info("Starting server on %s", listen_addr)
await server.start()
await server.wait_for_termination()
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
asyncio.run(serve())
See my other answer for how to implement the client.
It can be done asynchronously if your call to res.get can be done asynchronously (if it is defined with the async keyword).
While grpc.server says it requires a futures.ThreadPoolExecutor, it will actually work with any futures.Executor that calls the behaviors submitted to it on some thread other than the one on which they were passed. Were you to pass to grpc.server a futures.Executor implemented by you that only used one thread to carry out four hundred (or more) concurrent calls to EventReporting.ReportEvent, your server should avoid the kind of blocking that you describe.
In my opinion is good simple implementation async grpc server, same like http based on aiohttp.
import asyncio
from concurrent import futures
import functools
import inspect
import threading
from grpc import _server
def _loop_mgr(loop: asyncio.AbstractEventLoop):
asyncio.set_event_loop(loop)
loop.run_forever()
# If we reach here, the loop was stopped.
# We should gather any remaining tasks and finish them.
pending = asyncio.Task.all_tasks(loop=loop)
if pending:
loop.run_until_complete(asyncio.gather(*pending))
class AsyncioExecutor(futures.Executor):
def __init__(self, *, loop=None):
super().__init__()
self._shutdown = False
self._loop = loop or asyncio.get_event_loop()
self._thread = threading.Thread(target=_loop_mgr, args=(self._loop,),
daemon=True)
self._thread.start()
def submit(self, fn, *args, **kwargs):
if self._shutdown:
raise RuntimeError('Cannot schedule new futures after shutdown')
if not self._loop.is_running():
raise RuntimeError("Loop must be started before any function can "
"be submitted")
if inspect.iscoroutinefunction(fn):
coro = fn(*args, **kwargs)
return asyncio.run_coroutine_threadsafe(coro, self._loop)
else:
func = functools.partial(fn, *args, **kwargs)
return self._loop.run_in_executor(None, func)
def shutdown(self, wait=True):
self._loop.stop()
self._shutdown = True
if wait:
self._thread.join()
# --------------------------------------------------------------------------- #
async def _call_behavior(rpc_event, state, behavior, argument, request_deserializer):
context = _server._Context(rpc_event, state, request_deserializer)
try:
return await behavior(argument, context), True
except Exception as e: # pylint: disable=broad-except
with state.condition:
if e not in state.rpc_errors:
details = 'Exception calling application: {}'.format(e)
_server.logging.exception(details)
_server._abort(state, rpc_event.operation_call,
_server.cygrpc.StatusCode.unknown, _server._common.encode(details))
return None, False
async def _take_response_from_response_iterator(rpc_event, state, response_iterator):
try:
return await response_iterator.__anext__(), True
except StopAsyncIteration:
return None, True
except Exception as e: # pylint: disable=broad-except
with state.condition:
if e not in state.rpc_errors:
details = 'Exception iterating responses: {}'.format(e)
_server.logging.exception(details)
_server._abort(state, rpc_event.operation_call,
_server.cygrpc.StatusCode.unknown, _server._common.encode(details))
return None, False
async def _unary_response_in_pool(rpc_event, state, behavior, argument_thunk,
request_deserializer, response_serializer):
argument = argument_thunk()
if argument is not None:
response, proceed = await _call_behavior(rpc_event, state, behavior,
argument, request_deserializer)
if proceed:
serialized_response = _server._serialize_response(
rpc_event, state, response, response_serializer)
if serialized_response is not None:
_server._status(rpc_event, state, serialized_response)
async def _stream_response_in_pool(rpc_event, state, behavior, argument_thunk,
request_deserializer, response_serializer):
argument = argument_thunk()
if argument is not None:
# Notice this calls the normal `_call_behavior` not the awaitable version.
response_iterator, proceed = _server._call_behavior(
rpc_event, state, behavior, argument, request_deserializer)
if proceed:
while True:
response, proceed = await _take_response_from_response_iterator(
rpc_event, state, response_iterator)
if proceed:
if response is None:
_server._status(rpc_event, state, None)
break
else:
serialized_response = _server._serialize_response(
rpc_event, state, response, response_serializer)
print(response)
if serialized_response is not None:
print("Serialized Correctly")
proceed = _server._send_response(rpc_event, state,
serialized_response)
if not proceed:
break
else:
break
else:
break
_server._unary_response_in_pool = _unary_response_in_pool
_server._stream_response_in_pool = _stream_response_in_pool
if __name__ == '__main__':
server = grpc.server(AsyncioExecutor())
# Add Servicer and Start Server Here
link to original:
https://gist.github.com/seglberg/0b4487b57b4fd425c56ad72aba9971be
How do you delete messages using imap4.IMAP4Client? I cannot get the "deleted" tag correctly applied for using the "expunge" method.
I keep getting the following error:
Failure: twisted.mail.imap4.IMAP4Exception: Invalid system flag \
Sample code would be appreciated. This is what I have so far:
from twisted.internet import protocol, reactor
from twisted.mail import imap4
#Variables for connection
username = 'user#host.com'
password = 'mypassword'
host = 'imap.host.com'
port = 143
class IMAP4LocalClient(imap4.IMAP4Client):
def connectionMade(self):
self.login(username,password).addCallbacks(self._getMessages, self._ebLogin)
#reports any connection errors
def connectionLost(self,reason):
reactor.stop()
#drops the connection
def _ebLogin(self,result):
print result
self.transport.loseConnection()
def _programUtility(self,result):
print result
return self.logout()
def _cbExpungeMessage(self,result):
return self.expunge().addCallback(self._programUtility)
def _cbDeleteMessage(self,result):
return self.setFlags("1:5",flags=r"\\Deleted",uid=False).addCallback(self._cbExpungeMessage)
#gets the mailbox list
def _getMessages(self,result):
return self.list("","*").addCallback(self._cbPickMailbox)
#selects the inbox desired
def _cbPickMailbox(self,result):
mbox='INBOX.Trash'
return self.select(mbox).addCallback(self._cbExamineMbox)
def _cbExamineMbox(self,result):
return self.fetchMessage("1:*",uid=False).addCallback(self._cbDeleteMessage)
class IMAP4ClientFactory(protocol.ClientFactory):
def buildProtocol(self,addr):
return IMAP4LocalClient()
def clientConnectionFailed(self,connector,reason):
print reason
reactor.stop()
reactor.connectTCP(host,port,IMAP4ClientFactory())
reactor.run()
Changed to:
def _cbDeleteMessage(self,result):
return self.setFlags("1:5",flags=['\\Deleted'],uid=False).addCallback(self._cbExpungeMessage)
thanks to Jean-Paul Calderone and it worked, setFlags requires a list, not just a string.
I think there are two problems here.
First, you're passing a string as the flags parameter to setFlags. Notice the documentation for that parameter: The flags to set (type: Any iterable of str). Try a list containing one string, instead.
Second, \\Deleted is probably not a flag the server you're interacting with supports. The standard deleted flag in IMAP4 is \Deleted.
I'm trying to make an IRC bot using the twisted.words.protocols.irc module.
The bot will parse messages from a channel and parse them for command strings.
Everything works fine except when I need the bot to identify a nick by sending a whois command. The whois reply will not be handled until the privmsg method (the method from which I'm doing the parsing) returns.
example:
from twisted.words.protocols import irc
class MyBot(irc.IRClient):
..........
def privmsg(self, user, channel, msg):
"""This method is called when the client recieves a message"""
if msg.startswith(':whois '):
nick = msg.split()[1]
self.whois(nick)
print(self.whoislist)
def irc_RPL_WHOISCHANNELS(self, prefix, params):
"""This method is called when the client recieves a reply for whois"""
self.whoislist[prefix] = params
Is there a way to somehow make the bot wait for a reply after self.whois(nick)?
Perhaps use a thread (I don't have any experience with those).
Deferred is a core concept in Twisted, you must be familiar with it to use Twisted.
Basically, your whois checking function should return a Deferred that will be fired when you receive whois-reply.
I managed to fix this by running all handler methods as threads, and then setting a field, following
kirelagin's suggestion, before running a whois query, and modifying the method that recieves the data
to change the field when it recieves a reply. Its not the most elegant solution but it works.
Modified code:
class MyBot(irc.IRClient):
..........
def privmsg(self, user, channel, msg):
"""This method is called when the client recieves a message"""
if msg.startswith(':whois '):
nick = msg.split()[1]
self.whois_status = 'REQUEST'
self.whois(nick)
while not self.whois_status == 'ACK':
sleep(1)
print(self.whoislist)
def irc_RPL_WHOISCHANNELS(self, prefix, params):
"""This method is called when the client recieves a reply for whois"""
self.whoislist[prefix] = params
def handleCommand(self, command, prefix, params):
"""Determine the function to call for the given command and call
it with the given arguments.
"""
method = getattr(self, "irc_%s" % command, None)
try:
# all handler methods are now threaded.
if method is not None:
thread.start_new_thread(method, (prefix, params))
else:
thread.start_new_thread(self.irc_unknown, (prefix, command, params))
except:
irc.log.deferr()
def irc_RPL_WHOISCHANNELS(self, prefix, params):
"""docstring for irc_RPL_WHOISCHANNELS"""
self.whoislist[prefix] = params
def irc_RPL_ENDOFWHOIS(self, prefix, params):
self.whois_status = 'ACK'