I'm using tornado and tornado-redis but I can't close the connection to redis without getting an error. Look at this example:
import tornadoredis
import tornado.web
import tornado.gen
client = tornadoredis.Client()
client.connect()
class MainHandler(tornado.web.RequestHandler):
#tornado.web.asynchronous
#tornado.gen.engine
def get(self):
client.publish('test_channel', 'ahahah')
self.finish('Ok')
class ListenerHandler(tornado.web.RequestHandler):
#tornado.web.asynchronous
#tornado.gen.engine
def get(self):
self.client = tornadoredis.Client()
self.client.connect()
yield tornado.gen.Task(self.client.subscribe, 'test_channel')
self.client.listen(self.from_redis)
def from_redis(self, msg):
print msg.kind
if msg.kind == 'message':
self.write(str(msg.body))
self.aaaa()
if msg.kind == 'disconnect':
self.write('Redis error')
self.aaaa()
def aaaa(self):
print('aaaa')
self.finish()
if self.client.subscribed:
self.client.unsubscribe('test_channel')
self.client.disconnect()
def main():
app = tornado.web.Application([
(r"/", MainHandler),
(r"/listen", ListenerHandler),
])
app.listen(9898)
tornado.ioloop.IOLoop.instance().start()
if __name__ == "__main__":
main()
and test from command line with:
curl "http://yourip:9898/listen" &
curl "http://yourip:9898/"
All work fine except that the following error is returned when "self.client.disconnect()" is called:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/tornado/web.py", line 1115, in _stack_context_handle_exception
raise_exc_info((type, value, traceback))
File "/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py", line 302, in wrapped
ret = fn(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/tornado/gen.py", line 550, in inner
self.set_result(key, result)
File "/usr/local/lib/python2.7/dist-packages/tornado/gen.py", line 476, in set_result
self.run()
File "/usr/local/lib/python2.7/dist-packages/tornado/gen.py", line 505, in run
yielded = self.gen.throw(*exc_info)
File "/usr/local/lib/python2.7/dist-packages/tornadoredis/client.py", line 1070, in listen
data = yield gen.Task(self.connection.readline)
File "/usr/local/lib/python2.7/dist-packages/tornado/gen.py", line 533, in run
self.yield_point.start(self)
File "/usr/local/lib/python2.7/dist-packages/tornado/gen.py", line 371, in start
self.func(*self.args, **self.kwargs)
File "/usr/local/lib/python2.7/dist-packages/tornadoredis/connection.py", line 154, in
readline
raise ConnectionError('Tried to read from '
ConnectionError: Tried to read from non-existent connection
Do you have any suggestion to get the connection to redis closed in a clean and right way ?
Related
I'm trying to write a simple application that communicates using RPCs. I'm using python 3.7's xmlrpc.
This is my server code
MY_ADDR = ("localhost", int(sys.argv[1]))
HOST_ADDR = ("localhost", int(sys.argv[2]))
class RpcServer(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.port = MY_ADDR[1]
self.addr = MY_ADDR[0]
# serve other hosts using this
self.server = SimpleXMLRPCServer((self.addr, self.port))
self.server.register_function(self.recv_ops)
def run(self):
self.server.serve_forever()
def recv_ops(self, sender, op):
print("Sender ", sender, " sent: ", op)
pass
And this is what I'm using as my client's code
def send_ops(host_addr, op):
# contact the other host using this
proxy_addr = "http://{addr}:{port}/".format(addr=host_addr[0], port=host_addr[1])
client_proxy = xmlrpc.client.ServerProxy(proxy_addr, allow_none=True)
resp = client_proxy.recv_ops(MY_ADDR, op)
...
send_ops(HOST_ADDR, ("d", ii, last_line[ii])) # THE RPC CALL I MAKE
Despite setting allow_none=True, I keep getting this:
Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py", line 917, in _bootstrap_inner
self.run()
File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "nb.py", line 102, in editor
send_ops(HOST_ADDR, ("d", ii, last_line[ii]))
File "nb.py", line 63, in send_ops
resp = client_proxy.recv_ops(MY_ADDR, op)
File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/xmlrpc/client.py", line 1112, in __call__
return self.__send(self.__name, args)
File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/xmlrpc/client.py", line 1452, in __request
verbose=self.__verbose
File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/xmlrpc/client.py", line 1154, in request
return self.single_request(host, handler, request_body, verbose)
File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/xmlrpc/client.py", line 1170, in single_request
return self.parse_response(resp)
File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/xmlrpc/client.py", line 1342, in parse_response
return u.close()
File "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/xmlrpc/client.py", line 656, in close
raise Fault(**self._stack[0])
xmlrpc.client.Fault: <Fault 1: "<class 'TypeError'>:cannot marshal None unless allow_none is enabled">
What's tripping my is that the server on the other side actually receives the message (without any None)
Sender ['localhost', 8001] sent: ['d', 4, 'o']
What am I missing here? Any help would be appreciated.
Thanks!
In your server class, add allow_none=True to your SimpleXMLRPCServer instantiation.
self.server = SimpleXMLRPCServer((self.addr, self.port), allow_none=True)
The allow_none and encoding parameters are passed on to xmlrpc.client and control the XML-RPC responses that will be returned from the server.
I use celery 4.3.0 and rabbitmq 3.6.6. There is also a Haproxy for balancing rabbitmq, but right now it only uses one rabbitmq node.
Timeout client/server in Haproxy 24 hours.
Sometimes I get an error and the task execution status is not returned, while the tasks continue to run:
Received 0x00 while expecting 0xce
Traceback (most recent call last):
File "/var/www/serv/app/mod_axmsg/dispatch.py", line 376, in execute
result_value = func_meta.func(*task.args_list, **task.kwargs_dict)
File "/var/www/serv/app/mod_rmc/libs/health.py", line 91, in update_all_health
health.update_all_health()
File "/var/www/serv/app/mod_core/libs/health.py", line 515, in update_all_health
result.join()
File "/usr/local/serv/lib/python3.5/site-packages/celery/result.py", line 765, in join
interval=interval, no_ack=no_ack, on_interval=on_interval,
File "/usr/local/serv/lib/python3.5/site-packages/celery/result.py", line 226, in get
on_message=on_message,
File "/usr/local/serv/lib/python3.5/site-packages/celery/backends/asynchronous.py", line 188, in wait_for_pending
for _ in self._wait_for_pending(result, **kwargs):
File "/usr/local/serv/lib/python3.5/site-packages/celery/backends/asynchronous.py", line 255, in _wait_for_pending
on_interval=on_interval):
File "/usr/local/serv/lib/python3.5/site-packages/celery/backends/asynchronous.py", line 56, in drain_events_until
yield self.wait_for(p, wait, timeout=1)
File "/usr/local/serv/lib/python3.5/site-packages/celery/backends/asynchronous.py", line 65, in wait_for
wait(timeout=timeout)
File "/usr/local/serv/lib/python3.5/site-packages/celery/backends/rpc.py", line 63, in drain_events
return self._connection.drain_events(timeout=timeout)
File "/usr/local/serv/lib/python3.5/site-packages/kombu/connection.py", line 323, in drain_events
return self.transport.drain_events(self.connection, **kwargs)
File "/usr/local/serv/lib/python3.5/site-packages/kombu/transport/pyamqp.py", line 103, in drain_events
return connection.drain_events(**kwargs)
File "/usr/local/serv/lib/python3.5/site-packages/amqp/connection.py", line 505, in drain_events
while not self.blocking_read(timeout):
File "/usr/local/serv/lib/python3.5/site-packages/amqp/connection.py", line 510, in blocking_read
frame = self.transport.read_frame()
File "/usr/local/serv/lib/python3.5/site-packages/amqp/transport.py", line 280, in read_frame
'Received {0:#04x} while expecting 0xce'.format(ch))
amqp.exceptions.UnexpectedFrame: Received 0x00 while expecting 0xce
My code: celery.py
import logging
from app import db
from celery import Celery
LOG = logging.getLogger(__name__)
def make_celery(flask_app):
broker = flask_app.config['AMQP_URL']
celery = Celery('mod_celery', broker=broker, backend="rpc://")
celery.conf.update(flask_app.config)
class ContextTask(celery.Task):
def __call__(self, *args, **kwargs):
with flask_app.app_context():
return self.run(*args, **kwargs)
celery.Task = ContextTask
return celery
def in_tx(task_function):
def tx_wrapper(*args):
try:
task_function(*args)
db.session.commit()
except Exception as ex:
msg = 'Could not execute task {}. Reason: {}.' \
.format(task_function.__name__, str(ex))
LOG.error(msg)
db.session.rollback()
return tx_wrapper
And file tasks.py
from app import celery
from .celery import in_tx
#celery.task(name='app.mod_celery.update_client_health_task')
#in_tx
def update_client_health_task(client_id):
from app.mod_core.libs.health import update_client_health
update_client_health(client_id)
What could be the problem? Please don't offer me redis.
I'm not a developer and I didn't write it the code, but I really need to figure it out. I'm new to python
Thanks.
I'm trying to run two servers using web.py and initiating calls from one to another. Both servers start normally but when I try to call a url the below stack trace is thrown.
import web
urls = (
'/ping', 'Ping',
'/acqlock/+(.*)', 'Acquire',
)
class MSA(web.application):
def run(self, port=8081, *middleware):
func = self.wsgifunc(*middleware)
return web.httpserver.runsimple(func, ('127.0.0.1', port))
app = MSA(urls, globals())
if __name__ == "__main__":
app.run(port=8081)
class Acquire:
def GET(self, resource_name):
print resource_name
response = app.request('http://127.0.0.1:8080/acqlock/' + resource_name, method='GET')
return response
But I keep getting this error after calling the /acqlock.
Traceback (most recent call last):
File "C:\Python27\lib\site-packages\web\wsgiserver\__init__.py", line 1245, in communicate
req.respond()
File "C:\Python27\lib\site-packages\web\wsgiserver\__init__.py", line 775, in respond
self.server.gateway(self).respond()
File "C:\Python27\lib\site-packages\web\wsgiserver\__init__.py", line 2018, in respond
response = self.req.server.wsgi_app(self.env, self.start_response)
File "C:\Python27\lib\site-packages\web\httpserver.py", line 306, in __call__
return self.app(environ, xstart_response)
File "C:\Python27\lib\site-packages\web\httpserver.py", line 274, in __call__
return self.app(environ, start_response)
File "C:\Python27\lib\site-packages\web\application.py", line 279, in wsgi
result = self.handle_with_processors()
File "C:\Python27\lib\site-packages\web\application.py", line 249, in handle_with_processors
return process(self.processors)
File "C:\Python27\lib\site-packages\web\application.py", line 246, in process
raise self.internalerror()
File "C:\Python27\lib\site-packages\web\application.py", line 515, in internalerror
parent = self.get_parent_app()
File "C:\Python27\lib\site-packages\web\application.py", line 500, in get_parent_app
if self in web.ctx.app_stack:
AttributeError: 'ThreadedDict' object has no attribute 'app_stack'
Use requests library for this.
import requests
response = requests.request(method='GET', url ='http://127.0.0.1:8080/acqlock/' + resource_name)
Note: You have used port 8080 in url even though you have hosted the web.py in 8081
Consider the following Tornado (v 4.0.2) application, which is a little bit modified version of official hello world example:
import tornado.ioloop
import tornado.web
class MainHandler(tornado.web.RequestHandler):
def get(self):
self.set_status(400)
self.write("Hello, world")
application = tornado.web.Application([
(r"/", MainHandler),
])
if __name__ == "__main__":
application.listen(8888)
tornado.ioloop.IOLoop.instance().start()
As you can see, the only difference here is set_status call in MainHandler. Now, I save this code into app.py, then I open tests.py and I put there this simple unit test:
import tornado.ioloop
from tornado.httpclient import HTTPRequest
from tornado.testing import AsyncHTTPTestCase, gen_test
from app import application
class SimpleTest(AsyncHTTPTestCase):
def get_app(self):
return application
def get_new_ioloop(self):
return tornado.ioloop.IOLoop.instance()
#gen_test
def test_bad_request(self):
request = HTTPRequest(url=self.get_url('/'))
response = yield self.http_client.fetch(request)
self.assertEqual(response.code, 400)
When I run this test with python -m tornado.test.runtests tests I get the following result:
E
======================================================================
ERROR: test_bad_request (tests.SimpleTest)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/tornado/testing.py", line 118, in __call__
result = self.orig_method(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/tornado/testing.py", line 494, in post_coroutine
timeout=timeout)
File "/usr/local/lib/python2.7/dist-packages/tornado/ioloop.py", line 418, in run_sync
return future_cell[0].result()
File "/usr/local/lib/python2.7/dist-packages/tornado/concurrent.py", line 109, in result
raise_exc_info(self._exc_info)
File "/usr/local/lib/python2.7/dist-packages/tornado/gen.py", line 631, in run
yielded = self.gen.throw(*sys.exc_info())
File "tests.py", line 18, in test_bad_request
response = yield self.http_client.fetch(request)
File "/usr/local/lib/python2.7/dist-packages/tornado/gen.py", line 628, in run
value = future.result()
File "/usr/local/lib/python2.7/dist-packages/tornado/concurrent.py", line 111, in result
raise self._exception
HTTPError: HTTP 400: Bad Request
----------------------------------------------------------------------
Ran 1 test in 0.022s
FAILED (errors=1)
[E 140929 12:55:59 testing:687] FAIL
Obviously this is correct, because the handler sets 400 status code. But how can I test my application for such case? I think 4xx codes are useful, so I don't want to give them up. However I'm new to Tornado and I wasn't able to find a way to test them. Is there any?
Try this:
#gen_test
def test_bad_request(self):
request = HTTPRequest(url=self.get_url('/'))
with self.assertRaises(tornado.httpclient.HTTPError) as context:
yield self.http_client.fetch(request)
self.assertEqual(context.exception.code, 400)
See the documentation for assertRaises.
I get an exception on both client and server side when I run the first example at http://buildbot.twistedmatrix.com/builds/sphinx-html/291-15849/projects/web/howto/xmlrpc.html. The server code I used is below:
from twisted.web import xmlrpc, server
class Example(xmlrpc.XMLRPC):
"""An example object to be published."""
def xmlrpc_echo(self, x):
"""
Return all passed args.
"""
return x
def xmlrpc_add(self, a, b):
"""
Return sum of arguments.
"""
return a + b
def xmlrpc_fault(self):
"""
Raise a Fault indicating that the procedure should not be used.
"""
raise xmlrpc.Fault(123, "The fault procedure is faulty.")
if __name__ == '__main__':
from twisted.internet import reactor
r = Example()
reactor.listenTCP(7080, server.Site(r))
reactor.run()
Client side is below:
import xmlrpclib
s = xmlrpclib.Server('http://localhost:7080/')
print s.echo('Hello world')
The server side exception is:
Traceback (most recent call last):
File "/usr/lib/python2.6/dist-packages/twisted/web/xmlrpc.py", line 150, in render_POST
d.addCallback(self._cbRender, request, responseFailed)
File "/usr/lib/python2.6/dist-packages/twisted/internet/defer.py", line 260, in addCallback
callbackKeywords=kw)
File "/usr/lib/python2.6/dist-packages/twisted/internet/defer.py", line 249, in addCallbacks
self._runCallbacks()
File "/usr/lib/python2.6/dist-packages/twisted/internet/defer.py", line 441, in _runCallbacks
self.result = callback(self.result, *args, **kw)
--- <exception caught here> ---
File "/usr/lib/python2.6/dist-packages/twisted/web/xmlrpc.py", line 170, in _cbRender
allow_none=self.allowNone)
exceptions.TypeError: dumps() got an unexpected keyword argument 'allow_none'
Client side exception is:
Traceback (most recent call last):
File "./client.py", line 6, in <module>
print s.echo('Hello world')
File "/usr/local/lib/python2.6/dist-packages/xmlrpclib-1.0.1-py2.6.egg/xmlrpclib.py", line 986, in __call__
return self.__send(self.__name, args)
File "/usr/local/lib/python2.6/dist-packages/xmlrpclib-1.0.1-py2.6.egg/xmlrpclib.py", line 1239, in __request
verbose=self.__verbose
File "/usr/local/lib/python2.6/dist-packages/xmlrpclib-1.0.1-py2.6.egg/xmlrpclib.py", line 1037, in request
return self._parse_response(h.getfile(), sock)
File "/usr/local/lib/python2.6/dist-packages/xmlrpclib-1.0.1-py2.6.egg/xmlrpclib.py", line 1136, in _parse_response
p.close()
File "/usr/local/lib/python2.6/dist-packages/xmlrpclib-1.0.1-py2.6.egg/xmlrpclib.py", line 508, in close
self._parser.Parse("", 1) # end of data
xml.parsers.expat.ExpatError: no element found: line 1, column 0
Looks like you have an old version of xmlrpclib?
What version of python are you using?
Where is the xmlrpclib coming from that your xmlrpc server is using, and what version is it?
$ python -v
>>> import xmlrpclib
# /usr/lib/python2.6/xmlrpclib.pyc matches /usr/lib/python2.6/xmlrpclib.py
>>> xmlrpclib.__version__
'1.0.1'
>>> xmlrpclib.dumps((None,), allow_none=True)
'<params>\n<param>\n<value><nil/></value></param>\n</params>\n
i.e. this works for me. Perhaps you are somehow using an old version of xmlrpclib?