I use tornado (4.2.1) + momoko (2.2.0) + psycopg2 (2.6.1) for small web application and it works ok until PostgreSQL server close connection. Then after every db.execute() command I receive an error message:
Traceback (most recent call last):
File "C:\Python27\lib\site-packages\tornado-4.2.1-py2.7-win32.egg\tornado\web.py", line 1415, in _execute
result = yield result
File "C:\Python27\lib\site-packages\tornado-4.2.1-py2.7-win32.egg\tornado\gen.py", line 870, in run
value = future.result()
File "C:\Python27\lib\site-packages\tornado-4.2.1-py2.7-win32.egg\tornado\concurrent.py", line 215, in result
raise_exc_info(self._exc_info)
File "C:\Python27\lib\site-packages\tornado-4.2.1-py2.7-win32.egg\tornado\gen.py", line 876, in run
yielded = self.gen.throw(*exc_info)
File "server.py", line 63, in get
cursor = yield self.db.execute(query)
File "C:\Python27\lib\site-packages\tornado-4.2.1-py2.7-win32.egg\tornado\gen.py", line 870, in run
value = future.result()
File "C:\Python27\lib\site-packages\tornado-4.2.1-py2.7-win32.egg\tornado\concurrent.py", line 215, in result
raise_exc_info(self._exc_info)
File "D:\work\program-stat\momoko\connection.py", line 453, in when_available
future_or_result = method(conn, *args, **kwargs)
File "D:\work\program-stat\momoko\connection.py", line 743, in execute
cursor.execute(operation, parameters)
File "C:\Python27\lib\site-packages\psycopg2\extras.py", line 288, in execute
return super(NamedTupleCursor, self).execute(query, vars)
OperationalError: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
Here is a code:
import os
import tornado.httpserver
import tornado.ioloop
import tornado.options
import tornado.web
import momoko
from tornadotools.route import Route
from psycopg2.extras import NamedTupleCursor
import environments as env
tornado.options.define("port", default=9999, help="run on the given port", type=int)
tornado.options.define("pgsql_host", default=env.DB_HOST, help="database host")
tornado.options.define("pgsql_database", default=env.DB_DATABASE, help="database name")
tornado.options.define("pgsql_user", default=env.DB_LOGIN, help="database user")
tornado.options.define("pgsql_password", default=env.DB_PASSWORD, help="database password")
class Application(tornado.web.Application):
def __init__(self):
handlers = Route.routes()
settings = dict(
template_path=os.path.join(os.path.dirname(__file__), "templates"),
static_path=os.path.join(os.path.dirname(__file__), "static"),
debug=True,
)
tornado.web.Application.__init__(self, handlers, **settings)
# Have one global connection to DB across all handlers
self.db = momoko.Pool(
dsn='dbname=%s user=%s password=%s '
'host=%s port=5432' % (
tornado.options.options.pgsql_database,
tornado.options.options.pgsql_user,
tornado.options.options.pgsql_password,
tornado.options.options.pgsql_host),
cursor_factory=NamedTupleCursor,
)
self.db.connect()
class BaseHandler(tornado.web.RequestHandler):
#property
def db(self):
return self.application.db
#Route(r"/")
class HomeHandler(BaseHandler):
def get(self):
self.write("<ul>")
self.write("<li><a href='/test'>Test page</a>")
self.write("</ul>")
self.finish()
#Route(r"/test")
class MogrifyHandler(BaseHandler):
#tornado.web.asynchronous
#tornado.gen.coroutine
def get(self):
cursor = yield self.db.execute("SELECT %s as t;", (1,))
self.render("test.html", result=cursor.fetchall())
def main():
tornado.options.parse_command_line()
http_server = tornado.httpserver.HTTPServer(Application())
http_server.listen(tornado.options.options.port)
tornado.ioloop.IOLoop.instance().start()
if __name__ == "__main__":
main()
How could I handle this exception and reconnect to db automatically without restarting app?
Tornado/Momoko/AutoReload
Problem:
If the connection to the database Postgres distructed - momoko can not re-create the connection pool.
To solve this problem you need to restart the application. To automate this process, I wrote a small script that use the built-in tornado update mechanism.
Solution:
Test every 1 second that database connection is alive.
Add this check before you start ioloop.
import reload
check_connect = tornado.ioloop.PeriodicCallback(lambda: reload.reload_if_db_pool_is_dead(application.db), 1000)
check_connect.start()
https://gist.github.com/MaxRV/d7180e63d6b5396d904d446653aa7e88
you can add parameter:
raise_connect_errors=False,
When you create the connection pool:
self.db = momoko.Pool(
dsn='dbname=%s user=%s password=%s '
'host=%s port=5432' % (
tornado.options.options.pgsql_database,
tornado.options.options.pgsql_user,
tornado.options.options.pgsql_password,
tornado.options.options.pgsql_host),
cursor_factory=NamedTupleCursor,
raise_connect_errors=False,
)
But it is not always helpful.
Related
I have long running file I/O tasks which I'd like to be able to move into a daemon/server process. A CLI tool would be used to queue new jobs to run, query the status of running jobs, and wait for individual jobs. Python's multiprocessing.managers looks like a nice simple way to handle the IPC. I'd like to be able to construct a SyncManager.Event for the client to wait on without blocking the server, but attempting to do so results in triggers a "server not yet started" assertion. Ironically this assertion gets sent from the server to the client, so obviously the server is started, somewhere.
Here's the minimal example:
#!/usr/bin/env python3
import time
import sys
import concurrent.futures
from multiprocessing.managers import SyncManager
def do_work(files):
"""Simulate doing some work on a set of files."""
print(f"Starting work for {files}.")
time.sleep(2)
print(f"Finished work for {files}.")
# Thread pool to do work in.
pool = concurrent.futures.ProcessPoolExecutor(max_workers=1)
class Job:
job_counter = 1
def __init__(self, files):
"""Setup a job and queue work for files on our thread pool."""
self._job_number = self.job_counter
Job.job_counter += 1
print(f"manager._state.value = {manager._state.value}")
self._finished_event = manager.Event()
print(f"Queued job {self.number()}.")
future = pool.submit(do_work, files)
future.add_done_callback(lambda f : self._finished_event.set())
def number(self):
return self._job_number
def event(self):
"""Get an event which can be waited on for the job to complete."""
return self._finished_event
class MyManager(SyncManager):
pass
MyManager.register("Job", Job)
manager = MyManager(address=("localhost", 16000), authkey=b"qca-authkey")
if len(sys.argv) > 1 and sys.argv[1] == "server":
manager.start()
print(f"Manager listening at {manager.address}.")
while True:
time.sleep(1)
else:
manager.connect()
print(f"Connected to {manager.address}.")
job = manager.Job(["a", "b", "c"])
job.event().wait()
print("Done")
If I run the client I see:
$ ./mp-manager.py
Connected to ('localhost', 16000).
Traceback (most recent call last):
File "./mp-manager.py", line 54, in <module>
job = manager.Job(["a", "b", "c"])
File "/usr/lib/python3.8/multiprocessing/managers.py", line 740, in temp
token, exp = self._create(typeid, *args, **kwds)
File "/usr/lib/python3.8/multiprocessing/managers.py", line 625, in _create
id, exposed = dispatch(conn, None, 'create', (typeid,)+args, kwds)
File "/usr/lib/python3.8/multiprocessing/managers.py", line 91, in dispatch
raise convert_to_error(kind, result)
multiprocessing.managers.RemoteError:
---------------------------------------------------------------------------
Traceback (most recent call last):
File "/usr/lib/python3.8/multiprocessing/managers.py", line 210, in handle_request
result = func(c, *args, **kwds)
File "/usr/lib/python3.8/multiprocessing/managers.py", line 403, in create
obj = callable(*args, **kwds)
File "./mp-manager.py", line 24, in __init__
self._finished_event = manager.Event()
File "/usr/lib/python3.8/multiprocessing/managers.py", line 740, in temp
token, exp = self._create(typeid, *args, **kwds)
File "/usr/lib/python3.8/multiprocessing/managers.py", line 622, in _create
assert self._state.value == State.STARTED, 'server not yet started'
AssertionError: server not yet started
---------------------------------------------------------------------------
The server output is:
$ ./mp-manager.py server
Manager listening at ('127.0.0.1', 16000).
manager._state.value = 0
In Tornado's official documentation (https://www.tornadoweb.org/en/stable/guide/async.html) there is an example of synchronous fetch, but when I put it in get method of my IndexHandler it returns following error:
File "/home/stefan/.local/lib/python3.6/site-packages/tornado/web.py", line 1697, in _execute
result = method(*self.path_args, **self.path_kwargs)
File "ex1.py", line 19, in get
client = tornado.httpclient.HTTPClient()
File "/home/stefan/.local/lib/python3.6/site-packages/tornado/httpclient.py", line 107, in __init__
self._async_client = self._io_loop.run_sync(make_client)
File "/home/stefan/.local/lib/python3.6/site-packages/tornado/ioloop.py", line 526, in run_sync
self.start()
File "/home/stefan/.local/lib/python3.6/site-packages/tornado/platform/asyncio.py", line 148, in start
self.asyncio_loop.run_forever()
File "/usr/lib/python3.6/asyncio/base_events.py", line 428, in run_forever
'Cannot run the event loop while another loop is running')
RuntimeError: Cannot run the event loop while another loop is running
My code looks like:
import tornado.httpserver
import tornado.ioloop
import tornado.options
import tornado.web
import tornado.httpclient
from tornado.options import define, options
define("port", default=8000, help="run on the given port", type=int)
class IndexHandler(tornado.web.RequestHandler):
def get(self):
client = tornado.httpclient.HTTPClient()
response = client.fetch('https://www.google.com')
print(response.body)
if __name__ == "__main__":
tornado.options.parse_command_line()
app=tornado.web.Application(handlers=[(r"/", IndexHandler)],debug=True)
http_server = tornado.httpserver.HTTPServer(app)
http_server.listen(options.port)
tornado.ioloop.IOLoop.instance().start()
I believe I should make changes in main in order to make it work.
Because as you can see in your traceback, a "synchronous" fetch starts it's own ioloop under the hood. The easiest to fix is to rewrite your def get as a coroutine (either as async def get or with a #gen.coroutine decorator) and use asynchronous fetch
class IndexHandler(tornado.web.RequestHandler):
async def get(self):
client = tornado.httpclient.HTTPClient()
response = await client.fetch('https://www.google.com')
print(response.body)
Starting app.py, then killing the database and hitting /api/foo gives me:
peewee.OperationalError: could not connect to server: Connection refused
Bringing the database back up gives me and hitting /api/foo gives me:
peewee.OperationalError: terminating connection due to administrator
command\nSSL connection has been closed unexpectedly\n
And hitting /api/foo again gives me:
peewee.InterfaceError: connection already closed
Test case
test_case/__init__.py
#!/usr/bin/env python
from os import environ
from bottle import Bottle, request, response
from playhouse.db_url import connect
bottle_api = Bottle()
db = connect(environ['RDBMS_URI'])
from test_case.foo.models import Foo
db.connect() # Not needed, but do want to throw errors ASAP
db.create_tables([Foo], safe=True) # Create tables (if they don't exist)
from test_case.foo.routes import foo_api
bottle_api.merge(foo_api)
bottle_api.catchall = False
#bottle_api.hook('before_request')
def _connect_db():
print 'Connecting to db'
db.connect()
#bottle_api.hook('after_request')
def _close_db():
print 'Closing db'
if not db.is_closed():
db.close()
def error_catcher(environment, start_response):
try:
return bottle_api.wsgi(environment, start_response)
except Exception as e:
environment['PATH_INFO'] = '/api/error'
environment['api_error'] = e
return bottle_api.wsgi(environment, start_response)
#bottle_api.route('/api/error')
def global_error():
response.status = 500
return {'error': (lambda res: res[res.find("'") + 1:res.rfind("'")])(
str(request.environ['api_error'].__class__)),
'error_message': request.environ['api_error'].message}
test_case/__main__.py
from __init__ import bottle_api
# Or `from __init__ import bottle_api`; `from bottle import run`;
# Then `run(error_catcher, port=5555)`
bottle_api.run(port=5555)
test_case/foo/__init__.py
test_case/foo/models.py
from peewee import Model, CharField
from test_case import db
class Foo(Model):
id = CharField(primary_key=True)
class Meta(object):
database = db
test_case/foo/routes.py
from bottle import Bottle
from playhouse.shortcuts import model_to_dict
from test_case.foo.models import Foo
foo_api = Bottle()
#foo_api.get('/api/foo')
def retrieve_foos():
return {'foos': tuple(model_to_dict(foo) for foo in Foo.select())}
Github gist for easy cloning.
Update:
I believe the problem lies in how you've structured your imports and the way python loads and caches modules in sys.path.
I think that one of your modules is being imported and loaded twice and different parts of the codebase use different instances of the module.
Thus, the views in foo.routes, are using one instance of the database object, while the connection hooks are using another.
Instead of from __init__, what about trying from test_case import bottle_api? That is the one import statement that jumps out at me as a possible culprit.
I added the following to your code so I could run it from the command-line:
if __name__ == '__main__':
api.run()
Then I made a request to /api/foo and saw some fake data. I stopped the Postgresql server and got this error:
Traceback (most recent call last):
File "/usr/lib64/python2.7/wsgiref/handlers.py", line 85, in run
self.result = application(self.environ, self.start_response)
File "/home/charles/tmp/scrap/bottlez/lib/python2.7/site-packages/bottle.py", line 979, in __call__
return self.wsgi(environ, start_response)
File "/home/charles/tmp/scrap/bottlez/lib/python2.7/site-packages/bottle.py", line 954, in wsgi
out = self._cast(self._handle(environ))
File "/home/charles/tmp/scrap/bottlez/lib/python2.7/site-packages/bottle.py", line 857, in _handle
self.trigger_hook('before_request')
File "/home/charles/tmp/scrap/bottlez/lib/python2.7/site-packages/bottle.py", line 640, in trigger_hook
return [hook(*args, **kwargs) for hook in self._hooks[__name][:]]
File "bt.py", line 31, in _connect_db
db.connect()
File "/home/charles/tmp/scrap/bottlez/src/peewee/peewee.py", line 2967, in connect
self.initialize_connection(self.__local.conn)
File "/home/charles/tmp/scrap/bottlez/src/peewee/peewee.py", line 2885, in __exit__
reraise(new_type, new_type(*exc_value.args), traceback)
File "/home/charles/tmp/scrap/bottlez/src/peewee/peewee.py", line 2965, in connect
**self.connect_kwargs)
File "/home/charles/tmp/scrap/bottlez/src/peewee/peewee.py", line 3279, in _connect
conn = psycopg2.connect(database=database, **kwargs)
File "/home/charles/tmp/scrap/bottlez/lib/python2.7/site-packages/psycopg2/__init__.py", line 164, in connect
conn = _connect(dsn, connection_factory=connection_factory, async=async)
OperationalError: could not connect to server: Connection refused
Is the server running on host "localhost" (::1) and accepting
TCP/IP connections on port 5432?
could not connect to server: Connection refused
Is the server running on host "localhost" (127.0.0.1) and accepting
TCP/IP connections on port 5432?
When I restarted the server and made a subsequent request I got a normal response with my test data.
So, in short, I'm not sure what I may be missing but the code seems to be working correctly to me.
Postgresql 9.4, psycopg2 2.6, python 2.7.9, peewee 2.6.0
I'm using tornado and tornado-redis but I can't close the connection to redis without getting an error. Look at this example:
import tornadoredis
import tornado.web
import tornado.gen
client = tornadoredis.Client()
client.connect()
class MainHandler(tornado.web.RequestHandler):
#tornado.web.asynchronous
#tornado.gen.engine
def get(self):
client.publish('test_channel', 'ahahah')
self.finish('Ok')
class ListenerHandler(tornado.web.RequestHandler):
#tornado.web.asynchronous
#tornado.gen.engine
def get(self):
self.client = tornadoredis.Client()
self.client.connect()
yield tornado.gen.Task(self.client.subscribe, 'test_channel')
self.client.listen(self.from_redis)
def from_redis(self, msg):
print msg.kind
if msg.kind == 'message':
self.write(str(msg.body))
self.aaaa()
if msg.kind == 'disconnect':
self.write('Redis error')
self.aaaa()
def aaaa(self):
print('aaaa')
self.finish()
if self.client.subscribed:
self.client.unsubscribe('test_channel')
self.client.disconnect()
def main():
app = tornado.web.Application([
(r"/", MainHandler),
(r"/listen", ListenerHandler),
])
app.listen(9898)
tornado.ioloop.IOLoop.instance().start()
if __name__ == "__main__":
main()
and test from command line with:
curl "http://yourip:9898/listen" &
curl "http://yourip:9898/"
All work fine except that the following error is returned when "self.client.disconnect()" is called:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/tornado/web.py", line 1115, in _stack_context_handle_exception
raise_exc_info((type, value, traceback))
File "/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py", line 302, in wrapped
ret = fn(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/tornado/gen.py", line 550, in inner
self.set_result(key, result)
File "/usr/local/lib/python2.7/dist-packages/tornado/gen.py", line 476, in set_result
self.run()
File "/usr/local/lib/python2.7/dist-packages/tornado/gen.py", line 505, in run
yielded = self.gen.throw(*exc_info)
File "/usr/local/lib/python2.7/dist-packages/tornadoredis/client.py", line 1070, in listen
data = yield gen.Task(self.connection.readline)
File "/usr/local/lib/python2.7/dist-packages/tornado/gen.py", line 533, in run
self.yield_point.start(self)
File "/usr/local/lib/python2.7/dist-packages/tornado/gen.py", line 371, in start
self.func(*self.args, **self.kwargs)
File "/usr/local/lib/python2.7/dist-packages/tornadoredis/connection.py", line 154, in
readline
raise ConnectionError('Tried to read from '
ConnectionError: Tried to read from non-existent connection
Do you have any suggestion to get the connection to redis closed in a clean and right way ?
Why do I get an error while trying to implement LoopingCall function calling AMP commands?
from twisted.protocols.amp import AMP
from twisted.python.log import startLogging, err
from twisted.internet.task import LoopingCall
from twisted.internet import reactor
from sys import stdout
import commands
startLogging(stdout)
class MyAMP:
def __init__(self, host, port):
destination = TCP4ClientEndpoint(reactor, host, port)
self.protocol = AMP()
self.d = connectProtocol(destination, self.protocol)
def say(self):
return self.protocol.callRemote(commands.Say,
phrase='Hello world')
def loop(myamp):
myamp.say()
def main(host, port):
myamp = MyAMP(host, port)
lc = LoopingCall(loop, myamp=myamp)
lc.start(4.0)
reactor.run()
main('127.0.0.1', 12345)
Error while calling myamp.say() within loop:
2013-08-16 12:28:58-0400 [-] Starting factory <twisted.internet.endpoints.OneShotFactory instance at 0x92273ec>
2013-08-16 12:28:58-0400 [-] Unhandled error in Deferred:
2013-08-16 12:28:58-0400 [-] Unhandled Error
Traceback (most recent call last):
File "lib/client.py", line 35, in <module>
main('127.0.0.1', 12345)
File "lib/client.py", line 32, in main
lc.start(4.0)
File "/usr/local/lib/python2.7/site-packages/twisted/internet/task.py", line 173, in start
self()
File "/usr/local/lib/python2.7/site-packages/twisted/internet/task.py", line 218, in __call__
d = defer.maybeDeferred(self.f, *self.a, **self.kw)
--- <exception caught here> ---
File "/usr/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 137, in maybeDeferred
result = f(*args, **kw)
File "lib/client.py", line 26, in loop
myamp.say()
File "lib/client.py", line 22, in say
phrase='Hello world')
File "/usr/local/lib/python2.7/site-packages/twisted/protocols/amp.py", line 821, in callRemote
return co._doCommand(self)
File "/usr/local/lib/python2.7/site-packages/twisted/protocols/amp.py", line 1778, in _doCommand
self.requiresAnswer)
File "/usr/local/lib/python2.7/site-packages/twisted/protocols/amp.py", line 752, in _sendBoxCommand
box._sendTo(self.boxSender)
File "/usr/local/lib/python2.7/site-packages/twisted/protocols/amp.py", line 577, in _sendTo
proto.sendBox(self)
exceptions.AttributeError: 'NoneType' object has no attribute 'sendBox'
2013-08-16 12:28:58-0400 [Uninitialized] AMP connection established (HOST:IPv4Address(TCP, '127.0.0.1', 50457) PEER:IPv4Address(TCP, '127.0.0.1', 12345))
You're trying to callRemote before the connection is established. A LoopingCall will, by default, immediately run its function when you start it. Instead of doing lc.start(4.0), do lc.start(4.0, now=False). This will wait four second before the first call.
In the normal environment, where the network connection is rock stable, the way of #habnabit will be working, but in the real world, the connection latency can not be estimated as you expect. The better solution for this problem, the looping call must be executed after the amp client is connected like this.
from twisted.protocols.amp import AMP
from twisted.python.log import startLogging, err
from twisted.internet.task import LoopingCall
from twisted.internet import reactor, endpoints
from sys import stdout
import commands
startLogging(stdout)
class MyAMP:
def __init__(self, host, port):
destination = endpoints.TCP4ClientEndpoint(reactor, host, port)
self.protocol = AMP()
self.d = endpoints.connectProtocol(destination, self.protocol)
def loop (proto, ) :
return proto.callRemote(commands.get_user, key='Hello world')
def main(host, port):
def _cb_connected (proto, ) :
lc = LoopingCall(loop, proto, )
lc.start(4.0)
return
myamp = MyAMP(host, port)
myamp.d.addCallback(_cb_connected, )
reactor.run()
return
main('127.0.0.1', 12345, )