I am trying to turn logging off from sqlalchemy completely. I am creating a custom logging framework for Flask and errors from sqlalchemy/engine/base.py, sqlalchemy/engine/impl.py keep showing up..
I tried doing: logging.getLogger('SQLAlchemy.pool.base').addHandler(logging.NullHandler())
and
logging.getLogger('SQLAlchemy.pool.impl').addHandler(logging.NullHandler())
but they didn't work because they don't leverage python logging traditionally. Instead of having logger=logging.getLogger(__name__) they import from a log file in the library and so I then tried:
logging.getLogger('sqlalchemy.log').addHandler(logging.NullHandler()) instead, but that didn't work either.
I tried the following, but those didn't pan out either:
How to turn sqlalchemy logging off completely
https://docs.sqlalchemy.org/en/14/core/engines.html#more-on-the-echo-flag
Here is the code:
from sqlalchemy import create_engine
from snowflake.sqlalchemy import URL
#app.route("/")
#app.route("/home")
def home():
engine = create_engine('snowflake://{user}:{password}#{account}/'.format(user='user-name', password='password', account='account'), echo=False, echo_pool=False)
try:
connection = engine.connect()
results = connection.execute('select current_version()').fetchone()
print(results[0])
connection.close()
finally:
engine.dispose()
return render_template('home.html', posts=posts)
and here is the errors:
raceback (most recent call last):
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 606, in __connect
pool.logger.debug("Created new connection %r", connection)
TypeError: debug() takes 1 positional argument but 3 were given
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "sample_projects/app.py", line 69, in home
connection = engine.connect()
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 3165, in connect
return self._connection_cls(self, close_with_result=close_with_result)
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 96, in __init__
else engine.raw_connection()
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 3244, in raw_connection
return self._wrap_pool_connect(self.pool.connect, _connection)
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 3211, in _wrap_pool_connect
return fn()
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 307, in connect
return _ConnectionFairy._checkout(self)
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 767, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 425, in checkout
rec = pool._do_get()
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/pool/impl.py", line 146, in _do_get
self._dec_overflow()
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__
compat.raise_(
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/pool/impl.py", line 143, in _do_get
return self._create_connection()
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 253, in _create_connection
return _ConnectionRecord(self)
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 368, in __init__
self.__connect()
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 611, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/util/langhelpers.py", line 84, in __exit__
compat.raise_(value, with_traceback=traceback)
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 611, in __connect
pool.logger.debug("Error on connect(): %s", e)
TypeError: debug() takes 1 positional argument but 3 were given
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "sample_projects/env/lib/python3.8/site-packages/flask/app.py", line 2088, in __call__
return self.wsgi_app(environ, start_response)
File "sample_projects/env/lib/python3.8/site-packages/flask/app.py", line 2073, in wsgi_app
response = self.handle_exception(e)
File "sample_projects/env/lib/python3.8/site-packages/flask/app.py", line 2070, in wsgi_app
response = self.full_dispatch_request()
File "sample_projects/env/lib/python3.8/site-packages/flask/app.py", line 1515, in full_dispatch_request
rv = self.handle_user_exception(e)
File "sample_projects/env/lib/python3.8/site-packages/flask/app.py", line 1513, in full_dispatch_request
rv = self.dispatch_request()
File "sample_projects/env/lib/python3.8/site-packages/flask/app.py", line 1499, in dispatch_request
return self.ensure_sync(self.view_functions[rule.endpoint])(**req.view_args)
File "sample_projects/app.py", line 76, in home
engine.dispose()
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 2920, in dispose
self.pool.dispose()
File "sample_projects/env/lib/python3.8/site-packages/sqlalchemy/pool/impl.py", line 195, in dispose
self.logger.info("Pool disposed. %s", self.status())
TypeError: info() takes 1 positional argument but 3 were given
I would appreciate any insight that you might have. Ask me questions if I forgot to address something. Thank you if you read this far haha!
The following doesn't work because you're adding a handler on top of the existing handlers.
logger = logging.getLogger('sqlalchemy')
logger.addHandler(logging.NullHandler()) # creates an additional log handler
What you actually want to do is modify the existing handler.
logger = logging.getLogger('sqlalchemy')
logger.setLevel(logging.ERROR) # modifies the current log handler
Related
I have a webapp using Python, Flask and Flask-Security-Too. When running locally against a deployed database (also in Azure) It's working as intended. When the application is deployed to Azure it also works nicely. However, after a period of time it will start to throw errors like the one below:
2021-11-25T08:33:20.676633302Z [2021-11-25 08:33:20 +0000] [8] [ERROR] Error handling request /login
2021-11-25T08:33:20.676686905Z Traceback (most recent call last):
2021-11-25T08:33:20.676694906Z File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1202, in _execute_context
2021-11-25T08:33:20.676700406Z conn = self._revalidate_connection()
2021-11-25T08:33:20.676705406Z File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 469, in _revalidate_connection
2021-11-25T08:33:20.676710407Z raise exc.InvalidRequestError(
2021-11-25T08:33:20.676715107Z sqlalchemy.exc.InvalidRequestError: Can't reconnect until invalid transaction is rolled back
2021-11-25T08:33:20.676720007Z
2021-11-25T08:33:20.676724408Z The above exception was the direct cause of the following exception:
2021-11-25T08:33:20.676729108Z
2021-11-25T08:33:20.676733508Z Traceback (most recent call last):
2021-11-25T08:33:20.676738108Z File "/usr/local/lib/python3.8/site-packages/gunicorn/workers/sync.py", line 134, in handle
2021-11-25T08:33:20.676742809Z self.handle_request(listener, req, client, addr)
2021-11-25T08:33:20.676747309Z File "/usr/local/lib/python3.8/site-packages/gunicorn/workers/sync.py", line 175, in handle_request
2021-11-25T08:33:20.676752109Z respiter = self.wsgi(environ, resp.start_response)
2021-11-25T08:33:20.676756710Z File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 2464, in __call__
2021-11-25T08:33:20.676761610Z return self.wsgi_app(environ, start_response)
2021-11-25T08:33:20.676766110Z File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 2450, in wsgi_app
2021-11-25T08:33:20.676770711Z response = self.handle_exception(e)
2021-11-25T08:33:20.676775311Z File "/usr/local/lib/python3.8/site-packages/flask_cors/extension.py", line 165, in wrapped_function
2021-11-25T08:33:20.676780011Z return cors_after_request(app.make_response(f(*args, **kwargs)))
2021-11-25T08:33:20.676784712Z File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 1867, in handle_exception
2021-11-25T08:33:20.676789412Z reraise(exc_type, exc_value, tb)
2021-11-25T08:33:20.676794012Z File "/usr/local/lib/python3.8/site-packages/flask/_compat.py", line 39, in reraise
2021-11-25T08:33:20.676798612Z raise value
2021-11-25T08:33:20.676813213Z File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 2447, in wsgi_app
2021-11-25T08:33:20.676818114Z response = self.full_dispatch_request()
2021-11-25T08:33:20.676822214Z File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 1952, in full_dispatch_request
2021-11-25T08:33:20.676826514Z rv = self.handle_user_exception(e)
2021-11-25T08:33:20.676830615Z File "/usr/local/lib/python3.8/site-packages/flask_cors/extension.py", line 165, in wrapped_function
2021-11-25T08:33:20.676834915Z return cors_after_request(app.make_response(f(*args, **kwargs)))
2021-11-25T08:33:20.676840015Z File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 1821, in handle_user_exception
2021-11-25T08:33:20.676844515Z reraise(exc_type, exc_value, tb)
2021-11-25T08:33:20.676848616Z File "/usr/local/lib/python3.8/site-packages/flask/_compat.py", line 39, in reraise
2021-11-25T08:33:20.676852916Z raise value
2021-11-25T08:33:20.676874217Z File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 1950, in full_dispatch_request
2021-11-25T08:33:20.676878918Z rv = self.dispatch_request()
2021-11-25T08:33:20.676883218Z File "/usr/local/lib/python3.8/site-packages/flask/app.py", line 1936, in dispatch_request
2021-11-25T08:33:20.676888318Z return self.view_functions[rule.endpoint](**req.view_args)
2021-11-25T08:33:20.676892919Z File "/usr/local/lib/python3.8/site-packages/flask_security/decorators.py", line 436, in decorated
2021-11-25T08:33:20.676897419Z return fn(*args, **kwargs)
2021-11-25T08:33:20.676901819Z File "/usr/local/lib/python3.8/site-packages/flask_security/views.py", line 162, in login
2021-11-25T08:33:20.676906220Z if form.validate_on_submit():
2021-11-25T08:33:20.676910620Z File "/usr/local/lib/python3.8/site-packages/flask_wtf/form.py", line 100, in validate_on_submit
2021-11-25T08:33:20.676915220Z return self.is_submitted() and self.validate()
2021-11-25T08:33:20.676919720Z File "/usr/local/lib/python3.8/site-packages/flask_security/forms.py", line 450, in validate
2021-11-25T08:33:20.676924121Z self.user = find_user(self.email.data)
2021-11-25T08:33:20.676928321Z File "/usr/local/lib/python3.8/site-packages/flask_security/utils.py", line 829, in find_user
2021-11-25T08:33:20.676932921Z user = _datastore.find_user(
2021-11-25T08:33:20.676937122Z File "/usr/local/lib/python3.8/site-packages/flask_security/datastore.py", line 585, in find_user
2021-11-25T08:33:20.676941622Z return query.filter(subquery).first()
2021-11-25T08:33:20.676945822Z File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3429, in first
2021-11-25T08:33:20.676950422Z ret = list(self[0:1])
2021-11-25T08:33:20.676954623Z File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3203, in __getitem__
2021-11-25T08:33:20.676959023Z return list(res)
2021-11-25T08:33:20.676967724Z File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3535, in __iter__
2021-11-25T08:33:20.676972424Z return self._execute_and_instances(context)
2021-11-25T08:33:20.676976724Z File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3560, in _execute_and_instances
2021-11-25T08:33:20.676981224Z result = conn.execute(querycontext.statement, self._params)
2021-11-25T08:33:20.676985525Z File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1011, in execute
2021-11-25T08:33:20.676989925Z return meth(self, multiparams, params)
2021-11-25T08:33:20.676994525Z File "/usr/local/lib/python3.8/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
2021-11-25T08:33:20.676999126Z return connection._execute_clauseelement(self, multiparams, params)
2021-11-25T08:33:20.677003426Z File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1124, in _execute_clauseelement
2021-11-25T08:33:20.677007926Z ret = self._execute_context(
2021-11-25T08:33:20.677012127Z File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1206, in _execute_context
2021-11-25T08:33:20.677016627Z self._handle_dbapi_exception(
2021-11-25T08:33:20.677020827Z File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1510, in _handle_dbapi_exception
2021-11-25T08:33:20.677025327Z util.raise_(
2021-11-25T08:33:20.677029528Z File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 182, in raise_
2021-11-25T08:33:20.677034028Z raise exception
2021-11-25T08:33:20.677038128Z File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1202, in _execute_context
2021-11-25T08:33:20.677042729Z conn = self._revalidate_connection()
2021-11-25T08:33:20.677046929Z File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 469, in _revalidate_connection
2021-11-25T08:33:20.677051429Z raise exc.InvalidRequestError(
2021-11-25T08:33:20.677055629Z sqlalchemy.exc.StatementError: (sqlalchemy.exc.InvalidRequestError) Can't reconnect until invalid transaction is rolled back
2021-11-25T08:33:20.677061730Z [SQL: SELECT anon_1.user_id AS anon_1_user_id, anon_1.user_email AS anon_1_user_email, anon_1.user_username AS anon_1_user_username, anon_1.user_password AS anon_1_user_password, anon_1.user_active AS anon_1_user_active, anon_1.user_fs_uniquifier AS anon_1_user_fs_uniquifier, anon_1.user_confirmed_at AS anon_1_user_confirmed_at, anon_1.user_last_login_at AS anon_1_user_last_login_at, anon_1.user_current_login_at AS anon_1_user_current_login_at, anon_1.user_last_login_ip AS anon_1_user_last_login_ip, anon_1.user_current_login_ip AS anon_1_user_current_login_ip, anon_1.user_login_count AS anon_1_user_login_count, anon_1.user_tf_primary_method AS anon_1_user_tf_primary_method, anon_1.user_tf_totp_secret AS anon_1_user_tf_totp_secret, anon_1.user_tf_phone_number AS anon_1_user_tf_phone_number, anon_1.user_create_datetime AS anon_1_user_create_datetime, anon_1.user_update_datetime AS anon_1_user_update_datetime, anon_1.user_uuid AS anon_1_user_uuid, role_1.id AS role_1_id, role_1.name AS role_1_name, role_1.description AS role_1_description, role_1.permissions AS role_1_permissions, role_1.update_datetime AS role_1_update_datetime
2021-11-25T08:33:20.677072630Z FROM (SELECT TOP 1 [user].id AS user_id, [user].email AS user_email, [user].username AS user_username, [user].password AS user_password, [user].active AS user_active, [user].fs_uniquifier AS user_fs_uniquifier, [user].confirmed_at AS user_confirmed_at, [user].last_login_at AS user_last_login_at, [user].current_login_at AS user_current_login_at, [user].last_login_ip AS user_last_login_ip, [user].current_login_ip AS user_current_login_ip, [user].login_count AS user_login_count, [user].tf_primary_method AS user_tf_primary_method, [user].tf_totp_secret AS user_tf_totp_secret, [user].tf_phone_number AS user_tf_phone_number, [user].create_datetime AS user_create_datetime, [user].update_datetime AS user_update_datetime, [user].uuid AS user_uuid
2021-11-25T08:33:20.677078531Z FROM [user]
2021-11-25T08:33:20.677082731Z WHERE lower([user].email) = lower(?)) AS anon_1 LEFT OUTER JOIN (roles_users AS roles_users_1 JOIN role AS role_1 ON role_1.id = roles_users_1.role_id) ON anon_1.user_id = roles_users_1.user_id]
2021-11-25T08:33:20.677087431Z [parameters: [immutabledict({})]]
AFAIK, all the db.session's and user_datastore entities are either rolled back, commited and/or closed after they're used. Here's an example:
def create_user():
try:
current_app.logger.info('Creating admin user...')
setup_admin_user(current_app.user_datastore)
current_app.logger.info('Done!')
except IntegrityError as ie:
current_app.logger.warning('Failed to create admin user(s) as they already exists.')
db.session.rollback()
except Exception as e:
db.session.rollback()
raise e
finally:
db.session.close()
def setup_admin_user(usr_datastore):
if not usr_datastore.find_user(email="email#example.com"):
usr_datastore.create_user(email="email#example.com",
password=hash_password('password'),
roles=["admin"],
confirmed_at=datetime.datetime.utcnow())
usr_datastore.commit()
I also noticed that the parameters for the provided SQL are looking a bit suspicious. Sometimes there's a real parameter but the error message is still the same.
I'm confused as the query does work properly when executing it in the database and the Traceback does not mention any lines in the code I've written.
Any idea's on how I can fix this? I've tried bumping the versions for flask-sqlalchemy and flask-security-too to the latest stable versions.
This sometimes happens since often, cloud DB instances will close idle connections. Make sure you have the following in your config:
app.config["SQLALCHEMY_ENGINE_OPTIONS"] = {"pool_pre_ping": True}
and see if that helps.
Solved it by configuring a health check for the app service in Azure. The health check-endpoint will do a simple query to the database in order to keep it alive, which I believe was the problem.
def health_check():
role = current_app.user_datastore.find_role('dummy')
if role:
current_app.logger.info(f'[health_check] Found role {role.name}')
return jsonify(status='ok'), 200
return jsonify(status='failure'), 500
I would like to serialize many Gensim library Word2Vec models in MySQL database. For the purpose of my project I am utilizing: Python, Flask, SQLAlchemy, and MySQL+PyMySQL. Below you can see my Word2VecModel class which I am using in order to create word2vec table in MySQL DB:
from db.db import db
import db_models.influencer
class Word2VecModel(db.Model):
__tablename__ = "word2vec"
m_id = db.Column(db.Integer, primary_key=True)
m_username = db.Column(db.String(20), db.ForeignKey("influencer.m_username"))
m_binary = db.Column(db.LargeBinary)
influencer = db.relationship("InfluencerModel")
def __init__(self, m_binary, m_username):
self.m_binary = m_binary
self.m_username = m_username
def __repr__(self):
return '<Word2Vec {}>'.format(self.m_name)
#classmethod
def find_by_username(cls, m_username):
return cls.query.filter_by(name=m_username).first()
def save_to_db(self):
db.session.add(self)
db.session.commit()
def delete_from_db(self):
db.session.delete(self)
db.session.commit()
Whenever I try to execute:
word2vec_model = Word2VecModel(pickle.dumps(word2vec), username)
word2vec_model .save_to_db()
I get the following error message:
Traceback (most recent call last):
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 705, in _write_bytes
self._sock.sendall(data)
BrokenPipeError: [Errno 32] Broken pipe
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1278, in _execute_context
cursor, statement, parameters, context
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
cursor.execute(statement, parameters)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/cursors.py", line 163, in execute
result = self._query(query)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/cursors.py", line 321, in _query
conn.query(q)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 504, in query
self._execute_command(COMMAND.COM_QUERY, sql)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 773, in _execute_command
self.write_packet(sql[:packet_size])
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 634, in write_packet
self._write_bytes(data)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 710, in _write_bytes
"MySQL server has gone away (%r)" % (e,))
pymysql.err.OperationalError: (2006, "MySQL server has gone away (BrokenPipeError(32, 'Broken pipe'))")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "app.py", line 117, in <module>
word2vec.save_to_db()
File "/home/stefan/PycharmProjects/NLP - Influencer Text Analysis/src/flask_api/db_models/word2vec.py", line 25, in save_to_db
db.session.commit()
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/scoping.py", line 163, in do
return getattr(self.registry(), name)(*args, **kwargs)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 1042, in commit
self.transaction.commit()
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 504, in commit
self._prepare_impl()
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 483, in _prepare_impl
self.session.flush()
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2523, in flush
self._flush(objects)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2664, in _flush
transaction.rollback(_capture_exception=True)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 69, in __exit__
exc_value, with_traceback=exc_tb,
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 178, in raise_
raise exception
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2624, in _flush
flush_context.execute()
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/unitofwork.py", line 422, in execute
rec.execute(self)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/unitofwork.py", line 589, in execute
uow,
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/persistence.py", line 245, in save_obj
insert,
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/persistence.py", line 1136, in _emit_insert_statements
statement, params
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1014, in execute
return meth(self, multiparams, params)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1133, in _execute_clauseelement
distilled_params,
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1318, in _execute_context
e, statement, parameters, cursor, context
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1512, in _handle_dbapi_exception
sqlalchemy_exception, with_traceback=exc_info[2], from_=e
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 178, in raise_
raise exception
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1278, in _execute_context
cursor, statement, parameters, context
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
cursor.execute(statement, parameters)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/cursors.py", line 163, in execute
result = self._query(query)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/cursors.py", line 321, in _query
conn.query(q)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 504, in query
self._execute_command(COMMAND.COM_QUERY, sql)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 773, in _execute_command
self.write_packet(sql[:packet_size])
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 634, in write_packet
self._write_bytes(data)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 710, in _write_bytes
"MySQL server has gone away (%r)" % (e,))
Killed
I've read somewhere that increasing MySQL connection timeout might help, but not in my case. Here is my DB engine configuration:
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql+pymysql://root:''#localhost/********'
app.config['PROPAGATE_EXCEPTIONS'] = True
app.config["SQLALCHEMY_ENGINE_OPTIONS"] = {"pool_recycle": 120}
I am new to python and trying to use sqlalchemy statements for e.g read_sql_table.
I am using pycharm to execute below code.
When I am executing below code its not giving me any error :
import pandas as pd
import sqlalchemy
engine = sqlalchemy.create_engine('mysql+pymysql://root:*****#localhost:3306/STAGING')
print("BLAH")
OUTPUT:
BLAH
But when I am trying to execute below statement its giving me error
import pandas as pd
import sqlalchemy
engine = sqlalchemy.create_engine('mysql+pymysql://root:*****#localhost:3306/STAGING')
print("BLAH")
df = pd.read_sql_table("customers",engine)
ERROR:
BLAH Traceback (most recent call last): File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\sqlalchemy\pool.py", line 950, in _do_get
return self._pool.get(wait, self._timeout) File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\sqlalchemy\util\queue.py", line 145, in get
raise Empty sqlalchemy.util.queue.Empty
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File "C:/Users/suksengupta/PycharmProjects/Project_July2019/DB.py", line 6, in <module>
df = pd.read_sql_table("customers",engine) File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\pandas\io\sql.py", line 243, in read_sql_table
meta.reflect(only=[table_name], views=True) File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\sqlalchemy\sql\schema.py", line 3311, in reflect
with bind.connect() as conn: File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1778, in connect
return self._connection_cls(self, **kwargs) File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\sqlalchemy\engine\base.py", line 60, in __init__
self.__connection = connection or engine.raw_connection() File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1847, in raw_connection
return self.pool.unique_connection() File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\sqlalchemy\pool.py", line 280, in unique_connection
return _ConnectionFairy._checkout(self) File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\sqlalchemy\pool.py", line 644, in _checkout
fairy = _ConnectionRecord.checkout(pool) File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\sqlalchemy\pool.py", line 440, in checkout
rec = pool._do_get() File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\sqlalchemy\pool.py", line 963, in _do_get
return self._create_connection() File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\sqlalchemy\pool.py", line 285, in _create_connection
return _ConnectionRecord(self) File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\sqlalchemy\pool.py", line 411, in __init__
self.connection = self.__connect() File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\sqlalchemy\pool.py", line 538, in __connect
connection = self.__pool._creator() File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\sqlalchemy\engine\strategies.py", line 90, in connect
return dialect.connect(*cargs, **cparams) File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\sqlalchemy\engine\default.py", line 377, in connect
return self.dbapi.connect(*cargs, **cparams) File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\pymysql\__init__.py", line 88, in Connect
return Connection(*args, **kwargs) File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\pymysql\connections.py", line 644, in __init__
self._connect() File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\pymysql\connections.py", line 837, in _connect
self._get_server_information() File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\pymysql\connections.py", line 1072, in _get_server_information
self.server_charset = charset_by_id(lang).name File "C:\Users\suksengupta\PycharmProjects\Project_July2019\venv\lib\site-packages\pymysql\charset.py", line 34, in by_id
return self._by_id[id] KeyError: 255
In output I am expecting all the data present under Customers table
Trying to connect to SQL server using SQLAlchemy with pyodbc(freeTDS) as the driver; The connections succeeds if I use pyodbc directly:
>>> import pyodbc
>>> conn = pyodbc.connect('DSN=serverdsn;UID=user;PWD=password')
>>> crsr = conn.cursor()
>>> rows = crsr.execute("select ##VERSION").fetchall()
>>> print(rows)
[('Microsoft Azure SQL Data Warehouse - 10.0.9248.28 Sep 12 2017 01:08:55 Copyright (c) Microsoft Corporation', )]
>>> crsr.close()
>>> conn.close()
But when I use SQLAlchemy, it fails with a mysterious error:
>>> from sqlalchemy import create_engine
>>> e = create_engine("mssql+pyodbc://user:password#serverdsn")
>>> with e.connect() as con:
... rs = con.execute('select * from users')
... for row in rs:
... print(row)
...
Here is the full stack trace:
Traceback (most recent call last): File
"/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/pool.py",
line 1122, in _do_get
return self._pool.get(wait, self._timeout) File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/util/queue.py",
line 145, in get
raise Empty sqlalchemy.util.queue.Empty
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File "", line 1, in
File
"/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/engine/base.py",
line 2091, in connect
return self._connection_cls(self, **kwargs) File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/engine/base.py",
line 90, in init
if connection is not None else engine.raw_connection() File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/engine/base.py",
line 2177, in raw_connection
self.pool.unique_connection, _connection) File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/engine/base.py",
line 2147, in _wrap_pool_connect
return fn() File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/pool.py",
line 328, in unique_connection
return _ConnectionFairy._checkout(self) File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/pool.py",
line 766, in _checkout
fairy = _ConnectionRecord.checkout(pool) File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/pool.py",
line 516, in checkout
rec = pool._do_get() File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/pool.py",
line 1138, in _do_get
self._dec_overflow() File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/util/langhelpers.py",
line 66, in exit
compat.reraise(exc_type, exc_value, exc_tb) File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/util/compat.py",
line 187, in reraise
raise value File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/pool.py",
line 1135, in _do_get
return self._create_connection() File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/pool.py",
line 333, in _create_connection
return _ConnectionRecord(self) File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/pool.py",
line 461, in init
self.connect(first_connect_check=True) File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/pool.py",
line 661, in __connect
exec_once(self.connection, self) File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/event/attr.py",
line 246, in exec_once
self(*args, **kw) File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/event/attr.py",
line 256, in __call
fn(*args, **kw) File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/util/langhelpers.py",
line 1331, in go
return once_fn(*arg, **kw) File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/engine/strategies.py",
line 181, in first_connect
dialect.initialize(c) File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/connectors/pyodbc.py",
line 165, in initialize
super(PyODBCConnector, self).initialize(connection) File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/dialects/mssql/base.py",
line 1742, in initialize
super(MSDialect, self).initialize(connection) File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/engine/default.py",
line 256, in initialize
self.get_isolation_level(connection.connection) File "/Users/purecarscomputer/anaconda/envs/tensorflow/lib/python3.5/site-packages/sqlalchemy/dialects/mssql/base.py",
line 1735, in get_isolation_level
"tried views: %s; final error was: %s" % (views, err)) UnboundLocalError: local variable 'err' referenced before assignment
I've tried install and uninstall sqlalchemy and searched around on google, but not find a solution. Does anyone have similar problems and have a clue about what is happening?
OS information:
ProductName: Mac OS X
ProductVersion: 10.12.6
BuildVersion: 16G29
Here's the connection string I used to solve a connection problem with similar symptoms:
import urllib
from sqlalchemy import create_engine
# utilize existing odbc connection to create engine
params = urllib.quote_plus("DRIVER={}; SERVER=server; Database=database; UID=user; PWD=pw")
engine = create_engine("mssql+pyodbc:///?odbc_connect=%s" % params)
ref http://docs.sqlalchemy.org/en/latest/dialects/mssql.html#pass-through-exact-pyodbc-string
I am using Flask-KVSession (version 0.6.2). It does what it has to do. But when there are consecutive requests to webserver, following exception is repeated like a flood.
2016-06-10_05:41:58.97125 2016-06-10 05:41:58 [29567] [ERROR] Error handling request
2016-06-10_05:41:58.97127 Traceback (most recent call last):
2016-06-10_05:41:58.97128 File "/usr/lib/pymodules/python2.7/gunicorn/workers/async.py", line 45, in handle
2016-06-10_05:41:58.97128 self.handle_request(listener, req, client, addr)
2016-06-10_05:41:58.97129 File "/usr/lib/pymodules/python2.7/gunicorn/workers/ggevent.py", line 151, in handle_request
2016-06-10_05:41:58.97129 super(GeventWorker, self).handle_request(*args)
2016-06-10_05:41:58.97129 File "/usr/lib/pymodules/python2.7/gunicorn/workers/async.py", line 93, in handle_request
2016-06-10_05:41:58.97130 respiter = self.wsgi(environ, resp.start_response)
2016-06-10_05:41:58.97130 File "/usr/lib/python2.7/dist-packages/flask/app.py", line 1836, in __call__
2016-06-10_05:41:58.97140 return self.wsgi_app(environ, start_response)
2016-06-10_05:41:58.97141 File "<flask_app_deploy_file.py>", line 18, in __call__
2016-06-10_05:41:58.97142 File "/usr/lib/python2.7/dist-packages/werkzeug/contrib/fixers.py", line 144, in __call__
2016-06-10_05:41:58.97142 return self.app(environ, start_response)
2016-06-10_05:41:58.97142 File "/usr/lib/python2.7/dist-packages/flask/app.py", line 1813, in wsgi_app
2016-06-10_05:41:58.97143 ctx.push()
2016-06-10_05:41:58.97143 File "/usr/lib/python2.7/dist-packages/flask/ctx.py", line 321, in push
2016-06-10_05:41:58.97152 self.session = self.app.open_session(self.request)
2016-06-10_05:41:58.97161 File "/usr/lib/python2.7/dist-packages/flask/app.py", line 825, in open_session
2016-06-10_05:41:58.97162 return self.session_interface.open_session(self, request)
2016-06-10_05:41:58.97162 File "/usr/lib/pymodules/python2.7/flask_kvsession/__init__.py", line 159, in open_session
2016-06-10_05:41:58.97163 current_app.kvsession_store.get(sid_s)
2016-06-10_05:41:58.97163 EOFError
2016-06-10_05:41:58.97187 ERROR: Error handling request
What can be the cause for the EOFError?
I tried looking into the source code of flask_kvsession, couldn't figure out the exact problem. How that EOFError can be mitigated? Please help me out.