I want to add columns to a table in my database dynamically since I don't want to have to specify all columns when I set it up in the class
In order to solve this I am instead using alembic to add columns to a table but I am having problems.
In test.py script I have defined a class as below.
DATABASE_URL_dev = "postgresql+psycopg2://user:password#localhost:5432/testdb"
engine = create_engine(DATABASE_URL_dev)
Base = declarative_base(engine)
class my_class1(Base):
__tablename__ = "test1"
id = Column(Integer, primary_key=True)
list = Column(String)
def loadsession():
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
session = Session()
return session
session = loadsession()
Now I want to add a column to table test1 and I am thereby trying to use Alembic.
I followed the instructions at https://alembic.sqlalchemy.org/en/latest/tutorial.html and did the following:
I installed alembic by running pip install alembic
I created an environment using alembic init alembic
In the .ini file I set sqlalchemy.url = "postgresql+psycopg2://user:password#localhost:5432/testdb"
I created a migration script by running alembic revision -m "add a column "
I then edited the script by adding the following "op.add_column('test1', sa.Column('new_column', sa.String))" in upgrade()
I ran the first migration with "alembic upgrade head" This gave me the error: "sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) (Background on this error at: http://sqlalche.me/e/e3q8)"
Question
Have I set sqlalchemy.url correctly or what is else setting the OperationalError?
Entire traceback
Traceback (most recent call last):
File "c:\users\user\appdata\local\programs\python\python38-32\lib\runpy.py", line 193, in _run_module_as_main
return _run_code(code, main_globals, None,
File "c:\users\user\appdata\local\programs\python\python38-32\lib\runpy.py", line 86, in _run_code
exec(code, run_globals)
File "C:\Users\user\AppData\Local\Programs\Python\Python38-32\Scripts\alembic.exe\__main__.py", line 9, in <module>
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\alembic\config.py", line 577, in main
CommandLine(prog=prog).main(argv=argv)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\alembic\config.py", line 571, in main
self.run_cmd(cfg, options)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\alembic\config.py", line 548, in run_cmd
fn(
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\alembic\command.py", line 298, in upgrade
script.run_env()
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\alembic\script\base.py", line 489, in run_env
util.load_python_file(self.dir, "env.py")
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\alembic\util\pyfiles.py", line 98, in load_python_file
module = load_module_py(module_id, path)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\alembic\util\compat.py", line 184, in load_module_py
spec.loader.exec_module(module)
File "<frozen importlib._bootstrap_external>", line 783, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "y\env.py", line 77, in <module>
run_migrations_online()
File "y\env.py", line 65, in run_migrations_online
with connectable.connect() as connection:
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 2218, in connect
return self._connection_cls(self, **kwargs)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 103, in __init__
else engine.raw_connection()
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 2317, in raw_connection
return self._wrap_pool_connect(
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 2288, in _wrap_pool_connect
Connection._handle_dbapi_exception_noconnection(
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 1554, in _handle_dbapi_exception_noconnection
util.raise_(
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\util\compat.py", line 178, in raise_
raise exception
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 2285, in _wrap_pool_connect
return fn()
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\pool\base.py", line 303, in unique_connection
return _ConnectionFairy._checkout(self)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\pool\base.py", line 773, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\pool\impl.py", line 238, in _do_get
return self._create_connection()
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\pool\base.py", line 657, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.raise_(
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\util\compat.py", line 178, in raise_
raise exception
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\pool\base.py", line 652, in __connect
connection = pool._invoke_creator(self)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\engine\default.py", line 488, in connect
return self.dbapi.connect(*cargs, **cparams)
File "c:\users\c\appdata\local\programs\python\python38-32\lib\site-packages\psycopg2\__init__.py", line 126, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError)
(Background on this error at: http://sqlalche.me/e/e3q8)
Related
I have declared my connection function as below..
def create_psql_engine(db):
#conn_string = "postgresql+psycopg2://myuser:******#someserver:port/somedb"
conn_string = "postgresql+psycopg2://%s:%s#%s:%s/%s" % (
db.get('user'),
db.get('password'),
db.get('host'),
db.get('port'),
db.get('dbname')
)
conn_args = {
"sslmode": db.get('sslmode'),
"sslcompression": db.get('sslcompression'),
"sslrootcert": db.get('sslrootcert'),
"sslcert": db.get('sslcert'),
"sslkey": db.get('sslkey'),
"pool_pre_ping": True,
"pool_recycle": 300
}
try:
engine = create_engine(conn_string, connect_args=conn_args)
logging.info("sqlalchemy Engine to database created using psycopg2... ")
return engine
except psycopg2.Error as error:
logging.error(error)
And in my program I am connecting as below.
# Create PostgreSQL Engine
self.engine = abc.create_psql_engine(dbParam)
print('PSQL Engine Created to Table : {}'.format(self.tableName))
Then I use a pandas to_sql for inserting into DB
df.to_sql(self.table,self.engine,schema='abc',index=False,if_exists='append',chunksize=100)
It was working all this while with out the below 2 options.
"pool_pre_ping": True,
"pool_recycle": 300
But I was gettin connection interruption when handling large volume. And which is the reason I am trying out a solution with these 2 parameters.
However, I get this error , when I tried. Any clue what I am doing wrong.
sqlalchemy.exc.ProgrammingError: (psycopg2.ProgrammingError) invalid dsn: invalid connection option "pool_pre_ping"
Complete error log as below..
Traceback (most recent call last):
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3212, in _wrap_pool_connect
return fn()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 307, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 767, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 425, in checkout
rec = pool._do_get()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 146, in _do_get
self._dec_overflow()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 143, in _do_get
return self._create_connection()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 253, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 368, in __init__
self.__connect()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 611, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 605, in __connect
connection = pool._invoke_creator(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/create.py", line 578, in connect
return dialect.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 584, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/__init__.py", line 126, in connect
dsn = _ext.make_dsn(dsn, **kwargs)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/extensions.py", line 175, in make_dsn
parse_dsn(dsn)
psycopg2.ProgrammingError: invalid dsn: invalid connection option "pool_pre_ping"
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/opt/apps/python/loadtoPostgreSQL.py", line 275, in <module>
MX123D.processFiles(jobIdentifier)
File "/opt/apps/python/loadtoPostgreSQL.py", line 246, in processFiles
df.to_sql(self.table,self.engine,schema='abc',index=False,if_exists='append',chunksize=100)
File "/usr/local/lib64/python3.6/site-packages/pandas/core/generic.py", line 2615, in to_sql
method=method,
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 598, in to_sql
method=method,
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 1393, in to_sql
table.create()
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 721, in create
if self.exists():
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 708, in exists
return self.pd_sql.has_table(self.name, self.schema)
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 1431, in has_table
self.connectable.dialect.has_table, name, schema or self.meta.schema
File "<string>", line 2, in run_callable
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/deprecations.py", line 390, in warned
return fn(*args, **kwargs)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3074, in run_callable
with self.connect() as conn:
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3166, in connect
return self._connection_cls(self, close_with_result=close_with_result)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 96, in __init__
else engine.raw_connection()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3245, in raw_connection
return self._wrap_pool_connect(self.pool.connect, _connection)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3216, in _wrap_pool_connect
e, dialect, self
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 2070, in _handle_dbapi_exception_noconnection
sqlalchemy_exception, with_traceback=exc_infoÝ2¨, from_=e
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3212, in _wrap_pool_connect
return fn()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 307, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 767, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 425, in checkout
rec = pool._do_get()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 146, in _do_get
self._dec_overflow()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 143, in _do_get
return self._create_connection()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 253, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 368, in __init__
self.__connect()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 611, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 605, in __connect
connection = pool._invoke_creator(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/create.py", line 578, in connect
return dialect.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 584, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/__init__.py", line 126, in connect
dsn = _ext.make_dsn(dsn, **kwargs)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/extensions.py", line 175, in make_dsn
parse_dsn(dsn)
sqlalchemy.exc.ProgrammingError: (psycopg2.ProgrammingError) invalid dsn: invalid connection option "pool_pre_ping"
(Background on this error at: https://sqlalche.me/e/14/f405)
I myself found the issue.Sharing here so that it can help some one.
Issue was that the parameters were given in wrong place :-)
It should be given as below.
engine = create_engine(conn_string, connect_args=conn_args,pool_pre_ping=True,pool_recycle=300)
I would like to serialize many Gensim library Word2Vec models in MySQL database. For the purpose of my project I am utilizing: Python, Flask, SQLAlchemy, and MySQL+PyMySQL. Below you can see my Word2VecModel class which I am using in order to create word2vec table in MySQL DB:
from db.db import db
import db_models.influencer
class Word2VecModel(db.Model):
__tablename__ = "word2vec"
m_id = db.Column(db.Integer, primary_key=True)
m_username = db.Column(db.String(20), db.ForeignKey("influencer.m_username"))
m_binary = db.Column(db.LargeBinary)
influencer = db.relationship("InfluencerModel")
def __init__(self, m_binary, m_username):
self.m_binary = m_binary
self.m_username = m_username
def __repr__(self):
return '<Word2Vec {}>'.format(self.m_name)
#classmethod
def find_by_username(cls, m_username):
return cls.query.filter_by(name=m_username).first()
def save_to_db(self):
db.session.add(self)
db.session.commit()
def delete_from_db(self):
db.session.delete(self)
db.session.commit()
Whenever I try to execute:
word2vec_model = Word2VecModel(pickle.dumps(word2vec), username)
word2vec_model .save_to_db()
I get the following error message:
Traceback (most recent call last):
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 705, in _write_bytes
self._sock.sendall(data)
BrokenPipeError: [Errno 32] Broken pipe
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1278, in _execute_context
cursor, statement, parameters, context
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
cursor.execute(statement, parameters)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/cursors.py", line 163, in execute
result = self._query(query)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/cursors.py", line 321, in _query
conn.query(q)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 504, in query
self._execute_command(COMMAND.COM_QUERY, sql)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 773, in _execute_command
self.write_packet(sql[:packet_size])
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 634, in write_packet
self._write_bytes(data)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 710, in _write_bytes
"MySQL server has gone away (%r)" % (e,))
pymysql.err.OperationalError: (2006, "MySQL server has gone away (BrokenPipeError(32, 'Broken pipe'))")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "app.py", line 117, in <module>
word2vec.save_to_db()
File "/home/stefan/PycharmProjects/NLP - Influencer Text Analysis/src/flask_api/db_models/word2vec.py", line 25, in save_to_db
db.session.commit()
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/scoping.py", line 163, in do
return getattr(self.registry(), name)(*args, **kwargs)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 1042, in commit
self.transaction.commit()
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 504, in commit
self._prepare_impl()
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 483, in _prepare_impl
self.session.flush()
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2523, in flush
self._flush(objects)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2664, in _flush
transaction.rollback(_capture_exception=True)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 69, in __exit__
exc_value, with_traceback=exc_tb,
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 178, in raise_
raise exception
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2624, in _flush
flush_context.execute()
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/unitofwork.py", line 422, in execute
rec.execute(self)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/unitofwork.py", line 589, in execute
uow,
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/persistence.py", line 245, in save_obj
insert,
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/persistence.py", line 1136, in _emit_insert_statements
statement, params
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1014, in execute
return meth(self, multiparams, params)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1133, in _execute_clauseelement
distilled_params,
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1318, in _execute_context
e, statement, parameters, cursor, context
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1512, in _handle_dbapi_exception
sqlalchemy_exception, with_traceback=exc_info[2], from_=e
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 178, in raise_
raise exception
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1278, in _execute_context
cursor, statement, parameters, context
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
cursor.execute(statement, parameters)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/cursors.py", line 163, in execute
result = self._query(query)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/cursors.py", line 321, in _query
conn.query(q)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 504, in query
self._execute_command(COMMAND.COM_QUERY, sql)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 773, in _execute_command
self.write_packet(sql[:packet_size])
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 634, in write_packet
self._write_bytes(data)
File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 710, in _write_bytes
"MySQL server has gone away (%r)" % (e,))
Killed
I've read somewhere that increasing MySQL connection timeout might help, but not in my case. Here is my DB engine configuration:
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql+pymysql://root:''#localhost/********'
app.config['PROPAGATE_EXCEPTIONS'] = True
app.config["SQLALCHEMY_ENGINE_OPTIONS"] = {"pool_recycle": 120}
I am currently trying to create an SQLite database using python.
here is my code
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////c/PStuff/friend.db'
db = SQLAlchemy(app)
class ExampleTable(db.Model):
id = db.Column(db.Integer,primary_key=True)
I have already created a database called 'friend.db' in the PStuff folder. It is currently empty.
In cmd i used this to create the database:sqlite3 friend.db
I then used a python shell:from test3 import db
Lastly when i use:
db.create_all()
i get this error:
Traceback (most recent call last):
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2285, in _wrap_pool_connect
return fn()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 363, in connect
return _ConnectionFairy._checkout(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 773, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\impl.py", line 238, in _do_get
return self._create_connection()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 657, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "c:\PStuff\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.raise_(
File "c:\PStuff\lib\site-packages\sqlalchemy\util\compat.py", line 178, in raise_
raise exception
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 652, in __connect
connection = pool._invoke_creator(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\default.py", line 490, in connect
return self.dbapi.connect(*cargs, **cparams)
sqlite3.OperationalError: unable to open database file
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "c:\PStuff\lib\site-packages\flask_sqlalchemy\__init__.py", line 1033, in create_all
self._execute_for_all_tables(app, bind, 'create_all')
File "c:\PStuff\lib\site-packages\flask_sqlalchemy\__init__.py", line 1025, in _execute_for_all_tables
op(bind=self.get_engine(app, bind), **extra)
File "c:\PStuff\lib\site-packages\sqlalchemy\sql\schema.py", line 4320, in create_all
bind._run_visitor(
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2057, in _run_visitor
with self._optional_conn_ctx_manager(connection) as conn:
File "c:\PStuff\lib\contextlib.py", line 113, in __enter__
return next(self.gen)
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2049, in _optional_conn_ctx_manager
with self._contextual_connect() as conn:
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2251, in _contextual_connect
self._wrap_pool_connect(self.pool.connect, None),
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2288, in _wrap_pool_connect
Connection._handle_dbapi_exception_noconnection(
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 1554, in _handle_dbapi_exception_noconnection
util.raise_(
File "c:\PStuff\lib\site-packages\sqlalchemy\util\compat.py", line 178, in raise_
raise exception
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2285, in _wrap_pool_connect
return fn()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 363, in connect
return _ConnectionFairy._checkout(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 773, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\impl.py", line 238, in _do_get
return self._create_connection()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 657, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "c:\PStuff\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.raise_(
File "c:\PStuff\lib\site-packages\sqlalchemy\util\compat.py", line 178, in raise_
raise exception
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 652, in __connect
connection = pool._invoke_creator(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\default.py", line 490, in connect
return self.dbapi.connect(*cargs, **cparams)
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) unable to open database file
(Background on this error at: http://sqlalche.me/e/e3q8)
I am very new to SQL, any help would be greatly appreciated.
Try with Replace
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////c/PStuff/friend.db'
with
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///c/PStuff/friend.db'
based on SQLALCHEMY Documentation
sqlite:////db_absolute_path
sqlite:///db_relative_path
for windows try this thing (if you are putting database in same dir)
file_path = os.path.abspath(os.getcwd())+"\\friend.db"
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///'+file_path
I currently try to map an existing view within my postgreSQL DB to an SqlAlchemy ORM object in python. My view has the following name path_number_view and joins some tables together. It is wokring fine. But I am not able to map the view to an ORM object, when starting the application I get an error (belowe the code).
I followed these instructions,
Blog post
Docs: Reflecting Database Objects
Docs. Classical Mappings
but I not able to get that working.
Here is my code:
engine = create_engine("postgresql://****:*****#localhost/pi", pool_size=15, max_overflow=5, pool_recycle=1200)
Base = declarative_base(engine)
metadata = MetaData(engine)
path_number_view = Table('path_number_view', metadata,
Column('id', Integer, primary_key=True),
Column('path', String),
Column('threeDaysL', Integer),
Column('threeDaysC', Integer),
Column('tenDaysL', Integer),
Column('tenDaysC', Integer),
Column('number', Integer),
autoload=True
)
class PathNumberView(object):
__tablename__ = 'path_number_view'
def __init__(self, id, path, threeDaysL, threeDaysC, tenDaysL, tenDaysC, number):
self.id = id
self.path = path
self.threeDaysL = threeDaysL
self.threeDaysC = threeDaysC
self.tenDaysL = tenDaysL
self.tenDaysC = tenDaysC
self.number = number
mapper(PathNumberView, path_number_view )
Base.metadata.create_all(engine)
The error I get is not helpfull:
Traceback (most recent call last):
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2265, in _wrap_pool_connect
return fn()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 363, in connect
return _ConnectionFairy._checkout(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 760, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\impl.py", line 139, in _do_get
self._dec_overflow()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\compat.py", line 153, in reraise
raise value
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\impl.py", line 136, in _do_get
return self._create_connection()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 639, in __connect
connection = pool._invoke_creator(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\default.py", line 453, in connect
return self.dbapi.connect(*cargs, **cparams)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\psycopg2\__init__.py", line 126, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Program Files\JetBrains\PyCharm 2018.3.5\helpers\pycharm\_jb_unittest_runner.py", line 35, in <module>
main(argv=args, module=None, testRunner=unittestpy.TeamcityTestRunner, buffer=not JB_DISABLE_BUFFERING)
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\main.py", line 94, in __init__
self.parseArgs(argv)
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\main.py", line 141, in parseArgs
self.createTests()
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\main.py", line 148, in createTests
self.module)
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\loader.py", line 219, in loadTestsFromNames
suites = [self.loadTestsFromName(name, module) for name in names]
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\loader.py", line 219, in <listcomp>
suites = [self.loadTestsFromName(name, module) for name in names]
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\loader.py", line 153, in loadTestsFromName
module = __import__(module_name)
File "E:\Projekte\project\preprocessing\tests.py", line 6, in <module>
from preprocessing.data import TrainingDataProvider
File "E:\Projekte\project\preprocessing\data\__init__.py", line 7, in <module>
from entity import PathNumberView
File "E:\Projekte\project\entity\__init__.py", line 105, in <module>
Base.metadata.create_all(engine)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\sql\schema.py", line 4294, in create_all
ddl.SchemaGenerator, self, checkfirst=checkfirst, tables=tables
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2035, in _run_visitor
with self._optional_conn_ctx_manager(connection) as conn:
File "C:\Users\username\.conda\envs\tfGPU2\lib\contextlib.py", line 81, in __enter__
return next(self.gen)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2027, in _optional_conn_ctx_manager
with self._contextual_connect() as conn:
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2229, in _contextual_connect
self._wrap_pool_connect(self.pool.connect, None),
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2269, in _wrap_pool_connect
e, dialect, self
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 1536, in _handle_dbapi_exception_noconnection
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\compat.py", line 152, in reraise
raise value.with_traceback(tb)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2265, in _wrap_pool_connect
return fn()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 363, in connect
return _ConnectionFairy._checkout(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 760, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\impl.py", line 139, in _do_get
self._dec_overflow()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\compat.py", line 153, in reraise
raise value
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\impl.py", line 136, in _do_get
return self._create_connection()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 639, in __connect
connection = pool._invoke_creator(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\default.py", line 453, in connect
return self.dbapi.connect(*cargs, **cparams)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\psycopg2\__init__.py", line 126, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError)
(Background on this error at: http://sqlalche.me/e/e3q8)
The connection it self is working, I have 3 other tables (created with SQLAclhemy) which are working fine. I can query, update, delete no problems.
I am executing the following project in linux mint 17.2.
I installed the necessary modules and created the database server and created an additional file called 'dbconstants.py' with the database details.
username = 'postgres'
password = 'postgres'
dbname = 'mydb'
I am trying to execute the project partially as follows:
python -m plagcomps.extrinsic.extrinsic_testing
However, I am receving the following error
Traceback (most recent call last):
File "/usr/lib/python2.7/runpy.py", line 151, in _run_module_as_main
mod_name, loader, code, fname = _get_module_details(mod_name)
File "/usr/lib/python2.7/runpy.py", line 101, in _get_module_details
loader = get_loader(mod_name)
File "/usr/lib/python2.7/pkgutil.py", line 464, in get_loader
return find_loader(fullname)
File "/usr/lib/python2.7/pkgutil.py", line 474, in find_loader
for importer in iter_importers(fullname):
File "/usr/lib/python2.7/pkgutil.py", line 430, in iter_importers
__import__(pkg)
File "plagcomps/__init__.py", line 8, in <module>
import intrinsic
File "plagcomps/intrinsic/__init__.py", line 2, in <module>
from cluster import cluster
File "plagcomps/intrinsic/cluster.py", line 4, in <module>
import classify
File "plagcomps/intrinsic/classify.py", line 173, in <module>
Base.metadata.create_all(engine)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/sql/schema.py", line 3687, in create_all
tables=tables)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1855, in _run_visitor
with self._optional_conn_ctx_manager(connection) as conn:
File "/usr/lib/python2.7/contextlib.py", line 17, in __enter__
return self.gen.next()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1848, in _optional_conn_ctx_manager
with self.contextual_connect() as conn:
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 2039, in contextual_connect
self._wrap_pool_connect(self.pool.connect, None),
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 2078, in _wrap_pool_connect
e, dialect, self)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1405, in _handle_dbapi_exception_noconnection
exc_info
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 2074, in _wrap_pool_connect
return fn()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 376, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 713, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 480, in checkout
rec = pool._do_get()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 1060, in _do_get
self._dec_overflow()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 1057, in _do_get
return self._create_connection()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 323, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 449, in __init__
self.connection = self.__connect()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 607, in __connect
connection = self.__pool._invoke_creator(self)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/strategies.py", line 97, in connect
return dialect.connect(*cargs, **cparams)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 385, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/lib/python2.7/dist-packages/psycopg2/__init__.py", line 179, in connect
connection_factory=connection_factory, async=async)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) could not translate host name "mydb" to address: Name or service not known
UPDATE: I modified the dbconstants.py as follows:
username = 'postgres'
password = 'postgres'
dbname = 'localhost'
And then I receive the following error:
/usr/lib/pymodules/python2.7/matplotlib/__init__.py:1173: UserWarning: This call to matplotlib.use() has no effect
because the backend has already been chosen;
matplotlib.use() must be called *before* pylab, matplotlib.pyplot,
or matplotlib.backends is imported for the first time.
warnings.warn(_use_error_msg)
Traceback (most recent call last):
File "/usr/lib/python2.7/runpy.py", line 162, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/usr/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/eadaradhiraj/Downloads/plagcomps/extrinsic/extrinsic_testing.py", line 615, in <module>
save_to_db=False, ignore_high_obfuscation=False, show_false_negpos_info=False, get_best_of=False)
File "/home/eadaradhiraj/Downloads/plagcomps/extrinsic/extrinsic_testing.py", line 553, in test
mid = fingerprintstorage.get_mid(method, n, k, atom_type, hash_size)
File "plagcomps/extrinsic/fingerprintstorage.py", line 313, in get_mid
with _get_connection(autocommit=True) as conn:
File "plagcomps/extrinsic/fingerprintstorage.py", line 92, in _get_connection
conn = psycopg2.connect(user = username, password = password, database = dbname.split("/")[1], host="localhost", port = 5432)
IndexError: list index out of range
Just took a look at the project. In your dbconstants.py, set
dbname='localhost:5432/mydb'