Related
I have declared my connection function as below..
def create_psql_engine(db):
#conn_string = "postgresql+psycopg2://myuser:******#someserver:port/somedb"
conn_string = "postgresql+psycopg2://%s:%s#%s:%s/%s" % (
db.get('user'),
db.get('password'),
db.get('host'),
db.get('port'),
db.get('dbname')
)
conn_args = {
"sslmode": db.get('sslmode'),
"sslcompression": db.get('sslcompression'),
"sslrootcert": db.get('sslrootcert'),
"sslcert": db.get('sslcert'),
"sslkey": db.get('sslkey'),
"pool_pre_ping": True,
"pool_recycle": 300
}
try:
engine = create_engine(conn_string, connect_args=conn_args)
logging.info("sqlalchemy Engine to database created using psycopg2... ")
return engine
except psycopg2.Error as error:
logging.error(error)
And in my program I am connecting as below.
# Create PostgreSQL Engine
self.engine = abc.create_psql_engine(dbParam)
print('PSQL Engine Created to Table : {}'.format(self.tableName))
Then I use a pandas to_sql for inserting into DB
df.to_sql(self.table,self.engine,schema='abc',index=False,if_exists='append',chunksize=100)
It was working all this while with out the below 2 options.
"pool_pre_ping": True,
"pool_recycle": 300
But I was gettin connection interruption when handling large volume. And which is the reason I am trying out a solution with these 2 parameters.
However, I get this error , when I tried. Any clue what I am doing wrong.
sqlalchemy.exc.ProgrammingError: (psycopg2.ProgrammingError) invalid dsn: invalid connection option "pool_pre_ping"
Complete error log as below..
Traceback (most recent call last):
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3212, in _wrap_pool_connect
return fn()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 307, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 767, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 425, in checkout
rec = pool._do_get()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 146, in _do_get
self._dec_overflow()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 143, in _do_get
return self._create_connection()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 253, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 368, in __init__
self.__connect()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 611, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 605, in __connect
connection = pool._invoke_creator(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/create.py", line 578, in connect
return dialect.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 584, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/__init__.py", line 126, in connect
dsn = _ext.make_dsn(dsn, **kwargs)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/extensions.py", line 175, in make_dsn
parse_dsn(dsn)
psycopg2.ProgrammingError: invalid dsn: invalid connection option "pool_pre_ping"
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/opt/apps/python/loadtoPostgreSQL.py", line 275, in <module>
MX123D.processFiles(jobIdentifier)
File "/opt/apps/python/loadtoPostgreSQL.py", line 246, in processFiles
df.to_sql(self.table,self.engine,schema='abc',index=False,if_exists='append',chunksize=100)
File "/usr/local/lib64/python3.6/site-packages/pandas/core/generic.py", line 2615, in to_sql
method=method,
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 598, in to_sql
method=method,
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 1393, in to_sql
table.create()
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 721, in create
if self.exists():
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 708, in exists
return self.pd_sql.has_table(self.name, self.schema)
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 1431, in has_table
self.connectable.dialect.has_table, name, schema or self.meta.schema
File "<string>", line 2, in run_callable
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/deprecations.py", line 390, in warned
return fn(*args, **kwargs)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3074, in run_callable
with self.connect() as conn:
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3166, in connect
return self._connection_cls(self, close_with_result=close_with_result)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 96, in __init__
else engine.raw_connection()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3245, in raw_connection
return self._wrap_pool_connect(self.pool.connect, _connection)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3216, in _wrap_pool_connect
e, dialect, self
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 2070, in _handle_dbapi_exception_noconnection
sqlalchemy_exception, with_traceback=exc_infoÝ2¨, from_=e
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3212, in _wrap_pool_connect
return fn()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 307, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 767, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 425, in checkout
rec = pool._do_get()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 146, in _do_get
self._dec_overflow()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 143, in _do_get
return self._create_connection()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 253, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 368, in __init__
self.__connect()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 611, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 605, in __connect
connection = pool._invoke_creator(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/create.py", line 578, in connect
return dialect.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 584, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/__init__.py", line 126, in connect
dsn = _ext.make_dsn(dsn, **kwargs)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/extensions.py", line 175, in make_dsn
parse_dsn(dsn)
sqlalchemy.exc.ProgrammingError: (psycopg2.ProgrammingError) invalid dsn: invalid connection option "pool_pre_ping"
(Background on this error at: https://sqlalche.me/e/14/f405)
I myself found the issue.Sharing here so that it can help some one.
Issue was that the parameters were given in wrong place :-)
It should be given as below.
engine = create_engine(conn_string, connect_args=conn_args,pool_pre_ping=True,pool_recycle=300)
I am currently trying to create an SQLite database using python.
here is my code
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////c/PStuff/friend.db'
db = SQLAlchemy(app)
class ExampleTable(db.Model):
id = db.Column(db.Integer,primary_key=True)
I have already created a database called 'friend.db' in the PStuff folder. It is currently empty.
In cmd i used this to create the database:sqlite3 friend.db
I then used a python shell:from test3 import db
Lastly when i use:
db.create_all()
i get this error:
Traceback (most recent call last):
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2285, in _wrap_pool_connect
return fn()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 363, in connect
return _ConnectionFairy._checkout(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 773, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\impl.py", line 238, in _do_get
return self._create_connection()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 657, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "c:\PStuff\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.raise_(
File "c:\PStuff\lib\site-packages\sqlalchemy\util\compat.py", line 178, in raise_
raise exception
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 652, in __connect
connection = pool._invoke_creator(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\default.py", line 490, in connect
return self.dbapi.connect(*cargs, **cparams)
sqlite3.OperationalError: unable to open database file
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "c:\PStuff\lib\site-packages\flask_sqlalchemy\__init__.py", line 1033, in create_all
self._execute_for_all_tables(app, bind, 'create_all')
File "c:\PStuff\lib\site-packages\flask_sqlalchemy\__init__.py", line 1025, in _execute_for_all_tables
op(bind=self.get_engine(app, bind), **extra)
File "c:\PStuff\lib\site-packages\sqlalchemy\sql\schema.py", line 4320, in create_all
bind._run_visitor(
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2057, in _run_visitor
with self._optional_conn_ctx_manager(connection) as conn:
File "c:\PStuff\lib\contextlib.py", line 113, in __enter__
return next(self.gen)
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2049, in _optional_conn_ctx_manager
with self._contextual_connect() as conn:
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2251, in _contextual_connect
self._wrap_pool_connect(self.pool.connect, None),
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2288, in _wrap_pool_connect
Connection._handle_dbapi_exception_noconnection(
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 1554, in _handle_dbapi_exception_noconnection
util.raise_(
File "c:\PStuff\lib\site-packages\sqlalchemy\util\compat.py", line 178, in raise_
raise exception
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2285, in _wrap_pool_connect
return fn()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 363, in connect
return _ConnectionFairy._checkout(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 773, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\impl.py", line 238, in _do_get
return self._create_connection()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 657, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "c:\PStuff\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.raise_(
File "c:\PStuff\lib\site-packages\sqlalchemy\util\compat.py", line 178, in raise_
raise exception
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 652, in __connect
connection = pool._invoke_creator(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\default.py", line 490, in connect
return self.dbapi.connect(*cargs, **cparams)
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) unable to open database file
(Background on this error at: http://sqlalche.me/e/e3q8)
I am very new to SQL, any help would be greatly appreciated.
Try with Replace
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////c/PStuff/friend.db'
with
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///c/PStuff/friend.db'
based on SQLALCHEMY Documentation
sqlite:////db_absolute_path
sqlite:///db_relative_path
for windows try this thing (if you are putting database in same dir)
file_path = os.path.abspath(os.getcwd())+"\\friend.db"
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///'+file_path
I currently try to map an existing view within my postgreSQL DB to an SqlAlchemy ORM object in python. My view has the following name path_number_view and joins some tables together. It is wokring fine. But I am not able to map the view to an ORM object, when starting the application I get an error (belowe the code).
I followed these instructions,
Blog post
Docs: Reflecting Database Objects
Docs. Classical Mappings
but I not able to get that working.
Here is my code:
engine = create_engine("postgresql://****:*****#localhost/pi", pool_size=15, max_overflow=5, pool_recycle=1200)
Base = declarative_base(engine)
metadata = MetaData(engine)
path_number_view = Table('path_number_view', metadata,
Column('id', Integer, primary_key=True),
Column('path', String),
Column('threeDaysL', Integer),
Column('threeDaysC', Integer),
Column('tenDaysL', Integer),
Column('tenDaysC', Integer),
Column('number', Integer),
autoload=True
)
class PathNumberView(object):
__tablename__ = 'path_number_view'
def __init__(self, id, path, threeDaysL, threeDaysC, tenDaysL, tenDaysC, number):
self.id = id
self.path = path
self.threeDaysL = threeDaysL
self.threeDaysC = threeDaysC
self.tenDaysL = tenDaysL
self.tenDaysC = tenDaysC
self.number = number
mapper(PathNumberView, path_number_view )
Base.metadata.create_all(engine)
The error I get is not helpfull:
Traceback (most recent call last):
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2265, in _wrap_pool_connect
return fn()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 363, in connect
return _ConnectionFairy._checkout(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 760, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\impl.py", line 139, in _do_get
self._dec_overflow()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\compat.py", line 153, in reraise
raise value
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\impl.py", line 136, in _do_get
return self._create_connection()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 639, in __connect
connection = pool._invoke_creator(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\default.py", line 453, in connect
return self.dbapi.connect(*cargs, **cparams)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\psycopg2\__init__.py", line 126, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Program Files\JetBrains\PyCharm 2018.3.5\helpers\pycharm\_jb_unittest_runner.py", line 35, in <module>
main(argv=args, module=None, testRunner=unittestpy.TeamcityTestRunner, buffer=not JB_DISABLE_BUFFERING)
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\main.py", line 94, in __init__
self.parseArgs(argv)
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\main.py", line 141, in parseArgs
self.createTests()
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\main.py", line 148, in createTests
self.module)
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\loader.py", line 219, in loadTestsFromNames
suites = [self.loadTestsFromName(name, module) for name in names]
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\loader.py", line 219, in <listcomp>
suites = [self.loadTestsFromName(name, module) for name in names]
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\loader.py", line 153, in loadTestsFromName
module = __import__(module_name)
File "E:\Projekte\project\preprocessing\tests.py", line 6, in <module>
from preprocessing.data import TrainingDataProvider
File "E:\Projekte\project\preprocessing\data\__init__.py", line 7, in <module>
from entity import PathNumberView
File "E:\Projekte\project\entity\__init__.py", line 105, in <module>
Base.metadata.create_all(engine)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\sql\schema.py", line 4294, in create_all
ddl.SchemaGenerator, self, checkfirst=checkfirst, tables=tables
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2035, in _run_visitor
with self._optional_conn_ctx_manager(connection) as conn:
File "C:\Users\username\.conda\envs\tfGPU2\lib\contextlib.py", line 81, in __enter__
return next(self.gen)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2027, in _optional_conn_ctx_manager
with self._contextual_connect() as conn:
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2229, in _contextual_connect
self._wrap_pool_connect(self.pool.connect, None),
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2269, in _wrap_pool_connect
e, dialect, self
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 1536, in _handle_dbapi_exception_noconnection
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\compat.py", line 152, in reraise
raise value.with_traceback(tb)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2265, in _wrap_pool_connect
return fn()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 363, in connect
return _ConnectionFairy._checkout(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 760, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\impl.py", line 139, in _do_get
self._dec_overflow()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\compat.py", line 153, in reraise
raise value
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\impl.py", line 136, in _do_get
return self._create_connection()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 639, in __connect
connection = pool._invoke_creator(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\default.py", line 453, in connect
return self.dbapi.connect(*cargs, **cparams)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\psycopg2\__init__.py", line 126, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError)
(Background on this error at: http://sqlalche.me/e/e3q8)
The connection it self is working, I have 3 other tables (created with SQLAclhemy) which are working fine. I can query, update, delete no problems.
I'm on a Windows OS and Python 3.6.4 and I'm trying to connect to an InterSystem ODBC35 DSN Datasource using SQLAlchemy 1.2.5.
Using pyodbc to connect to the DSN Datasource works great but when using the SQLAlchemy create_engine method:
from sqlalchemy import create_engine
import pyodbc
engine = create_engine("mssql+pyodbc://user:pass#mydsn", echo=True)
cnxn = engine.connect()
rows = cnxn.execute("SELECT name FROM sys.tables").fetchall()
print(rows)
I get the following error:
2018-03-29 11:33:44,631 INFO sqlalchemy.engine.base.Engine SELECT CAST(SERVERPROPERTY('ProductVersion') AS VARCHAR)
2018-03-29 11:33:44,631 INFO sqlalchemy.engine.base.Engine ()
Traceback (most recent call last):
File "mentrix.py", line 28, in <module>
cnxn = engine.connect()
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\engine\base.py", line 2102, in connect
return self._connection_cls(self, **kwargs)
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\engine\base.py", line 90, in __init__
if connection is not None else engine.raw_connection()
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\engine\base.py", line 2188, in raw_connection
self.pool.unique_connection, _connection)
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\engine\base.py", line 2158, in _wrap_pool_connect
return fn()
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\pool.py", line 345, in unique_connection
return _ConnectionFairy._checkout(self)
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\pool.py", line 784, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\pool.py", line 532, in checkout
rec = pool._do_get()
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\pool.py", line 1189, in _do_get
self._dec_overflow()
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\util\langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\util\compat.py", line 187, in reraise
raise value
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\pool.py", line 1186, in _do_get
return self._create_connection()
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\pool.py", line 350, in _create_connection
return _ConnectionRecord(self)
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\pool.py", line 477, in __init__
self.__connect(first_connect_check=True)
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\pool.py", line 677, in __connect
exec_once(self.connection, self)
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\event\attr.py", line 274, in exec_once
self(*args, **kw)
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\event\attr.py", line 284, in __call__
fn(*args, **kw)
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\util\langhelpers.py", line 1334, in go
return once_fn(*arg, **kw)
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\engine\strategies.py", line 183, in first_connect
dialect.initialize(c)
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\dialects\mssql\base.py", line 1931, in initialize
super(MSDialect, self).initialize(connection)
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\engine\default.py", line 267, in initialize
self._get_default_schema_name(connection)
File "C:\Users\m.m\Envs\mentrix\lib\site-packages\sqlalchemy\dialects\mssql\base.py", line 1958, in _get_default_schema_name
if self.server_version_info < MS_2005_VERSION:
TypeError: '<' not supported between instances of 'str' and 'int'
Do you know why the code below is getting all the errors that appear in the traceback below? The code is to read the data from a table, if dont exist the table is created and the data is loaded in the postgresql table, but its not working. Do you know why? The version of python is 3.6 I dont know if can be something related to that.
python:
import pandas as pd
from scipy.spatial.distance import cosine
from sqlalchemy import create_engine, Table, MetaData
from flask import Flask, jsonify, make_response, request
engine = create_engine('postgresql:///test')
source_table_name = 'data'
song_aux_table_name = 'song_aux_matrix'
song_table_name = 'songr'
def init():
global insert, source_table, source_df, song_df, song_similarity_matrix
meta = MetaData()
try:
source_df = pd.read_sql_table(source_table_name, engine, index_col='index')
except ValueError as e:
print('SLoading from CSV, writing to DB.')
source_df = pd.read_csv('data.csv')
source_df.columns = [c.lower() for c in source_df.columns]
source_df.to_sql(source_table_name, engine)
print('Wrote to DB.')
source_table = Table(source_table_name, meta, autoload=True, autoload_with=engine)
insert = source_table.insert().values()
try:
song_similarity_matrix = pd.read_sql_table(song_aux_table_name, engine, index_col='index')
song__df = pd.read_sql_table(song_table_name, engine, index_col='index')
except ValueError as e:
print(e)
init()
Tracebak:
Traceback (most recent call last):
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 1122, in _do_get
return self._pool.get(wait, self._timeout)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\util\queue.py", line 145, in get
raise Empty
sqlalchemy.util.queue.Empty
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\base.py", line 2147, in _wrap_pool_connect
return fn()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 328, in unique_connection
return _ConnectionFairy._checkout(self)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 766, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 516, in checkout
rec = pool._do_get()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 1138, in _do_get
self._dec_overflow()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\util\langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\util\compat.py", line 187, in reraise
raise value
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 1135, in _do_get
return self._create_connection()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 333, in _create_connection
return _ConnectionRecord(self)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 461, in __init__
self.__connect(first_connect_check=True)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 651, in __connect
connection = pool._invoke_creator(self)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\strategies.py", line 105, in connect
return dialect.connect(*cargs, **cparams)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\default.py", line 393, in connect
return self.dbapi.connect(*cargs, **cparams)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\psycopg2\__init__.py", line 130, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: fe_sendauth: no password supplied
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "app.py", line 38, in <module>
init()
File "app.py", line 23, in init
source_df = pd.read_sql_table(source_table_name, engine, index_col='index')
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\pandas\io\sql.py", line 258, in read_sql_table
meta.reflect(only=[table_name], views=True)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\sql\schema.py", line 3855, in reflect
with bind.connect() as conn:
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\base.py", line 2091, in connect
return self._connection_cls(self, **kwargs)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\base.py", line 90, in __init__
if connection is not None else engine.raw_connection()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\base.py", line 2177, in raw_connection
self.pool.unique_connection, _connection)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\base.py", line 2151, in _wrap_pool_connect
e, dialect, self)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\base.py", line 1465, in _handle_dbapi_exception_noconnection
exc_info
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\util\compat.py", line 203, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\util\compat.py", line 186, in reraise
raise value.with_traceback(tb)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\base.py", line 2147, in _wrap_pool_connect
return fn()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 328, in unique_connection
return _ConnectionFairy._checkout(self)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 766, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 516, in checkout
rec = pool._do_get()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 1138, in _do_get
self._dec_overflow()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\util\langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\util\compat.py", line 187, in reraise
raise value
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 1135, in _do_get
return self._create_connection()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 333, in _create_connection
return _ConnectionRecord(self)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 461, in __init__
self.__connect(first_connect_check=True)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 651, in __connect
connection = pool._invoke_creator(self)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\strategies.py", line 105, in connect
return dialect.connect(*cargs, **cparams)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\default.py", line 393, in connect
return self.dbapi.connect(*cargs, **cparams)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\psycopg2\__init__.py", line 130, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) fe_sendauth: no password supplied
(env) C:\Users\Jon\Desktop\testproj>python app.py
Traceback (most recent call last):
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 1122, in _do_get
return self._pool.get(wait, self._timeout)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\util\queue.py", line 145, in get
raise Empty
sqlalchemy.util.queue.Empty
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\base.py", line 2147, in _wrap_pool_connect
return fn()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 328, in unique_connection
return _ConnectionFairy._checkout(self)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 766, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 516, in checkout
rec = pool._do_get()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 1138, in _do_get
self._dec_overflow()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\util\langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\util\compat.py", line 187, in reraise
raise value
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 1135, in _do_get
return self._create_connection()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 333, in _create_connection
return _ConnectionRecord(self)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 461, in __init__
self.__connect(first_connect_check=True)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 651, in __connect
connection = pool._invoke_creator(self)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\strategies.py", line 105, in connect
return dialect.connect(*cargs, **cparams)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\default.py", line 393, in connect
return self.dbapi.connect(*cargs, **cparams)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\psycopg2\__init__.py", line 130, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: fe_sendauth: no password supplied
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "app.py", line 32, in <module>
init()
File "app.py", line 16, in init
source_df = pd.read_sql_table(source_table_name, engine, index_col='index')
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\pandas\io\sql.py", line 258, in read_sql_table
meta.reflect(only=[table_name], views=True)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\sql\schema.py", line 3855, in reflect
with bind.connect() as conn:
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\base.py", line 2091, in connect
return self._connection_cls(self, **kwargs)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\base.py", line 90, in __init__
if connection is not None else engine.raw_connection()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\base.py", line 2177, in raw_connection
self.pool.unique_connection, _connection)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\base.py", line 2151, in _wrap_pool_connect
e, dialect, self)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\base.py", line 1465, in _handle_dbapi_exception_noconnection
exc_info
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\util\compat.py", line 203, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\util\compat.py", line 186, in reraise
raise value.with_traceback(tb)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\base.py", line 2147, in _wrap_pool_connect
return fn()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 328, in unique_connection
return _ConnectionFairy._checkout(self)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 766, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 516, in checkout
rec = pool._do_get()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 1138, in _do_get
self._dec_overflow()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\util\langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\util\compat.py", line 187, in reraise
raise value
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 1135, in _do_get
return self._create_connection()
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 333, in _create_connection
return _ConnectionRecord(self)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 461, in __init__
self.__connect(first_connect_check=True)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\pool.py", line 651, in __connect
connection = pool._invoke_creator(self)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\strategies.py", line 105, in connect
return dialect.connect(*cargs, **cparams)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\sqlalchemy\engine\default.py", line 393, in connect
return self.dbapi.connect(*cargs, **cparams)
File "C:\Users\Jon\Desktop\testproj\env\lib\site-packages\psycopg2\__init__.py", line 130, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) fe_sendauth: no password supplied