Related
I'm trying to connect to a Windows SQL Server from a Windows Server 2016 (with ODBC Driver 17 for SQL Server installed) through Sqlalchemy and Python 3.6.2.
As stated here I created a data source in the Odbc Data Source Administrator
I cannot use pyodbc but I'm able to connect and run queries using it.
I'm also able to connect to the database and run queries using SQL Server Management Studio from the Windows Server.
What I'm trying:
sqlalchemy.create_engine('mssql+pyodbc://DB_INSTANCE_ADDRESS/DB_NAME?driver=ODBC_DATA_SOURCE?Trusted_Connection=yes')
What I've got:
Traceback (most recent call last):
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\engine\base.py", line 3250, in _wrap_pool_connect
return fn()
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\base.py", line 310, in connect
return _ConnectionFairy._checkout(self)
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\base.py", line 868, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\base.py", line 476, in checkout
rec = pool._do_get()
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\impl.py", line 146, in _do_get
self._dec_overflow()
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\util\langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\util\compat.py", line 207, in raise_
raise exception
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\impl.py", line 143, in _do_get
return self._create_connection()
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\base.py", line 256, in _create_connection
return _ConnectionRecord(self)
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\base.py", line 371, in __init__
self.__connect()
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\base.py", line 666, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\util\langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\util\compat.py", line 207, in raise_
raise exception
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\base.py", line 661, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\engine\create.py", line 590, in connect
return dialect.connect(*cargs, **cparams)
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\engine\default.py", line 597, in connect
return self.dbapi.connect(*cargs, **cparams)
pyodbc.InterfaceError: ('IM002', '[IM002] [Microsoft][ODBC Driver Manager] Data source name not found and no default driver specified (0) (SQLDriverConnect)')
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<string>", line 2, in table_names
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\util\deprecations.py", line 401, in warned
return fn(*args, **kwargs)
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\engine\base.py", line 3220, in table_names
with self._optional_conn_ctx_manager(connection) as conn:
File "C:\Users\Public\Documents\Python\Python36\lib\contextlib.py", line 81, in __enter__
return next(self.gen)
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\engine\base.py", line 2972, in _optional_conn_ctx_manager
with self.connect() as conn:
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\engine\base.py", line 3204, in connect
return self._connection_cls(self, close_with_result=close_with_result)
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\engine\base.py", line 96, in __init__
else engine.raw_connection()
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\engine\base.py", line 3283, in raw_connection
return self._wrap_pool_connect(self.pool.connect, _connection)
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\engine\base.py", line 3254, in _wrap_pool_connect
e, dialect, self
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\engine\base.py", line 2101, in _handle_dbapi_exception_noconnection
sqlalchemy_exception, with_traceback=exc_info[2], from_=e
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\util\compat.py", line 207, in raise_
raise exception
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\engine\base.py", line 3250, in _wrap_pool_connect
return fn()
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\base.py", line 310, in connect
return _ConnectionFairy._checkout(self)
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\base.py", line 868, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\base.py", line 476, in checkout
rec = pool._do_get()
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\impl.py", line 146, in _do_get
self._dec_overflow()
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\util\langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\util\compat.py", line 207, in raise_
raise exception
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\impl.py", line 143, in _do_get
return self._create_connection()
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\base.py", line 256, in _create_connection
return _ConnectionRecord(self)
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\base.py", line 371, in __init__
self.__connect()
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\base.py", line 666, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\util\langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\util\compat.py", line 207, in raise_
raise exception
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\pool\base.py", line 661, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\engine\create.py", line 590, in connect
return dialect.connect(*cargs, **cparams)
File "C:\Users\Public\Documents\Python\Python36\lib\site-packages\sqlalchemy\engine\default.py", line 597, in connect
return self.dbapi.connect(*cargs, **cparams)
sqlalchemy.exc.InterfaceError: (pyodbc.InterfaceError) ('IM002', '[IM002] [Microsoft][ODBC Driver Manager] Data source name not found and no default driver specified (0) (SQLDriverConnect)')
(Background on this error at: https://sqlalche.me/e/14/rvf5)
I have declared my connection function as below..
def create_psql_engine(db):
#conn_string = "postgresql+psycopg2://myuser:******#someserver:port/somedb"
conn_string = "postgresql+psycopg2://%s:%s#%s:%s/%s" % (
db.get('user'),
db.get('password'),
db.get('host'),
db.get('port'),
db.get('dbname')
)
conn_args = {
"sslmode": db.get('sslmode'),
"sslcompression": db.get('sslcompression'),
"sslrootcert": db.get('sslrootcert'),
"sslcert": db.get('sslcert'),
"sslkey": db.get('sslkey'),
"pool_pre_ping": True,
"pool_recycle": 300
}
try:
engine = create_engine(conn_string, connect_args=conn_args)
logging.info("sqlalchemy Engine to database created using psycopg2... ")
return engine
except psycopg2.Error as error:
logging.error(error)
And in my program I am connecting as below.
# Create PostgreSQL Engine
self.engine = abc.create_psql_engine(dbParam)
print('PSQL Engine Created to Table : {}'.format(self.tableName))
Then I use a pandas to_sql for inserting into DB
df.to_sql(self.table,self.engine,schema='abc',index=False,if_exists='append',chunksize=100)
It was working all this while with out the below 2 options.
"pool_pre_ping": True,
"pool_recycle": 300
But I was gettin connection interruption when handling large volume. And which is the reason I am trying out a solution with these 2 parameters.
However, I get this error , when I tried. Any clue what I am doing wrong.
sqlalchemy.exc.ProgrammingError: (psycopg2.ProgrammingError) invalid dsn: invalid connection option "pool_pre_ping"
Complete error log as below..
Traceback (most recent call last):
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3212, in _wrap_pool_connect
return fn()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 307, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 767, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 425, in checkout
rec = pool._do_get()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 146, in _do_get
self._dec_overflow()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 143, in _do_get
return self._create_connection()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 253, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 368, in __init__
self.__connect()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 611, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 605, in __connect
connection = pool._invoke_creator(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/create.py", line 578, in connect
return dialect.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 584, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/__init__.py", line 126, in connect
dsn = _ext.make_dsn(dsn, **kwargs)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/extensions.py", line 175, in make_dsn
parse_dsn(dsn)
psycopg2.ProgrammingError: invalid dsn: invalid connection option "pool_pre_ping"
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/opt/apps/python/loadtoPostgreSQL.py", line 275, in <module>
MX123D.processFiles(jobIdentifier)
File "/opt/apps/python/loadtoPostgreSQL.py", line 246, in processFiles
df.to_sql(self.table,self.engine,schema='abc',index=False,if_exists='append',chunksize=100)
File "/usr/local/lib64/python3.6/site-packages/pandas/core/generic.py", line 2615, in to_sql
method=method,
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 598, in to_sql
method=method,
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 1393, in to_sql
table.create()
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 721, in create
if self.exists():
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 708, in exists
return self.pd_sql.has_table(self.name, self.schema)
File "/usr/local/lib64/python3.6/site-packages/pandas/io/sql.py", line 1431, in has_table
self.connectable.dialect.has_table, name, schema or self.meta.schema
File "<string>", line 2, in run_callable
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/deprecations.py", line 390, in warned
return fn(*args, **kwargs)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3074, in run_callable
with self.connect() as conn:
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3166, in connect
return self._connection_cls(self, close_with_result=close_with_result)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 96, in __init__
else engine.raw_connection()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3245, in raw_connection
return self._wrap_pool_connect(self.pool.connect, _connection)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3216, in _wrap_pool_connect
e, dialect, self
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 2070, in _handle_dbapi_exception_noconnection
sqlalchemy_exception, with_traceback=exc_infoÝ2¨, from_=e
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 3212, in _wrap_pool_connect
return fn()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 307, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 767, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 425, in checkout
rec = pool._do_get()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 146, in _do_get
self._dec_overflow()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/impl.py", line 143, in _do_get
return self._create_connection()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 253, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 368, in __init__
self.__connect()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 611, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 72, in __exit__
with_traceback=exc_tb,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/pool/base.py", line 605, in __connect
connection = pool._invoke_creator(self)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/create.py", line 578, in connect
return dialect.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 584, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/__init__.py", line 126, in connect
dsn = _ext.make_dsn(dsn, **kwargs)
File "/usr/local/lib64/python3.6/site-packages/psycopg2/extensions.py", line 175, in make_dsn
parse_dsn(dsn)
sqlalchemy.exc.ProgrammingError: (psycopg2.ProgrammingError) invalid dsn: invalid connection option "pool_pre_ping"
(Background on this error at: https://sqlalche.me/e/14/f405)
I myself found the issue.Sharing here so that it can help some one.
Issue was that the parameters were given in wrong place :-)
It should be given as below.
engine = create_engine(conn_string, connect_args=conn_args,pool_pre_ping=True,pool_recycle=300)
I am currently trying to create an SQLite database using python.
here is my code
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////c/PStuff/friend.db'
db = SQLAlchemy(app)
class ExampleTable(db.Model):
id = db.Column(db.Integer,primary_key=True)
I have already created a database called 'friend.db' in the PStuff folder. It is currently empty.
In cmd i used this to create the database:sqlite3 friend.db
I then used a python shell:from test3 import db
Lastly when i use:
db.create_all()
i get this error:
Traceback (most recent call last):
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2285, in _wrap_pool_connect
return fn()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 363, in connect
return _ConnectionFairy._checkout(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 773, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\impl.py", line 238, in _do_get
return self._create_connection()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 657, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "c:\PStuff\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.raise_(
File "c:\PStuff\lib\site-packages\sqlalchemy\util\compat.py", line 178, in raise_
raise exception
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 652, in __connect
connection = pool._invoke_creator(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\default.py", line 490, in connect
return self.dbapi.connect(*cargs, **cparams)
sqlite3.OperationalError: unable to open database file
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "c:\PStuff\lib\site-packages\flask_sqlalchemy\__init__.py", line 1033, in create_all
self._execute_for_all_tables(app, bind, 'create_all')
File "c:\PStuff\lib\site-packages\flask_sqlalchemy\__init__.py", line 1025, in _execute_for_all_tables
op(bind=self.get_engine(app, bind), **extra)
File "c:\PStuff\lib\site-packages\sqlalchemy\sql\schema.py", line 4320, in create_all
bind._run_visitor(
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2057, in _run_visitor
with self._optional_conn_ctx_manager(connection) as conn:
File "c:\PStuff\lib\contextlib.py", line 113, in __enter__
return next(self.gen)
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2049, in _optional_conn_ctx_manager
with self._contextual_connect() as conn:
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2251, in _contextual_connect
self._wrap_pool_connect(self.pool.connect, None),
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2288, in _wrap_pool_connect
Connection._handle_dbapi_exception_noconnection(
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 1554, in _handle_dbapi_exception_noconnection
util.raise_(
File "c:\PStuff\lib\site-packages\sqlalchemy\util\compat.py", line 178, in raise_
raise exception
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\base.py", line 2285, in _wrap_pool_connect
return fn()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 363, in connect
return _ConnectionFairy._checkout(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 773, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\impl.py", line 238, in _do_get
return self._create_connection()
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 657, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "c:\PStuff\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.raise_(
File "c:\PStuff\lib\site-packages\sqlalchemy\util\compat.py", line 178, in raise_
raise exception
File "c:\PStuff\lib\site-packages\sqlalchemy\pool\base.py", line 652, in __connect
connection = pool._invoke_creator(self)
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "c:\PStuff\lib\site-packages\sqlalchemy\engine\default.py", line 490, in connect
return self.dbapi.connect(*cargs, **cparams)
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) unable to open database file
(Background on this error at: http://sqlalche.me/e/e3q8)
I am very new to SQL, any help would be greatly appreciated.
Try with Replace
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////c/PStuff/friend.db'
with
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///c/PStuff/friend.db'
based on SQLALCHEMY Documentation
sqlite:////db_absolute_path
sqlite:///db_relative_path
for windows try this thing (if you are putting database in same dir)
file_path = os.path.abspath(os.getcwd())+"\\friend.db"
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///'+file_path
I currently try to map an existing view within my postgreSQL DB to an SqlAlchemy ORM object in python. My view has the following name path_number_view and joins some tables together. It is wokring fine. But I am not able to map the view to an ORM object, when starting the application I get an error (belowe the code).
I followed these instructions,
Blog post
Docs: Reflecting Database Objects
Docs. Classical Mappings
but I not able to get that working.
Here is my code:
engine = create_engine("postgresql://****:*****#localhost/pi", pool_size=15, max_overflow=5, pool_recycle=1200)
Base = declarative_base(engine)
metadata = MetaData(engine)
path_number_view = Table('path_number_view', metadata,
Column('id', Integer, primary_key=True),
Column('path', String),
Column('threeDaysL', Integer),
Column('threeDaysC', Integer),
Column('tenDaysL', Integer),
Column('tenDaysC', Integer),
Column('number', Integer),
autoload=True
)
class PathNumberView(object):
__tablename__ = 'path_number_view'
def __init__(self, id, path, threeDaysL, threeDaysC, tenDaysL, tenDaysC, number):
self.id = id
self.path = path
self.threeDaysL = threeDaysL
self.threeDaysC = threeDaysC
self.tenDaysL = tenDaysL
self.tenDaysC = tenDaysC
self.number = number
mapper(PathNumberView, path_number_view )
Base.metadata.create_all(engine)
The error I get is not helpfull:
Traceback (most recent call last):
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2265, in _wrap_pool_connect
return fn()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 363, in connect
return _ConnectionFairy._checkout(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 760, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\impl.py", line 139, in _do_get
self._dec_overflow()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\compat.py", line 153, in reraise
raise value
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\impl.py", line 136, in _do_get
return self._create_connection()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 639, in __connect
connection = pool._invoke_creator(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\default.py", line 453, in connect
return self.dbapi.connect(*cargs, **cparams)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\psycopg2\__init__.py", line 126, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Program Files\JetBrains\PyCharm 2018.3.5\helpers\pycharm\_jb_unittest_runner.py", line 35, in <module>
main(argv=args, module=None, testRunner=unittestpy.TeamcityTestRunner, buffer=not JB_DISABLE_BUFFERING)
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\main.py", line 94, in __init__
self.parseArgs(argv)
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\main.py", line 141, in parseArgs
self.createTests()
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\main.py", line 148, in createTests
self.module)
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\loader.py", line 219, in loadTestsFromNames
suites = [self.loadTestsFromName(name, module) for name in names]
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\loader.py", line 219, in <listcomp>
suites = [self.loadTestsFromName(name, module) for name in names]
File "C:\Users\username\.conda\envs\tfGPU2\lib\unittest\loader.py", line 153, in loadTestsFromName
module = __import__(module_name)
File "E:\Projekte\project\preprocessing\tests.py", line 6, in <module>
from preprocessing.data import TrainingDataProvider
File "E:\Projekte\project\preprocessing\data\__init__.py", line 7, in <module>
from entity import PathNumberView
File "E:\Projekte\project\entity\__init__.py", line 105, in <module>
Base.metadata.create_all(engine)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\sql\schema.py", line 4294, in create_all
ddl.SchemaGenerator, self, checkfirst=checkfirst, tables=tables
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2035, in _run_visitor
with self._optional_conn_ctx_manager(connection) as conn:
File "C:\Users\username\.conda\envs\tfGPU2\lib\contextlib.py", line 81, in __enter__
return next(self.gen)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2027, in _optional_conn_ctx_manager
with self._contextual_connect() as conn:
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2229, in _contextual_connect
self._wrap_pool_connect(self.pool.connect, None),
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2269, in _wrap_pool_connect
e, dialect, self
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 1536, in _handle_dbapi_exception_noconnection
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\compat.py", line 152, in reraise
raise value.with_traceback(tb)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\base.py", line 2265, in _wrap_pool_connect
return fn()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 363, in connect
return _ConnectionFairy._checkout(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 760, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\impl.py", line 139, in _do_get
self._dec_overflow()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\util\compat.py", line 153, in reraise
raise value
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\impl.py", line 136, in _do_get
return self._create_connection()
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\pool\base.py", line 639, in __connect
connection = pool._invoke_creator(self)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\sqlalchemy\engine\default.py", line 453, in connect
return self.dbapi.connect(*cargs, **cparams)
File "C:\Users\username\.conda\envs\tfGPU2\lib\site-packages\psycopg2\__init__.py", line 126, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError)
(Background on this error at: http://sqlalche.me/e/e3q8)
The connection it self is working, I have 3 other tables (created with SQLAclhemy) which are working fine. I can query, update, delete no problems.
This is the smallest example I can give that is a MRE
I am attempting to do the following:
Using pyodbc, read from a SQL Server instance (complete)
Then, print that data to verify it (complete)
Then, take that data, and insert it into a new table (or overwrite the table if it exists) <- FAILURE
The code is below:
import pyodbc
import pandas as pd
import sqlalchemy as sa
sqlConn = pyodbc.connect(
"DRIVER={SQL Server};"
"SERVER=servername;"
"DATABASE=dbname;"
"Trusted_Connection=yes;"
)
sql = """
SELECT TOP (1000) [PART]
,[STEP]
,[COMPLETIONTIME]
FROM [dbname].[dbo].[STEPS]
"""
engine = sa.create_engine('mssql+pyodbc://servername/dbname')
df = pd.read_sql(sql, sqlConn)
df.to_sql(name = 'Test', con = engine, if_exists = 'replace', index = False)
sqlConn.commit()
sqlConn.close()
I get the following for an error:
File "WiP.py", line 26, in <module>
df.to_sql(name = 'Test', con = engine, if_exists = 'replace', index = False)
File "C:\Python367-64\lib\site-packages\pandas\core\generic.py", line 2712, in to_sql
method=method,
File "C:\Python367-64\lib\site-packages\pandas\io\sql.py", line 518, in to_sql
method=method,
File "C:\Python367-64\lib\site-packages\pandas\io\sql.py", line 1319, in to_sql
table.create()
File "C:\Python367-64\lib\site-packages\pandas\io\sql.py", line 641, in create
if self.exists():
File "C:\Python367-64\lib\site-packages\pandas\io\sql.py", line 628, in exists
return self.pd_sql.has_table(self.name, self.schema)
File "C:\Python367-64\lib\site-packages\pandas\io\sql.py", line 1344, in has_table
self.connectable.dialect.has_table, name, schema or self.meta.schema
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\base.py", line 2162, in run_callable
with self._contextual_connect() as conn:
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\base.py", line 2242, in _contextual_connect
self._wrap_pool_connect(self.pool.connect, None),
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\base.py", line 2280, in _wrap_pool_connect
e, dialect, self
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\base.py", line 1547, in _handle_dbapi_exception_noconnection
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "C:\Python367-64\lib\site-packages\sqlalchemy\util\compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "C:\Python367-64\lib\site-packages\sqlalchemy\util\compat.py", line 152, in reraise
raise value.with_traceback(tb)
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\base.py", line 2276, in _wrap_pool_connect
return fn()
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 363, in connect
return _ConnectionFairy._checkout(self)
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 760, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\impl.py", line 139, in _do_get
self._dec_overflow()
File "C:\Python367-64\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "C:\Python367-64\lib\site-packages\sqlalchemy\util\compat.py", line 153, in reraise
raise value
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\impl.py", line 136, in _do_get
return self._create_connection()
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 639, in __connect
connection = pool._invoke_creator(self)
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\default.py", line 482, in connect
return self.dbapi.connect(*cargs, **cparams)
sqlalchemy.exc.InterfaceError: (pyodbc.InterfaceError) ('IM002', '[IM002] [Microsoft][ODBC Driver Manager] Data source name not found and no default driver specified (0) (SQLDriverConnect)')
(Background on this error at: http://sqlalche.me/e/rvf5)
I looked that up on this site, and got the following:
Exception raised for errors that are related to the database’s
operation and not necessarily under the control of the programmer,
e.g. an unexpected disconnect occurs, the data source name is not
found, a transaction could not be processed, a memory allocation error
occurred during processing, etc.
I have also consulted:
Connecting to Microsoft SQL server using Python
List sql tables in pandas.read_sql
https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_sql_table.html
https://github.com/mkleehammer/pyodbc/issues/300
Connecting to SQL Server 2012 using sqlalchemy and pyodbc
I also tried switching df.to_sql(name = 'Test', con = engine, if_exists = 'replace', index = False) to df.to_sql(name = 'Test', con = sqlConn, if_exists = 'replace', index = False) but got the exact same error.
What am I doing incorrectly? How can I wrote to a new table (or overwrite an existing one) from a Pandas dataframe?
UPDATE
It appears the connection is failing. The following:
import pyodbc
import pandas as pd
import sqlalchemy as sa
from sqlalchemy import create_engine
sqlConn = pyodbc.connect(
"DRIVER={SQL Server};"
"SERVER=servername;"
"DATABASE=dbname;"
"Trusted_Connection=yes;"
)
sql = """
SELECT TOP (1000) [ORDR_PART_NO]
,[OROP_ID]
,[COMPLETIONTIME]
FROM [dbname].[dbo].[OpsLookup]
"""
engine = sa.create_engine('mssql+pyodbc://servername/dbname')
cnxn = engine.connect()
result = cnxn.execute("SELECT TOP (1000) * FROM [dbname].[dbo].[STEPS]")
for row in result:
print(row)
cnxn.close()
Yields:
C:\Python367-64\lib\site-packages\sqlalchemy\connectors\pyodbc.py:79: SAWarning: No driver name specified; this is expected by PyODBC when using DSN-less connections
"No driver name specified; "
Traceback (most recent call last):
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\base.py", line 2276, in _wrap_pool_connect
return fn()
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 303, in unique_connection
return _ConnectionFairy._checkout(self)
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 760, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\impl.py", line 139, in _do_get
self._dec_overflow()
File "C:\Python367-64\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "C:\Python367-64\lib\site-packages\sqlalchemy\util\compat.py", line 153, in reraise
raise value
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\impl.py", line 136, in _do_get
return self._create_connection()
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 639, in __connect
connection = pool._invoke_creator(self)
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\default.py", line 482, in connect
return self.dbapi.connect(*cargs, **cparams)
pyodbc.InterfaceError: ('IM002', '[IM002] [Microsoft][ODBC Driver Manager] Data source name not found and no default driver specified (0) (SQLDriverConnect)')
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "WiP.py", line 21, in <module>
cnxn = engine.connect()
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\base.py", line 2209, in connect
return self._connection_cls(self, **kwargs)
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\base.py", line 103, in __init__
else engine.raw_connection()
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\base.py", line 2307, in raw_connection
self.pool.unique_connection, _connection
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\base.py", line 2280, in _wrap_pool_connect
e, dialect, self
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\base.py", line 1547, in _handle_dbapi_exception_noconnection
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "C:\Python367-64\lib\site-packages\sqlalchemy\util\compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "C:\Python367-64\lib\site-packages\sqlalchemy\util\compat.py", line 152, in reraise
raise value.with_traceback(tb)
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\base.py", line 2276, in _wrap_pool_connect
return fn()
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 303, in unique_connection
return _ConnectionFairy._checkout(self)
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 760, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\impl.py", line 139, in _do_get
self._dec_overflow()
File "C:\Python367-64\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "C:\Python367-64\lib\site-packages\sqlalchemy\util\compat.py", line 153, in reraise
raise value
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\impl.py", line 136, in _do_get
return self._create_connection()
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "C:\Python367-64\lib\site-packages\sqlalchemy\pool\base.py", line 639, in __connect
connection = pool._invoke_creator(self)
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "C:\Python367-64\lib\site-packages\sqlalchemy\engine\default.py", line 482, in connect
return self.dbapi.connect(*cargs, **cparams)
sqlalchemy.exc.InterfaceError: (pyodbc.InterfaceError) ('IM002', '[IM002] [Microsoft][ODBC Driver Manager] Data source name not found and no default driver specified (0) (SQLDriverConnect)')
(Background on this error at: http://sqlalche.me/e/rvf5)
engine = sa.create_engine('mssql+pyodbc://servername/dbname')
needed to be changed to
engine = sa.create_engine('mssql+pyodbc://servername/dbname?trusted_connection=yes&driver=ODBC Driver 13 for SQL Server')
Per: https://docs.sqlalchemy.org/en/13/dialects/mssql.html#hostname-connections