I am dealing with a huge table where I have to do query. I decided to do so by chunking my data based on user_id and every time read and write into the sql.
from sqlalchemy import create_engine
engine = create_engine('mysql+pymysql://')
q1 = "SELECT max(id) FROM users"
max_users = pd.read_sql(q1, engine)
max_users = max_users.iloc[0][0]
# since user_ids start from 1 to ... I make the split based on that
data = range(max_users)
chunks = [list(data[x:x+1000]) for x in range(0, len(data), 1000)]
def make_q(userid):
q2 = "SELECT alotofusers from bigtable WHERE userid in (" + str(','.join(str(e) for e in userid)) + ")"
from multiprocessing import Pool, TimeoutError
import time
import os
table_name = "user_type_tmp6"
def f(q):
df = pd.read_sql(q, engine)
df.to_sql(con=engine, name=table_name, if_exists='append')
pool = Pool(processes=10) # start 4 worker processes
pool.map(f, [make_q(item) for item in chunks[0:3]])
In fact my table only get populated by the first chunck but I get the following error
Exception during reset or similar
Traceback (most recent call last):
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/pool/base.py", line 680, in _finalize_fairy
fairy._reset(pool)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/pool/base.py", line 867, in _reset
pool._dialect.do_rollback(self)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/dialects/mysql/base.py", line 2302, in do_rollback
dbapi_connection.rollback()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 430, in rollback
self._read_ok_packet()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 394, in _read_ok_packet
pkt = self._read_packet()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 671, in _read_packet
% (packet_number, self._next_seq_id))
pymysql.err.InternalError: Packet sequence number wrong - got 48 expected 1
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1246, in _execute_context
cursor, statement, parameters, context
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 581, in do_execute
cursor.execute(statement, parameters)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 517, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 732, in _read_query_result
result.read()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 1075, in read
first_packet = self.connection._read_packet()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 671, in _read_packet
% (packet_number, self._next_seq_id))
pymysql.err.InternalError: Packet sequence number wrong - got 114 expected 1
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 733, in _rollback_impl
self.engine.dialect.do_rollback(self.connection)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/dialects/mysql/base.py", line 2302, in do_rollback
dbapi_connection.rollback()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 429, in rollback
self._execute_command(COMMAND.COM_QUERY, "ROLLBACK")
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 750, in _execute_command
raise err.InterfaceError("(0, '')")
pymysql.err.InterfaceError: (0, '')
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/multiprocessing/pool.py", line 121, in worker
result = (True, func(*args, **kwds))
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/multiprocessing/pool.py", line 44, in mapstar
return list(map(*args))
File "user_app_usage_type.py", line 90, in f
df = pd.read_sql(q, engine) # index_col = 'user_id'
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pandas/io/sql.py", line 436, in read_sql
chunksize=chunksize,
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pandas/io/sql.py", line 1218, in read_query
result = self.execute(*args)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pandas/io/sql.py", line 1087, in execute
return self.connectable.execute(*args, **kwargs)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 2182, in execute
return connection.execute(statement, *multiparams, **params)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 976, in execute
return self._execute_text(object_, multiparams, params)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1149, in _execute_text
parameters,
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1250, in _execute_context
e, statement, parameters, cursor, context
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1471, in _handle_dbapi_exception
self._autorollback()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py", line 79, in __exit__
compat.reraise(type_, value, traceback)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 153, in reraise
raise value
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1471, in _handle_dbapi_exception
self._autorollback()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 861, in _autorollback
self._root._rollback_impl()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 735, in _rollback_impl
self._handle_dbapi_exception(e, None, None, None, None)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1384, in _handle_dbapi_exception
exc_info,
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 152, in reraise
raise value.with_traceback(tb)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 733, in _rollback_impl
self.engine.dialect.do_rollback(self.connection)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/dialects/mysql/base.py", line 2302, in do_rollback
dbapi_connection.rollback()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 429, in rollback
self._execute_command(COMMAND.COM_QUERY, "ROLLBACK")
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 750, in _execute_command
raise err.InterfaceError("(0, '')")
sqlalchemy.exc.InterfaceError: (pymysql.err.InterfaceError) (0, '')
(Background on this error at: http://sqlalche.me/e/rvf5)
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "user_app_usage_type.py", line 109, in <module>
pool.map(f, [make_q(item) for item in chunks[0:3]])
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/multiprocessing/pool.py", line 268, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/multiprocessing/pool.py", line 657, in get
raise self._value
sqlalchemy.exc.InterfaceError: (pymysql.err.InterfaceError) (0, '')
(Background on this error at: http://sqlalche.me/e/rvf5)
I am guess I am doing the multiprocessing not correct ! Or perhaps the sqlalchemy is not aligned with the pooling.
Update
From my understand by reading this and this suggested by Ilja I updated my worked function as following
def f(q):
engine = create_engine('mysql+pymysql://')
df = pd.read_sql(q, engine, index_col = 'user_id')
df.fillna(0, inplace = True)
df.to_csv('tmp.csv')
df.to_sql(con=engine, name=table_name, if_exists='append' )
engine.dispose()
but now I get errors like
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1246, in _execute_context
cursor, statement, parameters, context
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 581, in do_execute
cursor.execute(statement, parameters)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 517, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 732, in _read_query_result
result.read()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 1075, in read
first_packet = self.connection._read_packet()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 684, in _read_packet
packet.check_error()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/protocol.py", line 220, in check_error
err.raise_mysql_exception(self._data)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/err.py", line 109, in raise_mysql_exception
raise errorclass(errno, errval)
pymysql.err.InternalError: (1050, "Table 'users_usage_frequency_oly_12' already exists")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/multiprocessing/pool.py", line 121, in worker
result = (True, func(*args, **kwds))
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/multiprocessing/pool.py", line 44, in mapstar
return list(map(*args))
File "user_login_type.py", line 103, in f
df.to_sql(con=engine, name=table_name, schema = 'datateam', if_exists='append' ) # dtype={'user_type': Enum('Browser', 'Hoarder', 'Mementor', 'Explorer', 'Lister', 'Scanner') }
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pandas/core/generic.py", line 2712, in to_sql
method=method,
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pandas/io/sql.py", line 518, in to_sql
method=method,
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pandas/io/sql.py", line 1319, in to_sql
table.create()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pandas/io/sql.py", line 656, in create
self._execute_create()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pandas/io/sql.py", line 638, in _execute_create
self.table.create()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 870, in create
bind._run_visitor(ddl.SchemaGenerator, self, checkfirst=checkfirst)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 2049, in _run_visitor
conn._run_visitor(visitorcallable, element, **kwargs)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1618, in _run_visitor
visitorcallable(self.dialect, self, **kwargs).traverse_single(element)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/sql/visitors.py", line 138, in traverse_single
return meth(obj, **kw)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/sql/ddl.py", line 826, in visit_table
include_foreign_key_constraints,
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 982, in execute
return meth(self, multiparams, params)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/sql/ddl.py", line 72, in _execute_on_connection
return connection._execute_ddl(self, multiparams, params)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1044, in _execute_ddl
compiled,
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1250, in _execute_context
e, statement, parameters, cursor, context
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1476, in _handle_dbapi_exception
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 152, in reraise
raise value.with_traceback(tb)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1246, in _execute_context
cursor, statement, parameters, context
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 581, in do_execute
cursor.execute(statement, parameters)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 517, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 732, in _read_query_result
result.read()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 1075, in read
first_packet = self.connection._read_packet()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 684, in _read_packet
packet.check_error()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/protocol.py", line 220, in check_error
err.raise_mysql_exception(self._data)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/err.py", line 109, in raise_mysql_exception
raise errorclass(errno, errval)
sqlalchemy.exc.InternalError: (pymysql.err.InternalError) (1050, "Table 'tmp_oly_12' already exists")
[SQL:
CREATE TABLE tmp_oly_12 (
user_id BIGINT,
total_logins BIGINT,
distinct_month BIGINT,
freq TEXT,
lastlogin DATETIME,
typelastlog TEXT
)
]
(Background on this error at: http://sqlalche.me/e/2j85)
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "user_login_type.py", line 125, in <module>
pool.map(f, [make_q(item) for item in chunks[0:3]])
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/multiprocessing/pool.py", line 268, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/multiprocessing/pool.py", line 657, in get
raise self._value
sqlalchemy.exc.InternalError: (pymysql.err.InternalError) (1050, "Table 'users_usage_frequency_oly_12' already exists")
[SQL:
CREATE TABLE tmp_oly_12 (
user_id BIGINT,
total_logins BIGINT,
distinct_month BIGINT,
freq TEXT,
lastlogin DATETIME,
I can see the table tmp_oly_12 is populated, not fully - but still I get this error ...
Related
I am using SQLAlchemy and mysql-connector. While executing a query, if I get mysql.connector.errors.DatabaseError I have used the exception provided by mysql.
The exception is going in the Exception block and not in the DatabaseError catch block.
Code
try:
row_exists = None
session_object = self.connect_to_database()
# check if the value exists or not
try:
if session_object.query(
exists().where(
ConnectorRequest.CR_Context_Key == input_context['Context_Key'])).scalar():
row_exists = True
else:
row_exists = False
except DatabaseError as mysql_error:
logger.error('Error connecting to the MYSQL Server.Invalid database IP or Port provided' +
f'\nError-{str(mysql_error)}', exc_info=True)
# close the session_object
session_object.close()
return row_exists
except Exception as error:
logger.error('Error in exists query of the database' +
f'\nException - {str(error)}', exc_info=True)
Error caught in Exception
(mysql.connector.errors.DatabaseError) 2003 (HY000): Can't connect to MySQL server on '10.10.1.00:3306' (101)
(Background on this error at: https://sqlalche.me/e/14/4xp6)
Stacktrace
{"asctime": "2023-02-01 13:53:45,195", "levelname": "ERROR", "module": "Helper", "funcName": "check_if_file_is_already_processed", "process": 22216,
"message": "Error in exists query of the database\nException - (mysql.connector.errors.DatabaseError) 2003 (HY000): Can't connect to MySQL server on '10.10.1.00:3306' (101)\n(Background on this error at: https://sqlalche.me/e/14/4xp6)",
"exc_info": "Traceback (most recent call last):\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/connection_cext.py\", line 232, in _open_connection\n self._cmysql.connect(**cnx_kwargs)\n_mysql_connector.MySQLInterfaceError: Can't connect to MySQL server on '10.10.1.00:3306' (101)\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py\", line 3361, in _wrap_pool_connect\n return fn()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 327, in connect\n return _ConnectionFairy._checkout(self)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 894, in _checkout\n fairy = _ConnectionRecord.checkout(pool)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 493, in checkout\n rec = pool._do_get()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/impl.py\", line 146, in _do_get\n self._dec_overflow()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py\", line 72, in __exit__\n with_traceback=exc_tb,\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py\", line 211, in raise_\n raise exception\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/impl.py\", line 143, in _do_get\n return self._create_connection()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 273, in _create_connection\n return _ConnectionRecord(self)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 388, in __init__\n self.__connect()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 691, in __connect\n pool.logger.debug(\"Error on connect(): %s\", e)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py\", line 72, in __exit__\n with_traceback=exc_tb,\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py\", line 211, in raise_\n raise exception\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 686, in __connect\n self.dbapi_connection = connection = pool._invoke_creator(self)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/create.py\", line 578, in connect\n return dialect.connect(*cargs, **cparams)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/default.py\", line 598, in connect\n return self.dbapi.connect(*cargs, **cparams)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/__init__.py\", line 272, in connect\n return CMySQLConnection(*args, **kwargs)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/connection_cext.py\", line 85, in __init__\n self.connect(**kwargs)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/abstracts.py\", line 1003, in connect\n self._open_connection()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/connection_cext.py\", line 235, in _open_connection\n sqlstate=exc.sqlstate)\nmysql.connector.errors.DatabaseError: 2003 (HY000): Can't connect to MySQL server on '10.10.1.00:3306' (101)\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/home/usr/Workarea-2/usr/pycharm_project/toyota_finance_service/python_module/folder_listener_connector/folder_listener_connector/Helper.py\", line 155, in check_if_file_is_already_processed\n ConnectorRequest.CR_Context_Key == input_context['Connector_Context_Key'])).scalar():\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py\", line 2893, in scalar\n ret = self.one()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py\", line 2870, in one\n return self._iter().one()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py\", line 2919, in _iter\n execution_options={\"_sa_orm_load_options\": self.load_options},\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/orm/session.py\", line 1713, in execute\n conn = self._connection_for_bind(bind)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/orm/session.py\", line 1553, in _connection_for_bind\n engine, execution_options\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/orm/session.py\", line 747, in _connection_for_bind\n conn = bind.connect()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py\", line 3315, in connect\n return self._connection_cls(self, close_with_result=close_with_result)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py\", line 96, in __init__\n else engine.raw_connection()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py\", line 3394, in raw_connection\n return self._wrap_pool_connect(self.pool.connect, _connection)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py\", line 3365, in _wrap_pool_connect\n e, dialect, self\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py\", line 2199, in _handle_dbapi_exception_noconnection\n sqlalchemy_exception, with_traceback=exc_info[2], from_=e\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py\", line 211, in raise_\n raise exception\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py\", line 3361, in _wrap_pool_connect\n return fn()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 327, in connect\n return _ConnectionFairy._checkout(self)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 894, in _checkout\n fairy = _ConnectionRecord.checkout(pool)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 493, in checkout\n rec = pool._do_get()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/impl.py\", line 146, in _do_get\n self._dec_overflow()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py\", line 72, in __exit__\n with_traceback=exc_tb,\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py\", line 211, in raise_\n raise exception\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/impl.py\", line 143, in _do_get\n return self._create_connection()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 273, in _create_connection\n return _ConnectionRecord(self)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 388, in __init__\n self.__connect()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 691, in __connect\n pool.logger.debug(\"Error on connect(): %s\", e)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py\", line 72, in __exit__\n with_traceback=exc_tb,\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py\", line 211, in raise_\n raise exception\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 686, in __connect\n self.dbapi_connection = connection = pool._invoke_creator(self)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/create.py\", line 578, in connect\n return dialect.connect(*cargs, **cparams)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/default.py\", line 598, in connect\n return self.dbapi.connect(*cargs, **cparams)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/__init__.py\", line 272, in connect\n return CMySQLConnection(*args, **kwargs)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/connection_cext.py\", line 85, in __init__\n self.connect(**kwargs)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/abstracts.py\", line 1003, in connect\n self._open_connection()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/connection_cext.py\", line 235, in _open_connection\n sqlstate=exc.sqlstate)\nsqlalchemy.exc.DatabaseError: (mysql.connector.errors.DatabaseError) 2003 (HY000): Can't connect to MySQL server on '10.10.1.00:3306' (101)\n(Background on this error at: https://sqlalche.me/e/14/4xp6)"}
The exception is sqlalchemy.exc.DatabaseError; SQLAlchemy wraps the error from the DB-API connector (here MySQL-Connector) but displays the details from it.
So you need to do
from sqlalchemy.exc import DatabaseError
...
try:
# something
except DatabaseError:
# handle exception
I have model written in declarative base of SQL Alchemy.
Class Roles(Base):
__tablename__ = "roles"
__table_args__ = (
Index("roles_name", "name", unique=True),
)
id = Column(Integer, primary_key=True, default=get_uuid())
name = Column(String(10), nullable=False)
As you may have noticed I have set the default value of primary key column id to get_uuid().
def get_uuid():
pk = uuid.uuid4().int >> 64
return pk
The above method return UUID as integer of bit size 64 or less. This is because the column id of this table is set to int and spanner can hold up to 64 bit.
So now to insert a row in this table -
>>> role = Roles()
>>> role.name = "Admin"
>>> session.add(role)
>>> session.commit()
This resulted in following exception -
Traceback (most recent call last):
File "/usr/local/lib/python3.10/site-packages/google/api_core/grpc_helpers.py", line 72, in error_remapped_callable
return callable_(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/grpc/_channel.py", line 946, in __call__
return _end_unary_response_blocking(state, call, False, None)
File "/usr/local/lib/python3.10/site-packages/grpc/_channel.py", line 849, in _end_unary_response_blocking
raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
status = StatusCode.FAILED_PRECONDITION
details = "Could not parse 18011687921562567628 as an integer"
debug_error_string = "UNKNOWN:Error received from peer ipv4:172.19.0.3:9010 {grpc_message:"Could not parse 18011687921562567628 as an integer", grpc_status:9, created_time:"2022-11-12T06:48:36.468914625+00:00"}"
>
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_dbapi/cursor.py", line 269, in execute
) = self.connection.run_statement(statement)
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_dbapi/connection.py", line 454, in run_statement
_execute_insert_heterogenous(
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_dbapi/_helpers.py", line 57, in _execute_insert_heterogenous
transaction.execute_update(
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_v1/transaction.py", line 302, in execute_update
response = api.execute_sql(
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_v1/services/spanner/client.py", line 1096, in execute_sql
response = rpc(
File "/usr/local/lib/python3.10/site-packages/google/api_core/gapic_v1/method.py", line 154, in __call__
return wrapped_func(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/google/api_core/retry.py", line 283, in retry_wrapped_func
return retry_target(
File "/usr/local/lib/python3.10/site-packages/google/api_core/retry.py", line 190, in retry_target
return target()
File "/usr/local/lib/python3.10/site-packages/google/api_core/timeout.py", line 99, in func_with_timeout
return func(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/google/api_core/grpc_helpers.py", line 74, in error_remapped_callable
raise exceptions.from_grpc_error(exc) from exc
google.api_core.exceptions.FailedPrecondition: 400 Could not parse 18011687921562567628 as an integer
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1900, in _execute_context
self.dialect.do_execute(
File "/usr/local/lib/python3.10/site-packages/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py", line 1013, in do_execute
cursor.execute(statement, parameters)
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_dbapi/cursor.py", line 70, in wrapper
return function(cursor, *args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_dbapi/cursor.py", line 289, in execute
raise IntegrityError(getattr(e, "details", e)) from e
google.cloud.spanner_dbapi.exceptions.IntegrityError: []
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py", line 1451, in commit
self._transaction.commit(_to_root=self.future)
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py", line 829, in commit
self._prepare_impl()
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py", line 808, in _prepare_impl
self.session.flush()
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py", line 3386, in flush
self._flush(objects)
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py", line 3525, in _flush
with util.safe_reraise():
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__
compat.raise_(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/compat.py", line 208, in raise_
raise exception
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py", line 3486, in _flush
flush_context.execute()
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/unitofwork.py", line 456, in execute
rec.execute(self)
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/unitofwork.py", line 630, in execute
util.preloaded.orm_persistence.save_obj(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/persistence.py", line 245, in save_obj
_emit_insert_statements(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/persistence.py", line 1238, in _emit_insert_statements
result = connection._execute_20(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1705, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/sql/elements.py", line 333, in _execute_on_connection
return connection._execute_clauseelement(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1572, in _execute_clauseelement
ret = self._execute_context(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1943, in _execute_context
self._handle_dbapi_exception(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 2124, in _handle_dbapi_exception
util.raise_(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/compat.py", line 208, in raise_
raise exception
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1900, in _execute_context
self.dialect.do_execute(
File "/usr/local/lib/python3.10/site-packages/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py", line 1013, in do_execute
cursor.execute(statement, parameters)
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_dbapi/cursor.py", line 70, in wrapper
return function(cursor, *args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_dbapi/cursor.py", line 289, in execute
raise IntegrityError(getattr(e, "details", e)) from e
sqlalchemy.exc.IntegrityError: (google.cloud.spanner_dbapi.exceptions.IntegrityError) []
[SQL: INSERT INTO roles (id, name) VALUES (%s, %s)]
[parameters: [18011687921562567628, 'Admin']]
(Background on this error at: https://sqlalche.me/e/14/gkpj)
What I understood for this is that the spanner is not willing to accept the generated UUID.
status = StatusCode.FAILED_PRECONDITION
details = "Could not parse 18011687921562567628 as an integer"
I have checked the method get_uuid(). It does return int value of but size 64 or less.
The README of this repo suggests creating a table's primary key as Integer and while in inserting a row in the database generate value of primary key in hex. I did exactly the same but it didn't work.
The generated int value is larger than the maximum INT64 value that is allowed in Cloud Spanner:
Max allowed: 9223372036854775807
Your value : 18011687921562567628
See https://cloud.google.com/spanner/docs/reference/standard-sql/data-types#integer_types for more information on the INT64 type.
I'm no Python expert, but my guess is that the int value that you are generating is interpreted as an unsigned int, while the INT64 data type in Cloud Spanner is signed.
EDIT: Add example to get signed value.
My understanding is that you can do the following to get a signed 64-bit integer value from a UUID in Python:
import ctypes
import uuid
ctypes.c_long(uuid.uuid4().int >> 64).value
I created a small program that uses numba on a method containing a 2D array (list of lists). This program runs just fine on Spyder, but when I try to run the exact same program on a remote Ubuntu server, I get a long error saying "unsupported nested memory-managed object" (listed below the program code).
The code:
from numba import jit
#jit(nopython=True)
def test():
num1 = 10
num2 = 5
array = [ [ 1 for i in range(num1) ] for j in range(num2) ]
#array = [2 for i in range(num1) ]
sum = 0
for i in range(0,num1):
for j in range(0,num2):
sum = sum + array[i][j]
print(sum)
test()
The error:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/numba/runtime/context.py", line 186, in _call_incref_decref
meminfo = data_model.get_nrt_meminfo(builder, value)
File "/usr/lib/python3/dist-packages/numba/datamodel/models.py", line 329, in get_nrt_meminfo
"unsupported nested memory-managed object")
NotImplementedError: unsupported nested memory-managed object
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/numba/errors.py", line 243, in new_error_context
yield
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 216, in lower_block
self.lower_inst(inst)
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 263, in lower_inst
self.storevar(val, inst.target.name)
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 928, in storevar
self.decref(fetype, old)
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 982, in decref
self.context.nrt.decref(self.builder, typ, val)
File "/usr/lib/python3/dist-packages/numba/runtime/context.py", line 208, in decref
self._call_incref_decref(builder, typ, typ, value, "NRT_decref")
File "/usr/lib/python3/dist-packages/numba/runtime/context.py", line 179, in _call_incref_decref
funcname, getters + (getter,))
File "/usr/lib/python3/dist-packages/numba/runtime/context.py", line 188, in _call_incref_decref
raise NotImplementedError("%s: %s" % (root_type, str(e)))
NotImplementedError: list(list(int64)): unsupported nested memory-managed object
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "2DArrayTest.py", line 15, in <module>
test()
File "/usr/lib/python3/dist-packages/numba/dispatcher.py", line 307, in _compile_for_args
return self.compile(tuple(argtypes))
File "/usr/lib/python3/dist-packages/numba/dispatcher.py", line 579, in compile
cres = self._compiler.compile(args, return_type)
File "/usr/lib/python3/dist-packages/numba/dispatcher.py", line 80, in compile
flags=flags, locals=self.locals)
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 740, in compile_extra
return pipeline.compile_extra(func)
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 360, in compile_extra
return self._compile_bytecode()
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 699, in _compile_bytecode
return self._compile_core()
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 686, in _compile_core
res = pm.run(self.status)
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 246, in run
raise patched_exception
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 238, in run
stage()
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 621, in stage_nopython_backend
self._backend(lowerfn, objectmode=False)
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 576, in _backend
lowered = lowerfn()
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 563, in backend_nopython_mode
self.flags)
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 858, in native_lowering_stage
lower.lower()
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 135, in lower
self.lower_normal_function(self.fndesc)
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 176, in lower_normal_function
entry_block_tail = self.lower_function_body()
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 201, in lower_function_body
self.lower_block(block)
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 216, in lower_block
self.lower_inst(inst)
File "/usr/lib/python3.6/contextlib.py", line 99, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/lib/python3/dist-packages/numba/errors.py", line 249, in new_error_context
six.reraise(type(newerr), newerr, sys.exc_info()[2])
File "/usr/lib/python3/dist-packages/numba/six.py", line 658, in reraise
raise value.with_traceback(tb)
File "/usr/lib/python3/dist-packages/numba/errors.py", line 243, in new_error_context
yield
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 216, in lower_block
self.lower_inst(inst)
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 263, in lower_inst
self.storevar(val, inst.target.name)
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 928, in storevar
self.decref(fetype, old)
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 982, in decref
self.context.nrt.decref(self.builder, typ, val)
File "/usr/lib/python3/dist-packages/numba/runtime/context.py", line 208, in decref
self._call_incref_decref(builder, typ, typ, value, "NRT_decref")
File "/usr/lib/python3/dist-packages/numba/runtime/context.py", line 179, in _call_incref_decref
funcname, getters + (getter,))
File "/usr/lib/python3/dist-packages/numba/runtime/context.py", line 188, in _call_incref_decref
raise NotImplementedError("%s: %s" % (root_type, str(e)))
numba.errors.LoweringError: Failed at nopython (nopython mode backend)
list(list(int64)): unsupported nested memory-managed object
File "2DArrayTest.py", line 7
[1] During: lowering "$56 = build_list(items=[])" at 2DArrayTest.py (7)
I have seen elsewhere that numba doesn't like 2D arrays and lists of lists because of memory. Is there a way to make it work on the remote server, since it works on Spyder? What is it that Spyder does differently to make it work?
I'm saving a table into SQL server and need it to replace existing table of the same name.
df1.to_sql('customer', schema = r'Marketing\xyz', con = engine, index = False, if_exists = 'replace')
worked but adding brackets at the schema will break it:
df1.to_sql('customer', schema = r'[Marketing\xyz]', con = engine, index = False, if_exists = 'replace')
with this error:
Traceback (most recent call last):
File "<ipython-input-11-69ec99f37fd6>", line 1, in <module>
df1.to_sql('customer', schema = r'[abc\xyz]', con = engine, index = False, if_exists = 'replace')
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\pandas\core\generic.py", line 2712, in to_sql
method=method,
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\pandas\io\sql.py", line 518, in to_sql
method=method,
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\pandas\io\sql.py", line 1319, in to_sql
table.create()
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\pandas\io\sql.py", line 656, in create
self._execute_create()
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\pandas\io\sql.py", line 638, in _execute_create
self.table.create()
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\sql\schema.py", line 860, in create
bind._run_visitor(ddl.SchemaGenerator, self, checkfirst=checkfirst)
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 2036, in _run_visitor
conn._run_visitor(visitorcallable, element, **kwargs)
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1607, in _run_visitor
visitorcallable(self.dialect, self, **kwargs).traverse_single(element)
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\sql\visitors.py", line 132, in traverse_single
return meth(obj, **kw)
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\sql\ddl.py", line 826, in visit_table
include_foreign_key_constraints,
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 988, in execute
return meth(self, multiparams, params)
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\sql\ddl.py", line 72, in _execute_on_connection
return connection._execute_ddl(self, multiparams, params)
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1050, in _execute_ddl
compiled,
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1248, in _execute_context
e, statement, parameters, cursor, context
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1466, in _handle_dbapi_exception
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\util\compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\util\compat.py", line 152, in reraise
raise value.with_traceback(tb)
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1244, in _execute_context
cursor, statement, parameters, context
File "C:\Users\xyz\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\default.py", line 552, in do_execute
cursor.execute(statement, parameters)
ProgrammingError: (pyodbc.ProgrammingError) ('42S01', "[42S01] [Microsoft][ODBC Driver 13 for SQL Server][SQL Server]There is already an object named 'customer' in the database. (2714) (SQLExecDirectW)")
[SQL:
CREATE TABLE [Marketing\xyz].customer (
[Customer] not NULL
)
]
(Background on this error at: http://sqlalche.me/e/f405)
Anyone know why adding the brackets in the schema would cause pandas fail to automatically drop the table for me? Thanks-
I'm using
sys.version
Out[19]: '3.7.4 (default, Aug 9 2019, 18:34:13) [MSC v.1915 64 bit (AMD64)]'
pd.__version__
Out[16]: '0.25.3'
I want insert records to 3 tables. The codes is below.
sessionDB.autocommit = False
# insert Address
addrRecord = Addres(street=street, city=city, zip_code=zip_code)
sessionDB.add(addrRecord)
sessionDB.flush()
# insert customer
cRecord = Customer(email=email, passwords=password, kind='individual', aID=addrRecord.aID)
sessionDB.add(cRecord)
sessionDB.flush()
# insert Hcustomer
homeCRecord = HomeCu(cID=cRecord.cID, fname=first_name, lname=last_name, age=age, marriage=marriage, remain=remain)
sessionDB.add(homeCRecord)
print (addrRecord.city, cRecord.kind, homeCRecord.lname)
sessionDB.commit()
The Addres's key is aID and Customer's key is cID. They are auto increment. In Customer, there is a foreign key constrained by Addres's aID. HomeCu's foreign key cID is constrained by Customer's cID.
So I want to insert Addres, Customer, HomeCu sequentially, during which I have to get the keys without doing commit. I used flush() to get the keys but failed at commit operation. Why during the commit operation, the value like email, password become None, but the print sentence is doing well? I have been struggling on this for hours. You help will be greatly appreciated. Thanks a lot!!!
Here is the error log:
Traceback (most recent call last):
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py",
line 1193, in _execute_context context)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/engine/default.py",
line 509, in do_execute cursor.execute(statement, parameters)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/cursors.py",
line 170, in execute result = self._query(query)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/cursors.py",
line 328, in _query conn.query(q)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/connections.py",
line 516, in query self._affected_rows =
self._read_query_result(unbuffered=unbuffered)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/connections.py",
line 727, in _read_query_result result.read()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/connections.py",
line 1066, in read first_packet = self.connection._read_packet()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/connections.py",
line 683, in _read_packet packet.check_error()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/protocol.py",
line 220, in check_error err.raise_mysql_exception(self._data)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/err.py",
line 109, in raise_mysql_exception raise errorclass(errno, errval)
pymysql.err.IntegrityError: (1048, "Column 'email' cannot be null")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Volumes/Code/Previous
Content/grad/2018_2019/db/final/app/modelController.py", line 164, in
registerIndividual('534 Tilling St', 'New York', '15233',
'123#qq.com', '123', 'Zimo', 'Tang', 23, 755.23, 0)
File
"/Volumes/Code/Previous
Content/grad/2018_2019/db/final/app/modelController.py", line 129, in
registerIndividual sessionDB.commit()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/session.py",
line 943, in commit self.transaction.commit()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/session.py",
line 467, in commit self._prepare_impl()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/session.py",
line 447, in _prepare_impl self.session.flush()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/session.py",
line 2254, in flush self._flush(objects)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/session.py",
line 2381, in _flush transaction.rollback(_capture_exception=True)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py",
line 66, in exit compat.reraise(exc_type, exc_value, exc_tb)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/util/compat.py",
line 249, in reraise raise value
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/session.py",
line 2345, in _flush flush_context.execute()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/unitofwork.py",
line 395, in execute rec.execute(self)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/unitofwork.py",
line 560, in execute uow
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/persistence.py",
line 181, in save_obj mapper, table, insert)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/persistence.py",
line 830, in _emit_insert_statements execute(statement,
multiparams)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py",
line 948, in execute return meth(self, multiparams, params)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/sql/elements.py",
line 269, in _execute_on_connection return
connection._execute_clauseelement(self, multiparams, params)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py",
line 1060, in _execute_clauseelement compiled_sql, distilled_params
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py",
line 1200, in _execute_context context)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py",
line 1413, in _handle_dbapi_exception exc_info
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/util/compat.py",
line 265, in raise_from_cause reraise(type(exception), exception,
tb=exc_tb, cause=cause)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/util/compat.py",
line 248, in reraise raise value.with_traceback(tb)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py",
line 1193, in _execute_context context)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/engine/default.py",
line 509, in do_execute cursor.execute(statement, parameters)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/cursors.py",
line 170, in execute result = self._query(query)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/cursors.py",
line 328, in _query conn.query(q)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/connections.py",
line 516, in query self._affected_rows =
self._read_query_result(unbuffered=unbuffered)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/connections.py",
line 727, in _read_query_result result.read()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/connections.py",
line 1066, in read first_packet = self.connection._read_packet()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/connections.py",
line 683, in _read_packet packet.check_error()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/protocol.py",
line 220, in check_error err.raise_mysql_exception(self._data)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/err.py",
line 109, in raise_mysql_exception raise errorclass(errno, errval)
sqlalchemy.exc.IntegrityError: (pymysql.err.IntegrityError) (1048,
"Column 'email' cannot be null") [SQL: 'INSERT INTO customer
(cID, email, passwords, kind, aID) VALUES (%(cID)s, %(email)s,
%(passwords)s, %(kind)s, %(aID)s)'] [parameters: {'cID': 31,
'email': None, 'passwords': None, 'kind': None, 'aID': None}]
(Background on this error at: http://sqlalche.me/e/gkpj)
Process finished with exit code 1
addrRecord = sessionDB.query(Addres).filter(Addres.street == street).filter(Addres.city == city) \
.filter(Addres.zip_code == zip_code).first()
cRecord = Customer(email=email, passwords=password, kind='individual', aID=addrRecord.aID)
homeCRecord = HomeCu(fname=first_name, lname=last_name, age=age, marriage=marriage, remain=remain)
homeCRecord.customer = cRecord
sessionDB.add(cRecord)
sessionDB.add(homeCRecord)
I didn' t use flush finally. I think using flush for multiple time will cause error. I add the relationship in instance. Hope this can help someone.