I want insert records to 3 tables. The codes is below.
sessionDB.autocommit = False
# insert Address
addrRecord = Addres(street=street, city=city, zip_code=zip_code)
sessionDB.add(addrRecord)
sessionDB.flush()
# insert customer
cRecord = Customer(email=email, passwords=password, kind='individual', aID=addrRecord.aID)
sessionDB.add(cRecord)
sessionDB.flush()
# insert Hcustomer
homeCRecord = HomeCu(cID=cRecord.cID, fname=first_name, lname=last_name, age=age, marriage=marriage, remain=remain)
sessionDB.add(homeCRecord)
print (addrRecord.city, cRecord.kind, homeCRecord.lname)
sessionDB.commit()
The Addres's key is aID and Customer's key is cID. They are auto increment. In Customer, there is a foreign key constrained by Addres's aID. HomeCu's foreign key cID is constrained by Customer's cID.
So I want to insert Addres, Customer, HomeCu sequentially, during which I have to get the keys without doing commit. I used flush() to get the keys but failed at commit operation. Why during the commit operation, the value like email, password become None, but the print sentence is doing well? I have been struggling on this for hours. You help will be greatly appreciated. Thanks a lot!!!
Here is the error log:
Traceback (most recent call last):
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py",
line 1193, in _execute_context context)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/engine/default.py",
line 509, in do_execute cursor.execute(statement, parameters)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/cursors.py",
line 170, in execute result = self._query(query)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/cursors.py",
line 328, in _query conn.query(q)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/connections.py",
line 516, in query self._affected_rows =
self._read_query_result(unbuffered=unbuffered)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/connections.py",
line 727, in _read_query_result result.read()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/connections.py",
line 1066, in read first_packet = self.connection._read_packet()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/connections.py",
line 683, in _read_packet packet.check_error()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/protocol.py",
line 220, in check_error err.raise_mysql_exception(self._data)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/err.py",
line 109, in raise_mysql_exception raise errorclass(errno, errval)
pymysql.err.IntegrityError: (1048, "Column 'email' cannot be null")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Volumes/Code/Previous
Content/grad/2018_2019/db/final/app/modelController.py", line 164, in
registerIndividual('534 Tilling St', 'New York', '15233',
'123#qq.com', '123', 'Zimo', 'Tang', 23, 755.23, 0)
File
"/Volumes/Code/Previous
Content/grad/2018_2019/db/final/app/modelController.py", line 129, in
registerIndividual sessionDB.commit()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/session.py",
line 943, in commit self.transaction.commit()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/session.py",
line 467, in commit self._prepare_impl()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/session.py",
line 447, in _prepare_impl self.session.flush()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/session.py",
line 2254, in flush self._flush(objects)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/session.py",
line 2381, in _flush transaction.rollback(_capture_exception=True)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py",
line 66, in exit compat.reraise(exc_type, exc_value, exc_tb)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/util/compat.py",
line 249, in reraise raise value
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/session.py",
line 2345, in _flush flush_context.execute()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/unitofwork.py",
line 395, in execute rec.execute(self)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/unitofwork.py",
line 560, in execute uow
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/persistence.py",
line 181, in save_obj mapper, table, insert)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/orm/persistence.py",
line 830, in _emit_insert_statements execute(statement,
multiparams)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py",
line 948, in execute return meth(self, multiparams, params)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/sql/elements.py",
line 269, in _execute_on_connection return
connection._execute_clauseelement(self, multiparams, params)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py",
line 1060, in _execute_clauseelement compiled_sql, distilled_params
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py",
line 1200, in _execute_context context)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py",
line 1413, in _handle_dbapi_exception exc_info
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/util/compat.py",
line 265, in raise_from_cause reraise(type(exception), exception,
tb=exc_tb, cause=cause)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/util/compat.py",
line 248, in reraise raise value.with_traceback(tb)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/engine/base.py",
line 1193, in _execute_context context)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sqlalchemy/engine/default.py",
line 509, in do_execute cursor.execute(statement, parameters)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/cursors.py",
line 170, in execute result = self._query(query)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/cursors.py",
line 328, in _query conn.query(q)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/connections.py",
line 516, in query self._affected_rows =
self._read_query_result(unbuffered=unbuffered)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/connections.py",
line 727, in _read_query_result result.read()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/connections.py",
line 1066, in read first_packet = self.connection._read_packet()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/connections.py",
line 683, in _read_packet packet.check_error()
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/protocol.py",
line 220, in check_error err.raise_mysql_exception(self._data)
File
"/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pymysql/err.py",
line 109, in raise_mysql_exception raise errorclass(errno, errval)
sqlalchemy.exc.IntegrityError: (pymysql.err.IntegrityError) (1048,
"Column 'email' cannot be null") [SQL: 'INSERT INTO customer
(cID, email, passwords, kind, aID) VALUES (%(cID)s, %(email)s,
%(passwords)s, %(kind)s, %(aID)s)'] [parameters: {'cID': 31,
'email': None, 'passwords': None, 'kind': None, 'aID': None}]
(Background on this error at: http://sqlalche.me/e/gkpj)
Process finished with exit code 1
addrRecord = sessionDB.query(Addres).filter(Addres.street == street).filter(Addres.city == city) \
.filter(Addres.zip_code == zip_code).first()
cRecord = Customer(email=email, passwords=password, kind='individual', aID=addrRecord.aID)
homeCRecord = HomeCu(fname=first_name, lname=last_name, age=age, marriage=marriage, remain=remain)
homeCRecord.customer = cRecord
sessionDB.add(cRecord)
sessionDB.add(homeCRecord)
I didn' t use flush finally. I think using flush for multiple time will cause error. I add the relationship in instance. Hope this can help someone.
Related
I am using SQLAlchemy and mysql-connector. While executing a query, if I get mysql.connector.errors.DatabaseError I have used the exception provided by mysql.
The exception is going in the Exception block and not in the DatabaseError catch block.
Code
try:
row_exists = None
session_object = self.connect_to_database()
# check if the value exists or not
try:
if session_object.query(
exists().where(
ConnectorRequest.CR_Context_Key == input_context['Context_Key'])).scalar():
row_exists = True
else:
row_exists = False
except DatabaseError as mysql_error:
logger.error('Error connecting to the MYSQL Server.Invalid database IP or Port provided' +
f'\nError-{str(mysql_error)}', exc_info=True)
# close the session_object
session_object.close()
return row_exists
except Exception as error:
logger.error('Error in exists query of the database' +
f'\nException - {str(error)}', exc_info=True)
Error caught in Exception
(mysql.connector.errors.DatabaseError) 2003 (HY000): Can't connect to MySQL server on '10.10.1.00:3306' (101)
(Background on this error at: https://sqlalche.me/e/14/4xp6)
Stacktrace
{"asctime": "2023-02-01 13:53:45,195", "levelname": "ERROR", "module": "Helper", "funcName": "check_if_file_is_already_processed", "process": 22216,
"message": "Error in exists query of the database\nException - (mysql.connector.errors.DatabaseError) 2003 (HY000): Can't connect to MySQL server on '10.10.1.00:3306' (101)\n(Background on this error at: https://sqlalche.me/e/14/4xp6)",
"exc_info": "Traceback (most recent call last):\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/connection_cext.py\", line 232, in _open_connection\n self._cmysql.connect(**cnx_kwargs)\n_mysql_connector.MySQLInterfaceError: Can't connect to MySQL server on '10.10.1.00:3306' (101)\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py\", line 3361, in _wrap_pool_connect\n return fn()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 327, in connect\n return _ConnectionFairy._checkout(self)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 894, in _checkout\n fairy = _ConnectionRecord.checkout(pool)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 493, in checkout\n rec = pool._do_get()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/impl.py\", line 146, in _do_get\n self._dec_overflow()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py\", line 72, in __exit__\n with_traceback=exc_tb,\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py\", line 211, in raise_\n raise exception\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/impl.py\", line 143, in _do_get\n return self._create_connection()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 273, in _create_connection\n return _ConnectionRecord(self)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 388, in __init__\n self.__connect()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 691, in __connect\n pool.logger.debug(\"Error on connect(): %s\", e)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py\", line 72, in __exit__\n with_traceback=exc_tb,\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py\", line 211, in raise_\n raise exception\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 686, in __connect\n self.dbapi_connection = connection = pool._invoke_creator(self)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/create.py\", line 578, in connect\n return dialect.connect(*cargs, **cparams)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/default.py\", line 598, in connect\n return self.dbapi.connect(*cargs, **cparams)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/__init__.py\", line 272, in connect\n return CMySQLConnection(*args, **kwargs)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/connection_cext.py\", line 85, in __init__\n self.connect(**kwargs)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/abstracts.py\", line 1003, in connect\n self._open_connection()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/connection_cext.py\", line 235, in _open_connection\n sqlstate=exc.sqlstate)\nmysql.connector.errors.DatabaseError: 2003 (HY000): Can't connect to MySQL server on '10.10.1.00:3306' (101)\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/home/usr/Workarea-2/usr/pycharm_project/toyota_finance_service/python_module/folder_listener_connector/folder_listener_connector/Helper.py\", line 155, in check_if_file_is_already_processed\n ConnectorRequest.CR_Context_Key == input_context['Connector_Context_Key'])).scalar():\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py\", line 2893, in scalar\n ret = self.one()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py\", line 2870, in one\n return self._iter().one()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/orm/query.py\", line 2919, in _iter\n execution_options={\"_sa_orm_load_options\": self.load_options},\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/orm/session.py\", line 1713, in execute\n conn = self._connection_for_bind(bind)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/orm/session.py\", line 1553, in _connection_for_bind\n engine, execution_options\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/orm/session.py\", line 747, in _connection_for_bind\n conn = bind.connect()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py\", line 3315, in connect\n return self._connection_cls(self, close_with_result=close_with_result)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py\", line 96, in __init__\n else engine.raw_connection()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py\", line 3394, in raw_connection\n return self._wrap_pool_connect(self.pool.connect, _connection)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py\", line 3365, in _wrap_pool_connect\n e, dialect, self\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py\", line 2199, in _handle_dbapi_exception_noconnection\n sqlalchemy_exception, with_traceback=exc_info[2], from_=e\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py\", line 211, in raise_\n raise exception\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py\", line 3361, in _wrap_pool_connect\n return fn()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 327, in connect\n return _ConnectionFairy._checkout(self)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 894, in _checkout\n fairy = _ConnectionRecord.checkout(pool)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 493, in checkout\n rec = pool._do_get()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/impl.py\", line 146, in _do_get\n self._dec_overflow()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py\", line 72, in __exit__\n with_traceback=exc_tb,\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py\", line 211, in raise_\n raise exception\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/impl.py\", line 143, in _do_get\n return self._create_connection()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 273, in _create_connection\n return _ConnectionRecord(self)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 388, in __init__\n self.__connect()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 691, in __connect\n pool.logger.debug(\"Error on connect(): %s\", e)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py\", line 72, in __exit__\n with_traceback=exc_tb,\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py\", line 211, in raise_\n raise exception\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/pool/base.py\", line 686, in __connect\n self.dbapi_connection = connection = pool._invoke_creator(self)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/create.py\", line 578, in connect\n return dialect.connect(*cargs, **cparams)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/sqlalchemy/engine/default.py\", line 598, in connect\n return self.dbapi.connect(*cargs, **cparams)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/__init__.py\", line 272, in connect\n return CMySQLConnection(*args, **kwargs)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/connection_cext.py\", line 85, in __init__\n self.connect(**kwargs)\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/abstracts.py\", line 1003, in connect\n self._open_connection()\n File \"/home/usr/Workarea-2/usr/python_proj_env/folder_listener_connector_venv/lib/python3.6/site-packages/mysql/connector/connection_cext.py\", line 235, in _open_connection\n sqlstate=exc.sqlstate)\nsqlalchemy.exc.DatabaseError: (mysql.connector.errors.DatabaseError) 2003 (HY000): Can't connect to MySQL server on '10.10.1.00:3306' (101)\n(Background on this error at: https://sqlalche.me/e/14/4xp6)"}
The exception is sqlalchemy.exc.DatabaseError; SQLAlchemy wraps the error from the DB-API connector (here MySQL-Connector) but displays the details from it.
So you need to do
from sqlalchemy.exc import DatabaseError
...
try:
# something
except DatabaseError:
# handle exception
I am dealing with a huge table where I have to do query. I decided to do so by chunking my data based on user_id and every time read and write into the sql.
from sqlalchemy import create_engine
engine = create_engine('mysql+pymysql://')
q1 = "SELECT max(id) FROM users"
max_users = pd.read_sql(q1, engine)
max_users = max_users.iloc[0][0]
# since user_ids start from 1 to ... I make the split based on that
data = range(max_users)
chunks = [list(data[x:x+1000]) for x in range(0, len(data), 1000)]
def make_q(userid):
q2 = "SELECT alotofusers from bigtable WHERE userid in (" + str(','.join(str(e) for e in userid)) + ")"
from multiprocessing import Pool, TimeoutError
import time
import os
table_name = "user_type_tmp6"
def f(q):
df = pd.read_sql(q, engine)
df.to_sql(con=engine, name=table_name, if_exists='append')
pool = Pool(processes=10) # start 4 worker processes
pool.map(f, [make_q(item) for item in chunks[0:3]])
In fact my table only get populated by the first chunck but I get the following error
Exception during reset or similar
Traceback (most recent call last):
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/pool/base.py", line 680, in _finalize_fairy
fairy._reset(pool)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/pool/base.py", line 867, in _reset
pool._dialect.do_rollback(self)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/dialects/mysql/base.py", line 2302, in do_rollback
dbapi_connection.rollback()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 430, in rollback
self._read_ok_packet()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 394, in _read_ok_packet
pkt = self._read_packet()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 671, in _read_packet
% (packet_number, self._next_seq_id))
pymysql.err.InternalError: Packet sequence number wrong - got 48 expected 1
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1246, in _execute_context
cursor, statement, parameters, context
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 581, in do_execute
cursor.execute(statement, parameters)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 517, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 732, in _read_query_result
result.read()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 1075, in read
first_packet = self.connection._read_packet()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 671, in _read_packet
% (packet_number, self._next_seq_id))
pymysql.err.InternalError: Packet sequence number wrong - got 114 expected 1
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 733, in _rollback_impl
self.engine.dialect.do_rollback(self.connection)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/dialects/mysql/base.py", line 2302, in do_rollback
dbapi_connection.rollback()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 429, in rollback
self._execute_command(COMMAND.COM_QUERY, "ROLLBACK")
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 750, in _execute_command
raise err.InterfaceError("(0, '')")
pymysql.err.InterfaceError: (0, '')
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/multiprocessing/pool.py", line 121, in worker
result = (True, func(*args, **kwds))
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/multiprocessing/pool.py", line 44, in mapstar
return list(map(*args))
File "user_app_usage_type.py", line 90, in f
df = pd.read_sql(q, engine) # index_col = 'user_id'
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pandas/io/sql.py", line 436, in read_sql
chunksize=chunksize,
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pandas/io/sql.py", line 1218, in read_query
result = self.execute(*args)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pandas/io/sql.py", line 1087, in execute
return self.connectable.execute(*args, **kwargs)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 2182, in execute
return connection.execute(statement, *multiparams, **params)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 976, in execute
return self._execute_text(object_, multiparams, params)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1149, in _execute_text
parameters,
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1250, in _execute_context
e, statement, parameters, cursor, context
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1471, in _handle_dbapi_exception
self._autorollback()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py", line 79, in __exit__
compat.reraise(type_, value, traceback)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 153, in reraise
raise value
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1471, in _handle_dbapi_exception
self._autorollback()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 861, in _autorollback
self._root._rollback_impl()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 735, in _rollback_impl
self._handle_dbapi_exception(e, None, None, None, None)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1384, in _handle_dbapi_exception
exc_info,
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 152, in reraise
raise value.with_traceback(tb)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 733, in _rollback_impl
self.engine.dialect.do_rollback(self.connection)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/dialects/mysql/base.py", line 2302, in do_rollback
dbapi_connection.rollback()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 429, in rollback
self._execute_command(COMMAND.COM_QUERY, "ROLLBACK")
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 750, in _execute_command
raise err.InterfaceError("(0, '')")
sqlalchemy.exc.InterfaceError: (pymysql.err.InterfaceError) (0, '')
(Background on this error at: http://sqlalche.me/e/rvf5)
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "user_app_usage_type.py", line 109, in <module>
pool.map(f, [make_q(item) for item in chunks[0:3]])
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/multiprocessing/pool.py", line 268, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/multiprocessing/pool.py", line 657, in get
raise self._value
sqlalchemy.exc.InterfaceError: (pymysql.err.InterfaceError) (0, '')
(Background on this error at: http://sqlalche.me/e/rvf5)
I am guess I am doing the multiprocessing not correct ! Or perhaps the sqlalchemy is not aligned with the pooling.
Update
From my understand by reading this and this suggested by Ilja I updated my worked function as following
def f(q):
engine = create_engine('mysql+pymysql://')
df = pd.read_sql(q, engine, index_col = 'user_id')
df.fillna(0, inplace = True)
df.to_csv('tmp.csv')
df.to_sql(con=engine, name=table_name, if_exists='append' )
engine.dispose()
but now I get errors like
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1246, in _execute_context
cursor, statement, parameters, context
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 581, in do_execute
cursor.execute(statement, parameters)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 517, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 732, in _read_query_result
result.read()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 1075, in read
first_packet = self.connection._read_packet()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 684, in _read_packet
packet.check_error()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/protocol.py", line 220, in check_error
err.raise_mysql_exception(self._data)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/err.py", line 109, in raise_mysql_exception
raise errorclass(errno, errval)
pymysql.err.InternalError: (1050, "Table 'users_usage_frequency_oly_12' already exists")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/multiprocessing/pool.py", line 121, in worker
result = (True, func(*args, **kwds))
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/multiprocessing/pool.py", line 44, in mapstar
return list(map(*args))
File "user_login_type.py", line 103, in f
df.to_sql(con=engine, name=table_name, schema = 'datateam', if_exists='append' ) # dtype={'user_type': Enum('Browser', 'Hoarder', 'Mementor', 'Explorer', 'Lister', 'Scanner') }
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pandas/core/generic.py", line 2712, in to_sql
method=method,
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pandas/io/sql.py", line 518, in to_sql
method=method,
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pandas/io/sql.py", line 1319, in to_sql
table.create()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pandas/io/sql.py", line 656, in create
self._execute_create()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pandas/io/sql.py", line 638, in _execute_create
self.table.create()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 870, in create
bind._run_visitor(ddl.SchemaGenerator, self, checkfirst=checkfirst)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 2049, in _run_visitor
conn._run_visitor(visitorcallable, element, **kwargs)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1618, in _run_visitor
visitorcallable(self.dialect, self, **kwargs).traverse_single(element)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/sql/visitors.py", line 138, in traverse_single
return meth(obj, **kw)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/sql/ddl.py", line 826, in visit_table
include_foreign_key_constraints,
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 982, in execute
return meth(self, multiparams, params)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/sql/ddl.py", line 72, in _execute_on_connection
return connection._execute_ddl(self, multiparams, params)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1044, in _execute_ddl
compiled,
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1250, in _execute_context
e, statement, parameters, cursor, context
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1476, in _handle_dbapi_exception
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 152, in reraise
raise value.with_traceback(tb)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1246, in _execute_context
cursor, statement, parameters, context
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 581, in do_execute
cursor.execute(statement, parameters)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 517, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 732, in _read_query_result
result.read()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 1075, in read
first_packet = self.connection._read_packet()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/connections.py", line 684, in _read_packet
packet.check_error()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/protocol.py", line 220, in check_error
err.raise_mysql_exception(self._data)
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/site-packages/pymysql/err.py", line 109, in raise_mysql_exception
raise errorclass(errno, errval)
sqlalchemy.exc.InternalError: (pymysql.err.InternalError) (1050, "Table 'tmp_oly_12' already exists")
[SQL:
CREATE TABLE tmp_oly_12 (
user_id BIGINT,
total_logins BIGINT,
distinct_month BIGINT,
freq TEXT,
lastlogin DATETIME,
typelastlog TEXT
)
]
(Background on this error at: http://sqlalche.me/e/2j85)
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "user_login_type.py", line 125, in <module>
pool.map(f, [make_q(item) for item in chunks[0:3]])
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/multiprocessing/pool.py", line 268, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "/Users/opt/anaconda3/envs/UserExperience/lib/python3.7/multiprocessing/pool.py", line 657, in get
raise self._value
sqlalchemy.exc.InternalError: (pymysql.err.InternalError) (1050, "Table 'users_usage_frequency_oly_12' already exists")
[SQL:
CREATE TABLE tmp_oly_12 (
user_id BIGINT,
total_logins BIGINT,
distinct_month BIGINT,
freq TEXT,
lastlogin DATETIME,
I can see the table tmp_oly_12 is populated, not fully - but still I get this error ...
I am creating a backend with multiple tables have user id as the primary field with is of UUID type in the postgresql database. But I am getting the following error:
Traceback (most recent call last):
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1179, in _execute_context
context = constructor(dialect, self, conn, *args)
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 743, in _init_compiled
for key in compiled_params
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 743, in <genexpr>
for key in compiled_params
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/dialects/postgresql/psycopg2.py", line 487, in process
value = _python_UUID(value)
File "/usr/lib64/python3.7/uuid.py", line 157, in __init__
hex = hex.replace('urn:', '').replace('uuid:', '')
AttributeError: 'UUID' object has no attribute 'replace'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/flask/app.py", line 1832, in full_dispatch_request
rv = self.dispatch_request()
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/flask/app.py", line 1818, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/flask_restful/__init__.py", line 458, in wrapper
resp = resource(*args, **kwargs)
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/flask/views.py", line 88, in view
return self.dispatch_request(*args, **kwargs)
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/flask_restful/__init__.py", line 573, in dispatch_request
resp = meth(*args, **kwargs)
File "/mnt/home/Programming/Projects/Social Network/Server/src/new_user.py", line 17, in post
self.database_helper(args)
File "/mnt/home/Programming/Projects/Social Network/Server/src/new_user.py", line 26, in database_helper
db.session.commit()
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/orm/scoping.py", line 162, in do
return getattr(self.registry(), name)(*args, **kwargs)
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 1026, in commit
self.transaction.commit()
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 493, in commit
self._prepare_impl()
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 472, in _prepare_impl
self.session.flush()
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 2458, in flush
self._flush(objects)
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 2596, in _flush
transaction.rollback(_capture_exception=True)
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py", line 68, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 129, in reraise
raise value
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 2556, in _flush
flush_context.execute()
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/orm/unitofwork.py", line 422, in execute
rec.execute(self)
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/orm/unitofwork.py", line 589, in execute
uow,
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/orm/persistence.py", line 245, in save_obj
insert,
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/orm/persistence.py", line 1066, in _emit_insert_statements
c = cached_connections[connection].execute(statement, multiparams)
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 988, in execute
return meth(self, multiparams, params)
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/sql/elements.py", line 287, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1107, in _execute_clauseelement
distilled_params,
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1182, in _execute_context
e, util.text_type(statement), parameters, None, None
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1466, in _handle_dbapi_exception
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 383, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 128, in reraise
raise value.with_traceback(tb)
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1179, in _execute_context
context = constructor(dialect, self, conn, *args)
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 743, in _init_compiled
for key in compiled_params
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 743, in <genexpr>
for key in compiled_params
File "/home/ayushs/.local/share/virtualenvs/Server-D_x4HQZH/lib/python3.7/site-packages/sqlalchemy/dialects/postgresql/psycopg2.py", line 487, in process
value = _python_UUID(value)
File "/usr/lib64/python3.7/uuid.py", line 157, in __init__
hex = hex.replace('urn:', '').replace('uuid:', '')
sqlalchemy.exc.StatementError: (builtins.AttributeError) 'UUID' object has no attribute 'replace'
Here is my model:
from database.shared_db import db
from sqlalchemy.dialects import postgresql
from database.shared_bycrypt import bcrypt
class AuthenticationModel(db.Model):
user_id = db.Column('UserId', postgresql.UUID, primary_key = True)
email = db.Column('Email', postgresql.TEXT, unique = False, nullable=False)
phone_no = db.Column('PhoneNo', postgresql.TEXT, unique = False, nullable=False)
password = db.Column('Password', postgresql.TEXT, unique = False, nullable=False)
def __repr__(self):
return '<User %r>' % self.user_id
#property
def password_raw(self):
raise AttributeError('password not readable')
#password_raw.setter
def password_raw(self, password):
self.password = bcrypt.generate_password_hash(password)
def verify_password(self, password):
return bcrypt.check_password_hash(self.password, password)
And here is what my database helper function:
def database_helper(self, args):
id = uuid4()
userAuth = AuthenticationModel(user_id=id, email=args['email'], phone_no=args['phone_no'], password_raw=args['password'])
userInfo = PersonalInfoModel(user_id=id, first_name=args['first_name'], last_name=args['last_name'], dob=args['dob'])
db.session.add(userAuth)
db.session.add(userInfo)
db.session.commit()
I found many examples of autogenerating UUID fields. However, if I want to create some user across multiple databases, how can I do that? Because I think I will need to be able to pass the user id to the other tables even if I autogenerate it for one of the tables. Or should I simply use a text field?
uuid4() is generating a UUID instance.
You want to pass a string for the user_id field. Convert it to a string
user_id = str(uuid4())
If you like to pass UUID instance around and get them back in the result, you should update the user_id field in your schema to:
user_id = db.Column('UserId', postgresql.UUID(as_uuid=True), primary_key=True)
I'm writting application to store leaked DBs and hashes in MySQL. I want to save some space, so I figured out it may be a good idea to store only unique hashes in Hashes table and put a reference from DBList table. In other words, if I want to get all hashes that belongs to db_id=1 I should ask for them using association table.
For testing purposes I've created these files:
create_fresh_db.py
mydb.py
regex_test.txt
My problem is that association table called dbtohash is filled properly only when I'm inserting unique hashes. If I'm trying to create different name for DB and using the same hashlist (regex_test.txt) an IntegrityError exception is raised. How can I fix that?
EDIT: Following suggestion given in comments. Here are relevant parts of my code. This is how I create many-to-many relationship and association table:
dbtohash = db.Table(
'dbtohash',
db.Column(
'hash_id',
db.Integer,
db.ForeignKey('Hashes.hash_id')),
db.Column(
'db_id',
db.Integer,
db.ForeignKey('DBList.db_id')))
class DBList(db.Model):
__tablename__ = 'DBList'
db_id = db.Column(db.Integer, primary_key=True)
db_name = db.Column(db.String(128), index=True, unique=True)
class Hashes(db.Model):
__tablename__ = 'Hashes'
hash_id = db.Column(db.Integer, primary_key=True)
hash_val = db.Column(db.String(1024), unique=True)
hash_salt = db.Column(db.String(256))
hash_plain = db.Column(db.String(256))
subscriptions = db.relationship(
'DBList', secondary=dbtohash, backref=db.backref(
'dbtohashref', lazy='dynamic'))
And this is how I insert data to it:
new list
# if we have some matched hashes...
if len(hash_pass) > 0:
# create new list
newlist = DBList(
db_name = '{}_{}'.format('db', datetime.utcnow()),
)
print('newlist: {}'.format(newlist))
# try to insert DB name to DBList table, raise exception if name is taken
try:
db.session.add(newlist)
db.session.commit()
db.session.flush()
except IntegrityError:
db.session.rollback()
print('this DB name is taken, choose another one', 'danger')
insert hashes to DB
failed=[]
for hash in hash_pass:
inserthash = Hashes (
hash_val = hash,
)
try:
db.session.add(inserthash)
db.session.commit()
db.session.flush()
hashid=inserthash.hash_id
except IntegrityError:
db.session.rollback()
failed.append(hash)
hashid=db.session.query(Hashes.hash_id).filter_by(hash_val=hash).one()[0]
# fill up association table
newlist.dbtohashref.append(inserthash)
db.session.commit()
if failed:
print('failed: {}'.format(failed))
problematic line is: newlist.dbtohashref.append(inserthash)
EDIT 2: full trace attached
Traceback (most recent call last):
File "mydb.py", line 88, in <module>
db.session.commit()
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/scoping.py", line 153, in do
return getattr(self.registry(), name)(*args, **kwargs)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 943, in commit
self.transaction.commit()
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 467, in commit
self._prepare_impl()
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 447, in _prepare_impl
self.session.flush()
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2254, in flush
self._flush(objects)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2380, in _flush
transaction.rollback(_capture_exception=True)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2344, in _flush
flush_context.execute()
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 391, in execute
rec.execute(self)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 556, in execute
uow
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 181, in save_obj
mapper, table, insert)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 866, in _emit_insert_statements
execute(statement, params)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 948, in execute
return meth(self, multiparams, params)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 269, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1060, in _execute_clauseelement
compiled_sql, distilled_params
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1200, in _execute_context
context)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1413, in _handle_dbapi_exception
exc_info
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1193, in _execute_context
context)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 507, in do_execute
cursor.execute(statement, parameters)
File "/home/user/envs/project/local/lib/python2.7/site-packages/pymysql/cursors.py", line 165, in execute
result = self._query(query)
File "/home/user/envs/project/local/lib/python2.7/site-packages/pymysql/cursors.py", line 321, in _query
conn.query(q)
File "/home/user/envs/project/local/lib/python2.7/site-packages/pymysql/connections.py", line 860, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/home/user/envs/project/local/lib/python2.7/site-packages/pymysql/connections.py", line 1061, in _read_query_result
result.read()
File "/home/user/envs/project/local/lib/python2.7/site-packages/pymysql/connections.py", line 1349, in read
first_packet = self.connection._read_packet()
File "/home/user/envs/project/local/lib/python2.7/site-packages/pymysql/connections.py", line 1018, in _read_packet
packet.check_error()
File "/home/user/envs/project/local/lib/python2.7/site-packages/pymysql/connections.py", line 384, in check_error
err.raise_mysql_exception(self._data)
File "/home/user/envs/project/local/lib/python2.7/site-packages/pymysql/err.py", line 107, in raise_mysql_exception
raise errorclass(errno, errval)
sqlalchemy.exc.IntegrityError: (pymysql.err.IntegrityError) (1062, u"Duplicate entry 'd38b70e33d7ae06c1696910eb6f93ca4' for key 'hash_val'") [SQL: u'INSERT INTO `Hashes` (hash_val, hash_salt, hash_plain) VALUES (%(hash_val)s, %(hash_salt)s, %(hash_plain)s)'] [parameters: {'hash_salt': None, 'hash_val': 'd38b70e33d7ae06c1696910eb6f93ca4', 'hash_plain': None}] (Background on this error at: http://sqlalche.me/e/gkpj)
I have to work on an already written code using SQLAlchemy, Marshmellow and Flask and im trying to add new users to a database. I'm still studying the code trying to figure out if it works or not, but anytime that i try to add a new entry to the db it calls me an exception
sqlalchemy.exc.IntegrityError: (psycopg2.IntegrityError) null value in column "id" violates not-null constraint
This is my SQLAlchemy model user.py
from .. import db
class User(db.Model):
__tablename__ = 'user'
id = db.Column(db.Integer, primary_key=True)
age = db.Column(db.Integer)
gender = db.Column(db.String)
region = db.Column(db.String)
date_created = db.Column(db.DateTime, default=db.func.current_timestamp())
def __init__(self, age, gender, region):
self.age = age
self.gender = gender
self.region = region
def __repr__(self):
return '<User {1}>'.format(self.id)
and this is the api that manage the POST operation
#api.route('/users', methods=['POST'])
def create_user():
print("debug")
json_data = request.get_json()
print(json_data)
if not json_data:
return jsonify({'message': 'Invalid Request'}), 400
# Validate and deserialize input
user,errors = user_schema.load(json_data)
if errors:
return jsonify(errors), 422
# save new user to db
new_user = User(user.age,user.gender,user.region)
print(user.age)
db.session.add(new_user)
if db.session:
db.session.commit()
initialize(0, user)
print("wtf")
return jsonify({"message": "Created new user.",
"user": json_data}), 201
and it always has problems with the commit
How can i resolve it ? Im losing my mind working on i
EDIT:
It seems that the db is connected because i can do db.session.add(user) without problems
This is the full traceback
Traceback (most recent call last):
File "/home/leonardo/.local/lib/python3.5/site-packages/flask/app.py",
line 2000, in __call__
return self.wsgi_app(environ, start_response)
File "/home/leonardo/.local/lib/python3.5/site-packages/flask/app.py",
line 1991, in wsgi_app
response = self.make_response(self.handle_exception(e))
File "/home/leonardo/.local/lib/python3.5/site-packages/flask/app.py",
line 1567, in handle_exception
reraise(exc_type, exc_value, tb)
File "/home/leonardo/.local/lib/python3.5/site-
packages/flask/_compat.py", line 33, in reraise
raise value
File "/home/leonardo/.local/lib/python3.5/site-packages/flask/app.py",
line 1988, in wsgi_app
response = self.full_dispatch_request()
File "/home/leonardo/.local/lib/python3.5/site-packages/flask/app.py",
line 1641, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/home/leonardo/.local/lib/python3.5/site-packages/flask/app.py",
line 1544, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/home/leonardo/.local/lib/python3.5/site-
packages/flask/_compat.py", line 33, in reraise
raise value
File "/home/leonardo/.local/lib/python3.5/site-packages/flask/app.py",
line 1639, in full_dispatch_request
rv = self.dispatch_request()
File "/home/leonardo/.local/lib/python3.5/site-packages/flask/app.py",
line 1625, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/home/leonardo/Università/Tirocinio/tim-recommender-
api/app/api/user.py", line 44, in create_user
db.session.commit()
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/orm/scoping.py", line 157, in do
return getattr(self.registry(), name)(*args, **kwargs)
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/orm/session.py", line 801, in commit
self.transaction.commit()
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/orm/session.py", line 392, in commit
self._prepare_impl()
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/orm/session.py", line 372, in _prepare_impl
self.session.flush()
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/orm/session.py", line 2019, in flush
self._flush(objects)
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/orm/session.py", line 2137, in _flush
transaction.rollback(_capture_exception=True)
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/util/compat.py", line 186, in reraise
raise value
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/orm/session.py", line 2101, in _flush
flush_context.execute()
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/orm/unitofwork.py", line 373, in execute
rec.execute(self)
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/orm/unitofwork.py", line 532, in execute
uow
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/orm/persistence.py", line 174, in save_obj
mapper, table, insert)
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/orm/persistence.py", line 800, in
_emit_insert_statements
execute(statement, params)
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/engine/base.py", line 914, in execute
return meth(self, multiparams, params)
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/sql/elements.py", line 323, in
_execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/engine/base.py", line 1010, in
_execute_clauseelement
compiled_sql, distilled_params
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/engine/base.py", line 1146, in _execute_context
context)
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/engine/base.py", line 1341, in
_handle_dbapi_exception
exc_info
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/util/compat.py", line 202, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/util/compat.py", line 185, in reraise
raise value.with_traceback(tb)
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/engine/base.py", line 1139, in _execute_context
context)
File "/home/leonardo/.local/lib/python3.5/site-
packages/sqlalchemy/engine/default.py", line 450, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.IntegrityError: (psycopg2.IntegrityError) null value in
column "id" violates not-null constraint
DETAIL: Failing row contains (null, 19, Male
, Basilicata , 2017-10-25
09:40:42.880058).
[SQL: 'INSERT INTO "user" (age, gender, region, date_created) VALUES
(%(age)s, %(gender)s, %(region)s, CURRENT_TIMESTAMP) RETURNING
"user".id'] [parameters: {'region': 'Basilicata', 'gender': 'Male',
'age': 19}]