I'm trying to select specific columns from a table like this:
users = Table('users', metadata, autoload=True)
s = users.select([users.c.email])
results = s.execute()
print results
and I'm getting this error:
> Traceback (most recent call last): File "my_mailer.py", line 35, in
> <module>
> s = users.select([users.c.email]) File "/task/__pips__/sqlalchemy/sql/selectable.py", line 175, in select
> return Select([self], whereclause, **params) File "/task/__pips__/sqlalchemy/sql/selectable.py", line 2082, in __init__
> self._whereclause = _literal_as_text(whereclause) File "/task/__pips__/sqlalchemy/sql/elements.py", line 2745, in
> _literal_as_text
> "SQL expression object or string expected." sqlalchemy.exc.ArgumentError: SQL expression object or string
> expected.
So I tried this:
users = Table('users', metadata, autoload=True)
s = users.select('email')
results = s.execute()
print results
And got this in response:
> Traceback (most recent call last): File "my_mailer.py", line 36, in
> <module>
> results = s.execute() File "/task/__pips__/sqlalchemy/sql/base.py", line 124, in execute
> return e._execute_clauseelement(self, multiparams, params) File "/task/__pips__/sqlalchemy/engine/base.py", line 1605, in
> _execute_clauseelement
> return connection._execute_clauseelement(elem, multiparams, params) File "/task/__pips__/sqlalchemy/engine/base.py", line 761,
> in _execute_clauseelement
> compiled_sql, distilled_params File "/task/__pips__/sqlalchemy/engine/base.py", line 874, in
> _execute_context
> context) File "/task/__pips__/sqlalchemy/engine/base.py", line 1023, in _handle_dbapi_exception
> exc_info File "/task/__pips__/sqlalchemy/util/compat.py", line 185, in raise_from_cause
> reraise(type(exception), exception, tb=exc_tb) File "/task/__pips__/sqlalchemy/engine/base.py", line 867, in
> _execute_context
> context) File "/task/__pips__/sqlalchemy/engine/default.py", line 388, in do_execute
> cursor.execute(statement, parameters) sqlalchemy.exc.ProgrammingError: (ProgrammingError) argument of WHERE
> must be type boolean, not type character varying LINE 3: WHERE email
Sure enough, the first argument here is the 'whereclause', not 'columns' like everywhere else, this is reflected in the documentation:
This argument is not present on the form of select() available on
Table.
Question: how can I select only specific columns using select on the table? And generally, why on earth the columns argument is not available on select on the table? I can't understand why somebody made the decision to make this different than the standard select.
Use general purpose select instead of Table.select:
stmt = select([users.c.email])
result = conn.execute(stmt)
With metadata you can do this:
metadata = MetaData(bind=engine)
tblusers = metadata.tables['users']
tblproducts = metadata.tables['products']
# example 1: only columns you want.
data_users = tblusers.select().with_only_columns([tblusers.c.id, tblusers.c.name]).execute()
# example 2: w/ where & order_by
data_products = tblproducts.select().with_only_columns([tblproducts.c.id, tblproducts.c.price, tblproductos.c.description]).where(tblproducts.c.stock > 0).order_by(tblproducts.c.brand).execute()
Be well...
SqlAlchemy Docs
Related
I have model written in declarative base of SQL Alchemy.
Class Roles(Base):
__tablename__ = "roles"
__table_args__ = (
Index("roles_name", "name", unique=True),
)
id = Column(Integer, primary_key=True, default=get_uuid())
name = Column(String(10), nullable=False)
As you may have noticed I have set the default value of primary key column id to get_uuid().
def get_uuid():
pk = uuid.uuid4().int >> 64
return pk
The above method return UUID as integer of bit size 64 or less. This is because the column id of this table is set to int and spanner can hold up to 64 bit.
So now to insert a row in this table -
>>> role = Roles()
>>> role.name = "Admin"
>>> session.add(role)
>>> session.commit()
This resulted in following exception -
Traceback (most recent call last):
File "/usr/local/lib/python3.10/site-packages/google/api_core/grpc_helpers.py", line 72, in error_remapped_callable
return callable_(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/grpc/_channel.py", line 946, in __call__
return _end_unary_response_blocking(state, call, False, None)
File "/usr/local/lib/python3.10/site-packages/grpc/_channel.py", line 849, in _end_unary_response_blocking
raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
status = StatusCode.FAILED_PRECONDITION
details = "Could not parse 18011687921562567628 as an integer"
debug_error_string = "UNKNOWN:Error received from peer ipv4:172.19.0.3:9010 {grpc_message:"Could not parse 18011687921562567628 as an integer", grpc_status:9, created_time:"2022-11-12T06:48:36.468914625+00:00"}"
>
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_dbapi/cursor.py", line 269, in execute
) = self.connection.run_statement(statement)
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_dbapi/connection.py", line 454, in run_statement
_execute_insert_heterogenous(
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_dbapi/_helpers.py", line 57, in _execute_insert_heterogenous
transaction.execute_update(
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_v1/transaction.py", line 302, in execute_update
response = api.execute_sql(
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_v1/services/spanner/client.py", line 1096, in execute_sql
response = rpc(
File "/usr/local/lib/python3.10/site-packages/google/api_core/gapic_v1/method.py", line 154, in __call__
return wrapped_func(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/google/api_core/retry.py", line 283, in retry_wrapped_func
return retry_target(
File "/usr/local/lib/python3.10/site-packages/google/api_core/retry.py", line 190, in retry_target
return target()
File "/usr/local/lib/python3.10/site-packages/google/api_core/timeout.py", line 99, in func_with_timeout
return func(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/google/api_core/grpc_helpers.py", line 74, in error_remapped_callable
raise exceptions.from_grpc_error(exc) from exc
google.api_core.exceptions.FailedPrecondition: 400 Could not parse 18011687921562567628 as an integer
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1900, in _execute_context
self.dialect.do_execute(
File "/usr/local/lib/python3.10/site-packages/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py", line 1013, in do_execute
cursor.execute(statement, parameters)
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_dbapi/cursor.py", line 70, in wrapper
return function(cursor, *args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_dbapi/cursor.py", line 289, in execute
raise IntegrityError(getattr(e, "details", e)) from e
google.cloud.spanner_dbapi.exceptions.IntegrityError: []
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py", line 1451, in commit
self._transaction.commit(_to_root=self.future)
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py", line 829, in commit
self._prepare_impl()
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py", line 808, in _prepare_impl
self.session.flush()
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py", line 3386, in flush
self._flush(objects)
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py", line 3525, in _flush
with util.safe_reraise():
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__
compat.raise_(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/compat.py", line 208, in raise_
raise exception
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/session.py", line 3486, in _flush
flush_context.execute()
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/unitofwork.py", line 456, in execute
rec.execute(self)
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/unitofwork.py", line 630, in execute
util.preloaded.orm_persistence.save_obj(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/persistence.py", line 245, in save_obj
_emit_insert_statements(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/orm/persistence.py", line 1238, in _emit_insert_statements
result = connection._execute_20(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1705, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/sql/elements.py", line 333, in _execute_on_connection
return connection._execute_clauseelement(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1572, in _execute_clauseelement
ret = self._execute_context(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1943, in _execute_context
self._handle_dbapi_exception(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 2124, in _handle_dbapi_exception
util.raise_(
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/util/compat.py", line 208, in raise_
raise exception
File "/usr/local/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 1900, in _execute_context
self.dialect.do_execute(
File "/usr/local/lib/python3.10/site-packages/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py", line 1013, in do_execute
cursor.execute(statement, parameters)
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_dbapi/cursor.py", line 70, in wrapper
return function(cursor, *args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/google/cloud/spanner_dbapi/cursor.py", line 289, in execute
raise IntegrityError(getattr(e, "details", e)) from e
sqlalchemy.exc.IntegrityError: (google.cloud.spanner_dbapi.exceptions.IntegrityError) []
[SQL: INSERT INTO roles (id, name) VALUES (%s, %s)]
[parameters: [18011687921562567628, 'Admin']]
(Background on this error at: https://sqlalche.me/e/14/gkpj)
What I understood for this is that the spanner is not willing to accept the generated UUID.
status = StatusCode.FAILED_PRECONDITION
details = "Could not parse 18011687921562567628 as an integer"
I have checked the method get_uuid(). It does return int value of but size 64 or less.
The README of this repo suggests creating a table's primary key as Integer and while in inserting a row in the database generate value of primary key in hex. I did exactly the same but it didn't work.
The generated int value is larger than the maximum INT64 value that is allowed in Cloud Spanner:
Max allowed: 9223372036854775807
Your value : 18011687921562567628
See https://cloud.google.com/spanner/docs/reference/standard-sql/data-types#integer_types for more information on the INT64 type.
I'm no Python expert, but my guess is that the int value that you are generating is interpreted as an unsigned int, while the INT64 data type in Cloud Spanner is signed.
EDIT: Add example to get signed value.
My understanding is that you can do the following to get a signed 64-bit integer value from a UUID in Python:
import ctypes
import uuid
ctypes.c_long(uuid.uuid4().int >> 64).value
I'm scraping reddit using praw and storing records in a pandas df. Using a combination of sqlalchemy & pymysql to connect to my AWS RDS db and to_sql to append records to an existing table. All seems to be working fine until I hit the to_sql method. It throws the following errors and i'm not really sure where to go from here. Any help or suggestions would be awesome!
engine = sqlalchemy.create_engine('mysql+pymysql://username:password#database...rds.amazonaws.com:3306/socialdata')
df_comment = pd.DataFrame(comment_table)
df_comment.to_sql(name='reddit_comments', con=engine, index=False, if_exists='append')
Traceback (most recent call last):
File "/Users/ty/Desktop/Python/reddit_scraper.py", line 121, in <module>
df_comment.to_sql(name='reddit_comments', con=engine, index=False, if_exists='append')
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pandas/core/generic.py", line 2605, in to_sql
sql.to_sql(
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pandas/io/sql.py", line 589, in to_sql
pandas_sql.to_sql(
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pandas/io/sql.py", line 1398, in to_sql
table.insert(chunksize, method=method)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pandas/io/sql.py", line 830, in insert
exec_insert(conn, keys, chunk_iter)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pandas/io/sql.py", line 747, in _execute_insert
conn.execute(self.table.insert(), data)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1011, in execute
return meth(self, multiparams, params)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1124, in _execute_clauseelement
ret = self._execute_context(
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1316, in _execute_context
self._handle_dbapi_exception(
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1514, in _handle_dbapi_exception
util.raise_(exc_info[1], with_traceback=exc_info[2])
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 182, in raise_
raise exception
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1256, in _execute_context
self.dialect.do_executemany(
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/sqlalchemy/dialects/mysql/mysqldb.py", line 148, in do_executemany
rowcount = cursor.executemany(statement, parameters)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pymysql/cursors.py", line 188, in executemany
return self._do_execute_many(q_prefix, q_values, q_postfix, args,
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pymysql/cursors.py", line 206, in _do_execute_many
v = values % escape(next(args), conn)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pymysql/cursors.py", line 120, in _escape_args
return {key: conn.literal(val) for (key, val) in args.items()}
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pymysql/cursors.py", line 120, in <dictcomp>
return {key: conn.literal(val) for (key, val) in args.items()}
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pymysql/connections.py", line 469, in literal
return self.escape(obj, self.encoders)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pymysql/connections.py", line 462, in escape
return converters.escape_item(obj, self.charset, mapping=mapping)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pymysql/converters.py", line 27, in escape_item
val = encoder(val, mapping)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pymysql/converters.py", line 123, in escape_unicode
return u"'%s'" % _escape_unicode(value)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pymysql/converters.py", line 78, in _escape_unicode
return value.translate(_escape_table)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/praw/models/reddit/base.py", line 35, in __getattr__
return getattr(self, attribute)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/praw/models/reddit/base.py", line 36, in __getattr__
raise AttributeError(
AttributeError: 'Redditor' object has no attribute 'translate'
One of the columns in your DataFrame contains a custom "Redditor" object which doesn't map to a corresponding SQL datatype. pymysql calls the object's translate function when it isn't something obvious like int float or string
If Redditor is just a wrapper object for a store of usernames and other metadata, then you can do something like remapping that column to the string / number representation of the Redditor object. If it is an object you've defined, you can add a translate() function to the Redditor class's definition to return the appropriate value. For example if Redditor.id contains the value that you want to store in the column :-
class Redditor():
def translate(self):
# Change self.id with the value you care about
return self.id
or in pandas before you save
df[REDDITOR_COLUMN] = df[REDDITOR_COLUMN].apply(lambda x: x.id)
I'm trying to build a query to count occurrences from distinct months in my SQL DB, using annotate and extract for this purpose.
Here is the code used:
Model
class Results(model.Model):
trip = models.BooleanField(default=False)
created_on = models.DateTimeField(default=datetime.now)
Query
from django.db.models.functions import Extract, ExtractYear
from django.db.models import Count
res = Results.objects.annotate(month=ExtractMonth('created_on'))
.values('month').annotate(count=Count('month'))
OR
res = Results.objects.annotate(month=Extract('created_on','month'))
.values('month').annotate(count=Count('month'))
Both return:
Error
Traceback (most recent call last):
File "<console>", line 1, in <module>
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\query.py", line 244, in __repr__
data = list(self[:REPR_OUTPUT_SIZE + 1])
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\query.py", line 268, in __iter__
self._fetch_all()
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\query.py", line 1186, in _fetch_all
self._result_cache = list(self._iterable_class(self))
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\query.py", line 106, in __iter__
for row in compiler.results_iter(chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size):
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\sql\compiler.py", line 1017, in results_iter
results = self.execute_sql(MULTI, chunked_fetch=chunked_fetch, chunk_size=chunk_size)
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\sql\compiler.py", line 1052, in execute_sql
sql, params = self.as_sql()
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\sql\compiler.py", line 449, in as_sql
extra_select, order_by, group_by = self.pre_sql_setup()
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\sql\compiler.py", line 50, in pre_sql_setup
self.setup_query()
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\sql\compiler.py", line 41, in setup_query
self.select, self.klass_info, self.annotation_col_map = self.get_select()
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\sql\compiler.py", line 244, in get_select
sql, params = self.compile(col, select_format=True)
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\sql\compiler.py", line 390, in compile
sql, params = node.as_sql(self, self.connection)
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\aggregates.py", line 76, in as_sql
return super().as_sql(compiler, connection, **extra_context)
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\expressions.py", line 618, in as_sql
data['expressions'] = data['field'] = arg_joiner.join(sql_parts)
TypeError: sequence item 0: expected str instance, tuple found
Can anybody see what is wrong in my approach?
I've followed the documentation for this query.
I'm using Django 2.1.5 and Python 3.7.0 on Windows.
If you see the documentation of ExtractMonth it is postgreSQL only function, If you use a postgreSQL function with MySQL Database, of course its going to give error.
I would suggest using the raw query, or changing the database to postgres.
I am trying to turn a pandas DataFrame into a SQL update statement
and this is my problem part...............
values = tuple(zip(*filtered_data[['site_code']].values.tolist()))[0]
query = """
UPDATE govtracker
SET pending = 'TE'
WHERE site_code = '%s')
"""
connection = engine.connect()
update_db_query = connection.execute(query, values)
I am trying to update a specific column called pending through an Object
and this is my traceback
Traceback (most recent call last):
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\sqlalchemy\engine\base.py", line 1224, in _execute_context
cursor, statement, parameters, context
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\sqlalchemy\engine\default.py", line 547, in do_executemany
cursor.executemany(statement, parameters)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\mysql\connector\cursor.py", line 659, in executemany
self.execute(operation, params)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\mysql\connector\cursor.py", line 551, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\mysql\connector\connection.py", line 490, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\mysql\connector\connection.py", line 395, in _handle_result
raise errors.get_exception(packet)
mysql.connector.errors.ProgrammingError: 1064 (42000): You have an error
in your SQL syntax; check the manual that corresponds to your MariaDB
server version for the right syntax to use near 'LCAIE30089'')' at line 3
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:/Users/DELL/PycharmProjects/MyALLRefProf/MyAutomationFuncs.py",
line 82, in <module>
update_db_query = connection.execute(query, values)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\sqlalchemy\engine\base.py", line 982, in execute
return self._execute_text(object_, multiparams, params)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\sqlalchemy\engine\base.py", line 1155, in _execute_text
parameters,
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\sqlalchemy\engine\base.py", line 1248, in _execute_context
e, statement, parameters, cursor, context
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\sqlalchemy\engine\base.py", line 1466, in _handle_dbapi_exception
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\sqlalchemy\util\compat.py", line 383, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\sqlalchemy\util\compat.py", line 128, in reraise
raise value.with_traceback(tb)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\sqlalchemy\engine\base.py", line 1224, in _execute_context
cursor, statement, parameters, context
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\sqlalchemy\engine\default.py", line 547, in do_executemany
cursor.executemany(statement, parameters)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\mysql\connector\cursor.py", line 659, in executemany
self.execute(operation, params)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\mysql\connector\cursor.py", line 551, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\mysql\connector\connection.py", line 490, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-
packages\mysql\connector\connection.py", line 395, in _handle_result
raise errors.get_exception(packet)
sqlalchemy.exc.ProgrammingError: (mysql.connector.errors.ProgrammingError)
1064 (42000): You have an error in your SQL syntax; check the manual that
corresponds to your MariaDB server version for the right syntax to use
near 'LCAIE30089'')' at line 3
[SQL:
UPDATE govtracker
SET pending = 'TE'
WHERE site_code = '%s')
]
[parameters: [('LCAIE30089',), ('LSINI16321',)]]
(Background on this error at: http://sqlalche.me/e/f405)
and the error in this line..........................
update_db_query = connection.execute(query, values)
and this is my whole code............
from sqlalchemy import create_engine
import pandas as pd
import os
import csv
import MySQLdb
from sqlalchemy import types, create_engine
# MySQL Connection
MYSQL_USER = 'root'
MYSQL_PASSWORD = 'Mharooney'
MYSQL_HOST_IP = '127.0.0.1'
MYSQL_PORT = 3306
MYSQL_DATABASE = 'mydb'
engine = create_engine
('mysql+mysqlconnector://'+MYSQL_USER+':'+MYSQL_PASSWORD+'#'+MYSQ
L_HOST_IP+':'+str(MYSQL_PORT)+'/'+MYSQL_DATABASE, echo=False)
mydir = (os.getcwd()).replace('\\', '/') + '/'
all_data = pd.read_sql('SELECT * FROM govtracker', engine)
# .drop(['#'], axis=1)
myOutlook_inBox = pd.read_csv(r'' + mydir + 'test.CSV', usecols=['Subject',
'Body', 'From: (Name)', 'To: (Name)'],
encoding='latin-1')
myOutlook_inBox.columns = myOutlook_inBox.columns.str.replace(' ', '')
#this object extract 5 chars and 5 numbers from specific column in csv
replaced_sbj_value = myOutlook_inBox['Subject']
.str.extract(pat='(L(?:DEL|CAI|SIN).\d{5})').dropna()
#this columns I want to filter in database
myOutlook_inBox["Subject"] = replaced_sbj_value
# this conditions filters and get and dublicate repeated data from outlook
exported file
# Condition 1: any mail from mowafy to te
frm_mwfy_to_te = myOutlook_inBox.loc[myOutlook_inBox['From:
(Name)'].str.contains("mowafy", na=False)
& myOutlook_inBox['To:
(Name)'].str.contains("te", na=False)].drop_duplicates(
keep=False)
#Condition 2: any mail from pp to mowafy
frm_pp_to_mwfy = \
myOutlook_inBox.loc[myOutlook_inBox['From:(Name)'].str.contains("Amr Mohamed
Abuella barakat", na=False)] \
| myOutlook_inBox.loc[myOutlook_inBox['From:(Name)'].str.contains("Reham",
na=False)] \
| myOutlook_inBox.loc[myOutlook_inBox['From:(Name)'].str.contains("Hany",
na=False)] \
| myOutlook_inBox.loc[myOutlook_inBox['From:(Name)'].str.contains("Ossama
Monir", na=False)] \
& myOutlook_inBox.loc[myOutlook_inBox['To:(Name)'].str.contains("mowafy",
na=False)].drop_duplicates(keep=False)
#Condition 3: any mail from TE to mowafy
frm_te_to_mwfy = \
myOutlook_inBox.loc[myOutlook_inBox['From:(Name)'].str.contains("te",
na=False)] & \
myOutlook_inBox.loc[myOutlook_inBox['To:(Name)'].str.contains("mowafy",
na=False)].drop_duplicates(keep=False)
frm_mwfy_to_te.Subject
filtered_data = all_data
.loc[all_data.site_code.str.contains('|'.join(frm_mwfy_to_te.Subject))]
print(myOutlook_inBox)
all_data.replace('\n', '', regex=True)
df = all_data.where((pd.notnull(all_data)), None)
print(df)
print("Success")
print(frm_mwfy_to_te.Subject)
print(filtered_data.site_code)
values = tuple(filtered_data[['site_code']].values)
query = """
UPDATE govtracker
SET pending = 'TE'
WHERE site_code in %s)
"""
connection = engine.connect()
update_db_query = connection.execute(query, values)
So how can I solve this problem..................................
The problem is in the values your are passing, since the sql's where condition expects a well formed tuple like
WHERE site_code in ('LCAIE30089', 'LSINI16321')
So the solution is:
update your values assignment to
values = tuple(zip(*filtered_data[['site_code']].values.tolist()]))[0]
Next instead of WHERE site_code = '%s' use WHERE site_code in %s
This should work for you.
Let me know if you are stuck in any part.
I refer to Neo4j create nodes and relationships from pandas dataframe with py2neo and code with belows.
But I got py2neo.database.status.CypherSyntaxError. Please verify below is right approach and let me know what is correct Cypher syntax.
My Code:
for line in reader:
print(line['word'], line['similar_word'], line['probability'] )
w1 = Node("Word", name = line['word'])
w2 = Node("Word", name = line['similar_word'])
graph.merge(w1|w2)
graph.run('''
MATCH (a:Paper),(b:Word)
WHERE (a.name = 'Paper10' AND b.name = {$word1})
CREATE (a)<-[o:ORIGINAL]-(b)
''', parameters = {'word1':line['word']})
py2neo.database.status.CypherSyntaxError:
Traceback (most recent call last): File "test.py", line 24, in <module>
''', parameters = {'word1':line['word']}) File "/root/miniconda3/lib/python3.6/site-packages/py2neo/database/__init__.py", line 731, in run
return self.begin(autocommit=True).run(statement, parameters, **kwparameters) File "/root/miniconda3/lib/python3.6/site-packages/py2neo/database/__init__.py", line 1277, in run
self.finish() File "/root/miniconda3/lib/python3.6/site-packages/py2neo/database/__init__.py", line 1296, in finish
self._sync() File "/root/miniconda3/lib/python3.6/site-packages/py2neo/database/__init__.py", line 1286, in _sync
connection.fetch() File "/root/miniconda3/lib/python3.6/site-packages/py2neo/packages/neo4j/v1/bolt.py", line 344, in fetch
handler(*fields) File "/root/miniconda3/lib/python3.6/site-packages/py2neo/database/__init__.py", line 961, in on_failure
raise GraphError.hydrate(metadata) py2neo.database.status.CypherSyntaxError: Invalid input '$': expected whitespace, an identifier, UnsignedDecimalInteger, a property key name or '}' (line 3, column 53 (offset: 78)) " WHERE (a.name = 'Paper10' AND b.name = {$word1})"
For referencing a parameter, you can either use the $ syntax, or the {}, but not both. Try this with just $word1.