I'm trying to write to an existing database consisting of multiple tables of the following form
total_usage_<application>:
id
version
date
where <application> runs over a number of strings, say "appl1", "appl2" etc. Now I would like to use SQLAlchemy to create a single class like
class DBEntry:
id = ''
application = ''
version = ''
date = ''
such that an instance foo of DBEntry gets mapped to the table "total_usage_" + foo.application. How can this be achieved?
OK, please take a look at example below, which is self-contained and might show you one way this can be done. It assumes that you know the app_name when you start your program, as well as it assumes that table names follow some naming convention, which obviously you can adapt to your needs or have it completely manually configured by overriding __tablename__ in each mapped class.
But the main idea is that the configuration is wrapped into a function (which can be done also using module import with pre-defined constant beforehand).
from sqlalchemy import create_engine, Column, Integer, String, ForeignKey
from sqlalchemy.orm import relationship, scoped_session, sessionmaker
def camel_to_under(name):
import re
s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
def configure_database(app_name):
""" #return: dictionary with all the classes mapped to proper tables for
specific application. """
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.ext.declarative import declarative_base
class Base(object):
# docs: http://docs.sqlalchemy.org/en/rel_0_9/orm/extensions/declarative.html#augmenting-the-base
#declared_attr
def __tablename__(cls):
return camel_to_under(cls.__name__) + "_" + app_name
def __repr__(self):
attrs = class_mapper(self.__class__).column_attrs # only columns
# attrs = class_mapper(self.__class__).attrs # show also relationships
return u"{}({})".format(self.__class__.__name__,
', '.join('%s=%r' % (k.key, getattr(self, k.key))
for k in sorted(attrs)
)
)
Base = declarative_base(cls=Base)
class Class1(Base):
id = Column(Integer, primary_key=True)
name = Column(String)
class Class1Child(Base):
id = Column(Integer, primary_key=True)
parent_id = Column(Integer, ForeignKey(Class1.id))
name = Column(String)
# relationships
parent = relationship(Class1, backref="children")
# return locals()
return {
"app_name": app_name,
"Base": Base,
"Class1": Class1,
"Class1Child": Class1Child,
}
def _test():
""" Little test for the app. """
engine = create_engine(u'sqlite:///:memory:', echo=True)
session = scoped_session(sessionmaker(bind=engine))
app_name = "app1"
x = configure_database(app_name)
# assign real names
app_name = x["app_name"]
Base = x["Base"]
Class1 = x["Class1"]
Class1Child = x["Class1Child"]
# create DB model (not required in production)
Base.metadata.create_all(engine)
# test data
cc = Class1Child(name="child-11")
c1 = Class1(name="some instance", children=[cc])
session.add(c1)
session.commit()
_test()
Related
I'm trying to do some path-checking logic on some values in a database. I'm having trouble implementing the class level expression for the hybrid method.
Here is some stripped down code:
from sqlalchemy import Column, Integer, String, create_engine, func, select
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session, aliased, sessionmaker
from sqlalchemy.ext.hybrid import hybrid_property, hybrid_method
from pathlib import Path
import sqlalchemy as sa
dbEngine = create_engine(
"sqlite:///.sql_app.db", connect_args={"check_same_thread": False}
)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=dbEngine)
Base = declarative_base()
class Folder(Base):
__tablename__ = "folder"
id = Column(Integer, primary_key=True)
value = Column(Integer, nullable=False)
fpath = Column(String, index=True)
#hybrid_method
def test_path(self, fpath):
a = Path(fpath)
b = Path(self.fpath)
return a in [b] + [p for p in b.parents]
#test_path.expression
def test_path(cls, fpath):
a = func.Path(fpath)
b = func.Path(cls.fpath)
# return a in [b] + [p for p in b.parents]
# What to do here
return (
select([func.Path(Folder.fpath)]).
label("fpath_in_folder")
)
#sa.event.listens_for(sa.engine.Engine, "connect")
def sqlite_engine_connect(dbapi_conn, connection_record) -> None:
dbapi_conn.create_function("Path", 1, Path)
def db_create_row(db: Session, value: int, fpath: str):
folder = Folder(value=value, fpath=fpath)
db.add(folder)
db.commit()
db.refresh(folder)
# Determine if the supplied path is a sub path of any records in the table
def db_query_row(db: Session, fpath: str):
records = db.query(Folder).filter(Folder.test_path(fpath) == True)
return records
if __name__ == "__main__":
Base.metadata.create_all(bind=dbEngine)
db = SessionLocal()
db_create_row(db, 5, "/folder 1/folder 2/file.ext")
records = db_query_row(db, "/folder 1")
print(records.count())
db.close()
I'm getting the error: Neither 'Function' object nor 'Comparator' object has an attribute 'parents'
So I have to create a SQL expression for this function, but I'm not sure how, or if it's even possible with accessing the parents property on the Path object.
SQLite can't handle Python instances, and SQLAlchemy Function support doesn't cover instance methods either.
First of all, you can't use Path() objects as a SQLite function, as explained in the sqlite3.Connection.create_function documentation:
The callable must return a type natively supported by SQLite.
The natively supported types are None, float, int, str or bytes values.
The error you see comes from your attempt to use b.parents in your expression; b is the Function object, the func.Path(...) call, and SQLAlchemy expects a function to return a SQL type, not a Path() object.
Instead of trying to shoehorn Path() objects into SQLite, you'll need to find another way to test if a path is a parent folder. You could use Column.startswith() here, provided you first ensure the paths don't end with /, by using the standard RTRIM() function:
#test_path.expression
def test_path(cls, fpath):
a = func.rtrim(fpath, "/")
b = func.rtrim(cls.fpath, "/")
return a == b or b.startswith(a + "/")
This will produce a SQL expression like this:
(
rtrim(?, '/') == rtrim(folder.fpath, '/')
OR rtrim(folder.fpath, '/') LIKE rtrim(?, '/') || '/' || '%'
)
I wrote a habit tracker app and used SQLAlchemy to store the data in an SQLite3 database. Now I'm writing the unit tests using Pytest for all the functions I wrote. Besides functions returning values, there are functions that create entries in the database by creating objects. Here's my object-relational mapper setup and the two main classes:
from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, Date
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
# Setting up SQLAlchemy to connect to the local SQLite3 database
Base = declarative_base()
engine = create_engine('sqlite:///:main:', echo=True)
Base.metadata.create_all(bind=engine)
Session = sessionmaker(bind=engine)
session = Session()
class Habit(Base):
__tablename__ = 'habit'
habit_id = Column('habit_id', Integer, primary_key=True)
name = Column('name', String, unique=True)
periodicity = Column('periodicity', String)
start_date = Column('start_date', Date)
class HabitEvent(Base):
__tablename__ = 'habit_event'
event_id = Column('event_id', Integer, primary_key=True)
date = Column('date', Date)
habit_id = Column('fk_habit_id', Integer, ForeignKey(Habit.habit_id))
One of the creating functions is the following:
def add_habit(name, periodicity):
if str(periodicity) not in ['d', 'w']:
print('Wrong periodicity. \nUse d for daily or w for weekly.')
else:
h = Habit()
h.name = str(name)
if str(periodicity) == 'd':
h.periodicity = 'Daily'
if str(periodicity) == 'w':
h.periodicity = 'Weekly'
h.start_date = datetime.date.today()
session.add(h)
session.commit()
print('Habit added.')
Here's my question: Since this functions doesn't return a value which can be matched with an expected result, I don't know how to test if the object was created. The same problem occurs to me, when I want to check if all objects were deleted using the following function:
def delete_habit(habitID):
id_list = []
id_query = session.query(Habit).all()
for i in id_query:
id_list.append(i.habit_id)
if habitID in id_list:
delete_id = int(habitID)
session.query(HabitEvent).filter(
HabitEvent.habit_id == delete_id).delete()
session.query(Habit).filter(Habit.habit_id == delete_id).delete()
session.commit()
print('Habit deleted.')
else:
print('Non existing Habit ID.')
If I understand correctly, you can utilize the get_habits function as part of the test for add_habit.
def test_add_habit():
name = 'test_add_habit'
periodicity = 'd'
add_habit(name, periodicity)
# not sure of the input or output from get_habits, but possibly:
results = get_habits(name)
assert name in results['name']
I have a simple polling script that polls entries based on new ID's in a MSSQL table. I'm using SQLAlchemy's ORM to create a table class and then query that table. I want to be able to add more tables "dynamically" without coding it directly into the method.
My polling function:
def poll_db():
query = db.query(
Transactions.ID).order_by(Transactions.ID.desc()).limit(1)
# Continually poll for new images to classify
max_id_query = query
last_max_id = max_id_query.scalar()
while True:
max_id = max_id_query.scalar()
if max_id > last_max_id:
print(
f"New row(s) found. "
f"Processing ids {last_max_id + 1} through {max_id}"
)
# Insert ML model
id_query = db.query(Transactions).filter(
Transactions.ID > last_max_id)
df_from_query = pd.read_sql_query(
id_query.statement, db.bind, index_col='ID')
print(f"New query was made")
last_max_id = max_id
time.sleep(5)
My table model:
import sqlalchemy as db
from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, Text
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import defer, relationship, query
from database import SessionLocal, engine
insp = db.inspect(engine)
db_list = insp.get_schema_names()
Base = declarative_base(cls=BaseModel)
class Transactions(Base):
__tablename__ = 'simulation_data'
sender_account = db.Column('sender_account', db.BigInteger)
recipient_account = db.Column('recipient_account', db.String)
sender_name = db.Column('sender_name', db.String)
recipient_name = db.Column('recipient_name', db.String)
date = db.Column('date', db.DateTime)
text = db.Column('text', db.String)
amount = db.Column('amount', db.Float)
currency = db.Column('currency', db.String)
transaction_type = db.Column('transaction_type', db.String)
fraud = db.Column('fraud', db.BigInteger)
swift_bic = db.Column('swift_bic', db.String)
recipient_country = db.Column('recipient_country', db.String)
internal_external = db.Column('internal_external', db.String)
ID = Column('ID', db.BigInteger, primary_key=True)
QUESTION
How can I pass the table class name "dynamically" in the likes of poll_db(tablename), where tablename='Transactions', and instead of writing similar queries for multiple tables, such as:
query = db.query(Transactions.ID).order_by(Transactions.ID.desc()).limit(1)
query2 = db.query(Transactions2.ID).order_by(Transactions2.ID.desc()).limit(1)
query3 = db.query(Transactions3.ID).order_by(Transactions3.ID.desc()).limit(1)
The tables will have identical structure, but different data.
I can't give you a full example right now (will edit later) but here's one hacky way to do it (the documentation will probably be a better place to check):
def dynamic_table(tablename):
for class_name, cls in Base._decl_class_registry.items():
if cls.__tablename__ == tablename:
return cls
Transactions2 = dynamic_table("simulation_data")
assert Transactions2 is Transactions
The returned class is the model you want. Keep in mind that Base can only access the tables that have been subclassed already so if you have them in other modules you need to import them first so they are registered as Base's subclasses.
For selecting columns, something like this should work:
def dynamic_table_with_columns(tablename, *columns):
cls = dynamic_table(tablename)
subset = []
for col_name in columns:
column = getattr(cls, col_name)
if column:
subset.append(column)
# in case no columns were given
if not subset:
return db.query(cls)
return db.query(*subset)
My problem is the following:
I have the two models Entry and Tag linked by a many-to-many relationship in SQLAlchemy. Now I want to delete every Tag that doesn't have any corresponding Entry after an Entry is deleted.
Example to illustrate what I want:
Entry 1 with tags python, java
Entry 2 with tags python, c++
With these two entries the database contains the tags python, java, and c++. If I now delete Entry 2 I want SQLAlchemy to automatically delete the c++ tag from the database. Is it possible to define this behavior in the Entry model itself or is there an even more elegant way?
Thanks.
this question was asked awhile back here: Setting delete-orphan on SQLAlchemy relationship causes AssertionError: This AttributeImpl is not configured to track parents
This is the "many-to-many orphan" problem. jadkik94 is close in that you should use events to catch this, but I try to recommend against using the Session inside of mapper events, though it works in this case.
Below, I take the answer verbatim from the other SO question, and replace the word "Role" with "Entry":
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import event
from sqlalchemy.orm import attributes
Base= declarative_base()
tagging = Table('tagging',Base.metadata,
Column('tag_id', Integer, ForeignKey('tag.id', ondelete='cascade'), primary_key=True),
Column('entry_id', Integer, ForeignKey('entry.id', ondelete='cascade'), primary_key=True)
)
class Tag(Base):
__tablename__ = 'tag'
id = Column(Integer, primary_key=True)
name = Column(String(100), unique=True, nullable=False)
def __init__(self, name=None):
self.name = name
class Entry(Base):
__tablename__ = 'entry'
id = Column(Integer, primary_key=True)
tag_names = association_proxy('tags', 'name')
tags = relationship('Tag',
secondary=tagging,
backref='entries')
#event.listens_for(Session, 'after_flush')
def delete_tag_orphans(session, ctx):
# optional: look through Session state to see if we want
# to emit a DELETE for orphan Tags
flag = False
for instance in session.dirty:
if isinstance(instance, Entry) and \
attributes.get_history(instance, 'tags').deleted:
flag = True
break
for instance in session.deleted:
if isinstance(instance, Entry):
flag = True
break
# emit a DELETE for all orphan Tags. This is safe to emit
# regardless of "flag", if a less verbose approach is
# desired.
if flag:
session.query(Tag).\
filter(~Tag.entries.any()).\
delete(synchronize_session=False)
e = create_engine("sqlite://", echo=True)
Base.metadata.create_all(e)
s = Session(e)
r1 = Entry()
r2 = Entry()
r3 = Entry()
t1, t2, t3, t4 = Tag("t1"), Tag("t2"), Tag("t3"), Tag("t4")
r1.tags.extend([t1, t2])
r2.tags.extend([t2, t3])
r3.tags.extend([t4])
s.add_all([r1, r2, r3])
assert s.query(Tag).count() == 4
r2.tags.remove(t2)
assert s.query(Tag).count() == 4
r1.tags.remove(t2)
assert s.query(Tag).count() == 3
r1.tags.remove(t1)
assert s.query(Tag).count() == 2
two almost identical SO questions qualifies this as something to have on hand so I've added it to the wiki at http://www.sqlalchemy.org/trac/wiki/UsageRecipes/ManyToManyOrphan.
I will let code speak for me:
from sqlalchemy import create_engine, exc, event
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import func, Table, Column, Integer, String, Float, Boolean, MetaData, ForeignKey
from sqlalchemy.orm import relationship, backref
# Connection
engine = create_engine('sqlite:///', echo=True)
Base = declarative_base(bind=engine)
Session = sessionmaker(bind=engine)
# Models
entry_tag_link = Table('entry_tag', Base.metadata,
Column('entry_id', Integer, ForeignKey('entries.id')),
Column('tag_id', Integer, ForeignKey('tags.id'))
)
class Entry(Base):
__tablename__ = 'entries'
id = Column(Integer, primary_key=True)
name = Column(String(255), nullable=False, default='')
tags = relationship("Tag", secondary=entry_tag_link, backref="entries")
def __repr__(self):
return '<Entry %s>' % (self.name,)
class Tag(Base):
__tablename__ = 'tags'
id = Column(Integer, primary_key=True)
name = Column(String(255), nullable=False)
def __repr__(self):
return '<Tag %s>' % (self.name,)
# Delete listener
def delete_listener(mapper, connection, target):
print "---- DELETING %s ----" % (target,)
print '-' * 20
for t in target.tags:
if len(t.entries) == 0:
print ' ' * 5, t, 'is to be deleted'
session.delete(t)
print '-' * 20
event.listen(Entry, 'before_delete', delete_listener)
# Utility functions
def dump(session):
entries = session.query(Entry).all()
tags = session.query(Tag).all()
print '*' * 20
print 'Entries', entries
print 'Tags', tags
print '*' * 20
Base.metadata.create_all()
session = Session()
t1, t2, t3 = Tag(name='python'), Tag(name='java'), Tag(name='c++')
e1, e2 = Entry(name='Entry 1', tags=[t1, t2]), Entry(name='Entry 2', tags=[t1, t3])
session.add_all([e1,e2])
session.commit()
dump(session)
raw_input("---- Press return to delete the second entry and see the result ----")
session.delete(e2)
session.commit()
dump(session)
This code above uses the after_delete event of the SQLAlchemy ORM events. This line does the magic:
event.listen(Entry, 'before_delete', delete_listener)
This says to listen to all deletes to an Entry item, and call our listener which will do what we want. However, the docs do not recommend changing the session inside the events (see the warning in the link I added). But as far as I can see, it works, so it's up to you to see if this works for you.
table:
id(integer primary key)
data(blob)
I use mysql and sqlalchemy.
To insert data I use:
o = Demo()
o.data = mydata
session.add(o)
session.commit()
I would like to insert to table like that:
INSERT INTO table(data) VALUES(COMPRESS(mydata))
How can I do this using sqlalchemy?
you can assign a SQL function to the attribute:
from sqlalchemy import func
object.data = func.compress(mydata)
session.add(object)
session.commit()
Here's an example using a more DB-agnostic lower() function:
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.ext.declarative import declarative_base
Base= declarative_base()
class A(Base):
__tablename__ = "a"
id = Column(Integer, primary_key=True)
data = Column(String)
e = create_engine('sqlite://', echo=True)
Base.metadata.create_all(e)
s = Session(e)
a1 = A()
a1.data = func.lower("SomeData")
s.add(a1)
s.commit()
assert a1.data == "somedata"
you can make it automatic with #validates:
from sqlalchemy.orm import validates
class MyClass(Base):
# ...
data = Column(BLOB)
#validates("data")
def _set_data(self, key, value):
return func.compress(value)
if you want it readable in python before the flush, you'd need to memoize it locally and use a descriptor to access it.