I'm trying to do some path-checking logic on some values in a database. I'm having trouble implementing the class level expression for the hybrid method.
Here is some stripped down code:
from sqlalchemy import Column, Integer, String, create_engine, func, select
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session, aliased, sessionmaker
from sqlalchemy.ext.hybrid import hybrid_property, hybrid_method
from pathlib import Path
import sqlalchemy as sa
dbEngine = create_engine(
"sqlite:///.sql_app.db", connect_args={"check_same_thread": False}
)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=dbEngine)
Base = declarative_base()
class Folder(Base):
__tablename__ = "folder"
id = Column(Integer, primary_key=True)
value = Column(Integer, nullable=False)
fpath = Column(String, index=True)
#hybrid_method
def test_path(self, fpath):
a = Path(fpath)
b = Path(self.fpath)
return a in [b] + [p for p in b.parents]
#test_path.expression
def test_path(cls, fpath):
a = func.Path(fpath)
b = func.Path(cls.fpath)
# return a in [b] + [p for p in b.parents]
# What to do here
return (
select([func.Path(Folder.fpath)]).
label("fpath_in_folder")
)
#sa.event.listens_for(sa.engine.Engine, "connect")
def sqlite_engine_connect(dbapi_conn, connection_record) -> None:
dbapi_conn.create_function("Path", 1, Path)
def db_create_row(db: Session, value: int, fpath: str):
folder = Folder(value=value, fpath=fpath)
db.add(folder)
db.commit()
db.refresh(folder)
# Determine if the supplied path is a sub path of any records in the table
def db_query_row(db: Session, fpath: str):
records = db.query(Folder).filter(Folder.test_path(fpath) == True)
return records
if __name__ == "__main__":
Base.metadata.create_all(bind=dbEngine)
db = SessionLocal()
db_create_row(db, 5, "/folder 1/folder 2/file.ext")
records = db_query_row(db, "/folder 1")
print(records.count())
db.close()
I'm getting the error: Neither 'Function' object nor 'Comparator' object has an attribute 'parents'
So I have to create a SQL expression for this function, but I'm not sure how, or if it's even possible with accessing the parents property on the Path object.
SQLite can't handle Python instances, and SQLAlchemy Function support doesn't cover instance methods either.
First of all, you can't use Path() objects as a SQLite function, as explained in the sqlite3.Connection.create_function documentation:
The callable must return a type natively supported by SQLite.
The natively supported types are None, float, int, str or bytes values.
The error you see comes from your attempt to use b.parents in your expression; b is the Function object, the func.Path(...) call, and SQLAlchemy expects a function to return a SQL type, not a Path() object.
Instead of trying to shoehorn Path() objects into SQLite, you'll need to find another way to test if a path is a parent folder. You could use Column.startswith() here, provided you first ensure the paths don't end with /, by using the standard RTRIM() function:
#test_path.expression
def test_path(cls, fpath):
a = func.rtrim(fpath, "/")
b = func.rtrim(cls.fpath, "/")
return a == b or b.startswith(a + "/")
This will produce a SQL expression like this:
(
rtrim(?, '/') == rtrim(folder.fpath, '/')
OR rtrim(folder.fpath, '/') LIKE rtrim(?, '/') || '/' || '%'
)
Related
I wrote a habit tracker app and used SQLAlchemy to store the data in an SQLite3 database. Now I'm writing the unit tests using Pytest for all the functions I wrote. Besides functions returning values, there are functions that create entries in the database by creating objects. Here's my object-relational mapper setup and the two main classes:
from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, Date
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
# Setting up SQLAlchemy to connect to the local SQLite3 database
Base = declarative_base()
engine = create_engine('sqlite:///:main:', echo=True)
Base.metadata.create_all(bind=engine)
Session = sessionmaker(bind=engine)
session = Session()
class Habit(Base):
__tablename__ = 'habit'
habit_id = Column('habit_id', Integer, primary_key=True)
name = Column('name', String, unique=True)
periodicity = Column('periodicity', String)
start_date = Column('start_date', Date)
class HabitEvent(Base):
__tablename__ = 'habit_event'
event_id = Column('event_id', Integer, primary_key=True)
date = Column('date', Date)
habit_id = Column('fk_habit_id', Integer, ForeignKey(Habit.habit_id))
One of the creating functions is the following:
def add_habit(name, periodicity):
if str(periodicity) not in ['d', 'w']:
print('Wrong periodicity. \nUse d for daily or w for weekly.')
else:
h = Habit()
h.name = str(name)
if str(periodicity) == 'd':
h.periodicity = 'Daily'
if str(periodicity) == 'w':
h.periodicity = 'Weekly'
h.start_date = datetime.date.today()
session.add(h)
session.commit()
print('Habit added.')
Here's my question: Since this functions doesn't return a value which can be matched with an expected result, I don't know how to test if the object was created. The same problem occurs to me, when I want to check if all objects were deleted using the following function:
def delete_habit(habitID):
id_list = []
id_query = session.query(Habit).all()
for i in id_query:
id_list.append(i.habit_id)
if habitID in id_list:
delete_id = int(habitID)
session.query(HabitEvent).filter(
HabitEvent.habit_id == delete_id).delete()
session.query(Habit).filter(Habit.habit_id == delete_id).delete()
session.commit()
print('Habit deleted.')
else:
print('Non existing Habit ID.')
If I understand correctly, you can utilize the get_habits function as part of the test for add_habit.
def test_add_habit():
name = 'test_add_habit'
periodicity = 'd'
add_habit(name, periodicity)
# not sure of the input or output from get_habits, but possibly:
results = get_habits(name)
assert name in results['name']
I have a table with JSON stored in a text column:
import json
from sqlalchemy import create_engine, Column, text, Integer, TEXT, TypeDecorator
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
engine = create_engine('sqlite:///:memory:')
engine.execute("create table t (t_id int not null primary key, attrs text not null)")
engine.execute("insert into t values (1, '{\"a\": 1, \"b\": 2}')")
Session = sessionmaker(bind=engine)
I defined a mapping to this table in SQLAlchemy, using the custom type defined in the SQLAlchemy docs under "Marshal JSON Strings":
Base = declarative_base()
# http://docs.sqlalchemy.org/en/rel_1_1/core/custom_types.html#marshal-json-strings
class JSONEncodedDict(TypeDecorator):
impl = TEXT
def process_bind_param(self, value, dialect):
if value is not None:
value = json.dumps(value)
return value
def process_result_value(self, value, dialect):
if value is not None:
value = json.loads(value)
return value
class T(Base):
__tablename__ = 't'
t_id = Column(Integer, primary_key=True)
attrs = Column(JSONEncodedDict)
If I query all Ts, attrs gets deserialized from JSON:
session = Session()
t = session.query(T).first()
assert type(t.attrs) == dict, repr(t.attrs)
But if I use a textual query / raw SQL, it isn't deserialized:
session = Session()
t = session.query(T).from_statement(text('select * from t')).first()
assert type(t.attrs) == dict, repr(t.attrs) # AssertionError: u'{"a": 1, "b": 2}'
How do I make SQLAlchemy deserialize the attrs column when querying with raw SQL?
The behavior is the same with other databases (MySQL, Postgres). The database I am using (MySQL 5.5) does not support native JSON types, so changing the column type is not an option.
You can tell TextClause (produced by text()) the column types using .columns():
from sqlalchemy import inspect
session = Session()
stmt = text('select * from t').columns(*inspect(T).columns)
t = session.query(T).from_statement(stmt).first()
assert type(t.attrs) == dict, repr(t.attrs)
Or, for SQLAlchemy<0.9, use the typemap argument:
from sqlalchemy import inspect
session = Session()
typemap = {c.name: c.type for c in inspect(T).columns}
stmt = text('select * from t', typemap=typemap)
t = session.query(T).from_statement(stmt).first()
assert type(t.attrs) == dict, repr(t.attrs)
I'm trying to write to an existing database consisting of multiple tables of the following form
total_usage_<application>:
id
version
date
where <application> runs over a number of strings, say "appl1", "appl2" etc. Now I would like to use SQLAlchemy to create a single class like
class DBEntry:
id = ''
application = ''
version = ''
date = ''
such that an instance foo of DBEntry gets mapped to the table "total_usage_" + foo.application. How can this be achieved?
OK, please take a look at example below, which is self-contained and might show you one way this can be done. It assumes that you know the app_name when you start your program, as well as it assumes that table names follow some naming convention, which obviously you can adapt to your needs or have it completely manually configured by overriding __tablename__ in each mapped class.
But the main idea is that the configuration is wrapped into a function (which can be done also using module import with pre-defined constant beforehand).
from sqlalchemy import create_engine, Column, Integer, String, ForeignKey
from sqlalchemy.orm import relationship, scoped_session, sessionmaker
def camel_to_under(name):
import re
s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
def configure_database(app_name):
""" #return: dictionary with all the classes mapped to proper tables for
specific application. """
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.ext.declarative import declarative_base
class Base(object):
# docs: http://docs.sqlalchemy.org/en/rel_0_9/orm/extensions/declarative.html#augmenting-the-base
#declared_attr
def __tablename__(cls):
return camel_to_under(cls.__name__) + "_" + app_name
def __repr__(self):
attrs = class_mapper(self.__class__).column_attrs # only columns
# attrs = class_mapper(self.__class__).attrs # show also relationships
return u"{}({})".format(self.__class__.__name__,
', '.join('%s=%r' % (k.key, getattr(self, k.key))
for k in sorted(attrs)
)
)
Base = declarative_base(cls=Base)
class Class1(Base):
id = Column(Integer, primary_key=True)
name = Column(String)
class Class1Child(Base):
id = Column(Integer, primary_key=True)
parent_id = Column(Integer, ForeignKey(Class1.id))
name = Column(String)
# relationships
parent = relationship(Class1, backref="children")
# return locals()
return {
"app_name": app_name,
"Base": Base,
"Class1": Class1,
"Class1Child": Class1Child,
}
def _test():
""" Little test for the app. """
engine = create_engine(u'sqlite:///:memory:', echo=True)
session = scoped_session(sessionmaker(bind=engine))
app_name = "app1"
x = configure_database(app_name)
# assign real names
app_name = x["app_name"]
Base = x["Base"]
Class1 = x["Class1"]
Class1Child = x["Class1Child"]
# create DB model (not required in production)
Base.metadata.create_all(engine)
# test data
cc = Class1Child(name="child-11")
c1 = Class1(name="some instance", children=[cc])
session.add(c1)
session.commit()
_test()
I am trying to move an object from one database into another. The mappings are the same but the tables are different. This is a merging tool where data from an old database need to be imported into a new one. Still, I think I am missing something fundamental about SQLAlchemy here. What is it?
from sqlalchemy import Column, Float, String, Enum
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import orm
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
DeclarativeBase = declarative_base()
class Datum (DeclarativeBase):
__tablename__ = "xUnitTestData"
Key = Column(String, primary_key=True)
Value = Column(Float)
def __init__ (self, k, v):
self.Key = k
self.Value = v
src_engine = create_engine('sqlite:///:memory:', echo=False)
dst_engine = create_engine('sqlite:///:memory:', echo=False)
DeclarativeBase.metadata.create_all(src_engine)
DeclarativeBase.metadata.create_all(dst_engine)
SessionSRC = sessionmaker(bind=src_engine)
SessionDST = sessionmaker(bind=dst_engine)
item = Datum('eek', 666)
session1 = SessionSRC()
session1.add(item)
session1.commit()
session1.close()
session2 = SessionDST()
session2.add(item)
session2.commit()
print item in session2 # >>> True
print session2.query(Datum).all() # >>> []
session2.close()
I'm not really aware about what happens under the hood, but in the ORM pattern an object matches to a particular row in a particular table. If you try to add the same object to two different tables in two different databases, that doesn't sound like a good practice even if the table definition is exactly the same.
What I'd do to workaround this problem is just create a new object that is a copy of the original object and add it to the database:
session1 = SessionSRC()
session1.add(item)
session1.commit()
new_item = Datum(item.Key, item.Value)
session2 = SessionDST()
session2.add(new_item)
session2.commit()
print new_item in session2 # >>> True
print session2.query(Datum).all() # >>> [<__main__.Datum object at 0x.......>]
session2.close()
session1.close()
Note that session1 isn't closed immediately to be able to read the original object attributes while creating the new object.
Can someone explain, how can I avoid application freezing when I, for example, have a list of entities and an ability to move to detail pages.
So, I open the list, and one sqlalchemy session starts, then I open one detail page and another one goes, then another one, and application freeze, because one session blocks another.
I cannot use one session for whole application, because I then can't say, that something was edited on form by just checking out session.dirty, new, deleted attributes and application state handling becomes the hell of fragile unreadable code.
Do I need to implement some another kind of session handling policy?
Do I need to tune sqlalchemy mapping or sql server?
Here is the minimal working example:
from sqlalchemy import MetaData, Table, Column, FetchedValue, ForeignKey, create_engine
from sqlalchemy.types import BigInteger, String
from sqlalchemy.orm import mapper, relationship, sessionmaker, Session
class Ref(object):
id = None
name = None
id_parent = None
class TableMapper(object):
def __init__(self, metadata, mapped_type):
self._table = None
self._mapped_type = mapped_type
def get_table(self):
return self._table
def set_table(self, table):
assert isinstance(table, Table)
self._table = table
class RefTableMapper(TableMapper):
def __init__(self, metadata):
TableMapper.__init__(self, metadata, Ref)
self.set_table(Table('Ref', metadata,
Column('id', BigInteger,
primary_key = True, nullable = False),
Column('name', String),
Column('id_parent', BigInteger,
ForeignKey('Ref.id'))
))
def map_table(self):
r_parent = relationship(Ref,
uselist = False,
remote_side = [self._table.c.id],
primaryjoin = (
self._table.c.id_parent == self._table.c.id))
mapper(Ref, self._table,
properties = {'parent': r_parent})
return self._table
class Mapper(object):
def __init__(self, url, echo = False):
self._engine = create_engine(url, echo = echo)
self._metadata = MetaData(self._engine)
self._Session = sessionmaker(bind = self._engine, autoflush = False)
ref_t = RefTableMapper(self._metadata).map_table()
def create_session(self):
return self._Session()
if __name__ == '__main__':
mapp = Mapper(r'mssql://username:pwd#Server\SQLEXPRESS/DBName', True)
s = mapp.create_session()
rr = s.query(Ref).all()
s1 = mapp.create_session()
merged = s1.merge(rr)
merged.flush()
s2 = mapp.create_session()
rr1 = s2.query(Ref).all() #application freezes!
SQL Server's default isolation mode locks entire tables very aggressively. (The above example seems like perhaps you're emitting an UPDATE and then emitting SELECT in a different transaction while the previous transaction is pending, though session.merge() does not accept a list and the contents of the table aren't specified above so its difficult to say).
Anyway, it's typical practice to enable multi-version concurrency control (SQL server calls it "row versioning") so that it has reasonable ability to lock individual rows against each other instead of full tables:
ALTER DATABASE MyDatabase SET ALLOW_SNAPSHOT_ISOLATION ON
ALTER DATABASE MyDatabase SET READ_COMMITTED_SNAPSHOT ON
Detail on this is available at http://msdn.microsoft.com/en-us/library/ms175095.aspx .