I have a table with JSON stored in a text column:
import json
from sqlalchemy import create_engine, Column, text, Integer, TEXT, TypeDecorator
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
engine = create_engine('sqlite:///:memory:')
engine.execute("create table t (t_id int not null primary key, attrs text not null)")
engine.execute("insert into t values (1, '{\"a\": 1, \"b\": 2}')")
Session = sessionmaker(bind=engine)
I defined a mapping to this table in SQLAlchemy, using the custom type defined in the SQLAlchemy docs under "Marshal JSON Strings":
Base = declarative_base()
# http://docs.sqlalchemy.org/en/rel_1_1/core/custom_types.html#marshal-json-strings
class JSONEncodedDict(TypeDecorator):
impl = TEXT
def process_bind_param(self, value, dialect):
if value is not None:
value = json.dumps(value)
return value
def process_result_value(self, value, dialect):
if value is not None:
value = json.loads(value)
return value
class T(Base):
__tablename__ = 't'
t_id = Column(Integer, primary_key=True)
attrs = Column(JSONEncodedDict)
If I query all Ts, attrs gets deserialized from JSON:
session = Session()
t = session.query(T).first()
assert type(t.attrs) == dict, repr(t.attrs)
But if I use a textual query / raw SQL, it isn't deserialized:
session = Session()
t = session.query(T).from_statement(text('select * from t')).first()
assert type(t.attrs) == dict, repr(t.attrs) # AssertionError: u'{"a": 1, "b": 2}'
How do I make SQLAlchemy deserialize the attrs column when querying with raw SQL?
The behavior is the same with other databases (MySQL, Postgres). The database I am using (MySQL 5.5) does not support native JSON types, so changing the column type is not an option.
You can tell TextClause (produced by text()) the column types using .columns():
from sqlalchemy import inspect
session = Session()
stmt = text('select * from t').columns(*inspect(T).columns)
t = session.query(T).from_statement(stmt).first()
assert type(t.attrs) == dict, repr(t.attrs)
Or, for SQLAlchemy<0.9, use the typemap argument:
from sqlalchemy import inspect
session = Session()
typemap = {c.name: c.type for c in inspect(T).columns}
stmt = text('select * from t', typemap=typemap)
t = session.query(T).from_statement(stmt).first()
assert type(t.attrs) == dict, repr(t.attrs)
Related
I'm trying to do some path-checking logic on some values in a database. I'm having trouble implementing the class level expression for the hybrid method.
Here is some stripped down code:
from sqlalchemy import Column, Integer, String, create_engine, func, select
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session, aliased, sessionmaker
from sqlalchemy.ext.hybrid import hybrid_property, hybrid_method
from pathlib import Path
import sqlalchemy as sa
dbEngine = create_engine(
"sqlite:///.sql_app.db", connect_args={"check_same_thread": False}
)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=dbEngine)
Base = declarative_base()
class Folder(Base):
__tablename__ = "folder"
id = Column(Integer, primary_key=True)
value = Column(Integer, nullable=False)
fpath = Column(String, index=True)
#hybrid_method
def test_path(self, fpath):
a = Path(fpath)
b = Path(self.fpath)
return a in [b] + [p for p in b.parents]
#test_path.expression
def test_path(cls, fpath):
a = func.Path(fpath)
b = func.Path(cls.fpath)
# return a in [b] + [p for p in b.parents]
# What to do here
return (
select([func.Path(Folder.fpath)]).
label("fpath_in_folder")
)
#sa.event.listens_for(sa.engine.Engine, "connect")
def sqlite_engine_connect(dbapi_conn, connection_record) -> None:
dbapi_conn.create_function("Path", 1, Path)
def db_create_row(db: Session, value: int, fpath: str):
folder = Folder(value=value, fpath=fpath)
db.add(folder)
db.commit()
db.refresh(folder)
# Determine if the supplied path is a sub path of any records in the table
def db_query_row(db: Session, fpath: str):
records = db.query(Folder).filter(Folder.test_path(fpath) == True)
return records
if __name__ == "__main__":
Base.metadata.create_all(bind=dbEngine)
db = SessionLocal()
db_create_row(db, 5, "/folder 1/folder 2/file.ext")
records = db_query_row(db, "/folder 1")
print(records.count())
db.close()
I'm getting the error: Neither 'Function' object nor 'Comparator' object has an attribute 'parents'
So I have to create a SQL expression for this function, but I'm not sure how, or if it's even possible with accessing the parents property on the Path object.
SQLite can't handle Python instances, and SQLAlchemy Function support doesn't cover instance methods either.
First of all, you can't use Path() objects as a SQLite function, as explained in the sqlite3.Connection.create_function documentation:
The callable must return a type natively supported by SQLite.
The natively supported types are None, float, int, str or bytes values.
The error you see comes from your attempt to use b.parents in your expression; b is the Function object, the func.Path(...) call, and SQLAlchemy expects a function to return a SQL type, not a Path() object.
Instead of trying to shoehorn Path() objects into SQLite, you'll need to find another way to test if a path is a parent folder. You could use Column.startswith() here, provided you first ensure the paths don't end with /, by using the standard RTRIM() function:
#test_path.expression
def test_path(cls, fpath):
a = func.rtrim(fpath, "/")
b = func.rtrim(cls.fpath, "/")
return a == b or b.startswith(a + "/")
This will produce a SQL expression like this:
(
rtrim(?, '/') == rtrim(folder.fpath, '/')
OR rtrim(folder.fpath, '/') LIKE rtrim(?, '/') || '/' || '%'
)
Given a table with the following schema:
create table json_data (
id integer PRIMARY KEY NOT NULL,
default_object VARCHAR(10) NOT NULL,
data jsonb NOT NULL
);
For each of entity in the table I want to retrieve value of data['first']['name'] field, or if it's null value of data[json_data.default_object]['name'], or if the latter is also null then return some default value. In "pure" SQL I can write the following code to satisfy my needs:
insert into
json_data(
id,
default_object,
data
)
values(
0,
'default',
'{"first": {"name": "first_name_1"}, "default": {"name": "default_name_1"}}'
),
(
1,
'default',
'{"first": {}, "default": {"name": "default_name_2"}}'
);
select
id,
coalesce(
json_data.data -> 'first' ->> 'name',
json_data.data -> json_data.default_object ->> 'name',
'default_value'
) as value
from
json_data;
I tried to "translate" the "model" above into an SQLAlchemy entity:
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.hybrid import hybrid_property
Base = declarative_base()
class JsonObject(Base):
__tablename__ = 'json_data'
id = sa.Column(sa.Integer, primary_key=True)
default_object = sa.Column(sa.String(10), nullable=False)
data = sa.Column(postgresql.JSONB, nullable=False)
#hybrid_property
def name(self) -> str:
obj = self.data.get('first')
default_obj = self.data.get(self.default_object)
return (obj.get('name') if obj else default_obj.get('name')) or default_obj.get('name')
#name.setter
def name(self, value: str):
obj = self.data.setdefault('first', dict())
obj['name'] = value
#name.expression
def name(self):
return sa.func.coalesce(
self.data[('first', 'name')].astext,
self.data[(self.default_object, 'name')].astext,
'default_value',
)
But it seems that expression for the name hybrid property doesn't work as I expect. If I query entities by name property, like:
query = session.query(JsonObject).filter(JsonObject.name == 'name')
The query is expanded by SQLAlchemy into a something like this:
SELECT json_data.id AS json_data_id, json_data.default_object AS json_data_default_object, json_data.data AS json_data_data
FROM json_data
WHERE coalesce((json_data.data #> %(data_1)s), (json_data.data #> %(data_2)s), %(coalesce_1)s) = %(coalesce_2)s
It uses path operator instead of index operator. What should I do to make SQLAlchemy create an expression such as I wrote in the beginning of the question?
Ok, the solution I found is quite straightforward. As SQLAlchemy documentation tells:
Index operations return an expression object whose type defaults to JSON by default, so that further JSON-oriented instructions may be called upon the result type.
Therefore we can use "chained" python indexing operators. So the following code looks legit to me:
class JsonObject(Base):
# Almost the same stuff, except for the following:
#name.expression
def name(self):
return sa.func.coalesce(
self.data['first']['name'].astext,
self.data[self.default_object]['name'].astext,
'default_value',
)
My goal is to read data off of an excel sheet and create a database on a SQL server. I am trying to write a sample code using SQLalchemy and I am new to it. The code that I have so far is:
import time
from sqlalchemy import create_engine, Column, Integer, Date, String, Table, MetaData,table
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
engine = create_engine('sqlite:///:memory:', echo = False)
Base = declarative_base()
class blc(Base):
__tablename__ = 'BLC_Databse'
date = Column(String, primary_key = True)
RES = Column(String)
BTTLCOLUMN = Column(String)
CS_HR = Column(Integer)
Base.metadata.create_all(engine)
sample = blc(date=time.strftime("%m/%d/%y") , RES = 'BDY_21', BTTLCOLUMN = '2075', CS_HR = 563)
Session = sessionmaker(bind=engine)
session = Session()
sample2 = blc(date=time.strftime("%m/%d/%y") , RES = 'BDY_21', BTTLCOLUMN = '2076', CS_HR = 375)
session.add(sample2)
session.commit()
with session.no_autoflush:
result = session.query(blc).filter_by(RES = 'BDY_21').first()
print(result)
When I am performing a filter query (which I am assuming it is similar to where clause in SQL) it gives <__main__.blc object at 0x00705770> error
Eventually, I plan to have the insert clause on a loop and it will read data from an excel sheet.
Result is an object that references the class blc. To get the desired column, I had to do result.ColName.
I am trying to obtain a row from DB, modify that row and save it again.
Everything by using SqlAlchemy
My code
from sqlalchemy import Column, DateTime, Integer, String, Table, MetaData
from sqlalchemy.orm import mapper
from sqlalchemy import create_engine, orm
metadata = MetaData()
product = Table('product', metadata,
Column('id', Integer, primary_key=True),
Column('name', String(1024), nullable=False, unique=True),
)
class Product(object):
def __init__(self, id, name):
self.id = id
self.name = name
mapper(Product, product)
db = create_engine('sqlite:////' + db_path)
sm = orm.sessionmaker(bind=db, autoflush=True, autocommit=True, expire_on_commit=True)
session = orm.scoped_session(sm)
result = session.execute("select * from product where id = :id", {'id': 1}, mapper=Product)
prod = result.fetchone() #there are many products in db so query is ok
prod.name = 'test' #<- here I got AttributeError: 'RowProxy' object has no attribute 'name'
session .add(prod)
session .flush()
Unfortunately it does not work, because I am trying to modify RowProxy object. How can I do what I want (load, change and save(update) row) in SqlAlchemy ORM way?
I assume that your intention is to use Object-Relational API.
So to update row in db you'll need to do this by loading mapped object from the table record and updating object's property.
Please see code example below.
Please note I've added example code for creating new mapped object and creating first record in table also there is commented out code at the end for deleting the record.
from sqlalchemy import Column, DateTime, Integer, String, Table, MetaData
from sqlalchemy.orm import mapper
from sqlalchemy import create_engine, orm
metadata = MetaData()
product = Table('product', metadata,
Column('id', Integer, primary_key=True),
Column('name', String(1024), nullable=False, unique=True),
)
class Product(object):
def __init__(self, id, name):
self.id = id
self.name = name
def __repr__(self):
return "%s(%r,%r)" % (self.__class__.name,self.id,self.name)
mapper(Product, product)
db = create_engine('sqlite:////temp/test123.db')
metadata.create_all(db)
sm = orm.sessionmaker(bind=db, autoflush=True, autocommit=True, expire_on_commit=True)
session = orm.scoped_session(sm)
#create new Product record:
if session.query(Product).filter(Product.id==1).count()==0:
new_prod = Product("1","Product1")
print "Creating new product: %r" % new_prod
session.add(new_prod)
session.flush()
else:
print "product with id 1 already exists: %r" % session.query(Product).filter(Product.id==1).one()
print "loading Product with id=1"
prod = session.query(Product).filter(Product.id==1).one()
print "current name: %s" % prod.name
prod.name = "new name"
print prod
prod.name = 'test'
session.add(prod)
session.flush()
print prod
#session.delete(prod)
#session.flush()
PS SQLAlchemy also provides SQL Expression API that allows to work with table records directly without creating mapped objects. In my practice we are using Object-Relation API in most of the applications, sometimes we use SQL Expressions API when we need to perform low level db operations efficiently such as inserting or updating thousands of records with one query.
Direct links to SQLAlchemy documentation:
Object Relational Tutorial
SQL Expression Language Tutorial
table:
id(integer primary key)
data(blob)
I use mysql and sqlalchemy.
To insert data I use:
o = Demo()
o.data = mydata
session.add(o)
session.commit()
I would like to insert to table like that:
INSERT INTO table(data) VALUES(COMPRESS(mydata))
How can I do this using sqlalchemy?
you can assign a SQL function to the attribute:
from sqlalchemy import func
object.data = func.compress(mydata)
session.add(object)
session.commit()
Here's an example using a more DB-agnostic lower() function:
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.ext.declarative import declarative_base
Base= declarative_base()
class A(Base):
__tablename__ = "a"
id = Column(Integer, primary_key=True)
data = Column(String)
e = create_engine('sqlite://', echo=True)
Base.metadata.create_all(e)
s = Session(e)
a1 = A()
a1.data = func.lower("SomeData")
s.add(a1)
s.commit()
assert a1.data == "somedata"
you can make it automatic with #validates:
from sqlalchemy.orm import validates
class MyClass(Base):
# ...
data = Column(BLOB)
#validates("data")
def _set_data(self, key, value):
return func.compress(value)
if you want it readable in python before the flush, you'd need to memoize it locally and use a descriptor to access it.