I'm trying to filter my flask-sqlalchemy query via strings. My procedure for creating the string results in the following array:
["user.name == 'Sid'", "user.role == 'admin'"]
It gets formed with something like this:
for i in myarray:
filter_string = "%s.%s == '%s'" % (self.model.__tablename__, i[0], i[2])
or_filters.append(filter_string)
Here is how I'm using it:
db = SQLAlchemy(app)
...
class myclass:
def __init(self, model):
self.model = model
self.q = db.session.query(self.model)
def get(self, cfg):
...
# the following line works
myfilter = [User.name == 'Sid', User.role == 'admin']
# the following line does not work, but I need it to. How to modify into the line above?
myfilter = ["user.name == 'Sid'", "user.role == 'admin'"]
if myfilter is not None:
self.q = self.apply_filter(myfilter)
items = self.q.all()
return items
def apply_filter(self, ftr):
self.q.filter(or_(*ftr))
Ilja had a good solution
myfilter = getattr(self.model, i[0]).is_(i[2])
if myfilter is not None:
self.q = self.apply_filter(myfilter)
Related
I am using Django and PostgreSQL (psycopg2) for one of our web REST API projects.
Basically, the app is driven by the django-rest-framework library for all REST API-centric tasks such as authentication, permission, serialization and API views. However, since our database tables are not created thru Django's migration system (rather created manually and directly thru DBeaver), our modeling and serialization can actually be considered as highly customized and is no longer following Django's ORM standard (although we try to write our custom ORM design to feel as close as Django's as possible so that the pattern will still feel familiar).
The way I communicate CRUD actions to and from the database is by creating one (1) custom manager class mapped to each custom model class that it's supposed to manage. So in my manager I have like get(), insert(), update(), delete(), force_delete() methods which contain logic for actually sending queries to the database.
For all methods responsible for fetching data i.e. get() all() filter(), they communicate through a database view instead of directly sending a query to the concerned table that may contain JOINs which may be too expensive for the DB server.
This design works fine for us, but I still ask myself if this is actually ideal or at least an acceptable approach specially on real-world, daily consumption of our API by potentially millions of clients.
Or is there any best practice which I should strictly follow? Or any better approach anyone may suggest?
Here are the sample classes for one of our API resources - API version:
DB table
Model Class
class ApiVersionModel:
objects = ApiVersionManager()
def __init__(self):
self.version = None
self.version_info = None
self.date_released = None
self.development = None
self.production = None
def save(self, initial=False):
if not initial:
self.objects.update(self)
else:
self.objects.insert(self)
def delete(self, force_delete=False):
if force_delete:
self.objects.delete(self)
else:
self.objects.soft_delete(self)
Serializer Class
class ApiVersionSerializer(serializers.Serializer):
version = serializers.CharField(max_length=15)
version_info = serializers.CharField(max_length=255, required=False, allow_null=True)
date_released = serializers.DateField()
development = serializers.BooleanField(default=True, required=False, allow_null=True)
production = serializers.BooleanField(default=False, required=False, allow_null=True)
def create(self, validated_data):
c = ApiVersionModel()
c.version = validated_data.get("version")
c.version_info = validated_data.get("version_info")
c.date_released = validated_data.get("date_released")
c.development = validated_data.get("development")
c.production = validated_data.get("production")
c.save(initial=True)
return c
def update(self, c, validated_data):
c.version = validated_data.get("version")
c.version_info = validated_data.get("version_info")
c.date_released = validated_data.get("date_released")
c.development = validated_data.get("development")
c.production = validated_data.get("production")
c.save()
return c
def delete(self, c, validated_data, force_delete=False):
c.version = validated_data.get("version")
c.version_info = validated_data.get("version_info")
c.date_released = validated_data.get("date_released")
c.development = validated_data.get("development")
c.production = validated_data.get("production")
c.delete(force_delete=force_delete)
return c
Manager Class
import traceback
from config.utils import (raw_sql_select, raw_sql_select_enhanced, raw_sql_insert, raw_sql_update, raw_sql_delete)
from unit2_app.utils.ModelUtil import where
class ApiVersionManager():
def __init__(self):
pass
#staticmethod
def all(**kwargs):
query = None
x = None
where_clause = where(**kwargs)
query = ("""
SELECT *
FROM sample_schema.api_version {};
""".format(where_clause))
x = raw_sql_select_enhanced(query, "slave", list(kwargs.values()))
if x is not None:
objects = []
from unit2_app.models.Sys import ApiVersionModel
for col in x[1]:
c = ApiVersionModel()
c.version = col.version
c.version_info = col.version_info
c.date_released = col.date_released
c.development = col.development
c.production = col.production
objects.append(c)
return [] if len(objects) == 0 else objects
return []
#staticmethod
def get(**kwargs):
query = None
x = None
where_clause = where(**kwargs)
query = ("""
SELECT *
FROM sample_schema.api_version {};
""".format(where_clause))
x = raw_sql_select_enhanced(query, "slave", list(kwargs.values()))
if x is not None:
objects = []
from unit2_app.models.Sys import ApiVersionModel
for col in x[1]:
c = ApiVersionModel()
c.version = col.version
c.version_info = col.version_info
c.date_released = col.date_released
c.development = col.development
c.production = col.production
objects.append(c)
return None if len(objects) == 0 else objects[0]
return None
#staticmethod
def filter(**kwargs):
query = None
x = None
where_clause = where(**kwargs)
query = ("""
SELECT *
FROM sample_schema.api_version {};
""".format(where_clause))
x = raw_sql_select_enhanced(query, "slave", list(kwargs.values()))
if x is not None:
objects = []
from unit2_app.models.Sys import ApiVersionModel
for col in x[1]:
c = ApiVersionModel()
c.version = col.version
c.version_info = col.version_info
c.date_released = col.date_released
c.development = col.development
c.production = col.production
objects.append(c)
return [] if len(objects) == 0 else objects
return []
#staticmethod
def insert(c):
try:
query = ("""
START TRANSACTION;
INSERT INTO sample_schema.api_version
(version, version_info, date_released, development, production)
VALUES (%(version)s, %(version_info)s, %(date_released)s, %(development)s, %(production)s);
""")
raw_sql_insert(query, "master", c.__dict__)
except Exception:
traceback.print_exc()
raise Exception("Unexpected manager exception has been encountered.")
#staticmethod
def update(c):
try:
query = ("""
START TRANSACTION;
UPDATE sample_schema.api_version SET
version_info = %(version_info)s,
date_released = %(date_released)s,
development = %(development)s,
production = %(production)s
WHERE version = %(version)s;
""")
raw_sql_update(query, "master", c.__dict__)
except Exception:
raise Exception("Unexpected manager exception has been encountered.")
#staticmethod
def delete(c):
try:
print(c.__dict__)
query = ("""
START TRANSACTION;
DELETE FROM sample_schema.api_version WHERE version=%(version)s;
""")
raw_sql_delete(query, "master", c.__dict__)
except Exception:
raise Exception("Something went wrong with the database manager.")
#staticmethod
def soft_delete(c):
pass
API View Class
class APIView_ApiVersion(views.APIView):
try:
serializer_class = ApiVersionSerializer
permission_classes = (IsAuthenticatedOrReadOnly,)
authentication_classes = ()
except:
traceback.print_exc()
def get_queryset(self, **fltr):
return self.serializer_class(ApiVersionModel.objects.all(**fltr), many=True).data
def get(self, request, **kwargs):
try:
fltr = {k:v[0] for k,v in dict(self.request.GET).items()}
return_data = None
url_path_param_version = None
return_data = self.get_queryset(**fltr)
# perform filtering for version if <version> path param
# ... is present in the URL
if request.resolver_match.kwargs and request.resolver_match.kwargs["version"]:
url_path_param_version = request.resolver_match.kwargs["version"]
return_data = ApiVersionModel.objects.get(version=url_path_param_version, **fltr)
else:
return_data = ApiVersionModel.objects.all(**fltr)
if isinstance(return_data, list):
if len(return_data) > 0:
return Response({
"success": True,
"message": "API version has been fetched successfully.",
"data": self.serializer_class(return_data, many=True).data
}, status=status.HTTP_200_OK)
else:
return Response({
"success": True,
"message": HTTPNotFound.resource_empty(None, obj="API version"),
"data": []
}, status=status.HTTP_200_OK)
else:
if return_data:
return Response({
"success": True,
"message": "API version has been fetched successfully.",
"data": self.serializer_class(return_data).data
}, status=status.HTTP_200_OK)
else:
return Response({
"success": False,
"message": HTTPNotFound.object_unknown(None, obj="API version")
}, status=HTTPNotFound.status_code)
except Exception as e:
return Response({
"success": False,
"message": str(HTTPServerError.unknown_error(None)) + " DETAIL: {}".format(str(e))
}, status=HTTPServerError.status_code)
# Other METHODS also go here i.e. post(), update(), etc.
Custom utility for dynamic API resource filtering / dynamic WHERE CLAUSE
Since our ORM is highly customized, it hinders us from using DRF's inbuilt filtering classes. So I created my own simple utility to optionally allow filtering of SELECT queries via a query string. When applied, the value that the where() method generates gets injected into the DB query in our custom managers.
def filter(key, val):
f = []
c = ""
operator = "="
if len(key.split('__')) > 1:
dus = key.split('__')
for x in range(len(dus)):
f.append(str('{}' if x == 0 else "'{}'").format(dus[x]))
else:
f.append(key)
c = c.join(f) + " {} {} ".format(operator, str(val))
if len(key.split('___')) > 1:
tus = key.split('___')
for x in range(len(tus)):
if tus[x] == "lt":
operator = "<"
elif tus[x] == "lte":
operator = "<="
elif tus[x] == "gt":
operator = ">"
elif tus[x] == "gte":
operator = ">="
for y in f:
if tus[x] in y:
f.remove(y)
y = ""
if len(f) > 2:
for x in range(len(f)):
if x < len(f)-2:
y += f[x] + "->"
elif x <= len(f)-2:
y += f[x] + "->>"
elif x >= len(f)-2:
y += f[x]
else:
y += "->>".join(f)
if val is not None:
if isinstance(val, bool):
c = y + " {} '{}' ".format(operator, str(val).lower())
else:
c = y + " {} '{}' ".format(operator, str(val))
else:
c = y + " IS NULL "
return c
def where(**kwargs):
fields = []
if bool(kwargs):
for key, val in kwargs.items():
# fields.append(filter_nest(key, val))
fields.append(filter(key,val))
return ' WHERE ' + ' AND '.join(fields) if bool(kwargs) else ""
class Bin():
def __init__(self):
self.bin = {}
def add_to_bin(self, medId , medName):
self.bin[medId] = medName
def remove_by_id(self, id):
self.bin.pop(id)
def clean_bin(self):
self.bin.clear()
def check_ids(self):
list(self.bin.keys())
def check_names(self):
list(self.bin.values())
def check_inventory(self):
list(self.bin.items())
if __name__ == "__main__":
bin1 = Bin()
bin1.add_to_bin(100, "advil")
bin1.add_to_bin(200, "tylenol")
bin1.add_to_bin(300, "pepto-bismol")
bin1.check_inventory()
What am I doing wrong? I am so confused.
I am trying to create a medical storage system with multiple dictionaries. Whenever I try to run the code, it does not return anything.
Firstly there is no inheritance in your code. It is just a class and object code. Second you need to return data from your methods.
class Bin():
def __init__(self):
self.bin = {}
def add_to_bin(self, medId , medName):
self.bin[medId] = medName
def remove_by_id(self, id):
self.bin.pop(id)
def clean_bin(self):
self.bin.clear()
def check_ids(self):
return list(self.bin.keys())
def check_names(self):
return list(self.bin.values())
def check_inventory(self):
return list(self.bin.items())
if __name__ == "__main__":
bin1 = Bin()
bin1.add_to_bin(100, "advil")
bin1.add_to_bin(200, "tylenol")
bin1.add_to_bin(300, "pepto-bismol")
inventory = bin1.check_inventory()
print(inventory)
ids = bin1.check_ids()
print(ids)
names = bin1.check_names()
print(names)
Hi I'm having a problem in this classes I created the parent class extracao_nia with the method aplica_extracao for having the similar part of the execution that I use in others class and the diferent part is in the transform method definined in the children class
but I'm having an issue that the variables that I defined as list() are Null variable when I execute the code:
AttributeError: 'NoneType' object has no attribute 'append'
class extracao_nia:
def __init__(self, d=1, h=1, m=15):
self._data_base = "database"
self.UM_DIA = datetime.timedelta(days=d)
self.UMA_HORA = datetime.timedelta(hours=h)
self.INTERVALO = datetime.timedelta(minutes=m)
#property
def data_base(self):
return self._data_base
def aplica_extracao(self, SQL):
fim_intervalo = self.inicio + self.INTERVALO#
pbar = self.cria_prog_bar(SQL)#
while (fim_intervalo <= self.FIM):#
self.connector.execute(SQL,(self.inicio.strftime('%Y-%m-%d %H:%M'),fim_intervalo.strftime('%Y-%m-%d %H:%M')))#
for log in self.connector:#
self.transforma(log)
self.inicio = fim_intervalo
fim_intervalo = self.inicio + self.INTERVALO
class usuarios_unicos(extracao_nia):
def __init__(self, d=1, h=1, m=15, file='nodes.json'):
self._data_base = "database"
self.UM_DIA = datetime.timedelta(days=d)
self.UMA_HORA = datetime.timedelta(hours=h)
self.INTERVALO = datetime.timedelta(minutes=m)
self.file = file
self.ids = list()
self.nodes = list()
self.list_cpf = list()
def transforma(self, log):
context = json.loads(log[0])['context']
output = json.loads(log[0])['output']
try:
nr_cpf = context['dadosDinamicos']['nrCpf']
conversation_id = context['conversation_id']
nodes_visited = output['output_watson']['nodes_visited']
i = self.ids.index(conversation_id)
atual = len(self.nodes[i])
novo = len(nodes_visited)
if novo > atual:
nodes[i] = nodes_visited
except KeyError:
pass
except ValueError:
self.ids.append(conversation_id)
self.nodes = self.nodes.append(nodes_visited)
self.list_cpf = self.list_cpf.append(nr_cpf)
list.append returns None since it is an in-place operation, so
self.nodes = self.nodes.append(nodes_visited)
will result in self.nodes being assigned None. Instead you can just use
self.nodes += nodes_visited
I am trying to replicate something similar to model queries in Django.
# database.py
class ModelFactory(object):
def __init__(self, table):
self.table = table
def fields(self, *args):
str_columns = ''
for count, arg in enumerate(args):
if count == 0:
str_columns += arg
else:
str_columns += ', %s' % arg
self.str_columns = str_columns
def wheres(self, **kwargs):
str_wheres = ''
for count, (key, value) in enumerate(kwargs.items()):
if count == 0:
str_wheres += 'WHERE %s = %s' % (key, value)
else:
str_wheres += ' AND %s = %s' % (key, value)
self.str_wheres = str_wheres
My idea is to use it as follows:
from database import ModelFactory
myInstance = ModelFactory('myTable')
myQuery = myInstance.fields('column1', 'column2').wheres(column1 = 5)
I am not sure if I need a new class or function inside ModelFactory class that would take the results from 'fields' and 'wheres' to compile a SQL string to query? Like the following line:
cur.execute('SELECT column1, column2 from myTable WHERE column1 = 5')
I am also not sure if calling class.function1.function2 is correct? Django has the 'objects' word, e.g. Instance.objects.filter().exclude()
I have tried to change the code base as follows:
# database.py
class ModelFactory(object):
def __init__(self, table):
self.table = table
def objects(self):
def fields(self, **kwargs):
return self.f(**kwargs)
def wheres(self, *args):
return self.w(*args)
def f(self, *args):
str_columns = ''
for count, arg in enumerate(args):
if count == 0:
str_columns += arg
else:
str_columns += ', %s' % arg
self.str_columns = str_columns
def w(self, **kwargs):
str_wheres = ''
for count, (key, value) in enumerate(kwargs.items()):
if count == 0:
str_wheres += 'WHERE %s = %s' % (key, value)
else:
str_wheres += ' AND %s = %s' % (key, value)
self.str_wheres = str_wheres
But when I try the following:
from database import ModelFactory
myInstance = ModelFactory('myTable')
myQuery = myInstance.objects.fields('column1', 'column2').wheres(column1 = 5)
I get an AttributeError: 'function' object has no attribute 'fields'
If you want to chain object's method calls, you need to return that object from method. i.e. add return self to your methods.
So your class declaration probably should be something like the following:
class ModelFactory(object):
def __init__(self, table):
self.table = table
def fields(self, *args):
self.str_columns = ', '.join(args)
return self
def wheres(self, **kwargs):
str_wheres = ' AND '.join('{} = {}'.format(k, v) for k, v in kwargs.items())
self.str_wheres = 'WHERE {}'.format(str_wheres)
return self
def execute(self):
// ATTN! This code is prone to SQL injection. Do not use!
cur.execute('SELECT {columns} FROM {table} {wheres}'.format(columns=self.str_columns, table=self.table, wheres=self.wheres))
The problem is that you still have objects as a function:
def objects(self):
Currently, this means that you'll need to call it as a function - your myQuery line would need to look something like:
myQuery = myInstance.objects().fields(...
However, that still wouldn't be enough because fields and wheres are both only scoped within the objects function.
If you want to push down this route, then you will need to create a class that you instantiate inside your model under the attribute objects - Django uses QuerySet for this. If you check out the source you'll see how much magic is required to make something "simple" like objects work.
So a simple alternative?
You will have to make a new class similar to QuerySet which can provide the fields and wheres chainable functions - let's call it WernerfeuerSet:
class WernerfeuerSet:
def __init__(self, table):
self.table = table
def fields(self, **kwargs):
pass
def wheres(self, **kwargs):
pass
And now instantiate that within your ModelFactory - something like:
class ModelFactory:
def __init__(self, table):
self.objects = WernerfeuerSet(table)
Now your original query should be possible because objects is an attribute of the ModelFactory, rather than a function:
myQuery = myInstance.objects.fields('column1', 'column2').wheres(column1 = 5)
What could be reasons for this code not to add to the dictionary(assume that it is going into the else block):
if dup_agent_log.modify_stamp > merge_agent_log.modify_stamp:
merge_agent_log.set_status_log(dup_agent_log.status, None, dup_agent_log.agents_id, dup_agent_log.source, dup_agent_log.status_type)
else:
print merge_agent_log.data # {}
now = str(datetime.datetime.now())
merge_agent_log.data[now] = [{"status": dup_agent_log.status, "change_agent": dup_agent_log.change_agent, "source": dup_agent_log.source}]
print "after", merge_agent_log.data # after {}
transaction.savepoint()
The AgentStatusLog() class(model):
class AgentStatusLog(Base):
__tablename__ = "agent_status_log"
id = Column(
Integer,
Sequence("agent_status_log_id", optional=True),
primary_key=True
)
status_id = Column(Integer, ForeignKey(StatusLookup.id))
_status = relationship(StatusLookup, uselist=False)
status = Column(Unicode(100), index=True)
previous_status = Column(Unicode(50), index=True)
effective_stamp = Column(DateTime, index=True)
modify_stamp = Column(DateTime, index=True)
agents_id = Column(Integer, ForeignKey(Agents.id))
agent = relationship(Agents, primaryjoin=Agents.id == agents_id, uselist=False)
_data = Column("data", Unicode(2048))
status_type = Column(Unicode(40), index=True)
change_agents_id = Column(Integer, ForeignKey(Agents.id))
change_agent = relationship(Agents, primaryjoin=Agents.id == change_agents_id, uselist=False)
source = Column(Unicode(100), index=True)
#property
def data(self):
if not self._data:
return {}
return json.loads(self._data)
#data.setter
def data(self, value):
self._data = json.dumps(value)
def set_data(self, field, value):
data = self.data
data[field] = value
self.data = data
def get_data(self):
if not self._data:
return {}
return json.loads(self._data)
def unset_data(self, field):
data = self.get_data()
if field in data:
del data[field]
self.data = data
#classmethod
def by_id(cls, id):
return Session.query(cls).filter(cls.id == id).first()
#classmethod
def by_agents_id(cls, aid):
return Session.query(cls).filter(cls.agents_id == aid).first()
#classmethod
def set_status_log(cls, status, change_agent=None, agent_id=None, source=None, status_type=None):
if agent_id:
sl = Session.query(cls).filter(cls.agents_id == agent_id).first()
sl.modify_stamp = func.now()
sl.source = source
if sl.status:
sl.previous_status = sl.status
sl.data[str(datetime.datetime.now())] = [{"status": sl.status, "change_agent": change_agent, "source": source}]
sl.status = status
sl.change_agents_id = change_agent
sl.status_type = status_type
transaction.commit()
Maybe I'm just over looking something simple, but this boggles my mind... When I do the same thing in my python shell, it works...
merge_agent_log is a SQLAlchemy object.
What possible could possibly cause this cause this?
Am I missing something? I have be having at this for two days...
Is there in any case an entry to a dict would fail using data["foo"] = "bar"?
No dict has failed here, it just that you created more than one dict and did not realize it.
Let's analyze merge_agent_log.data[now] = something closely: first the property getter is called. It reads _data and creates a new dict with json.loads. Then you modify this new dict, adding something for key now.
But you do not modify _data!
So the next time you call the property getter with merge_agent_log.data, you read the same string from _data and create another dict with the same contents you did before.
Try this:
data = merge_agent_log.data
data[now] = something
merge_agent_log.data = data # calls property setter, which modifies _data