I am trying to convert the rows returned in a SQLAlchemy query to dictionaries. When I try to use the ._asdict() method, I am only getting a key-value pair for the first column in my results.
Is there something else I should do to create a key-value pair in the dictionary for all columns in the result row?
class Project(db.Model):
__tablename__ = 'entries'
id = db.Column(db.Integer, primary_key=True)
time_start = db.Column(db.DateTime(timezone=False))
time_end = db.Column(db.DateTime(timezone=False))
name = db.Column(db.String(256), nullable=True)
analyst = db.Column(db.String(256), nullable=True)
def __init__(id, time_start, time_end, project_name, analyst):
self.id = id
self.time_start = time_start
self.time_end = time_end
self.name = name
self.analyst = analyst
latest_projects = db.session.query((func.max(Project.time_end)), Project.analyst).group_by(Project.analyst)
for row in latest_projects.all():
print (row._asdict())
{'analyst': 'Bob'}
{'analyst': 'Jane'}
{'analyst': 'Fred'}
I was expecting to see results like this...
{'analyst': 'Bob', 'time_end': '(2018, 11, 21, 14, 55)'}
{'analyst': 'Jane', 'time_end': '(2017, 10, 21, 08, 00)'}
{'analyst': 'Fred', 'time_end': '(2016, 09, 06, 01, 35)'}
You haven't named the func.max() column, so there is no name to use as a key in the resulting dictionary. Aggregate function columns are not automatically named, even when aggregating a single column; that you based that column on on the time_end column doesn't matter here.
Give that column a label:
latest_projects = db.session.query(
func.max(Project.time_end).label('time_end'),
Project.analyst
).group_by(Project.analyst)
Demo:
>>> latest_projects = db.session.query(
... func.max(Project.time_end).label('time_end'),
... Project.analyst
... ).group_by(Project.analyst)
>>> for row in latest_projects.all():
... print (row._asdict())
...
{'time_end': datetime.datetime(2018, 11, 21, 14, 55), 'analyst': 'Bob'}
{'time_end': datetime.datetime(2016, 9, 6, 1, 35), 'analyst': 'Fred'}
{'time_end': datetime.datetime(2017, 10, 21, 8, 0), 'analyst': 'Jane'}
Related
My models
class Machine(models.Model):
machineName = models.CharField(verbose_name="Machine Name", max_length=20, blank=False, null=False)
class SalesReport(models.Model):
machine = models.ForeignKey(Machine, on_delete=models.CASCADE, null=False, blank=False)
deviceDate = models.CharField(max_length=200, null=True, blank=True)
serverDate = models.DateTimeField(auto_now_add=True)
totalPrice = models.FloatField()
I have 3 machines, I wanted to get the total sales from each machines for the last 7 days.
my query is
from django.db.models import Sum, Value as V
from django.db.models.functions import Coalesce
SalesReport.objects.values("serverDate__date", "machine__machineName").annotate(
... sales=Coalesce(Sum("totalPrice"),V(0))).filter(
... serverDate__gte=week_start,
... serverDate__lte=week_end)
Which gives the following result,
[{'serverDate__date': datetime.date(2020, 7, 22), 'machine__machineName': 'machine__1', 'sales': 15.0},
{'serverDate__date': datetime.date(2020, 7, 28), 'machine__machineName': 'machine__1', 'sales': 145.0},
{'serverDate__date': datetime.date(2020, 7, 28), 'machine__machineName': 'machine__2', 'sales': 270.0},
{'serverDate__date': datetime.date(2020, 7, 28), 'machine__machineName': 'machine__3', 'sales': 255.0}]
What i am trying to get is
[{'serverDate__date': datetime.date(2020, 7, 22), 'machine__machineName': 'machine__1', 'sales': 15.0},
{'serverDate__date': datetime.date(2020, 7, 22), 'machine__machineName': 'machine__2', 'sales': 0.0},
{'serverDate__date': datetime.date(2020, 7, 22), 'machine__machineName': 'machine__3', 'sales': 0.0},
{'serverDate__date': datetime.date(2020, 7, 28), 'machine__machineName': 'machine__1', 'sales': 145.0},
{'serverDate__date': datetime.date(2020, 7, 28), 'machine__machineName': 'machine__2', 'sales': 270.0},
{'serverDate__date': datetime.date(2020, 7, 28), 'machine__machineName': 'machine__3', 'sales': 255.0}]
I am trying to do it with Coalesce, but i'm getting it wrong .
*I'm using mysql as db. a db specific query is also fine .
Since it is more SQL question I add a more specific answer
SELECT m.machineName, s.price
FROM machine m LEFT OUTER JOIN (
SELECT machine_id id, sum(totalPrice) price
FROM salesreport
WHERE serverDate BETWEEN DATE_SUB(curdate(), INTERVAL 1 WEEK) and curdate()
GROUP BY by machine_id) s on m.id = s.id
If you want the serverDate as outpout you have to apply an aggregate function (Max, Min) since it is located in your SalesReport table.
It depends what serverDate stands for. If it is the date when you bought the machine then it should be in machine table and it can be selected directly from machine table (and the WHERE BETWEEN clause must exist the sub-select and also apply on machine table). If it is a salesDate then it has to be in SalesReport and you must apply an aggregate function on it. ie: You can have potentially 7 dates over a week...
SELECT m.machineName, s.MaxserverDate, s.price
FROM machine m LEFT OUTER JOIN (
SELECT machine_id id, max(serverDate) MaxserverDate, sum(totalPrice) price
FROM salesreport
WHERE serverDate BETWEEN DATE_SUB(curdate(), INTERVAL 1 WEEK) and curdate()
GROUP BY by machine_id) s on m.id = s.id
The thing is that you don't have any sales for some dates. It is more a DB specific issue than a django ORM one. I would suggest to use raw sql with a left outer join on your machine table => take all the machine and list sales when present.
machine = Machine.objects.raw('''
SELECT machine.id, machine.name, sales.sid FROM app_machinelist as machine
LEFT JOIN (select sales_id as sid
from app_sales
where profile_id = {0}) sales
ON sales.sid = machine.id
ORDER BY machine.name ASC
'''.format(myuser.id))
This example works but for security reason, it is better to pass your parameters through a dictionary
machine = Machine.objects.raw(mysql, params)
Where
params = {'profile_id': pk, 'startdate': startdate, 'enddate': enddate}
mysql = '''
SELECT machine.id, machine.name, sales.sid FROM app_machinelist as machine
LEFT JOIN (select sales_id as sid
from app_sales
where profile_id = %(profile_id)s) sales
ON sales.sid = machine.id
ORDER BY machine.name ASC
'''
I created some mapped objects using the declarative style in SQLAlchemy. I have a mapping called ThermafuserReading which has a composed primary key made up of the Time_stamp column which is DateTime and ThermafuserId column which is an Integer and also acts as a Foreign Key to another table called Thermafuser. This is the definition of the class
class ThermafuserReading(Base):
"""Class to map to the Thermafuser Readings table in the HVAC DB"""
__tablename__ = 'Thermafuser_Reading'
_timestamp = Column('Time_stamp', DateTime, primary_key = True)
_thermafuserId = Column('ThermafuserId', Integer, ForeignKey("Thermafuser.ThermafuserId"), primary_key = True)
_roomOccupied = Column('RoomOccupied', Boolean)
_zoneTemperature = Column('ZoneTemperature', Float)
_supplyAir = Column('SupplyAir', Float, nullable=True)
_airflowFeedback = Column('AirflowFeedback', Float, nullable=True)
_CO2Input = Column('CO2Input', Float, nullable=True)
_maxAirflow = Column('MaxAirflow', Float, nullable=True)
_minAirflow = Column('MinAirflow', Float, nullable=True)
_unoccupiedHeatingSetpoint = Column('UnoccupiedHeatingSetpoint', Float, nullable=True)
_unoccupiedCoolingSetpoint = Column('UnoccupiedCoolingSetpoint', Float, nullable=True)
_occupiedCoolingSetpoint = Column('OccupiedCoolingSetpoint', Float, nullable=True)
_occupiedHeatingSetpoint = Column('OccupiedHeatingSetpoint', Float, nullable=True)
_terminalLoad = Column('TerminalLoad', Float, nullable=True)
#Relationship between Thermafuser Reading and Thermafuser
_thermafuser = relationship("Thermafuser", back_populates = "_thermafuserReadings", cascade = "all, delete-orphan", single_parent = True)
I am creating a session in the following way
sqlengine = sqlalchemy.create_engine("mysql+mysqldb://user:password#localhost:3306/HVAC")
Session = sessionmaker(bind=sqlengine)
session = Session()
At some point in my code I am creating a list called readings of Thermafuser Readings and adding such list the session via session.add_all(readings)
This are some example elements printed from the list readings:
<ThermafuserReading(thermafuserId = '21', timestamp = '2016-12-31 23:30:00')>
<ThermafuserReading(thermafuserId = '21', timestamp = '2016-12-31 23:35:00')>
<ThermafuserReading(thermafuserId = '21', timestamp = '2016-12-31 23:40:00')>
<ThermafuserReading(thermafuserId = '21', timestamp = '2016-12-31 23:45:00')>
<ThermafuserReading(thermafuserId = '21', timestamp = '2016-12-31 23:50:00')>
<ThermafuserReading(thermafuserId = '21', timestamp = '2016-12-31 23:55:00')>
<ThermafuserReading(thermafuserId = '21', timestamp = '2017-01-01 00:00:00')>
The problem is that the session is only keeping the last item in this list, eventhough I did session.add_all(readings) e.g. This is what the session has inside:
<ThermafuserReading(thermafuserId = '21', timestamp = '2017-01-01 00:00:00')>
I know the session keeps track of objects which have the same primary key and thus inserts only one instance of such objects in the sesssion but in this case the primary key (thermafuserId, timestamp) is different at each instance. I dont know why the session is only adding the last element of my list while neglecting the other elements.
Any idea?
EDIT:
I kept doing some tests and found out the reason why only the last element of the list is being added to the session. The problem lies in the identity_key for each of the objects in my list readings. This is the code I used for my tests:
for reading in readings:
print(reading, mapper.identity_key_from_instance(reading))
and this are some of the results
<ThermafuserReading(thermafuserId = '21', timestamp = '2017-01-14 23:15:00')> (<class 'hvacDBMapping.ThermafuserReading'>, (datetime.datetime(2017, 1, 15, 0, 0), 21))
<ThermafuserReading(thermafuserId = '21', timestamp = '2017-01-14 23:20:00')> (<class 'hvacDBMapping.ThermafuserReading'>, (datetime.datetime(2017, 1, 15, 0, 0), 21))
<ThermafuserReading(thermafuserId = '21', timestamp = '2017-01-14 23:25:00')> (<class 'hvacDBMapping.ThermafuserReading'>, (datetime.datetime(2017, 1, 15, 0, 0), 21))
<ThermafuserReading(thermafuserId = '21', timestamp = '2017-01-14 23:30:00')> (<class 'hvacDBMapping.ThermafuserReading'>, (datetime.datetime(2017, 1, 15, 0, 0), 21))
<ThermafuserReading(thermafuserId = '21', timestamp = '2017-01-14 23:35:00')> (<class 'hvacDBMapping.ThermafuserReading'>, (datetime.datetime(2017, 1, 15, 0, 0), 21))
<ThermafuserReading(thermafuserId = '21', timestamp = '2017-01-14 23:40:00')> (<class 'hvacDBMapping.ThermafuserReading'>, (datetime.datetime(2017, 1, 15, 0, 0), 21))
<ThermafuserReading(thermafuserId = '21', timestamp = '2017-01-14 23:45:00')> (<class 'hvacDBMapping.ThermafuserReading'>, (datetime.datetime(2017, 1, 15, 0, 0), 21))
<ThermafuserReading(thermafuserId = '21', timestamp = '2017-01-14 23:50:00')> (<class 'hvacDBMapping.ThermafuserReading'>, (datetime.datetime(2017, 1, 15, 0, 0), 21))
<ThermafuserReading(thermafuserId = '21', timestamp = '2017-01-14 23:55:00')> (<class 'hvacDBMapping.ThermafuserReading'>, (datetime.datetime(2017, 1, 15, 0, 0), 21))
<ThermafuserReading(thermafuserId = '21', timestamp = '2017-01-15 00:00:00')> (<class 'hvacDBMapping.ThermafuserReading'>, (datetime.datetime(2017, 1, 15, 0, 0), 21))
As you can observe, the function sqlalchemy.orm.util.identity_key_from_instance() is not creating the identity keys correctly for my datetime objects.
Can somebody help me clarify why?
EDIT
This is a simplified code that illustrates the problem. No connection to the database in this code. The code where this problem first appeared is much more involved and posting it will only create confusion, but this code reproduces the error.
Session = sessionmaker()
session = Session()
mapper = inspect(ThermafuserReading)
#Open the csv file
csvFilePath = "/Users/davidlaredorazo/Box Sync/Data/Zone4/1C1A/1C1A 2016-12-31.csv"
with open(csvFilePath, 'r') as csvfile:
reader = csv.reader(csvfile)
componentId = 1
count = 0
reading = ThermafuserReading(None, componentId)
for row in reader:
if count == 0:
count += 1
continue
#print(row)
timestamp = parse(row[0], None, ignoretz = True)
reading.timestamp = timestamp
new_object = copy.copy(reading)
new_object.timestamp = timestamp
print(new_object, mapper.identity_key_from_instance(new_object))
session.add(new_object)
print("new elements")
for new in session.new:
print(new, mapper.identity_key_from_instance(new_object))
As univerio mentioned in the comments. What I was doing wrong was using copy.copy to copy instances of my mapped objects, this was messing with _sa_instance_state. The solution was to create an "ad hoc" copy function for my instances. Here is the copy function I used, this, indeed solved the problem.
def copy_sqla_object(obj, omit_fk=True):
"""Given an SQLAlchemy object, creates a new object (FOR WHICH THE OBJECT
MUST SUPPORT CREATION USING __init__() WITH NO PARAMETERS), and copies
across all attributes, omitting PKs, FKs (by default), and relationship
attributes."""
cls = type(obj)
mapper = class_mapper(cls)
newobj = cls() # not: cls.__new__(cls)
pk_keys = set([c.key for c in mapper.primary_key])
rel_keys = set([c.key for c in mapper.relationships])
prohibited = pk_keys | rel_keys
if omit_fk:
fk_keys = set([c.key for c in mapper.columns if c.foreign_keys])
prohibited = prohibited | fk_keys
for k in [p.key for p in mapper.iterate_properties if p.key not in prohibited]:
try:
value = getattr(obj, k)
setattr(newobj, k, value)
except AttributeError:
pass
return newobj
You can see a more detailed discussion of this issue in
https://groups.google.com/forum/#!topic/sqlalchemy/HVSxndh23m0
I have a question about my script. I want to know all people who have more than 16 years from my Database. I want to check this when user triggers the function.
I have this function :
def Recensement_array(request) :
date = datetime.now().year
print date # I get year from now
birthday = Identity.objects.values_list('birthday', flat=True) # Return list with all birthday values
for element in birthday :
if date - element < 117 :
print "ok < 117"
else :
print "ok > 117"
From print date I get :
2017
From print birthday I get :
<QuerySet [datetime.date(1991, 12, 23), datetime.date(1900, 9, 12), datetime.date(1900, 9, 12), datetime.date(1900, 9, 12), datetime.date(1900, 9, 12), datetime.date(1089, 9, 22), datetime.date(1900, 9, 12), datetime.date(1900, 9, 12), datetime.date(1089, 9, 22), datetime.date(1089, 9, 22), datetime.date(1089, 9, 22), datetime.date(1089, 9, 22), datetime.date(1990, 12, 12)]>
So my goal is to substract date with birthday and compare if date - birthday = 16 years, I print element, else nothing.
I get two problems :
How extract only year from birthday ?
Then the comparison method is between int and tuple up to now. If I could extract only year from birthday, it should work right ?
Thank you
EDIT :
For example I want to get all people who had 16 years old since the begining of this year or will get 16 years old before the first year :
def Recensement_array(request) :
today = datetime.now()
age_16 = (today - relativedelta(years=16))
result = Identity.objects.filter(birthday__range=[age_16, today]).order_by('lastname')
paginator = Paginator(result, 3)
page = request.GET.get('page', 1)
try:
result = paginator.page(page)
except PageNotAnInteger:
result = paginator.page(1)
except EmptyPage:
result = paginator.page(paginator.num_pages)
context = {
"Identity":Identity,
"age_16":age_16,
"datetime" : datetime,
"result" : result,
"PageNotAnInteger":PageNotAnInteger,
}
return render(request, 'Recensement_resume.html', context)
If you need filter records with some specific year you can just use __year method of date field:
age_16 = (today - relativedelta(years=16))
result = Identity.objects.filter(birthday__year=age_16.year).order_by('lastname')
I've got a little problem:
I have a String in my Database which is called actions.
Now, I'm writing a method, which gets that string from the database
(that works), and then I want to turn that string into a list.
I know actions.split(), but this didn't work out so well for me, because if my string looks like this:
actions = [
{u'action': 'visit_testing', u'timestamp': datetime.datetime(2016, 2, 12, 13, 32, 14)},
{u'action': 'visit_foo', u'timestamp': datetime.datetime(2016, 2, 12, 13, 37, 50)}
]
I can't use actions.split(', ') because it would mess up the dictionaries inside.
Till now I've got the following code:
timestamp = datetime.now().replace(microsecond=0)
dict = {'timestamp': timestamp, 'action': action}
if self.actions:
actions_string = str(self.actions)
actions_stripped = actions_string.strip('[')
actions_stripped = actions_stripped.strip(']')
actions_splitted = actions_stripped.split(', ')
new_action_list = []
buffer = ''
for string in actions_splitted:
if '{' in string:
buffer = str(string)
elif '}' in string:
buffer = buffer + ', ' + str(string)
new_action_list.append(str(buffer))
buffer = ''
else:
buffer = buffer + ', ' + str(string)
self.actions = str(buffer)
self.last_action = datetime.now().replace(microsecond=0)
self.save()
else:
self.actions = '['+str(dict)+']'
self.last_action = datetime.now().replace(microsecond=0)
self.save()
Addition: If I run the method when actions is empty, it gives me a list with one dictionary, but if I run it when it already has something in it, if sets actions to "".
You should be using the json module to store valid JSON in your database. You can create a valid action list from that string using exec. But please beware that using exec or eval is a potentially dangerous practice.
import datetime
stuff = '''
actions = [{u'action': 'visit_testing', u'timestamp': datetime.datetime(2016, 2, 12, 13, 32, 14)}, {u'action': 'visit_foo', u'timestamp': datetime.datetime(2016, 2, 12, 13, 37, 50)}]
'''
exec(stuff)
print(actions)
print(actions[0]['timestamp'])
output
[{u'action': 'visit_testing', u'timestamp': datetime.datetime(2016, 2, 12, 13, 32, 14)}, {u'action': 'visit_foo', u'timestamp': datetime.datetime(2016, 2, 12, 13, 37, 50)}]
2016-02-12 13:32:14
Use json library.
import json
my_dict_or_list = json.loads(your_string)
then work with Python objects. You will gain so much time :-D
I found a way that works for me:
timestamp = datetime.datetime.now().replace(microsecond=0)
if self.actions:
new_dict = {"timestamp": timestamp, "action": action}
#tmp_actions = json.loads(self.actions)
tmp_actions = self.actions
exec(tmp_actions)
actions.append(new_dict)
self.actions = str('actions = '+str(actions))
self.last_action = datetime.datetime.now().replace(microsecond=0)
self.save()
else:
exec('''actions = ['''+str({"timestamp": timestamp, "action": action})+''']''')
self.actions = 'actions = '+str(actions)
self.last_action = datetime.datetime.now().replace(microsecond=0)
self.save()
Thanks for all the help.
I have something like this where trade_date, effective_date and termination_date are date values:
tradedates = dict(((k, k.strftime('%Y-%m-%d'))
for k in (trade_date,effective_date,termination_date)))
I get this:
{datetime.date(2005, 7, 25): '2005-07-25',
datetime.datetime(2005, 7, 27, 11, 26, 38): '2005-07-27',
datetime.datetime(2010, 7, 26, 11, 26, 38): '2010-07-26'}
What I would like is:
{'trade_date':'2005-07-25','effective_date':'2005-07-27','termination_date':'2010-07-26'}
How do I achieve this?
Using vars:
>>> import datetime
>>>
>>> trade_date = datetime.date(2005, 7, 25)
>>> effective_date = datetime.datetime(2005, 7, 27, 11, 26, 38)
>>> termination_date = datetime.datetime(2010, 7, 26, 11, 26, 38)
>>>
>>> d = vars() # You can access the variable as d['name']
>>> tradedates = {
... name: d[name].strftime('%Y-%m-%d')
... for name in ('trade_date', 'effective_date', 'termination_date')
... }
>>> tradedates
{'effective_date': '2005-07-27', 'termination_date': '2010-07-26', 'trade_date': '2005-07-25'}
For something that size, I'd create the dict directly:
result = {
'trade_date': format(trade_date, '%Y-%m-%d'),
'effective_date': format(effective_date, '%Y-%m-%d'),
# etc....
}
I am not sure if I got your question right. But let me explain what I understood and my answer for that:
You know the variable names: trade_date,effective_date,termination_date
And they have data in them
You could easily do:
tradedates = dict()
for k in ('trade_date','effective_date','termination_date'):
tradedates[k] = eval(k).strftime('%Y-%m-%d') // eval will evaluate them as a variable name not as a string.
This will give you a final dict something like:
{
'trade_date': <date_string_according_to_the_format_above>
'effective_date': <date_string_according_to_the_format_above>
'termination_date': <date_string_according_to_the_format_above>
}