I'm just trying to look through a queryset of objects and write to a csv export:
#expose('/csv-export')
def csv_export(self):
batch_num = request.args.get('batch_num')
if not batch_num:
flash('Invalid batch id', 'danger')
abort(404)
si = io.StringIO()
cw = csv.writer(si)
# array
my_objects = MyObject.query.filter_by(batch_num=batch_num).all()
row_headers = my_objects[0].serialize()
cw.writerow(row_headers)
object_rows = []
for my_object in my_objects:
for k, v in vars(my_object).items():
object_rows.append(v)
cw.writerows(object_rows)
output = make_response(si.getvalue())
output.headers["Content-Disposition"] = "attachment; filename=export.csv"
output.headers["Content-type"] = "text/csv"
return output
If I inspect what object_rows is it looks like this:
[<sqlalchemy.orm.state.InstanceState object at 0x7f92a791a390>, 2, None, '3312363552684', 551, None, '24daae41-82f5-42bf-b12f-9762554ee394', <sqlalchemy.orm.state.InstanceState object at 0x7f92a791a588>, ]
What am I doing wrong?
for my_object in my_objects:
for k, v in vars(my_object).items():
object_rows.append(v)
SQLAlchemy creates an attribute on each instance of an instrumented ORM object called _sa_instance_state which is used by SQLAlchemy internally. When you iterate over the object’s __dict__ using vars() this key/value pair is traversed along with all other attribute key/values.
So the solution is to filter it out. A common way to do this is to filter keys out based on whether they start with a leading underscore, e.g:
for my_object in my_objects:
for k, v in vars(my_object).items():
if not k.startswith(‘_’):
object_rows.append(v)
This will filter out all private attributes but you can be more specific with your filtering if there are other private attribute values that you want to capture.
Related
I have a config file with loads of paths and I want to organize them in a way. So I decided using types.SimpleNamespace to do that like:
paths = SimpleNamespace()
paths.base = '...'
paths.dataset.encoded = '...'
and I got:
AttributeError: 'types.SimpleNamespace' object has no attribute 'dataset'
I tried to define paths.dataset even though I didn't need it yet it didn't work:
paths = SimpleNamespace()
paths.base = '...'
paths.dataset = '...'
paths.dataset.encoded = '...'
AttributeError: 'str' object has no attribute 'encoded'
I also tried this:
_ = {
'base': '...',
'dataset': {
'encoded': '...',
}
}
paths = SimpleNamespace(**_)
and here is the result:
>>> paths.dataset.encoded # Error
AttributeError: 'dict' object has no attribute 'encoded'
>>> paths.dataset['encoded'] # works
'...'
This means that SimpleNamespace only works for one layer namespacing, right?
Is there another solution to this? I mean a solution other than using SimpleNamespace for every layer like this:
dataset = SimpleNamespace()
dataset.encoded = '...'
paths = SimpleNamespace()
paths.base = '???'
paths.dataset = dataset
>>> paths.base
'???'
>>> paths.dataset.encoded
'...'
Any ideas?
I came up with this solution:
def create_namespace(dictionary: dict):
"""Create a namespace of given dictionary
the difference between create_namespace and python's types.SimpleNamespace
is that the former will create name space recursively, but the later will
create the namespace in one layer indentation. See the examples to see the
difference.
Parameters
----------
dictionary : dict
A dict to be converted to a namespace object
Returns
-------
types.SimpleNamespace
A combination of SimpleNamespaces that will have an multilayer
namespace
Examples
--------
>>> dictionary = {
... 'layer1_a': '1a',
... 'layer1_b': {
... 'layer2_a': '2a',
... },
... }
>>> # types.SimpleNamespace
>>> simple = SimpleNamespace(**dictionary)
>>> simple.layer1_a
'1a'
>>> simple.layer1_b.layer2_a
AttributeError: 'dict' object has no attribute 'layer2_a'
# because layer1_b is still a dictionary
>>> # create_namespace
>>> complex = create_namespace(dictionary)
>>> complex.layer1_a
'1a'
>>> complex.layer1_a.layer2_a
'2a'
"""
space = {}
for key, value in dictionary.items():
if isinstance(value, dict):
value = create_namespace(value)
space[key] = value
return SimpleNamespace(**space)
but I think there is a better way that I'm not seeing. I appreciate any comments on this.
I am using django v1.10.2
I am trying to create dynamic reports whereby I store fields and conditions and the main ORM model information into database.
My code for the generation of the dynamic report is
class_object = class_for_name("app.models", main_model_name)
results = (class_object.objects.filter(**conditions_dict)
.values(*display_columns)
.order_by(*sort_columns)
[:50])
So main_model_name can be anything.
This works great except that sometimes associated models of the main_model have choicefield.
So for one of the reports main_model is Pallet.
Pallet has many PalletMovement.
My display columns are :serial_number, created_at, pallet_movement__location
The first two columns are fields that belong to Pallet model.
The last one is from PalletMovement
What happens is that PalletMovement model looks like this:
class PalletMovement(models.Model):
pallet = models.ForeignKey(Pallet, related_name='pallet_movements',
verbose_name=_('Pallet'))
WAREHOUSE_CHOICES = (
('AB', 'AB-Delaware'),
('CD', 'CD-Delaware'),
)
location = models.CharField(choices=WAREHOUSE_CHOICES,
max_length=2,
default='AB',
verbose_name=_('Warehouse Location'))
Since the queryset will return me the raw values, how can I make use of the choicefield in PalletMovement model to ensure that the pallet_movement__location gives me the display of AB-Delaware or CD-Delaware?
Bear in mind that the main_model can be anything depending on what I store in the database.
Presumably, I can store more information in the database to help me do the filtering and presentation of data even better.
The values() method returns a dictionary of key-value pairs representing your field name and a corresponding value.
For example:
Model:
class MyModel(models.Model):
name = models.CharField()
surname = models.CharField()
age = models.IntegerField()
...
Query:
result = MyModel.objects.filter(surname='moutafis').values('name', 'surname')
Result:
< Queryset [{'name': 'moutafis', 'surname': 'john'}] >
You can now manipulate this result as you would a normal dictionary:
if main_model_name is 'PalletMovement':
# Make life easier
choices = dict(PalletMovement.WAREHOUSE_CHOICES)
for item in result:
item.update({
pallet_movement__location: verbal_choice.get(
pallet_movement__location, pallet_movement__location)
})
You can even make this into a function for better re-usability:
def verbalize_choices(choices_dict, queryset, search_key):
result = queryset
for item in result:
item.update({ search_key: choices_dict.get(search_key, search_key) })
return result
verbal_result = verbalize_choices(
dict(PalletMovement.WAREHOUSE_CHOICES),
result,
'pallet_movement__location'
)
I suggest the use of the update() and get() methods because they will save you from potential errors, like:
The search_key does not exist in the choice_dict then get() will return the value of the search_key
update() will try to update the given key-value pair if exists, else it will add it to the dictionary.
If the usage of the above will be in the template representation of your data, you can create a custom template filter instead:
#register.filter(name='verbalize_choice')
def choice_to_verbal(choice):
return dict(PalletMovement.WAREHOUSE_CHOICES)[choice]
Have an extra look here: Django: How to access the display value of a ChoiceField in template given the actual value and the choices?
You would use get_foo_display
In your template:
{{ obj.get_location_display }}
or
{{ obj.pallet_movement.get_location_display }}
[Edit:] As pointed out in the comments this will not work when calling values()
an alternative to create a templatetag is :
{{form.choicefield.1}}
This shows the value of the initial data of the foreign key field instead the id.
The universal solution for any main_model_name is by Django Model _meta API introspection: class_object._meta.get_field(field_name).choices
That is:
choice_dicts = {}
for field_name in display_columns:
choice_dicts[field_name] = {
k: v for k, v in class_object._meta.get_field(field_name).choices
}
out = []
for row in results:
out.append({name: choice_dicts[name].get(value, value)
for name, value in row.items()
})
The rest is a trivial example, mostly copied code from the question
>>> pallet = app.models.Pallet.objects.create()
>>> palletm = app.models.PalletMovement.objects.create(pallet=pallet, location='AB')
>>>
>>> main_model_name = 'PalletMovement'
>>> conditions_dict = {}
>>> display_columns = ['pallet_id', 'location']
>>> sort_columns = []
>>>
>>> class_object = class_for_name("app.models", main_model_name)
>>> results = (class_object.objects.filter(**conditions_dict)
... .values(*display_columns)
... .order_by(*sort_columns)
... )[:50]
>>>
>>> # *** INSERT HERE ALL CODE THAT WAS ABOVE ***
>>>
>>> print(out)
[{'location': 'AB-Delaware', 'pallet_id': 1}]
It works equally with 'pallet_id' or with 'pallet' in display_columns. Even that "_meta" starts with underscore, it is a documented API.
I'm attempting to use python and boto to print a list of instances and IPs from Amazon EC2.
I'm used to PHP's nice multidimensional arrays and the similar JSON syntax but I'm having a lot of trouble in python. I tried using AutoVivification as mentioned in What's the best way to initialize a dict of dicts in Python? but am not having luck with access objects in it.
Here is my code:
import sys
import os
import boto
import string
import urllib2
from pprint import pprint
from inspect import getmembers
from datetime import datetime
class AutoVivification(dict):
"""Implementation of perl's autovivification feature."""
def __getitem__(self, item):
try:
return dict.__getitem__(self, item)
except KeyError:
value = self[item] = type(self)()
return value
conn = boto.connect_ec2_endpoint(ec2_url, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
tags = conn.get_all_tags()
myInstances = AutoVivification()
for tag in tags:
if ( tag.res_type == 'instance' and tag.name == 'Name'):
if( tag.res_id ):
myInstances[tag.res_id]['myid'] = tag.res_id
myInstances[tag.res_id]['name'] = tag.value
addrs = conn.get_all_addresses()
for a in addrs:
if( a.instance_id ):
myInstances[a.instance_id]['ip'] = a.public_ip
pprint( myInstances )
for i in myInstances:
print i.name.rjust(25), i.myid
If I do pprint( myInstances ) then I am able to see the multidimensional dict that I have created, but i am not able to access the sub-arrays with i.myid - I get errors like:
AttributeError: 'unicode' object has no attribute 'myid'
AttributeError: 'unicode' object has no attribute 'name'
Doing pprint( myInstances) gives me something like:
{u'i-08148364': {'myid': u'i-18243322', 'name': u'nagios', 'ip': u'1.2.3.4'}}
so I don't understand why I can't access these items.
Your problem is just in how you're trying to access the items:
for i in myInstances:
# i iterates over the KEYS in myInstances
print i.name.rjust(25), i.myid
This attempts, for each key in myInstances, to print i.name.rjust(25) and so on. What you want is to access the value of the given keys (and to use the right Python syntax for accessing dictionary elements):
for i in myInstances:
# i iterates over the KEYS in myInstances
print myInstances[i]["name"].rjust(25), myInstances[i]["myid"]
Or if you don't need the keys at all, just iterate over the values in the first place:
for i in myInstances.values():
# i iterates over the VALUES in myInstances
print i["name"].rjust(25), i["myid"]
Or finally, per request, if you really want to iterate over keys and values at once:
for k, v in myInstances.iteritems():
print k, v["name"].rjust(25), v["myid"]
I use this code to define my class in GAE Python:
class Pair(db.Model):
find = db.StringProperty()
replace = db.StringProperty()
rule = db.StringProperty()
tags = db.StringListProperty()
created = db.DateTimeProperty()
updated = db.DateTimeProperty(auto_now=True)
Then I use this code to serialize objects of that class with simplejson:
class PairEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, Pair):
return [str(obj.created), str(obj.updated), obj.find, obj.replace, obj.tags, obj.rule]
Finally I use this code to output the result as the response:
pairsquery = GqlQuery("SELECT * FROM Pair")
pairs = pairsquery.fetch(1000)
pairsList = []
for pair in pairs:
pairsList.append(json.dumps(pair, cls=PairEncoder))
serialized = json.dumps({
'pairs': pairsList,
'count': pairsquery.count()
})
self.response.out.write(serialized)
Here is a sample result I get:
{"count": 2, "pairs": ["[\"2010-12-06 12:32:48.140000\", \"2010-12-06 12:32:48.140000\", \"random string\", \"replacement\", [\"ort\", \"common\", \"movies\"], \"remove\"]", "[\"2010-12-06 12:37:07.765000\", \"2010-12-06 12:37:07.765000\", \"random string\", \"replacement\", [\"ort\", \"common\", \"movies\"], \"remove\"]"]}
All seems to be fine, except one thing - I need the fields in the response to have names from the class Pair, so there won't be just values but the names of the corresponding fields too. How can I do that?
class PairEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, Pair):
return {"created": str(obj.created), "updated:": str(obj.updated), "find": obj.find, "replace": obj.replace, "tags": obj.tags, "rule": obj.rule}
return json.JSONEncoder.default(self, obj)
But you are 'double encoding' here - i.e. encoding the pairs, adding that string to an object and encoding that too. If you 'double decode' on the other end it should work - but it's not the 'proper' way to do things.
I supposed I found a better simple solution for this, instead of serializing it with simplejson, I just created a method inside Pair class that looks like this:
def return_dict(self):
return {'find':self.find, 'replace':self.replace, 'rule':self.rule, 'tags':self.tags}
and does all I need. Thanks!
Is it possible to do something similar to this with a list, dictionary or something else?
data_dict = {
'title' : 'awesome title',
'body' : 'great body of text',
}
Model.objects.create(data_dict)
Even better if I can extend it:
Model.objects.create(data_dict, extra='hello', extra2='world')
If title and body are fields in your model, then you can deliver the keyword arguments in your dictionary using the ** operator.
Assuming your model is called MyModel:
# create instance of model
m = MyModel(**data_dict)
# don't forget to save to database!
m.save()
As for your second question, the dictionary has to be the final argument. Again, extra and extra2 should be fields in the model.
m2 =MyModel(extra='hello', extra2='world', **data_dict)
m2.save()
For any model DummyModel you can use DummyModel.objects.create(**data_dict). Does not require save after the create.
Not directly an answer to the question, but I find this code helped me create the dicts that save nicely into the correct answer. The type conversions made are required if this data will be exported to json.
I hope this helps:
#mod is a django database model instance
def toDict( mod ):
import datetime
from decimal import Decimal
import re
#Go through the object, load in the objects we want
obj = {}
for key in mod.__dict__:
if re.search('^_', key):
continue
#Copy my data
if isinstance( mod.__dict__[key], datetime.datetime ):
obj[key] = int(calendar.timegm( ts.utctimetuple(mod.__dict__[key])))
elif isinstance( mod.__dict__[key], Decimal ):
obj[key] = float( mod.__dict__[key] )
else:
obj[key] = mod.__dict__[key]
return obj
def toCsv( mod, fields, delim=',' ):
import datetime
from decimal import Decimal
#Dump the items
raw = []
for key in fields:
if key not in mod.__dict__:
continue
#Copy my data
if isinstance( mod.__dict__[key], datetime.datetime ):
raw.append( str(calendar.timegm( ts.utctimetuple(mod.__dict__[key]))) )
elif isinstance( mod.__dict__[key], Decimal ):
raw.append( str(float( mod.__dict__[key] )))
else:
raw.append( str(mod.__dict__[key]) )
return delim.join( raw )