How do I dump the data in a queryset of Mongo documents to a .json file using python?
I've tried to use django serializer, but didn't work as the fields are not accessed in django the same way as they are accessed in mongo.
for model in models:
json_serializer.serialize(model.objects.all(), indent=2, stream=output_file_mongo)
I also tried to use python JSON encode/decoder,
import json
for model in mongo_models:
output_file_mongo.write(json.dumps(model.objects.all()))
I get an exception
File "/usr/lib/python2.7/json/__init__.py", line 231, in dumps
return _default_encoder.encode(obj)
File "/usr/lib/python2.7/json/encoder.py", line 201, in encode
chunks = self.iterencode(o, _one_shot=True)
File "/usr/lib/python2.7/json/encoder.py", line 264, in iterencode
return _iterencode(o, 0)
File "/usr/lib/python2.7/json/encoder.py", line 178, in default
raise TypeError(repr(o) + " is not JSON serializable")
TypeError: [<MongoModelA: MongoModelA object>] is not JSON serializable
The django serializer does not know how to handle mongoengine objects, You'll most likely have to write your own json encoder to map them to a simple dictionary:
class MyEncoder(json.JSONEncoder):
def encode_object(self, obj):
return { 'id':unicode(obj.id), 'other_property': obj.other_property }
def default(self, obj):
if hasattr(obj, '__iter__'):
return [ self.encode_object(x) for x in obj ]
else:
return self.encode_object(obj)
then call it like so:
import json
import MyEncoder
json_string = json.dumps(model.objects.all(), cls=MyEncoder)
Related
So I was trying to make an ajax call in django but this happend
Internal Server Error: /ajax/new_notification/
Traceback (most recent call last):
File "C:\Users\berna\AppData\Local\Programs\Python\Python38-32\lib\site-packages\django\core\handlers\exception.py", line 34, in inner
response = get_response(request)
File "C:\Users\berna\AppData\Local\Programs\Python\Python38-32\lib\site-packages\django\core\handlers\base.py", line 115, in _get_response
response = self.process_exception_by_middleware(e, request)
File "C:\Users\berna\AppData\Local\Programs\Python\Python38-32\lib\site-packages\django\core\handlers\base.py", line 113, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\Users\berna\Desktop\Python & Javascript\Web development\MiFamiliaEsUnDesastre\mifamiliaesundesastre\chat\views.py", line 71, in new_notification
return JsonResponse(user, safe=False)
File "C:\Users\berna\AppData\Local\Programs\Python\Python38-32\lib\site-packages\django\http\response.py", line 561, in __init__
data = json.dumps(data, cls=encoder, **json_dumps_params)
File "C:\Users\berna\AppData\Local\Programs\Python\Python38-32\lib\json\__init__.py", line 234, in dumps
return cls(
File "C:\Users\berna\AppData\Local\Programs\Python\Python38-32\lib\json\encoder.py", line 199, in encode
chunks = self.iterencode(o, _one_shot=True)
File "C:\Users\berna\AppData\Local\Programs\Python\Python38-32\lib\json\encoder.py", line 257, in iterencode
return _iterencode(o, 0)
File "C:\Users\berna\AppData\Local\Programs\Python\Python38-32\lib\site-packages\django\core\serializers\json.py", line 104, in default
return super().default(o)
File "C:\Users\berna\AppData\Local\Programs\Python\Python38-32\lib\json\encoder.py", line 179, in default
raise TypeError(f'Object of type {o.__class__.__name__} '
TypeError: Object of type User is not JSON serializable
HTTP GET /ajax/new_notification/ 500 [0.15, 127.0.0.1:64660]
the code that is raising this error is here, also I am new in django so maybe the solution can be ver obvious
def new_notification(request):
user = request.user
user.profile.notifications = user.profile.notifications + 1
user.save()
return JsonResponse(user, safe=False)
thank you for the help
Well, the error is rather self-explanatory:
TypeError: Object of type User is not JSON serializable
Unfortunately, user from within the request object (not necessarily 'user' model, correct me if I'm wrong here) is not JSON-serializable by default. Simplest solution would be to map only necessary fields:
return JsonResponse({ "notifications": profile.notifications }, safe=True)
But that's rather naive. Instead, maybe django serializers could do the job:
from django.core import serializers
def new_notification(request):
user = request.user
user.profile.notifications = user.profile.notifications + 1
user.save()
return JsonResponse(serializers.serialize('json', user), safe=False)
django version: 1.11, python version: 3.6.3
I found this stackoverflow question:
Unit Testing a Django Form with a FileField
and I like how there isn't an actual image/external file used for the unittest; however I tried these approaches:
from django.test import TestCase
from io import BytesIO
from PIL import Image
from my_app.forms import MyForm
from django.core.files.uploadedfile import InMemoryUploadedFile
class MyModelTest(TestCase):
def test_valid_form_data(self):
im_io = BytesIO() # BytesIO has to be used, StrinIO isn't working
im = Image.new(mode='RGB', size=(200, 200))
im.save(im_io, 'JPEG')
form_data = {
'some_field': 'some_data'
}
image_data = {
InMemoryUploadedFile(im_io, None, 'random.jpg', 'image/jpeg', len(im_io.getvalue()), None)
}
form = MyForm(data=form_data, files=image_data)
self.assertTrue(form.is_valid())
however, this always results in the following error message:
Traceback (most recent call last):
File "/home/my_user/projects/my_app/products/tests/test_forms.py", line 44, in test_valid_form_data
self.assertTrue(form.is_valid())
File "/home/my_user/.virtualenvs/forum/lib/python3.6/site-packages/django/forms/forms.py", line 183, in is_valid
return self.is_bound and not self.errors
File "/home/my_user/.virtualenvs/forum/lib/python3.6/site-packages/django/forms/forms.py", line 175, in errors
self.full_clean()
File "/home/my_user/.virtualenvs/forum/lib/python3.6/site-packages/django/forms/forms.py", line 384, in full_clean
self._clean_fields()
File "/home/my_user/.virtualenvs/forum/lib/python3.6/site-packages/django/forms/forms.py", line 396, in _clean_fields
value = field.widget.value_from_datadict(self.data, self.files, self.add_prefix(name))
File "/home/my_user/.virtualenvs/forum/lib/python3.6/site-packages/django/forms/widgets.py", line 423, in value_from_datadict
upload = super(ClearableFileInput, self).value_from_datadict(data, files, name)
File "/home/my_user/.virtualenvs/forum/lib/python3.6/site-packages/django/forms/widgets.py", line 367, in value_from_datadict
return files.get(name)
AttributeError: 'set' object has no attribute 'get'
Why? I understand that .get() is a dictionary method, but I fail to see where it created a set.
image_data should be dict, without providing key {value} will create set object. You need to define it like this {key: value}. Fix to this:
image_data = {
'image_field': InMemoryUploadedFile(im_io, None, 'random.jpg', 'image/jpeg', len(im_io.getvalue()), None)
}
Without External File Code
from django.core.files.uploadedfile import SimpleUploadedFile
testfile = (
b'\x47\x49\x46\x38\x39\x61\x01\x00\x01\x00\x00\x00\x00\x21\xf9\x04'
b'\x01\x0a\x00\x01\x00\x2c\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02'
b'\x02\x4c\x01\x00\x3b')
avatar = SimpleUploadedFile('small.gif', testfile, content_type='image/gif')
In my construct below, I am trying to pass a JSON object through my web service. As a new requirement I have to pass a dictionary object which is sent in the code below. Can you please guide me how to add the dictionary to JSON object?
if plain_text is not None:
blob = TextBlob(plain_text)
sentiment = TextBlob(plain_text)
sent = {}
for sentence in blob.sentences:
sent[sentence] =sentence.sentiment.polarity
print sent
return json.dumps(
{'input' : plain_text,
'Polarity': sentiment.polarity,
#'sent': json.dumps(sent) # this is where I am stuck as this 'sent' is a dict
},
indent=4)
If I uncomment the line I get the below error:
Exception:
TypeError('keys must be a string',)
Traceback:
Traceback (most recent call last):
File "C:\Python27\lib\site-packages\bottle-0.12.7-py2.7.egg\bottle.py", line 862, in _handle
return route.call(**args)
File "C:\Python27\lib\site-packages\bottle-0.12.7-py2.7.egg\bottle.py", line 1729, in wrapper
rv = callback(*a, **ka)
File "C:\Users\hp\Desktop\RealPy\WebServices\bottle\server_bckup.py", line 53, in sentimentEngine
'sent': json.dumps(sent),
File "C:\Python27\lib\json\__init__.py", line 231, in dumps
return _default_encoder.encode(obj)
File "C:\Python27\lib\json\encoder.py", line 201, in encode
chunks = self.iterencode(o, _one_shot=True)
File "C:\Python27\lib\json\encoder.py", line 264, in iterencode
return _iterencode(o, 0)
TypeError: keys must be a string
In JSON, dictionary keys must be strings. You have a Python dictionary sent, that you want to serialize into JSON. This fails, as your dictionary sent has keys that are not strings, but textblob.blob.Sentence instances.
If it makes sense, you can change your code to read:
for sentence in blob.sentences:
sent[str(sentence)] = sentence.sentiment.polarity
Or, you can customize the Python JSON encoder to know how to serialize TextBlob Sentences.
I have this in my views.py
response_dict = {
'status': status,
'message': message
}
return HttpResponse(simplejson.dumps(response_dict),
mimetype='application/javascript')
Since I start using this import:
from django.utils.translation import ugettext_lazy as _
at this line:
message = _('This is a test message')
I get this error:
File "/home/chris/work/project/prokject/main/views.py", line 830, in fooFunc
return HttpResponse(simplejson.dumps(response_dict),
File "/usr/local/lib/python2.7/json/__init__.py", line 243, in dumps
return _default_encoder.encode(obj)
File "/usr/local/lib/python2.7/json/encoder.py", line 207, in encode
chunks = self.iterencode(o, _one_shot=True)
File "/usr/local/lib/python2.7/json/encoder.py", line 270, in iterencode
return _iterencode(o, 0)
File "/usr/local/lib/python2.7/json/encoder.py", line 184, in default
raise TypeError(repr(o) + " is not JSON serializable")
TypeError: <django.utils.functional.__proxy__ object at 0x7f42d581b590> is not JSON serializable
Why? What am I doing wrong?
You can also create you own JSON encoder that will force __proxy__ to unicode.
From https://docs.djangoproject.com/en/1.8/topics/serialization/
from django.utils.functional import Promise
from django.utils.encoding import force_text
from django.core.serializers.json import DjangoJSONEncoder
class LazyEncoder(DjangoJSONEncoder):
def default(self, obj):
if isinstance(obj, Promise):
return force_text(obj)
return super(LazyEncoder, self).default(obj)
So now your code can look like:
response_dict = {
'status': status,
'message': _('Your message')
}
return HttpResponse(json.dumps(response_dict, cls=LazyEncoder),
mimetype='application/javascript')
It's not a string yet, and Python's JSON encoder doesn't know about ugettext_lazy, so you'll have to force it to become a string with something like
response_dict = {
'status': status,
'message': unicode(message)
}
You can custom JSONEncoder or use str as default serialize method:
# Created by BaiJiFeiLong#gmail.com at 2022/3/28
import json
from json import JSONEncoder
from typing import Any
import django
from django.conf import settings
from django.utils.translation import gettext_lazy as _
settings.configure()
django.setup()
class MyEncoder(JSONEncoder):
def default(self, o: Any) -> Any:
if getattr(type(o), "_delegate_text", False):
return str(o)
return super().default(o)
print(json.dumps(dict(hello=_("world")), default=str))
print(json.dumps(dict(hello=_("world")), cls=MyEncoder))
Here is the sample string i am receiving from one of the web services,
body=%7B%22type%22%3A%22change%22%2C%22url%22%3A%22http%3A%2F%2Fapi.pachube.com%2Fv2%2Ftriggers%2F4100%22%2C%22environment%22%3A%7B%22feed%22%3A%22http%3A%2F%2Fapi.pachube.com%2Fv2%2Ffeeds%2F36133%22%2C%22title%22%3A%22Current+Cost+Bridge%22%2C%22description%22%3Anull%2C%22id%22%3A36133%7D%2C%22threshold_value%22%3Anull%2C%22timestamp%22%3A%222012-01-05T09%3A27%3A01Z%22%2C%22triggering_datastream%22%3A%7B%22url%22%3A%22http%3A%2F%2Fapi.pachube.com%2Fv2%2Ffeeds%2F36133%2Fdatastreams%2F1%22%2C%22value%22%3A%7B%22value%22%3A%22523%22%2C%22max_value%22%3A1269.0%2C%22min_value%22%3A0.0%7D%2C%22id%22%3A%221%22%2C%22units%22%3A%7B%22symbol%22%3A%22W%22%2C%22type%22%3A%22derivedUnits%22%2C%22label%22%3A%22watts%22%7D%7D%2C%22id%22%3A4100%7D
Here is the code,
class Feeds():
def GET(self):
print "Get request is accepted."
return render.index(None)
def POST(self):
print "Post request is accepted."
print (web.data())
Now when that web-service posts the above given data, how will i convert it to readable format? Then, i need to convert it to JSON object and use further. So, how will i convert it?
When i try this code,
json_data = json.loads(web.data())
print json_data['body']
return render.index(json_data['body'])
It gives me an error,
enter code Traceback (most recent call last):
File "/usr/local/lib/python2.6/dist-packages/web/application.py", line 237, in process
return self.handle()
File "/usr/local/lib/python2.6/dist-packages/web/application.py", line 228, in handle
return self._delegate(fn, self.fvars, args)
File "/usr/local/lib/python2.6/dist-packages/web/application.py", line 409, in _delegate
return handle_class(cls)
File "/usr/local/lib/python2.6/dist-packages/web/application.py", line 385, in handle_class
return tocall(*args)
File "/home/ubuntu/pachubeConsumer/src/controllers/feeds.py", line 17, in POST
json_data = json.loads(web.data())
File "/usr/lib/python2.6/json/__init__.py", line 307, in loads
return _default_decoder.decode(s)
File "/usr/lib/python2.6/json/decoder.py", line 319, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib/python2.6/json/decoder.py", line 338, in raw_decode
raise ValueError("No JSON object could be decoded")
ValueError: No JSON object could be decoded here
Where am i going wrong ??
Thanks in advance.
First you need to unquote the body
import urllib
body="%7B%22type%22%3A%22change%22%2C%22url%22%3A%22http%3A%2F%2Fapi.pachube.com%2Fv2%2Ftriggers%2F4100%22%2C%22environment%22%3A%7B%22feed%22%3A%22http%3A%2F%2Fapi.pachube.com%2Fv2%2Ffeeds%2F36133%22%2C%22title%22%3A%22Current+Cost+Bridge%22%2C%22description%22%3Anull%2C%22id%22%3A36133%7D%2C%22threshold_value%22%3Anull%2C%22timestamp%22%3A%222012-01-05T09%3A27%3A01Z%22%2C%22triggering_datastream%22%3A%7B%22url%22%3A%22http%3A%2F%2Fapi.pachube.com%2Fv2%2Ffeeds%2F36133%2Fdatastreams%2F1%22%2C%22value%22%3A%7B%22value%22%3A%22523%22%2C%22max_value%22%3A1269.0%2C%22min_value%22%3A0.0%7D%2C%22id%22%3A%221%22%2C%22units%22%3A%7B%22symbol%22%3A%22W%22%2C%22type%22%3A%22derivedUnits%22%2C%22label%22%3A%22watts%22%7D%7D%2C%22id%22%3A4100%7D"
unquoted = urllib.unquote(body)
Then just load the JSON like normal
import json
pythonDict = json.loads(unquoted)
you need to unescape the query string first using urllib.unquote()
import urllib
unescaped = urllib.unquote(web.data())
then you may use json.loads to convert it into json.
json_data = json.loads(unescaped)
Python 3 urllib unquote move to parse:
from urllib import parse
parse.unquote(web.data())