We Keep Coding

Python Pandas DataFrame of objects to json does not export all properties

i have a pandas dataframe that contains Objects of Cell, this is the class:
class Cell:
def __init__(self, data, coordinates, cell_id):
self._cell_id = cell_id
self._coordinates = coordinates
self._data = data
#property
def data(self) -> str:
return self._data
#property
def coordinates(self) -> dict:
return self._coordinates
#property
def cell_id(self) -> str:
return self._cell_id
#property
def empty(self) -> bool:
if self._data == 0:
return False
return self._data is None or self._data == ""
def set_data(self, data):
self._data = data
def __repr__(self):
return self._data
every item in the dataframe is represented by this class.
but when trying to transform this class to json, for some reason not all values from the class are exported to the json.
this is the command i am using:
json.loads(df.to_json())
and the results looks like this:
{
"header_col1": {
"1": {
"cell_id": "705f7e89-abd4-4f7b-adfb-802010016546",
"data": "7579"
},
"2": {
"cell_id": "f04f3168-95d1-45f6-bb41-fc572736aeb7",
"data": "6207"
}
},
"header_col2": {
"1": {
"cell_id": "03fce322-a6d4-40ba-abbe-4f2558e6805f",
"data": "some text"
},
"2": {
"cell_id": "7b373ba4-e062-4881-99c6-2beaa0a20bcb",
"data": "some text"
},
}
}
i expected it to export all other fields of the Cell object as well, i don't understand why it exports only those 2 specific fields..
to reproduce the issue you can try using this:
mat = []
cords = {"x": 0, "y": 0, "w": 0, "h": 0}
for i in range(5):
row = []
for b in range(3):
row.append(Cell(cell_id=f"id_{b}", coordinates=cords, data=f"data_{b}"))
mat.append(row)
df = pd.DataFrame(mat)
json.loads(df.to_json())

the problem exists on pandas 1.3 on python3.9
upgrading to pandas 1.43 solves the issue

Extract the mongodb json array element into json python

I have mongodb document as an array and need to create json file in python.
mongodb document looks like
db.mappedfields.insertMany(
[
{ sourceAttribute: "first_name",
domainAttribute: "First_Name"},
{ sourceAttribute: "last_name",
domainAttribute: "Last_Name"}
]
Code tried
class create_dict(dict):
def __init__(self):
self = dict()
def add(self, key, value):
self[key] = value
mydict = create_dict()
i = 1
for key in mycol.find():
mydict.add(i, ({key['sourceAttribute']:key['domainAttribute']}))
i = i+1
json_data = json.dumps(mydict, indent=2, sort_keys=True)
Output getting
{
"1": {
"first_name": "First_Name"
},
"2": {
"last_name": "Last_Name"
}
}
Expected Output
{
"first_name": "First_Name",
"last_name": "Last_Name"
}

import json
import pymongo
class create_dict(dict):
# __init__ function
def __init__(self):
self = dict()
# Function to add key:value
def add(self, key, value):
self[key] = value
mydict = create_dict()
i = 1
for key in mycol.find():
mydict.add(i, ({key['input']:key['output']}))
i = i+1
json_data = json.dumps(mydict, indent=2, sort_keys=True)
json_data1 = json.loads(json_data)
dictlist = []
for key, value in json_data1.items():
dictlist.append(value)
print(dictlist)
result = {}
for d in dictlist:
result.update(d)
print(result)

Union type cannot resolve Object Type at Runtime

I am setting up a GraphQL Server with Python using Starlette and Graphene and ran into a problem I cannot find a solution for. The Graphene Documentation does not go into detail regarding the union type, which I am trying to implement.
I set up a minimum example based on the graphene documentation which you can run to replicate this problem
import os
import uvicorn
from graphene import ObjectType, Field, List, String, Int, Union
from graphene import Schema
from starlette.applications import Starlette
from starlette.graphql import GraphQLApp
from starlette.routing import Route
mock_data = {
"episode": 3,
"characters": [
{
"type": "Droid",
"name": "R2-D2",
"primaryFunction": "Astromech"
},
{
"type": "Human",
"name": "Luke Skywalker",
"homePlanet": "Tatooine"
},
{
"type": "Starship",
"name": "Millennium Falcon",
"length": 35
}
]
}
class Human(ObjectType):
name = String()
homePlanet = String()
class Droid(ObjectType):
name = String()
primary_function = String()
class Starship(ObjectType):
name = String()
length = Int()
class Characters(Union):
class Meta:
types = (Human, Droid, Starship)
class SearchResult(ObjectType):
characters = List(Characters)
episode = Int()
class RootQuery(ObjectType):
result = Field(SearchResult)
#staticmethod
def resolve_result(_, info):
return mock_data
graphql_app = GraphQLApp(schema=Schema(query=RootQuery))
routes = [
Route("/graphql", graphql_app),
]
api = Starlette(routes=routes)
if __name__ == "__main__":
uvicorn.run(api, host="127.0.0.1", port=int(os.environ.get("PORT", 8080)))
If you then go to http://localhost:8080/graphq and enter the following query
query Humans{
result {
episode
characters {
... on Human {
name
}
}
}
}
I get this error
{
"data": {
"result": {
"episode": 3,
"characters": null
}
},
"errors": [
{
"message": "Abstract type Characters must resolve to an Object type at runtime for field SearchResult.characters with value \"[{'type': 'Droid', 'name': 'R2-D2', 'primaryFunction': 'Astromech'}, {'type': 'Human', 'name': 'Luke Skywalker', 'homePlanet': 'Tatooine'}, {'type': 'Starship', 'name': 'Millennium Falcon', 'length': 35}]\", received \"None\".",
"locations": [
{
"line": 4,
"column": 5
}
]
}
]
}
which I am now stuck with. Maybe someone has done this already and can help out? How can I resolve this at runtime. I have already tried different approaches for example I changed classes Character and RootQuery:
class Character(Union):
class Meta:
types = (Human, Droid, Starship)
def __init__(self, data, *args, **kwargs):
super().__init__(*args, **kwargs)
self.data = data
self.type = data.get("type")
def resolve_type(self, info):
if self.type == "Human":
return Human
if self.type == "Droid":
return Droid
if self.type == "Starship":
return Starship
class RootQuery(ObjectType):
result = Field(SearchResult)
#staticmethod
def resolve_result(_, info):
return {**mock_data, "characters": [Character(character) for character in mock_data.get('characters')]}
resulting in
{
"data": {
"result": {
"episode": 3,
"characters": [
{},
{
"name": null
},
{}
]
}
}
}
Any ideas would be very appreciated!

jkimbo answered the question here:
class Character(Union):
class Meta:
types = (Human, Droid, Starship)
#classmethod
def resolve_type(cls, instance, info):
if instance["type"] == "Human":
return Human
if instance["type"] == "Droid":
return Droid
if instance["type"] == "Starship":
return Starship
class RootQuery(ObjectType):
result = Field(SearchResult)
def resolve_result(_, info):
return mock_data
Note I'm just returning mock_data and I've updated the resolve_type method to switch based on the data. The Union type uses the same resolve_type method as Interface to figure out what type to resolve to at runtime: https://docs.graphene-python.org/en/latest/types/interfaces/#resolving-data-objects-to-types

Using OrderedDict() to print python class members

I am trying to simulate the problem statement using the below program:
import json
class System:
def __init__(self):
self.model = "abc"
self.fwVersion = "123"
self.prevfwVersion = "456"
self.safemodeVersion = "5756"
def __setitem__(self, key, val):
self.__dict__[key] = val
def __getitem__(self, key):
return self.__dict__[key]
def toJSON(self):
return self.__dict__
class Mainwall:
def __init__(self):
self.system = System()
def __setitem__(self, key, val):
self.__dict__[key] = val
def __getitem__(self, key):
return self.__dict__[key]
def toJSON(self):
return self.__dict__
class ComplexEncoder(json.JSONEncoder):
def default(self, obj):
if hasattr(obj, 'toJSON'):
return obj.toJSON()
else:
return json.JSONEncoder.default(self, obj)
fw = Mainwall()
def my_print():
print(json.dumps(fw.toJSON(), cls=ComplexEncoder, indent=4))
if __name__ == '__main__':
my_print()
Since python dictionary does not preserve the insertion order , the output of the above program is always will have the different key order.
Say, first time it prints:
{
"system": {
"safemodeVersion": "5756",
"prevfwVersion": "456",
"fwVersion": "123",
"model": "abc"
}
}
Second time it prints:
{
"system": {
"fwVersion": "123",
"prevfwVersion": "456",
"safemodeVersion": "5756",
"model": "abc"
}
}
But, in the output I would like to preserve the order in which the class members are initialized. i.e., Exactly as below:
{
"system": {
"model": "abc",
"fwVersion": "123",
"prevfwVersion": "456",
"safemodeVersion": "5756",
}
}
How to achieve the expected output for the same example using OrderedDict() or some other method?

There is a work-around I made for you. I invite you to look into the System() class. I created an OrderedDict() instead of four self attributes. Then, in you method toJSON(self):, instead of returning the self.__dict__ attributes, I am returning the OrderedDict() I set earlier.
class System:
def __init__(self, model='abc', fwVersion='123', prevfwVersion='456', safemodeVersion='5756'):
self.my_ordered_dict = OrderedDict()
self.my_ordered_dict['model'] = model
self.my_ordered_dict['fwVersion'] = fwVersion
self.my_ordered_dict['prevfwVersion'] = prevfwVersion
self.my_ordered_dict['safemodeVersion'] = safemodeVersion
# self.model = "abc"
# self.fwVersion = "123"
# self.prevfwVersion = "456"
# self.safemodeVersion = "5756"
def __setitem__(self, key, val):
self.__dict__[key] = val
def __getitem__(self, key):
return self.__dict__[key]
def toJSON(self):
return self.my_ordered_dict
This System() class instead of the one above, with the same code, outputs...
{
"system": {
"model": "abc",
"fwVersion": "123",
"prevfwVersion": "456",
"safemodeVersion": "5756"
}
}

upgrading to python 3.6 solved the problem.

Flask-RESTful - don't return object property instead of returning null

Let's say I've got a clients table with id, name and email fields. An email field is optional.
The code looks like this:
client_fields = {
'id' : fields.String,
'name' : fields.String,
'email' : fields.String
}
And for displaying:
class ClientList(Resource):
#marshal_with(client_fields)
def get(self):
return model.Client.query.all()
When email is not provided, API returns JSON like this:
{
"id": "1",
"name": "John Doe",
"email": null
}
But instead I want it to return this object:
{
"id": "1",
"name": "John Doe"
}
Which basically means that instead of a property with null value I want it to return no such property at all. Is there a way to achieve that?

I would use the marshal function instead of the marshal_with decorator:
class ClientList(Resource):
def get(self):
clients = []
for client in model.Client.query.all():
if client.email:
clients.append(marshal(client_fields))
else:
clients.append(marshal(client_fields_no_email))
return clients
Or even better
class ClientList(Resource):
def get(self):
return [client_marshal(client) for client in model.Client.query.all()]
with
def client_marshal(client):
if client.email:
return {'id' : fields.String,
'name' : fields.String,
'email' : fields.String}
else:
return {'id' : fields.String,
'name' : fields.String}

There are two ways it can be done, pre-marshalling and post-marshalling modification. Pre-marshalling removes any default values given to field names in the client_fields dict but post-marshalling preserves them.
In pre-marshalling method, you have to pass a modified fields dict to marshal function if client's email is None.
For example;
import json
from flask_restful import fields, marshal, marshal_with
class Client(object):
def __init__(self, id, name, email=None):
self.id = id
self.name = name
self.email = email
client_fields = {
'id': fields.String,
'name': fields.String,
'email': fields.String
}
def get():
clients =[Client(1, 'Tom'), Client(2, 'John', 'john#example.com')]
return [marshal(client, client_fields if client.email else {k: v for k, v in client_fields.items() if k != 'email'}) for client in clients]
print(json.dumps(get()))
Output;
[{"id": "1", "name": "Tom"}, {"email": "john#example.com", "id": "2", "name": "John"}]
In post-marshalling you have to remove the email field of the OrderedDict returned by marshal_with if it is None.
The de_none function by default removes all fields those are None or you have to explicitly pass the field names if that's not desired and you have to pass envelope argument too if marshal_with takes the same.
from functools import wraps
import json
from flask_restful import fields, marshal, marshal_with
client_fields = {
'id': fields.String,
'name': fields.String,
#'email': fields.String(default='user#example.com')
'email': fields.String
}
class Client(object):
def __init__(self, id, name, email=None):
self.id = id
self.name = name
self.email = email
def de_none(envelope=None, *fields):
def decorator(func):
def dict_remove(d):
if fields:
for field in fields:
if d[field] is None:
d.pop(field)
else:
for k, v in d.items():
if v is None:
d.pop(k)
#wraps(func)
def decorated(*args, **kwargs):
data = result = func(*args, **kwargs)
if isinstance(result, tuple):
data = result[0]
if envelope:
data = data[envelope]
if isinstance(data, (list, tuple)):
for d in data:
dict_remove(d)
else:
dict_remove(data)
return result
return decorated
return decorator
#de_none()
#marshal_with(client_fields)
def get():
#return [Client(1, 'Tom'), Client(2, 'john', 'john#example.com')], 200, {'Etag': 'blah'}
#return [Client(1, 'Tom'), Client(2, 'john', 'john#example.com')]
#return Client(1, 'Tom'), 200, {'Etag': 'foo'}
return Client(1, 'Tom')
print(json.dumps(get()))
#de_none()
#marshal_with(client_fields)
def get():
return Client(2, 'John', 'john#example.com'), 201, {'Etag': 'ok'}
print(json.dumps(get()))
Output;
{"id": "1", "name": "Tom"}
{"email": "john#example.com", "id": "2", "name": "John"}
UPDATE Request hooks
The app.after_request decorator can be used to modify response object. Any default values given to fields are preserved.
The remove_none_fields request hook takes fields parameter which can be None to remove all fields with None value or a list of field names to selectively remove.
import json
from flask import Flask, Response
from flask_restful import fields, marshal_with, Api, Resource
app = Flask(__name__)
api = Api(app)
class Client(object):
def __init__(self, id, name, email=None):
self.id = id
self.name = name
self.email = email
client_fields = {
'id': fields.String,
'name': fields.String,
'email': fields.String,
'age': fields.String
}
class ClientList(Resource):
#marshal_with(client_fields)
def get(self):
clients =[Client(1, 'Tom'), Client(2, 'John', 'john#example.com')]
return clients, 200
#app.after_request
def remove_none_fields(resp, fields=('email',)):
"""
removes all None fields
"""
if not 'application/json' in resp.content_type:
return resp
def dict_remove(d, fields):
if fields:
for field in fields:
if d[field] is None:
d.pop(field)
else:
for k, v in tuple(d.items()):
if v is None:
d.pop(k)
data = json.loads(resp.get_data())
if isinstance(data, list):
for obj in data:
dict_remove(obj, fields)
else:
dict_remove(data, fields)
resp.set_data(json.dumps(data, indent=1))
resp.content_length = resp.calculate_content_length()
return resp
api.add_resource(ClientList, '/')
if __name__ == '__main__':
app.run(debug=True)
Output;
[
{
"age": null,
"name": "Tom",
"id": "1"
},
{
"age": null,
"email": "john#example.com",
"name": "John",
"id": "2"
}
]
update patching flask_restful.marshal
I filter out None values in a genexp inside marshal function and replace flask_restful.marshal with marshal defined here.
from collections import OrderedDict
from flask import Flask
import flask_restful
from flask_restful import fields, marshal_with, Api, Resource
app = Flask(__name__)
api = Api(app)
class Client(object):
def __init__(self, id, name, email=None):
self.id = id
self.name = name
self.email = email
client_fields = {
'id': fields.String,
'name': fields.String,
'email': fields.String,
}
def marshal(data, fields, envelope=None):
def make(cls):
if isinstance(cls, type):
return cls()
return cls
if isinstance(data, (list, tuple)):
return (OrderedDict([(envelope, [marshal(d, fields) for d in data])])
if envelope else [marshal(d, fields) for d in data])
items = ((k, marshal(data, v) if isinstance(v, dict)
else make(v).output(k, data))
for k, v in fields.items())
#filtering None
items = ((k,v) for k, v in items if v is not None)
return OrderedDict([(envelope, OrderedDict(items))]) if envelope else OrderedDict(items)
flask_restful.marshal = marshal
class ClientList(Resource):
#marshal_with(client_fields)
def get(self):
clients =[Client(1, 'Tom'), Client(2, 'John', 'john#example.com')]
return clients, 200
api.add_resource(ClientList, '/')
if __name__ == '__main__':
app.run(debug=True)
Output;
[
{
"id": "1",
"name": "Tom"
},
{
"email": "john#example.com",
"id": "2",
"name": "John"
}
]

You should use the skip_none property of the #marshal decorator. Its much more convenient than the approaches suggested in the other answers.
#marshal(some_model, skip_none=True)
def get():
...
The documentation can be found here: https://flask-restplus.readthedocs.io/en/stable/marshalling.html
This is also possible using Flask Restx :)