We Keep Coding

"AttributeError: 'list' object has no attribute 'items'" While importing Json

I get the following error when i try to use the "deleted" part of my json file to change one field (ReportedStatusField) for each "Facility" in my Database that matches with the id's inside "deleted":
File "C:\users\management\commands\deactivate_facilities.py", line 42, in handle
for key, data in data_object.items():
AttributeError: 'list' object has no attribute 'items'
It's basically a list of UUID's of Facilities that closed so i change the status this way with a json file that gets imported with a url.
import requests
import json
from users.models import Facility
from django.core.management.base import BaseCommand
IMPORT_URL = 'https://domain/file.json'
class Command(BaseCommand):
def import_facility_from_file(self, data):
key = data.get('key', None)
if Facility.objects.filter(key=UUID):
msg = "\n\nFacility closed: {}\n{}".format(key, str())
print(msg)
facility, facility_closed = Facility.objects.update_or_create(key=UUID,
defaults={
'ReportedStatusField': 'closed'
}
)
def handle(self, *args, **options):
headers = {'Content-Type': 'application/json'}
response = requests.get(
url=IMPORT_URL,
headers=headers,
)
response.raise_for_status()
data = response.json()
for key, data_object in data.items():
if key in ["deleted"]:
for key, data in data_object.items():
self.import_facility_from_file(data)
My JSON
{
"added":
{"125hk24h5kjh43k5":
{
"UUID":"125hk24h5kjh43k5",
"Name":"Test Facility 1",
"AddressInfo":
{"PrimaryAddress":"1234 Drive RD"},
"ImporterLastModifiedTimestamp":1643721420}},
// This is the "list" of deleted Facilities
"deleted":["235hk24h5kjh43k5,235hk345789h43k5"],
"modified":{"995hk24h5kjh43k5":
{
"UUID":"995hk24h5kjh43k5",
"Name":"Test Facility 2",
"AddressInfo":
{"PrimaryAddress":"2345 Test RD"},
"ImporterLastModifiedTimestamp":1643721420}
}
}

Whenever we use 'items' method on the dictionary, it will return the tuple type element in the list.
and we cannot run 'item' method on the list
for key, data_object in data.items():
if key in ["deleted"]:
for data in data_object:
self.import_facility_from_file(data)

Getting type error in python

I am using a class based service in python and I get error whenever I want to use it. Unable to figure out the reason.
#!/usr/bin/python
# -*- coding: utf-8 -*-
from xml.dom import minidom
from pysimplesoap.client import SoapClient
from pysimplesoap.helpers import sort_dict
MEDIA_ROOT = '/User/sunand/documents/resumes/'
parser = ResumeParser()
names = parser.get_names(MEDIA_ROOT)
print names
class ParserClient(SoapClient):
""" Extends the soap client to encode the response with utf-8 encoding.
"""
def wsdl_call(
self,
method,
*args,
**kwargs
):
""" Override wsdl_call method to make sure unmarshall is not called.
"""
operation = self.get_operation(method)
# get i/o type declarations:
inp = operation['input']
header = operation.get('header')
if 'action' in operation:
self.action = operation['action']
# construct header and parameters
if header:
self.__call_headers = sort_dict(header, self.__headers)
(method, params) = self.wsdl_call_get_params(method, inp,
*args, **kwargs)
response = self.call(method, *params)
return response
def send(self, method, xml):
""" Overrides the send method to get the actual xml content.
"""
content = super(ParserClient, self).send(method, xml)
self.result = content
return content
class ResumeParser(object):
""" Connects to the Resume Parser's XML api to get parsed data.
"""
def __init__(self, simple=True, timeout=60):
""" Initializes the ResumeParser class.
"""
self.wsdl = \
'http://jobsite.onlineresumeparser.com/rPlusParseResume.asmx?WSDL'
self.secret = 'my-secret-key' # Enter key here
self.encoding = 'base64'
self.simple = simple
self.client = ParserClient(wsdl=self.wsdl, timeout=timeout)
self.names = []
def get_file_content(self, file_path):
""" Return the encoded content for the given file.
"""
file_obj = open(os.path.abspath(file_path), 'r')
content = file_obj.read().encode(self.encoding)
file_obj.close()
return content
def get_names(self, path):
"""
Given a path to a folder that contains resume files this method
will parse the resumes and will return the names of the candidates
as a list.
"""
opt = os.path
resumes = [opt.join(path, r) for r in os.listdir(path)
if opt.isfile(opt.join(path, r))]
# Parse information for each resume.
for resume in resumes:
try:
xml_data = self.get_xml(resume)
name = self.get_name_from_xml(xml_data)
if name:
self.names.append(name)
except Exception, err:
# print name
print 'Error parsing resume: %s' % str(err)
return list(set(self.names))
def get_name_from_xml(self, data):
""" Returns the full name from the xml data given.
"""
xmldata = minidom.parseString(data)
name = xmldata.getElementsByTagName('CANDIDATE_FULL_NAME')
name = name[0].childNodes[0].data.title()
return name
def get_xml(self, filepath):
""" Fetches and returns the xml for the given file from the api.
"""
filename = os.path.basename(filepath)
extension = os.path.splitext(filepath)[1]
base64 = self.get_file_content(filepath)
filedata = {
'B64FileZippedContent': base64,
'FileName': filename,
'InputType': extension,
'UserID': 1,
'secretKey': self.secret,
}
get = \
(self.client.GetSimpleXML if self.simple else self.client.getHRXML)
get(**filedata)
return self.process_raw_xml()
def process_raw_xml(self, data=None):
""" Processes and returns the clean XML.
"""
raw = (data if data else self.client.result)
parsed = minidom.parseString(raw)
result = parsed.getElementsByTagName('GetSimpleXMLResult')[0]
text_node = result.childNodes[0]
data = text_node.data.encode('UTF-8')
return data
Upon running the code I am getting an error
TypeError: wsdl_call_get_params() got an unexpected keyword argument 'secretKey'
What am I doing wrong?

It looks like you are incorrectly overriding wsdl_call.
Firstly, we can see that SoapClient (which you extend in ParserClient), has a __getattr__ function that fetches pseudo-attributes of the SoapClient.
def __getattr__(self, attr):
"Return a pseudo-method that can be called"
if not self.services: # not using WSDL?
return lambda self=self, *args, **kwargs: self.call(attr,*args,**kwargs)
else: # using WSDL:
return lambda *args, **kwargs: self.wsdl_call(attr,*args,**kwargs)
You can see that this function is using wsdl_call to help it map functions to unknown attributes.
The specific pseudo-method that is causing the problem is in your code (or appears to be):
filedata = {
'B64FileZippedContent': base64,
'FileName': filename,
'InputType': extension,
'UserID': 1,
'secretKey': self.secret, # <-- the secretKey key word argument
}
get = \
(self.client.GetSimpleXML if self.simple else self.client.getHRXML)
get(**filedata)
# here client is an instance of your `ParserClient` (and `SoapClient`).
This above bit took me a while to track down. With a full stack trace I would have found it much quicker. Please always post stack traces (when there is one) in future when asking for help.
How to solve this
Provide a concrete implementation of GetSimpleXML and getHRXML. This will solve the immediate problem, but not the larger problem.
Rewrite wsdl_call
The rewritten section of code should check the value of the method argument and either do what you want, or delegate to the SoapClient implementation.
eg.
def wsdl_call(self, method, *args, **kwargs):
if method == "some_method":
return self._my_wsdl_call(method, *args, **kwargs)
else:
return super(ParserClient, self).wsdl_call(method, *args, **kwargs)
def _my_wsdl_call(self, method, *args, **kwargs):
...

Pickleing error: connot pickle Request object

I know That it is not possible to pickle a pyramid request object, but I cant seem to find where I am sending the Request object.
Consider the following:
#task
def do_consignment_task(store, agent):
print "GOTHERE IN TASK"
s = sqlahelper.get_session()
consign = store.gen_consignment()
ca = Agents.by_id(store.consignment_agents_id)
consign.consignment_agents_id = ca.id
consign.consignment_teamleader_id = ca.ou[0].lead_agents_id
consign.consignment_timestamp = func.now()
consign.created_by_agent_id = agent.id
consign.complete_stamp = func.now()
consign.sims = store.sims
consign.status = "SUCCESS"
print "GOT BEFORE LOOP "
for sim in store.sims:
if sim in consign.sims:
continue
else:
consign.sims.append(sim)
s.add(consign)
transaction.savepoint()
print "GOT AFTER SAVEPOINT"
for sim in consign.sims:
is_reconsign = sim.consignment_agent or sim.consignment_teamlead
if is_reconsign:
if not sim.consignment_history:
sim.consignment_history = []
sim.consignment_history.append(dict(
stamp=sim.consignment_timestamp,
consignment_agent_id=sim.consignment_agents_id,
consignment_teamleader_id=sim.consignment_teamleader_id,
by_agent_id=agent.id
))
s.query(
Sims
).filter(
Sims.iccid == sim.iccid
).update(
{
"consignment_agents_id": consign.consignment_agents_id,
"consignment_history": sim.consignment_history,
"consignment_teamleader_id": ca.ou[0].lead_agents_id,
"consignment_timestamp": func.now(),
"modify_stamp": func.now(),
"consignments_id": consign.id
},
synchronize_session=False
)
print "GOT BEFORE COMMIT"
transaction.savepoint()
print "THIS IS THE ID ID ID ID ID ID : ", consign.id
I call this function like:
if self.store.finalise:
try:
store = self.store
agent = self.agent
do_consignment_task.delay(store, agent)
transaction.commit()
self.check_and_purge()
return "Consignmnet is being processed"
except Exception, exc:
self.check_and_purge()
self.log.exception(exc)
exc_error = "CONSIGNERR:", exc.message
raise USSDFailure(exc_error)
else:
self.store.status = "CANCELLED"
if "fullconfirm" in self.session:
del self.session["fullconfirm"]
self.check_and_purge()
return "CONSIGNMENT Cancelled"
When I run this code I get the following error:
EncodeError: Can't pickle <class 'pyramid.util.Request'>: attribute lookup pyramid.util.Request failed
I am not sending self or request objects - at least not that I can see.
How can solve this problem? Am I sending a request object, because I can not see one?
The traceback can be seen here
EDIT:
okay So I have tried to change the data I send to the function - I am not passing a sqlalchemy object and I am making a copy of the store object, that changes my code to:
#task
def do_consignment_task(agent_id, **store):
print "GOTHERE IN TASK"
s = sqlahelper.get_session()
cObj = USSDConsignmentsObject()
consign = cObj.gen_consignment()
ca = Agents.by_id(store.consignment_agents_id)
consign.consignment_agents_id = ca.id
consign.consignment_teamleader_id = ca.ou[0].lead_agents_id
consign.consignment_timestamp = func.now()
consign.created_by_agent_id = agent_id
# etc
and:
if self.store.finalise:
try:
# del self.service
store = self.store.__dict__.copy()
agent_id = self.agent.id
print store
print agent_id
# print help(store)
do_consignment_task.delay(agent_id, **store)
transaction.commit()
#etc
This however still gives me the same error :|

Try not to serialise a Pyramid request object. When you interact with a celery task you should think of it as an independent process.
Provide it all the information it needs to do it's work. Be aware that you need to serialise that information.
So self.store possibly contains attribute references that may be unrealistic to serialise.
Perhaps create a method on the store object that returns a clean dictionary object.
def serialize(self):
data = {}
data["element1"] = self.element1
data["element2"] = self.element2
data["element3"] = self.element3
return data
Then when you want to call the delay method make sure to use store.serialize() instead of store or the dict.

Django sending data to front-end. 'dict' object has no attribute '_meta'

I have these methods:
def get_all_from_database():
urls = Url.objects.all()
ips = Ip.objects.all()
context = {
'urls': serializers.serialize('json', urls),
'ip': serializers.serialize('json', ips)
}
return context
and the method that sends data to using ajax:
def send_results(request):
if request.is_ajax():
address = request.POST.get('url')
process_data(address, email_to, email_from)
context = get_all_from_database()
return HttpResponse(json.dumps(context), content_type='application/json')
But this raises error : INTERNAL SERVER ERROR 500 'dict' object has no attribute '_meta'.
Wheres the mistake, and how to correct it ?

You cant use serializers.serialize method with dict list that you got from values call:
urls = Url.objects.all().values('address', 'cnt')
Use default queryset:
urls = Url.objects.all()
ips = Ip.objects.all()

In you example context['urls'] value already in json format, and you cant use json.dumps() for json data.
You can use this example:
json.dumps({
'urls': Urls.objects.all().values_list('address', 'cnt'),
'ips': Ip.objects.all().values_list('address', 'cnt')
}), 'json')

urls = Url.objects.all().values('address', 'cnt')
ips = Ip.objects.all().values('address', 'cnt')
The above lines returns dict objects, try:
urls = Url.objects.all().values('address', 'cnt').values_list()
ips = Ip.objects.all().values('address', 'cnt').values_list()
Then you will have urls as a list containing the tuples:
[(address_1, cnt_1), (address_2, cnt_2), ...]
see: QuerySet API reference

I think it should be :
res=json.dumps({
'urls': list(Urls.objects.all().values_list('address', 'cnt')),
'ips': list(Ip.objects.all().values_list('address', 'cnt'))
}), 'json')
return HttpResponse(res,content_type="application/json")

Why doesn't my tastypie cache get called?

I'm looking at the tastypie caching docs and trying to set up my own simple caching thing, but the cache doesn't seem to get called. When I visit http://localhost:8000/api/poll/?format=json, I get my tastypie generated json, but I don't get the output from the cache class.
from tastypie.resources import ModelResource
from tastypie.cache import NoCache
from .models import Poll
class JSONCache(NoCache):
def _load(self):
print 'loading cache'
data_file = open(settings.TASTYPIE_JSON_CACHE, 'r')
return json.load(data_file)
def _save(self, data):
print 'saving to cache'
data_file = open(settings.TASTYPIE_JSON_CACHE, 'w')
return json.dump(data, data_file)
def get(self, key):
print 'jsoncache.get'
data = self._load()
return data.get(key, None)
def set(self, key, value, timeout=60):
print 'jsoncache.set'
data = self._load()
data[key] = value
self._save(data)
class PollResource(ModelResource):
class Meta:
queryset = Poll.objects.all()
resource_name = 'poll'
cache = JSONCache()

It seems that Tastypie doesn't automatically cache lists, tastypie.resources around line 1027:
def get_list(self, request, **kwargs):
# ...
# TODO: Uncached for now. Invalidation that works for everyone may be
# impossible.
objects = self.obj_get_list(
request=request, **self.remove_api_resource_names(kwargs))
# ...
, whereas with details (around line 1050):
def get_detail(self, request, **kwargs):
# ...
try:
obj = self.cached_obj_get(
request=request, **self.remove_api_resource_names(kwargs))
# ...
... note that in the former snippet obj_get_list is called instead of cached_obj_get_list. Perhaps overriding get_list and using cached_obj_get_list would allow you to use cache here as well?
Now you probably would get output from your class for http://localhost:8000/api/poll/<pk>/?format=json (detail view) but not for http://localhost:8000/api/poll/?format=json (list view) by default.