Pickleing error: connot pickle Request object - python

I know That it is not possible to pickle a pyramid request object, but I cant seem to find where I am sending the Request object.
Consider the following:
#task
def do_consignment_task(store, agent):
print "GOTHERE IN TASK"
s = sqlahelper.get_session()
consign = store.gen_consignment()
ca = Agents.by_id(store.consignment_agents_id)
consign.consignment_agents_id = ca.id
consign.consignment_teamleader_id = ca.ou[0].lead_agents_id
consign.consignment_timestamp = func.now()
consign.created_by_agent_id = agent.id
consign.complete_stamp = func.now()
consign.sims = store.sims
consign.status = "SUCCESS"
print "GOT BEFORE LOOP "
for sim in store.sims:
if sim in consign.sims:
continue
else:
consign.sims.append(sim)
s.add(consign)
transaction.savepoint()
print "GOT AFTER SAVEPOINT"
for sim in consign.sims:
is_reconsign = sim.consignment_agent or sim.consignment_teamlead
if is_reconsign:
if not sim.consignment_history:
sim.consignment_history = []
sim.consignment_history.append(dict(
stamp=sim.consignment_timestamp,
consignment_agent_id=sim.consignment_agents_id,
consignment_teamleader_id=sim.consignment_teamleader_id,
by_agent_id=agent.id
))
s.query(
Sims
).filter(
Sims.iccid == sim.iccid
).update(
{
"consignment_agents_id": consign.consignment_agents_id,
"consignment_history": sim.consignment_history,
"consignment_teamleader_id": ca.ou[0].lead_agents_id,
"consignment_timestamp": func.now(),
"modify_stamp": func.now(),
"consignments_id": consign.id
},
synchronize_session=False
)
print "GOT BEFORE COMMIT"
transaction.savepoint()
print "THIS IS THE ID ID ID ID ID ID : ", consign.id
I call this function like:
if self.store.finalise:
try:
store = self.store
agent = self.agent
do_consignment_task.delay(store, agent)
transaction.commit()
self.check_and_purge()
return "Consignmnet is being processed"
except Exception, exc:
self.check_and_purge()
self.log.exception(exc)
exc_error = "CONSIGNERR:", exc.message
raise USSDFailure(exc_error)
else:
self.store.status = "CANCELLED"
if "fullconfirm" in self.session:
del self.session["fullconfirm"]
self.check_and_purge()
return "CONSIGNMENT Cancelled"
When I run this code I get the following error:
EncodeError: Can't pickle <class 'pyramid.util.Request'>: attribute lookup pyramid.util.Request failed
I am not sending self or request objects - at least not that I can see.
How can solve this problem? Am I sending a request object, because I can not see one?
The traceback can be seen here
EDIT:
okay So I have tried to change the data I send to the function - I am not passing a sqlalchemy object and I am making a copy of the store object, that changes my code to:
#task
def do_consignment_task(agent_id, **store):
print "GOTHERE IN TASK"
s = sqlahelper.get_session()
cObj = USSDConsignmentsObject()
consign = cObj.gen_consignment()
ca = Agents.by_id(store.consignment_agents_id)
consign.consignment_agents_id = ca.id
consign.consignment_teamleader_id = ca.ou[0].lead_agents_id
consign.consignment_timestamp = func.now()
consign.created_by_agent_id = agent_id
# etc
and:
if self.store.finalise:
try:
# del self.service
store = self.store.__dict__.copy()
agent_id = self.agent.id
print store
print agent_id
# print help(store)
do_consignment_task.delay(agent_id, **store)
transaction.commit()
#etc
This however still gives me the same error :|

Try not to serialise a Pyramid request object. When you interact with a celery task you should think of it as an independent process.
Provide it all the information it needs to do it's work. Be aware that you need to serialise that information.
So self.store possibly contains attribute references that may be unrealistic to serialise.
Perhaps create a method on the store object that returns a clean dictionary object.
def serialize(self):
data = {}
data["element1"] = self.element1
data["element2"] = self.element2
data["element3"] = self.element3
return data
Then when you want to call the delay method make sure to use store.serialize() instead of store or the dict.

Related

KeyError: records when trying to return record count from API

I am trying to check how many records a player has using the Hypixel API friends endpoint (api.hypixel.net/friends)
It keeps giving me a key error when trying to count the records. Here is what the API gives me:
{
"success": true,
"records": [{"_id":"5806841c0cf247f13be18b9d","uuidSender":"71ef88df5f7e482fb472f344965beba8","uuidReceiver":"976129e438b54a839944b1c0703d4da3","started":1476822044856},{"_id":"59589dde0cf250df95af825e","uuidSender":"71ef88df5f7e482fb472f344965beba8","uuidReceiver":"2c5bfce120c04ef1bfb3b798fe0d650e","started":1498979806692},{"_id":"5a444d820cf2604c12e0f6bd","uuidSender":"71ef88df5f7e482fb472f344965beba8","uuidReceiver":"ea703151981a409f8d0ff7cb782ab1c1","started":1514425730370},{"_id":"5aa595800cf24bd1104381cc","uuidSender":"71ef88df5f7e482fb472f344965beba8","uuidReceiver":"27af346e5bde40f0a665e808a331576f","started":1520801152354},{"_id":"5e2511f90cf289be2d0ea273","uuidSender":"71ef88df5f7e482fb472f344965beba8","uuidReceiver":"24a90aca074c4293a656f5fda047f816","started":1579487737061},{"_id":"5e36448c0cf2174287f94af7","uuidSender":"71ef88df5f7e482fb472f344965beba8","uuidReceiver":"1nmHNU1mERJEG452cx1FazPx3RpAuZ9vW","started":1580614796544},{"_id":"5e3f54190cf2ab010c5a21ce","uuidSender":"71ef88df5f7e482fb472f344965beba8","uuidReceiver":"63817f05823945a2b58f4ba1de5589a3","started":1581208601957},{"_id":"55bec5e0c8f2e017bca39176","uuidSender":"1nmHNU1mERJEG452cx1FazPx3RpAuZ9vW","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1438565856136},{"_id":"55e21268c8f21846db2f2566","uuidSender":"8fca5ebf02f74a369b13f3407ad4a9bc","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1440879208434},{"_id":"56c26b190cf2d1a91ec25e83","uuidSender":"1nmHNU1mERJEG452cx1FazPx3RpAuZ9vW","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1455581977070},{"_id":"5755edc20cf2db67507e3a2e","uuidSender":"0ce4597de5484c0e82a067fa0bf171df","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1465249218466},{"_id":"5851ed8d0cf2b9563974034d","uuidSender":"d3dd0059775e46b1b1a63b94a10d2450","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1481764237412},{"_id":"5d087a1f0cf2d7aebbf96fd6","uuidSender":"d6695d1ea7ae480bb2129a3b7d0269ad","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1560836639346},{"_id":"5d703a880cf299e651b71cbb","uuidSender":"cbd9e9009ee94d159d52dff284ad7bf8","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1567636104118},{"_id":"5deeff7f0cf2d87bd75df1a0","uuidSender":"c53c4524ad174fd78134223fddedc484","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1575944063334},{"_id":"5e0638f20cf24f983d2cb02c","uuidSender":"1nmHNU1mERJEG452cx1FazPx3RpAuZ9vW","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1577466098097},{"_id":"5e1e70e70cf2795e4f1322be","uuidSender":"85090a8b495d4856817fd6df1d4da0bd","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1579053287837},{"_id":"5e1fbe4b0cf2795e4f14a36f","uuidSender":"1nmHNU1mERJEG452cx1FazPx3RpAuZ9vW","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1579138635859},{"_id":"5e25910d0cf2a892e569db39","uuidSender":"63ac73044aca4b2f908baff858cf34b9","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1579520269103},{"_id":"5e712f890cf2d292e1148ba6","uuidSender":"1nmHNU1mERJEG452cx1FazPx3RpAuZ9vW","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1584476041946},{"_id":"5e819f550cf2675e4372109b","uuidSender":"8cd38d8f97a24090a43e2e0ce898c521","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1585553237302},{"_id":"5e82c5ad0cf2675e437308aa","uuidSender":"d618457dd6044256bdb287b0df8137d4","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1585628589448},{"_id":"5eae3e420cf26efdbd55b592","uuidSender":"a936c3468bec4a2685199a398212b62d","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1588477506714},{"_id":"5ebd9da40cf22f431e9164d8","uuidSender":"1nmHNU1mERJEG452cx1FazPx3RpAuZ9vW","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1589484964296}]
}
Here is my code:
def get_friend_count(name):
getUUID = f"https://api.mojang.com/users/profiles/minecraft/{name}"
res = requests.get(getUUID)
data = res.json()
if data["id"] is None:
return None
returnUuid = (data["id"])
url1 = f"https://api.hypixel.net/friends?key={API_KEY}&uuid=" + returnUuid
res2 = requests.get(url1)
data2 = res2.json()
if data2["records"] is None:
return None
friend_count = len(data["records"])
return "Friends: " + friend_count
getUUID gets the UUID from the requested username and then uses the UUID to get the players Hypixel stats.
Any help is appreciated, thanks!
Mojang API: https://wiki.vg/Mojang_API
Hypixel API: https://github.com/HypixelDev/PublicAPI/tree/master/Documentation
Did you just forget to request.get(url1)?
In any case, from the mojang API doc, it seems the endpoint you query (https://api.mojang.com/users/profiles/minecraft/) never returns a "records" key ... thus the error when you do if data["records"] is None

Unable to access Python instance variables inside Celery Tasks

I want to have a MongoDB operation in a celery task but im facing some issues while accessing the instance variables.
Heres the code
app = CeleryConnector().app
class CeleryTasks:
def __init__(self, port, docID, dbName, collName, data):
self.port = port
self.docID = docID
self.dbName = dbName
self.collName = collName
self.data = data
self.client = MongoClient(host='localhost', port=port, username='admin', password='password')
self.db = self.client[dbName]
print ("CeleryTasks:init")
#app.task
def createDoc(dbName, collName, data):
print ("CeleryTasks:CreateDoc")
if 'refs' not in data:
return
# Here is the error I dont know how to access the client variable here.
#db = client[dbName]
print(data['refs'])
for id in data['refs']:
doc = db[collName].find_one({'_id': id})
if doc is None:
insertedID = db[collName].insert_one({
"_id": id
})
print (insertedID)
#app.task(bind=True)
def createDoc(self, dbName, collName, data):
print ("CeleryTasks:CreateDoc")
if 'refs' not in data:
return
print(data['refs'])
for id in data['refs']:
doc = self.db[collName].find_one({'_id': id})
if doc is None:
insertedID = self.db[collName].insert_one({
"_id": id
})
print (insertedID)
As we cannot pass non JSON serializable objects to a Task so passing db or client is not an option.
Problem with the first function
I dont know how to access client inside it. Tried a few things but failed.
Problem with the second function
It gives an error doc = self.db[collName].find_one({'_id': id})
AttributeError: 'createDoc' object has no attribute 'db'
etc etc
How to make this work or how to access instance variables inside celery tasks?

Instancemethod object is not iterable (AirPi) (Python)

I got ERROR:Exception during output: 'instancemethod' object is not iterable when debugging this AirPi code from https://github.com/haydnw/AirPi/blob/master/outputs/ubidots.py
This suppose to upload my sensor data to the Ubidots server.
*I'd put my correct token and variable ID inside the configuration file for this AirPi.
requiredSpecificParams = ["token"]
optionalSpecificParams = ["showcost",
"ID-BMP085-temp",
"ID-BMP085-pres",
"ID-DHT22-hum",
"ID-DHT22-temp",
"ID-LDR",
"ID-TGS2600",
"ID-MiCS-2710",
"ID-MiCS-5525",
"ID-Microphone",
"ID-Raingauge"
]
def __init__(self, config):
super(Ubidots, self).__init__(config)
self.token = self.params["token"]
if "showcost" in self.params:
self.showcost = self.params["showcost"]
else:
self.showcost = False
self.ubivariables = {}
for key, value in self.params.iteritems():
if key[:3] == "ID-":
if value:
self.ubivariables[key[3:]] = value
def output_data(self, datapoints, dummy):
"""Output data.
Output data in the format stipulated by the plugin. Calibration
is carried out first if required.
Because this particular plugin (ubidots) does not show time, the
third argument (normally called 'sampletime') is called 'dummy'
to facilitate compliance with pylint.
Args:
self: self.
datapoints: A dict containing the data to be output.
dummy: datetime representing the time the sample was taken.
Returns:
boolean True if data successfully output to Ubidots; False if
not
"""
if self.params["calibration"]:
datapoints = self.cal.calibrate(datapoints)
payload = []
for point in datapoints:
for ubivariablename, ubivariableid in self.ubivariables.iteritems():
if point["sensor"] == ubivariablename:
if point["value"] is not None:
thisvalue = {}
thisvalue["variable"] = ubivariableid
thisvalue["value"] = point["value"]
payload.append(thisvalue)
break
headers = {'Accept': 'application/json; indent=4', 'Content-Type': 'application/json', 'X-Auth-Token': self.token}
url = "http://things.ubidots.com/api/v1.6/collections/values"
req = None
cost = 0
try:
req = requests.post(url, data=json.dumps(payload), headers=headers)
except Exception, e:
print("ERROR: Failed to contact the Ubidots service.")
print("ERROR: " + str(e))
return False
for response in req.json:
if response["status_code"] is not 201:
print("ERROR: Ubidots responded with an error for one of the values.")
return False
else:
cost += 1
if self.showcost:
print("Ubidots upload cost " + str(cost) + " dots.")
return True
for response in req.json:
According to the documentation, json is a method and must be called, so this should be:
for response in req.json():
In the future it is helpful to include just as much of your code as is necessary to reproduce the problem, and to include the complete error message with traceback.

With ec2 python API boto, how to get spot instance_id from SpotInstanceRequest?

When using boto, Amazon aws python API.
ec2_connection.request_spot_instances(...)
# This will return an ResultSet of SpotInstanceRequest
How can I get instance_ids from the SpotInstanceRequest?
UPDATE: I did it this way, after a lot playing and googleing, hope this help:
ec2_connection.get_all_spot_instance_requests(request_ids=[my_spot_request_id, ])
This will return the updated SpotInstanceRequest, when the instance is ready, we can get *instance_id* from it.
I did something similar: check periodically to see if the spot instance request id returned by
ec2_connection.request_spot_instances(...)
is matched to an instance in the results of
conn.get_all_spot_instance_requests(...)
:
conn = boto.ec2.connect_to_region(region_name=region_name, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
req = conn.request_spot_instances(price=MAX_SPOT_BID,instance_type=instance_type,image_id=AMI_ID,availability_zone_group=region_name,key_name=KEY_PAIR_PEM[:-4],security_groups=security_groups)
job_instance_id = None
while job_instance_id == None:
print "checking job instance id for this spot request"
job_sir_id = req[0].id # spot instance request = sir, job_ is the relevant aws item for this job
reqs = conn.get_all_spot_instance_requests()
for sir in reqs:
if sir.id == job_sir_id:
job_instance_id = sir.instance_id
print "job instance id: " + str(job_instance_id)
break
time.sleep(SPINUP_WAIT_TIME)
spot_instance_requests = aws.ec2_get_connection().request_spot_instances(...)
MAX_MINUTES = 180
spot_instance_request_ids = [sir.id for sir in spot_instance_requests]
for _ in range(MAX_MINUTES):
log.info('waiting for spot instances to start', request_ids=spot_instance_request_ids, seconds=60)
time.sleep(60)
spot_instance_requests = aws.ec2_get_connection().get_all_spot_instance_requests(
request_ids=spot_instance_request_ids)
if any(sir.instance_id for sir in spot_instance_requests):
log.info('spot instance started. waiting...', seconds=60*5)
time.sleep(60*5)
break
else:
raise Exception("Spot instances didn't start in {0} minutes!".format(MAX_MINUTES))

Successful use of pidbox.Mailbox?

Has anyone used pidbox.Mailbox ?
I am attempting to do something similar to that example, but that documentation is way out of date. I have managed to get something that pubishes messages to a django transport but from there they never are successfuly received.
I was hoping that someone knows how to use this, and could show me an example of how to successfuly call/cast.
Here is what I have(dummy node really does nothing just prints or lists):
#node/server
mailbox = pidbox.Mailbox("test", type="direct")
connection = BrokerConnection(transport="django")
bound = mailbox(connection)
state = {"node": DummyNode(),
"connection": connection
}
node = bound.Node(state = state)
#node.handler
def list( state, **kwargs):
print 'list called'
return state["node"].list()
#node.handler
def connection_info(state, **kwargs):
return {"connection": state["connection"].info()}
#node.handler
def print_msg(state, **kwargs):
print 'Node handler!'
state["node"].print_msg(kwargs)
consumer = node.listen(channel = connection.channel())
try:
while not self.killed:
print 'Consumer Waiting'
connection.drain_events()
finally:
consumer.cancel()
And a simple client.
#client:
mailbox = pidbox.Mailbox("test", type="direct")
connection = BrokerConnection(transport="django")
bound = mailbox(connection)
bound.cast(["localhost"], "print_msg", {'msg' : 'Message for you'})
info = bound.call(["test_application"],"list", callback=callback)
The answer to this is apparently no. If you come across this post I highly recommend writing your own. There is too little documentation for pidbox, and the documentation that is there is out of date.

Categories