I want to have a MongoDB operation in a celery task but im facing some issues while accessing the instance variables.
Heres the code
app = CeleryConnector().app
class CeleryTasks:
def __init__(self, port, docID, dbName, collName, data):
self.port = port
self.docID = docID
self.dbName = dbName
self.collName = collName
self.data = data
self.client = MongoClient(host='localhost', port=port, username='admin', password='password')
self.db = self.client[dbName]
print ("CeleryTasks:init")
#app.task
def createDoc(dbName, collName, data):
print ("CeleryTasks:CreateDoc")
if 'refs' not in data:
return
# Here is the error I dont know how to access the client variable here.
#db = client[dbName]
print(data['refs'])
for id in data['refs']:
doc = db[collName].find_one({'_id': id})
if doc is None:
insertedID = db[collName].insert_one({
"_id": id
})
print (insertedID)
#app.task(bind=True)
def createDoc(self, dbName, collName, data):
print ("CeleryTasks:CreateDoc")
if 'refs' not in data:
return
print(data['refs'])
for id in data['refs']:
doc = self.db[collName].find_one({'_id': id})
if doc is None:
insertedID = self.db[collName].insert_one({
"_id": id
})
print (insertedID)
As we cannot pass non JSON serializable objects to a Task so passing db or client is not an option.
Problem with the first function
I dont know how to access client inside it. Tried a few things but failed.
Problem with the second function
It gives an error doc = self.db[collName].find_one({'_id': id})
AttributeError: 'createDoc' object has no attribute 'db'
etc etc
How to make this work or how to access instance variables inside celery tasks?
Related
I'm trying to use Redis in my FastAPI -application and struggling to update a value when publishing to a channel.
This is my Redis client setup:
import logging
import redis as _redis
logger = logging.getLogger(__name__)
REDIS_URL = "redis://redis:6379"
class RedisClient:
__instance = None
client: _redis.Redis
def __new__(cls) -> "RedisClient":
if cls.__instance is None:
cls.__instance = object.__new__(cls)
try:
logger.info("Connecting to Redis")
client = _redis.Redis.from_url(REDIS_URL)
cls.client = client
except Exception:
logger.error("Unable to connect to Redis")
logger.info("Connected to Redis")
return cls.__instance
def disconnect(self):
logger.info("Closing Redis connection")
self.client.connection_pool.disconnect()
redis = RedisClient().client
redis_pubsub = redis.pubsub()
This is my POST route what I'm using to test updating the value:
#router.post("/", response_model=Any)
async def post(tracing_no: int) -> Any:
data = {"tracing_no": tracing_no, "result": 200, "client_id": "10"}
data = json.dumps(data)
result = redis.publish(TestChannel.TestTopic.value, data)
return redis.get(TestChannel.TestTopic.value)
This code returns always "{\"tracing_no\": 5, \"result\": 200, \"client_id\": \"10\"}", no matter what number I give as tracking_no when I send the request. I think 5 was the first number I used when testing and that is in my dump.rdb -file.
Any ideas why my tracking_no isn't updating?
In my Flask app, I'm using Dependency Injection and here's what my app looks like. I have a service which uses S3 as a datastore and I'm trying to instantiate my app with the service injected (which is injected with the S3 client). However, it doesn't look like the S3 client is correctly instantiated or I'm doing something wildly different.
containers.py
class Container(containers.DeclarativeContainer):
wiring_config = containers.WiringConfiguration(modules=[".routes", ".scheduler"])
config = providers.Configuration()
s3_config = dict()
s3_config["path"], s3_config["filters"] = config.get("s3_bucket"), [("member_status_nm", "=", "ACTIVE")]
s3_repository = providers.Singleton(S3Repository, s3_config)
my_service = providers.Factory(
MyService, config, S3Repository
)
Here's my S3Repository:
import logging
import sys
import time
import some_library as lib
class S3Repository:
def __init__(self, s3_config):
self.path, self.columns, self.filters = \
s3_config.get("path", ""), s3_config.get("columns", []), s3_config.get("filters", [])
def fetch(self):
# execute fetch
result = lib.some_fetch_method(self.path, self.columns, self.filters)
return result
and MyService:
import #all relevant imports here
class MyService:
def __init__(self, config: dict, s3_repository: S3Repository) -> None:
logging.info("HealthSignalService(): initializing")
self.config = config["app"]["health_signal_service"]
# prepare s3_repository for the service
self.s3_repository = s3_repository
self.s3_repository.columns, self.s3_repository.filters, self.s3_repository.path = \
["x","y"], ["x1","y1"], "file_path"
def fetch_data(self) -> None:
try:
summary_result = self.s3_repository.fetch()
except (FileNotFoundError, IOError) as e:
print("failure")
return summary_result
def get_data(memberId):
sth = self.fetch_data()
return sth.get(memberId)
and finally tying it together in my routes.py:
#inject
#auth.login_required
def get_signals(
my_service: MyService = Provide[
Container.my_service
],
):
content = request.json
member_id = content["memberId"]
result = my_service.get_signals(member_id)
return jsonify(result)
When I hit my API endpoint I see this error:
summary_result = self.s3_repository.fetch()
TypeError: fetch() missing 1 required positional argument: 'self'
How do I correctly initialize my S3 client while using dependency injection?
I'm working on a project very similar to this one: GitHub
I have a class:
class DBfunctions:
def __init__(self, dbname = '../example.db'):
self.debname = dbname
self.conn = sqlite3.connect(dbname)
def search_db(self, telegram_id):
telegram_id = (telegram_id,)
sql = 'SELECT * FROM user WHERE id = ?;'
row = self.conn.execute(sql,telegram_id)
return row
def newuser_db(self, tele_id, name, nick):
par = (tele_id, name, nick, 0)
sql = 'INSERT INTO user VALUES(?,?,?,?);'
self.conn.execute(sql, par)
self.conn.commit()
than i have the main project:
from file import DBfunctions
db = DBfunction()
def start(update: Update, context: CallbackContext): #befor edit: somethingtodo
flag = db.search_db(update.effective_user.id) # here problems start
if flag == None:
db.newuser_db(update.effective_user.id, update.effective_user.first_name, update.effective_user.username)
update.message.reply_text(
'Hi!',
reply_markup=markup,
)
else:
update.message.reply_text(
'Hey! Welcome back!',
reply_markup=markup,
)
def main():
db.setup() # this function is to create tables if not exist yet
dispatcher.add_handler(CommandHandler('start', start))
# other function but nothing correlated
if __name__ == '__main__':
main()
And than the error appears:
File "filefolder/file.py", line 29, in search_db
row = self.conn.execute(sql,telegram_id)
sqlite3.ProgrammingError: SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 15004 and this is thread id 11036.
I can't figure out what i can do to fix it... and don't understand what is different from the project that I find on github (linked)
I know That it is not possible to pickle a pyramid request object, but I cant seem to find where I am sending the Request object.
Consider the following:
#task
def do_consignment_task(store, agent):
print "GOTHERE IN TASK"
s = sqlahelper.get_session()
consign = store.gen_consignment()
ca = Agents.by_id(store.consignment_agents_id)
consign.consignment_agents_id = ca.id
consign.consignment_teamleader_id = ca.ou[0].lead_agents_id
consign.consignment_timestamp = func.now()
consign.created_by_agent_id = agent.id
consign.complete_stamp = func.now()
consign.sims = store.sims
consign.status = "SUCCESS"
print "GOT BEFORE LOOP "
for sim in store.sims:
if sim in consign.sims:
continue
else:
consign.sims.append(sim)
s.add(consign)
transaction.savepoint()
print "GOT AFTER SAVEPOINT"
for sim in consign.sims:
is_reconsign = sim.consignment_agent or sim.consignment_teamlead
if is_reconsign:
if not sim.consignment_history:
sim.consignment_history = []
sim.consignment_history.append(dict(
stamp=sim.consignment_timestamp,
consignment_agent_id=sim.consignment_agents_id,
consignment_teamleader_id=sim.consignment_teamleader_id,
by_agent_id=agent.id
))
s.query(
Sims
).filter(
Sims.iccid == sim.iccid
).update(
{
"consignment_agents_id": consign.consignment_agents_id,
"consignment_history": sim.consignment_history,
"consignment_teamleader_id": ca.ou[0].lead_agents_id,
"consignment_timestamp": func.now(),
"modify_stamp": func.now(),
"consignments_id": consign.id
},
synchronize_session=False
)
print "GOT BEFORE COMMIT"
transaction.savepoint()
print "THIS IS THE ID ID ID ID ID ID : ", consign.id
I call this function like:
if self.store.finalise:
try:
store = self.store
agent = self.agent
do_consignment_task.delay(store, agent)
transaction.commit()
self.check_and_purge()
return "Consignmnet is being processed"
except Exception, exc:
self.check_and_purge()
self.log.exception(exc)
exc_error = "CONSIGNERR:", exc.message
raise USSDFailure(exc_error)
else:
self.store.status = "CANCELLED"
if "fullconfirm" in self.session:
del self.session["fullconfirm"]
self.check_and_purge()
return "CONSIGNMENT Cancelled"
When I run this code I get the following error:
EncodeError: Can't pickle <class 'pyramid.util.Request'>: attribute lookup pyramid.util.Request failed
I am not sending self or request objects - at least not that I can see.
How can solve this problem? Am I sending a request object, because I can not see one?
The traceback can be seen here
EDIT:
okay So I have tried to change the data I send to the function - I am not passing a sqlalchemy object and I am making a copy of the store object, that changes my code to:
#task
def do_consignment_task(agent_id, **store):
print "GOTHERE IN TASK"
s = sqlahelper.get_session()
cObj = USSDConsignmentsObject()
consign = cObj.gen_consignment()
ca = Agents.by_id(store.consignment_agents_id)
consign.consignment_agents_id = ca.id
consign.consignment_teamleader_id = ca.ou[0].lead_agents_id
consign.consignment_timestamp = func.now()
consign.created_by_agent_id = agent_id
# etc
and:
if self.store.finalise:
try:
# del self.service
store = self.store.__dict__.copy()
agent_id = self.agent.id
print store
print agent_id
# print help(store)
do_consignment_task.delay(agent_id, **store)
transaction.commit()
#etc
This however still gives me the same error :|
Try not to serialise a Pyramid request object. When you interact with a celery task you should think of it as an independent process.
Provide it all the information it needs to do it's work. Be aware that you need to serialise that information.
So self.store possibly contains attribute references that may be unrealistic to serialise.
Perhaps create a method on the store object that returns a clean dictionary object.
def serialize(self):
data = {}
data["element1"] = self.element1
data["element2"] = self.element2
data["element3"] = self.element3
return data
Then when you want to call the delay method make sure to use store.serialize() instead of store or the dict.
from pox.core import core
import pox.openflow.libopenflow_01 as of
import re
import datetime
from sqlalchemy import create_engine, ForeignKey
from sqlalchemy import Column, Date, Integer, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship, backref
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.sql.expression import exists
log = core.getLogger()
engine = create_engine('sqlite:///nwtopology.db', echo=False)
Base = declarative_base()
Session = sessionmaker(bind=engine)
session = Session()
########################################################################
class SourcetoPort(Base):
""""""
__tablename__ = 'source_to_port'
id = Column(Integer, primary_key=True)
port_no = Column(Integer)
src_address = Column(String,index=True)
#----------------------------------------------------------------------
def __init__(self, src_address,port_no):
""""""
self.src_address = src_address
self.port_no = port_no
########################################################################
#create tables
Base.metadata.create_all(engine)
class Tutorial (object):
def __init__ (self, connection):
self.connection = connection
connection.addListeners(self)
# Use this table to keep track of which ethernet address is on
# which switch port (keys are MACs, values are ports).
self.mac_to_port = {}
self.matrix={}
#This will keep track of the traffic matrix.
#matrix[i][j]=number of times a packet from i went to j
def send_packet (self, buffer_id, raw_data, out_port, in_port):
#print "calling send_packet"
#Sends a packet out of the specified switch port.
msg = of.ofp_packet_out()
msg.in_port = in_port
msg.data = raw_data
# Add an action to send to the specified port
action = of.ofp_action_output(port = out_port)
msg.actions.append(action)
# Send message to switch
self.connection.send(msg)
def act_like_hub (self, packet, packet_in):
#flood packet on all ports
self.send_packet(packet_in.buffer_id, packet_in.data,
of.OFPP_FLOOD, packet_in.in_port)
def act_like_switch (self, packet, packet_in):
"""
Implement switch-like behavior.
"""
# Learn the port for the source MAC
#print "RECIEVED FROM PORT ",packet_in.in_port , "SOURCE ",packet.src
# create a Session
#Session = sessionmaker(bind=engine)
#session = Session()
self.mac_to_port[packet.src]=packet_in.in_port
#if self.mac_to_port.get(packet.dst)!=None:
#print "count for dst",session.query(SourcetoPort).filter_by(src_address=str(packet.dst)).count(),str(packet.dst)
#if session.query(SourcetoPort).filter_by(src_address=str(packet.dst)).count():
if session.query(exists().where(SourcetoPort.src_address == str(packet.dst))).scalar() is not None:
#send this packet
print "got info from the database"
q_res = session.query(SourcetoPort).filter_by(src_address=str(packet.dst)).one()
self.send_packet(packet_in.buffer_id, packet_in.data,q_res.port_no, packet_in.in_port)
#create a flow modification message
msg = of.ofp_flow_mod()
#set the fields to match from the incoming packet
msg.match = of.ofp_match.from_packet(packet)
#send the rule to the switch so that it does not query the controller again.
msg.actions.append(of.ofp_action_output(port=q_res.port_no))
#push the rule
self.connection.send(msg)
else:
#flood this packet out as we don't know about this node.
print "flooding the first packet"
self.send_packet(packet_in.buffer_id, packet_in.data,
of.OFPP_FLOOD, packet_in.in_port)
#self.matrix[(packet.src,packet.dst)]+=1
entry = SourcetoPort(src_address=str(packet.src) , port_no=packet_in.in_port)
#add the record to the session object
session.add(entry)
#add the record to the session object
session.commit()
def _handle_PacketIn (self, event):
"""
Handles packet in messages from the switch.
"""
packet = event.parsed # This is the parsed packet data.
if not packet.parsed:
log.warning("Ignoring incomplete packet")
return
packet_in = event.ofp # The actual ofp_packet_in message.
#self.act_like_hub(packet, packet_in)
self.act_like_switch(packet, packet_in)
def launch ():
"""
Starts the component
"""
def start_switch (event):
log.debug("Controlling %s" % (event.connection,))
Tutorial(event.connection)
core.openflow.addListenerByName("ConnectionUp", start_switch)
When I run the above code I get the following error:
The problem that I am facing is for some reason if I use
if session.query(exists().where(SourcetoPort.src_address == str(packet.dst))).scalar() is not None:
in place of count query.
#if session.query(SourcetoPort).filter_by(src_address=str(packet.dst)).count():
The querying from the database
q_res = session.query(SourcetoPort).filter_by(src_address=str(packet.dst)).first()
self.send_packet(packet_in.buffer_id, packet_in.data,q_res.port_no, packet_in.in_port)
is giving the following error:
DEBUG:core:POX 0.1.0 (betta) going up...
DEBUG:core:Running on CPython (2.7.3/Aug 1 2012 05:14:39)
DEBUG:core:Platform is Linux-3.5.0-23-generic-x86_64-with-Ubuntu-12.04-precise
INFO:core:POX 0.1.0 (betta) is up.
DEBUG:openflow.of_01:Listening on 0.0.0.0:6633
INFO:openflow.of_01:[00-00-00-00-00-02 1] connected
DEBUG:tutorial:Controlling [00-00-00-00-00-02 1]
got info from the database
ERROR:core:Exception while handling Connection!PacketIn...
Traceback (most recent call last):
File "/home/karthik/pox/pox/lib/revent/revent.py", line 234, in raiseEventNoErrors
return self.raiseEvent(event, *args, **kw)
File "/home/karthik/pox/pox/lib/revent/revent.py", line 281, in raiseEvent
rv = event._invoke(handler, *args, **kw)
File "/home/karthik/pox/pox/lib/revent/revent.py", line 159, in _invoke
return handler(self, *args, **kw)
File "/home/karthik/pox/tutorial.py", line 118, in _handle_PacketIn
self.act_like_switch(packet, packet_in)
File "/home/karthik/pox/tutorial.py", line 86, in act_like_switch
self.send_packet(packet_in.buffer_id, packet_in.data,q_res.port_no, packet_in.in_port)
AttributeError: 'NoneType' object has no attribute 'port_no'
got info from the database
ERROR:core:Exception while handling Connection!PacketIn...
This line:
if session.query(exists().where(SourcetoPort.src_address == str(packet.dst))).scalar() is not None:
Is always true. The reason is that scalar() returns None only if there are no rows. However your query looks like SELECT EXISTS (SELECT * FROM source_to_port WHERE source_to_port.src_address=?). This will always return exactly one row with one column. The result will thus be True or False, never None.
Moving on to the line before the line that throws your exception: first() returns None if there are no matches, so q_res is None. Since q_res is None, q_res.port_no on the next line raises an exception.
(Note you can use one() if you want an exception to be thrown if there is no match.)
If you are expecting a match, double-check your data and your filter_by() condition to make sure they are doing what you think they should.
However I recommend that you use one query instead of two using first() or one(). With first(), you branch based on q_res being None or not:
q_res = session.query(SourcetoPort).filter_by(src_address=str(packet.dst)).first()
if q_res is not None:
print "got info from the database"
self.send_packet(....)
...
else:
print "flooding the first packet"
...
Or with one(), you put your "flooding" branch in an exception handler:
from sqlalchemy.orm.exc import (NoResultFound, MultipleResultsFound)
try:
q_res = session.query(SourcetoPort).filter_by(src_address=str(packet.dst)).one()
except NoResultFound:
print "flooding the first packet"
...
# except MultipleResultsFound:
# print "More than one result found! WUT?!"
else:
print "got info from the database"
...
A difference between these two approaches is that one() will ensure there is one and only one result, whereas first() doesn't care if there are multiple results.