How can I make #celery.task available on user resources package?
I'm new to python and confused about the "circular imports error" can somebody explain how it works? and how to handle this kind of errors in flask application. This is the part where I always get stuck.
This is my project current folder structure
code
|_resources
| |_user.py
|
|_utils
| |_flask_celery.py
|
|_flask_app.py
--- SOURCE CODE ---
flask_app.py
from flask import Flask
from flask_cors import CORS
from flask_restful import Api
from util.flask_celery import make_celery
from routes.endpoint import urls
from resources.user import Users
app = Flask(__name__)
CORS(app, resources={r"/*": {"origins": "*"}})
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql:///username:StrongPassword#localhost:3306/db'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['CELERY_BROKER_URL'] = 'amqp//admin:StrongPassword#localhost:5672'
app.config['CELERY_RESULT_BACKEND'] = 'db+mysql:///username:StrongPassword#localhost:3306/db'
app.config['PROPAGATE_EXCEPTIONS'] = True
api = Api(app)
celery = make_celery(app)
url = urls()
api.add_resource(Users, url.get('users'))
if __name__ == "__main__":
# sqlalchemy
from db import db
db.__init__(app)
app.run(host='0.0.0.0', debug=True, port=5000)
flask_celery.py
from celery import Celery
def make_celery(celery_app):
celery = Celery(
celery_app.import_name,
backend=celery_app.config['CELERY_RESULT_BACKEND'],
broker=celery_app.config['CELERY_BROKER_URL']
)
celery.conf.update(celery_app.config)
class ContextTask(celery.Task):
def __call__(self, *args, **kwargs):
with celery_app.app_context():
return self.run(*args, **kwargs)
celery.Task = ContextTask
return celery
user.py
from flask_restful import Resource, reqparse
from app import celery # <-- this cause the error
from util.zk_connector import zk_connect
from util.zk_error import error
class Users(Resource):
def post(self, ip, comkey):
zk = zk_connect(ip=ip, password=int(comkey))
try:
session = zk.connect()
session.disable_device()
users = session.get_users()
print(users)
session.enable_device()
session.disconnect()
return {'message': 'success'}, 200
except Exception as e:
return error(e)
#celery.task(name='user.reverse')
def reverse(string):
return string[::-1]
Error:
Traceback (most recent call last):
File ".\flask_app.py", line 6, in <module>
from resources.user import User
File "C:\Users\Gelo\Documents\Brand new clean arch pyzk\code\resources\user.py", line 2, in <module>
from flask_app import celery
File "C:\Users\Gelo\Documents\Brand new clean arch pyzk\code\flask_app.py", line 6, in <module>
from resources.user import User
ImportError: cannot import name 'User' from partially initialized module 'resources.user' (most likely due to a circular import) (C:\Users\Gelo\Documents\Brand new clean arch pyzk\code\resources\user.py)
Related
This is my inside flaskapp/__init__.py for creating my Flask app, with it I can access Session inside any module in the package, by just importing Session from flaskapp/db.py:
import os
from flask import Flask
def create_app(test_config=None):
app = Flask(__name__, instance_relative_config=True)
app.config.from_mapping(
SECRET_KEY=b'some_secret_key',
DEBUG=True,
SQLALCHEMY_DATABASE_URI=f'sqlite:///{os.path.join(app.instance_path, "flaskapp.sqlite")}',
)
if test_config is None:
app.config.from_pyfile('config.py', silent=True)
else:
app.config.from_mapping(test_config)
try:
os.makedirs(app.instance_path)
except OSError:
pass
with app.app_context():
from flaskapp.routes import home, auth
from flaskapp.db import init_db
init_db()
return app
This is my flaskapp/db.py:
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from flask import current_app
engine = create_engine(current_app.config['SQLALCHEMY_DATABASE_URI'], echo=True)
Session = sessionmaker(bind=engine)
def init_db():
import flaskapp.models as models
models.Base.metadata.create_all(engine)
def drop_db():
import flaskapp.models as models
models.Base.metadata.drop_all(engine)
With that, I can access the database in any other module, for example, flaskapp/auth.py:
from flask import flash, session
from sqlalchemy import select
from flaskapp.db import Session
from flaskapp.models import User
def log_user(username: str, password: str) -> bool:
with Session() as db_session:
stmt1 = select(User).where(User.username == username)
query_user = db_session.execute(stmt1).first()
if not query_user:
flash('Some error message')
# Some password verifications and other things
session['username'] = query_user[0].username
flash('Successfully logged in')
return True
Until that point, I don't have any problem, the problem comes when I try to do unit testing with unittest, I can't set the test environment and I don't know how can I use the Session object defined in flaskapp/db.py for testing in a separate database. Everything I've tried until now gets me an error, this is my tests/__init__.py:
import unittest
from flaskapp import create_app
from flaskapp.db import Session
from flaskapp.models import User
class BaseTestClass(unittest.TestCase):
def setUp(self):
self.app = create_app(test_config={
'TESTING': True,
'DEBUG': True,
'APP_ENV': 'testing',
# I pass the test database URI expecting the engine to use it
'SQLALCHEMY_DATABASE_URI': 'sqlite:///testdb.sqlite',
})
self.client = self.app.test_client()
# Context
with self.app.app_context():
self.populate_db()
def tearDown(self):
pass
def populate_db(self):
with Session() as db_session:
db_session.add(User(
username='Harry',
email='harry#yahoo.es',
password = 'Harry123.'
))
db_session.commit()
When I try to use the Session object inside populate_db() I get this error:
=====================================================================
ERROR: tests.test_auth (unittest.loader._FailedTest.tests.test_auth)
----------------------------------------------------------------------
ImportError: Failed to import test module: tests.test_auth
Traceback (most recent call last):
File "/usr/local/lib/python3.11/unittest/loader.py", line 407, in _find_test_path
module = self._get_module_from_name(name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/unittest/loader.py", line 350, in _get_module_from_name
__import__(name)
File "/home/chus/usb/soft/proy/SAGC/tests/test_auth.py", line 1, in <module>
from flaskapp.db import Session
File "/home/chus/usb/soft/proy/SAGC/flaskapp/db.py", line 5, in <module>
engine = create_engine(current_app.config['SQLALCHEMY_DATABASE_URI'], echo=True)
^^^^^^^^^^^^^^^^^^
File "/home/chus/usb/soft/proy/SAGC/venv/lib/python3.11/site-packages/werkzeug/local.py", line 316, in __get__
obj = instance._get_current_object() # type: ignore[misc]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/chus/usb/soft/proy/SAGC/venv/lib/python3.11/site-packages/werkzeug/local.py", line 513, in _get_current_object
raise RuntimeError(unbound_message) from None
RuntimeError: Working outside of application context.
This typically means that you attempted to use functionality that needed
the current application. To solve this, set up an application context
with app.app_context(). See the documentation for more information.
----------------------------------------------------------------------
Ran 1 test in 0.000s
FAILED (errors=1)
Can someone help me out? I've tried everything to make the tests use the app context in order to populate the test database and make queries but it appears to be conflicting with it being created in create_app() or something.
Firsly, the error you posted is about tests/test_auth.py which you did not include.
However, your code is tightly coupling the flask app and the db.py module with the use of flask.current_app to obtain the engine url.
So the engine creation will only work if there is an app context, which is probably absent when you do a top level import (line 1) of from flaskapp.db import Session in tests/test_auth.py.
A solution could be creating the engine and sessionmaker (or scoped_session) with your create_app and injecting the engine or Session in your functions (look up dependency injection principle):
def init_db(engine):
models.Base.metadata.create_all(engine)
def drop_db(engine):
models.Base.metadata.drop_all(engine)
def log_user(session, username: str, password: str) -> bool:
stmt1 = select(User).where(User.username == username)
query_user = session.execute(stmt1).first()
# skipping the rest
Or look into Flask-SQLAlchemy.
This is my main.py file :
from flask_redis import FlaskRedis
from controllers.utils import redis_conn # get Redis URL
import connexion
BASE_PATH = "/"
def create_app(*specs):
_app = connexion.App(__name__)
for s in specs:
logger.info("Adding specs {}".format(s))
_app.add_api(s, validate_responses=True)
return _app
app = create_app("specs.yaml")
rd_app = app.app
rd_app.config['REDIS_URL'] = redis_conn()
redis_client = FlaskRedis(rd_app)
if __name__ == '__main__':
app.run(host='127.0.0.1', port=5000, debug = True)
It seems the Redis has issue and producing this error :
ImportError: cannot import name 'redis_client' from partially
initialized module 'main' (most likely due to a circular import)
I can't find any tutorial that use connexion with Redis.
Example of usage get_fruit.py:
from main import redis_client
def get_fruit(colour, shape, taste):
hash_name = rd_collections[colour+'_'+shape]
key_name = '{}:{}'.format(hash_name, taste)
response_redis = redis_client.get(name=key_name)
if response_redis is None:
result = get_fruit_name(colour, shape, taste)
logger.debug("Updating Redis by adding {}".format(location_id))
redis_client.set(name=key_name, value=json.dumps(result['fruit_id']), ex=60*60)
result = OrderedDict({'Result': result})
return result
else:
...
UPDATE:
Attempted as suggested :
def create_app(*specs):
"""
Running apps using connexion
"""
_app = connexion.App(__name__)
rd_app = _app.app
rd_app.config['REDIS_URL'] = redis_conn()
rd_client = FlaskRedis(rd_app)
for s in specs:
logger.info("Adding specs {}".format(s))
_app.add_api(s, validate_responses=True)
return _app, rd_client
app, redis_client = create_app("specs.yaml")
But still producing the same error.
The standard way, which avoids the circular import, is the following one.
Create a config.py file with your configuration.
from controllers.utils import redis_conn # get Redis URL
class MyConfig(object):
REDIS_URL = redis_conn()
Create a wsgi.py file. This will be the starting point of your app.
from config import MyConfig
import app_factory
app = app_factory.create_app(MyConfig)
if __name__ == "__main__":
with app.app_context():
app.run()
Create an app_factory.py file with the App Factory pattern:
redis_client = FlaskRedis()
def create_app(config):
app = Flask(__name__)
app.config.from_object(config)
redis_client.init_app(app)
return app
Create a file routes.py with your routes:
from app_factory import redis_client
#bp.route('/')
def index():
return redis_client.get('potato')
I am fairly new to python development and have no idea about flask, I have been assigned a project that is developed using flask. After working for couple of weeks i am now able to resolve all the dependencies and project is now compiled successfully. But when I run the project using flask run and then enter the url in browser it throws "flask.cli.NoAppException". How can I run my project I have tried like this.
set FLASK_APP=init.py
set FLASK_ENV=develpment
flask run
Serving Flask app "init.py" (lazy loading)
Environment: develpment
Debug mode: on
Restarting with stat
Debugger is active!
Debugger PIN: 202-733-235
Running on http://127.0.0.1:5000/ (Press CTRL+C to quit)
here is the trackback
"FLASK_APP=myappnam:name to specify one.
Traceback (most recent call last)
File "C:\Program Files\Python38\Lib\site-packages\flask\_compat.py", line 39, in reraise
raise value
File "C:\Program Files\Python38\Lib\site-packages\flask\cli.py", line 97, in find_best_app
raise NoAppException(
flask.cli.NoAppException: Failed to find Flask application or factory in module "myappnam". Use "FLASK_APP=myappnam:name to specify one.
The debugger caught an exception in your WSGI application. You can now look at the traceback which led to the error.
To switch between the interactive traceback and the plaintext one, you can click on the "Traceback" headline. From the text traceback you can also create a paste of it. For code execution mouse-over the frame you want to debug and click on the console icon on the right side.
You can execute arbitrary Python code in the stack frames and there are some extra helpers available for introspection:
dump() shows all variables in the frame
dump(obj) dumps all that's known about the object
here is my --init--.py file
import os
import logging
import gevent
import datetime
import rollbar
from gevent.queue import Queue
from gevent.event import AsyncResult
import zmq.green as zmq
from werkzeug.contrib.fixers import ProxyFix
# Greens the postgress connector
try:
import psycogreen.gevent
psycogreen.gevent.patch_psycopg()
except ImportError:
pass
from rauth.service import OAuth2Service
from flask import Flask, Request
from flask_sqlalchemy import SQLAlchemy
from flask_bcrypt import Bcrypt
from flask_login import LoginManager, current_user
from flask_assets import Environment
from flask_uploads import UploadSet, configure_uploads, IMAGES
from app.util import setup_logging
from app.exceptions import TimeoutError, BackendError
import app.exceptions
flask_app = None
# Have to use an actor pattern because we cannot allow more than one request to
# be pending at a time.
class Backend(gevent.Greenlet):
def __init__(self):
super(Backend, self).__init__()
self.inbox = Queue()
self.zmq_context = zmq.Context()
self.zmq_socket = None
self.init_socket()
def init_socket(self):
zmq_socket = self.zmq_socket
if zmq_socket is not None:
zmq_socket.close(0)
zmq_socket = self.zmq_context.socket(zmq.REQ)
zmq_socket.connect(flask_app.config["SERVER_ZMQ_URI"])
self.zmq_socket = zmq_socket
def process(self, request):
zmq_socket = self.zmq_socket
poller = zmq.Poller()
poller.register(zmq_socket, zmq.POLLIN)
zmq_socket.send_json({
"command": request["command"],
"arguments": request["arguments"]
})
sockets = dict(poller.poll(10 * 1000))
if zmq_socket not in sockets:
self.init_socket()
result = request["result"]
result.set_exception(TimeoutError("The request to the backend timed out."))
return
received = zmq_socket.recv_json()
result = request["result"]
if received["success"]:
result.set(received["result"])
else:
result.set_exception(BackendError(received["result"]))
def _run(self):
while True:
self.process(self.inbox.get())
def send(self, command, **kwargs):
result = AsyncResult()
self.inbox.put({
"command": command,
"arguments": kwargs,
"result": result
})
return result.get()
class RollbarRequest(Request):
#property
def rollbar_person(self):
if current_user.is_anonymous:
return {
"id": 0,
"username": "anonymous"
}
return {
"id": current_user.id,
"username": current_user.name,
"email": current_user.email_address
}
def create_app(*args, **kwargs):
global flask_app
global l
app_mode = os.environ.get("APP_MODE")
assert app_mode is not None, "APP_MODE environment variable must be set"
flask_app = Flask(__name__)
flask_app.request_class = RollbarRequest
flask_app.config.from_object("config.mode_{}".format(app_mode))
flask_app.config["APP_MODE"] = app_mode
setup_logging(flask_app.config["LOGGING_LEVEL"])
l = logging.getLogger(__name__)
l.info("starting in mode {}".format(app_mode))
if not flask_app.config["DEBUG"]:
rollbar.init(
flask_app.config["ROLLBAR_API_KEY"],
app_mode,
allow_logging_basic_config=False
)
flask_app.jinja_env.globals.update(
current_year=lambda: datetime.datetime.now().year
)
# Have to do this so that redirects work in proxy mode behind NGINX.
if not flask_app.debug:
flask_app.wsgi_app = ProxyFix(flask_app.wsgi_app)
flask_app.db = SQLAlchemy(flask_app)
flask_app.bcrypt = Bcrypt(flask_app)
flask_app.assets = Environment(flask_app)
flask_app.images = UploadSet("images", IMAGES)
configure_uploads(flask_app, flask_app.images)
flask_app.photos = UploadSet("photos", IMAGES)
configure_uploads(flask_app, flask_app.photos)
login_manager = LoginManager()
login_manager.login_view = "signin"
login_manager.login_message_category = "alert" # Need newer release of Flask-Login for this to work.
login_manager.init_app(flask_app)
flask_app.facebook = OAuth2Service(
name="facebook",
base_url="https://graph.facebook.com/v2.8/",
client_id=flask_app.config["FACEBOOK_CLIENT_ID"],
client_secret=flask_app.config["FACEBOOK_CLIENT_SECRET"]
)
from app import views
from app import models
from app import commands
flask_app.backend = Backend()
flask_app.backend.start()
app.exceptions.register(flask_app)
return flask_app
and this is my project structure
I am using pycharm and widows 10.
In your create_app function you probably do not want to use the global keyword for late initialisation of the app.
In the examples you provided the create_app function is never called, and so the app instance is never created. It is more common to use the create_app function as follows:
def create_app():
app = flask.Flask(__name__)
# do some setup
return app
app = create_app()
The app instance should also be called app and not flask_app. Of course you can call it whatever you want, but by default flask looks for app. To specify your own change FLASK_APP=__init__.py to FLASK_APP=__init__:flask_app
I struggle with the implementation of the Flask-MQTT lib to my app. SQLAlchemy etc works fine, but
flask-mqtt throws the error AttributeError: module 'app.mqtt' has no attribute 'init_app'.
In the offical documentatio of Flask-MQTT they build up the create_app() Method the same way
( https://flask-mqtt.readthedocs.io/en/latest/usage.html )
Would be greate if someone can help me! Thank you very much
__init__.py
from flask import Flask
from flask_restful import Api
from flask_mqtt import Mqtt
from flask_sqlalchemy import SQLAlchemy
from flask_marshmallow import Marshmallow
from flask_jwt_extended import JWTManager
mqtt = Mqtt()
api = Api()
db = SQLAlchemy()
ma = Marshmallow()
jwt = JWTManager()
def create_app(config):
app = Flask(__name__)
app.config.from_object(config.DevelopmentConfig)
mqtt.init_app(app)
db.init_app(app)
api.init_app(app)
ma.init_app(app)
jwt.init_app(app)
return app
from app.mqtt import mqttclient
run.py
from app import create_app
import config
from flask_migrate import Migrate
app = create_app(config)
migrate = Migrate(app, db)
app.config['MQTT_BROKER_URL'] = 'hivemq'
app.config['MQTT_BROKER_PORT'] = 1883
app.config['MQTT_USERNAME'] = ''
app.config['MQTT_PASSWORD'] = ''
app.config['MQTT_KEEPALIVE'] = 5
app.config['MQTT_TLS_ENABLED'] = False
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5003, threaded=True)
ERROR:
mqttservice | Traceback (most recent call last):
mqttservice | File "run.py", line 5, in <module>
mqttservice | app = create_app(config)
mqttservice | File "/code/app/__init__.py", line 18, in create_app
mqttservice | mqtt.init_app(app)
mqttservice | AttributeError: module 'app.mqtt' has no attribute 'init_app'
In that code fragment, mqtt has two meanings. The first is the variable assigned in
mqtt = Mqtt()
The second is a namespace (module)
from app.mqtt import mqttclient
The tell is in the error
AttributeError: module 'app.mqtt' has no attribute 'init_app'
which is happening because the import is overwriting the initial value, so by the time of that .init_app(), mqtt isn't what you expected.
You're going to have to change one of those names.
Trying to run my python file updater.py to SSH to a server and run some commands every few set intervals or so. I'm using APScheduler to run the function update_printer() from __init__.py. Initially I got a working outside of application context error but someone suggested that I just import app from __init__.py. However it isn't working out so well. I keep getting a cannot import name 'app' error.
app.py
from queue_app import app
if __name__ == '__main__':
app.run(debug=True)
__init__.py
from flask import Flask, render_template
from apscheduler.schedulers.background import BackgroundScheduler
from queue_app.updater import update_printer
app = Flask(__name__)
app.config.from_object('config')
#app.before_first_request
def init():
sched = BackgroundScheduler()
sched.start()
sched.add_job(update_printer, 'interval', seconds=10)
#app.route('/')
def index():
return render_template('index.html')
updater.py
import paramiko
import json
from queue_app import app
def update_printer():
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(app.config['SSH_SERVER'], username = app.config['SSH_USERNAME'], password = app.config['SSH_PASSWORD'])
...
File Structure
queue/
app.py
config.py
queue_app/
__init__.py
updater.py
Error
Traceback (most recent call last):
File "app.py", line 1, in <module>
from queue_app import app
File "/Users/name/queue/queue_app/__init__.py", line 3, in <module>
from queue_app.updater import update_printer
File "/Users/name/queue/queue_app/updater.py", line 3, in <module>
from queue_app import app
ImportError: cannot import name 'app'
What do I need to do be able to get to the app.config from updater.py and avoid a "working outside of application context error" if ran from APScheduler?
It's a circular dependency, as you import updater in your __init__.py file. In my Flask setup, app is created in app.py.