Possible Python Flask Memory Leak - python

Hoping someone can help me address this issue.
I've got a very simple Flask server running in a docker container which accepts a new frame every time the /api/feed_image endpoint is called. The image passed through that endpoint then streams to the index.html from /video_feed.
The application works as expected initially, however, when I watch the resources being consumed using docker stats the memory usage slowly climbs until the entire system crashes.
Does anyone know what I could have missed or have any suggestions?
Thanks!
import os
import sys
from azure.iot.device.aio import IoTHubModuleClient
from flask import Response
from flask import request
from flask import Flask
from flask import render_template
import datetime
import time
import numpy as np
import cv2
import queue
frameQueue = queue.Queue()
app = Flask(__name__)
#app.route("/")
def index():
return render_template("index.html")
#app.route("/video_feed")
def video_feed():
print(len(frameQueue))
return Response(generate(), mimetype="multipart/x-mixed-replace; boundary=frame")
#app.route("/api/feed_image", methods=['POST'])
def read_image():
r = request
nparr = np.fromstring(r.data, np.uint8)
frameQueue.put(cv2.imdecode(nparr, cv2.IMREAD_COLOR))
def generate():
global frameQueue
while True:
outputFrame = frameQueue.get()
if outputFrame is None:
continue
(flag, encodedImage) = cv2.imencode(".jpg", outputFrame)
if not flag:
continue
yield(b'--frame\r\n' b'Content-Type: image/jpeg\r\n\r\n' + bytearray(encodedImage) + b'\r\n')
if __name__ == "__main__":
app.run(host='0.0.0.0', port='5001', debug=True, threaded=True, use_reloader=False)

After a long time I solved my own problem.
Turns out because I wasn't returning a Response object at the end of read_image(), Flask was generating an exception, which must have been caching in the background:
172.18.0.4 - - [22/Apr/2021 00:09:25] "POST /api/feed_image HTTP/1.1" 500 -
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2464, in __call__
return self.wsgi_app(environ, start_response)
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2450, in wsgi_app
response = self.handle_exception(e)
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1867, in handle_exception
reraise(exc_type, exc_value, tb)
File "/usr/local/lib/python3.7/site-packages/flask/_compat.py", line 39, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1953, in full_dispatch_request
return self.finalize_request(rv)
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1968, in finalize_request
response = self.make_response(rv)
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2098, in make_response
"The view function did not return a valid response. The"
TypeError: The view function did not return a valid response. The function either returned None or ended without a return statement.
Adding a return statement solved the problem.
#app.route("/api/feed_image", methods=['POST'])
def read_image():
...
return Response(status=201)
Perhaps someone would have an explanation as to why this would happen?
Either way, I hope this helps someone!

Related

Flask app NameError: name 'Markup' is not defined

I have been really stumped on this, been out of this game of web dev/python for a bit. I had a previously working Azure app service running this website, we did a minor HTML spelling change and re deployed. I am assuming some dependency got updated and now its broken. I don't know if this is a packaging version issue or if I have a missing import for my flask app.
I am getting a NameError: name 'Markup' is not defined error when trying to load a static html page. My app starts up just fine but I can't load the actual web pages.
Full Traceback
Traceback (most recent call last):
File "C:\Users\St**\PythonProjects\**Site-Portfolio\lib\site-packages\flask\app.py", line 2095, in __call__
return self.wsgi_app(environ, start_response)
File "C:\Users\St**\PythonProjects\**Site-Portfolio\lib\site-packages\flask\app.py", line 2080, in wsgi_app
response = self.handle_exception(e)
File "C:\Users\St**\PythonProjects\**Site-Portfolio\lib\site-packages\flask_cors\extension.py", line 165, in wrapped_function
return cors_after_request(app.make_response(f(*args, **kwargs)))
File "C:\Users\St**\PythonProjects\**Site-Portfolio\lib\site-packages\flask\app.py", line 2077, in wsgi_app
response = self.full_dispatch_request()
File "C:\Users\St**\PythonProjects\**Site-Portfolio\lib\site-packages\flask\app.py", line 1525, in full_dispatch_request
rv = self.handle_user_exception(e)
File "C:\Users\St**\PythonProjects\**Site-Portfolio\lib\site-packages\flask_cors\extension.py", line 165, in wrapped_function
return cors_after_request(app.make_response(f(*args, **kwargs)))
File "C:\Users\St**\PythonProjects\**Site-Portfolio\lib\site-packages\flask\app.py", line 1523, in full_dispatch_request
rv = self.dispatch_request()
File "C:\Users\St**\PythonProjects\**Site-Portfolio\lib\site-packages\flask\app.py", line 1509, in dispatch_request
return self.ensure_sync(self.view_functions[rule.endpoint])(**req.view_args)
File "C:\Users\St**\PycharmProjects\**Site-portfolio\application.py", line 32, in index
return render_template("//index.html")
File "C:\Users\St**\PythonProjects\**Site-Portfolio\lib\site-packages\flask\templating.py", line 147, in render_template
ctx.app.update_template_context(context)
File "C:\Users\St**\PythonProjects\**Site-Portfolio\lib\site-packages\flask\app.py", line 756, in update_template_context
context.update(func())
File "C:\Users\St**\PythonProjects\**Site-Portfolio\lib\site-packages\flask_recaptcha.py", line 59, in get_code
return dict(recaptcha=Markup(self.get_code()))
NameError: name 'Markup' is not defined
127.0.0.1 - - [08/Apr/2022 17:02:51] "GET /?__debugger__=yes&cmd=resource&f=style.css HTTP/1.1" 304 -
127.0.0.1 - - [08/Apr/2022 17:02:51] "GET /?__debugger__=yes&cmd=resource&f=debugger.js HTTP/1.1" 304 -
127.0.0.1 - - [08/Apr/2022 17:02:51] "GET /?__debugger__=yes&cmd=resource&f=console.png HTTP/1.1" 304 -
Process finished with exit code 0
Here is my code:
from flask import Flask, request, render_template, flash
from flask_mail import Mail, Message
from flask_cors import CORS, cross_origin
from flask_recaptcha import ReCaptcha
import requests
import json
import os
app = Flask(__name__, static_folder='static', static_url_path='')
recaptcha = ReCaptcha(app=app)
app.config['RECAPTCHA_ENABLED'] = True
app.config['RECAPTCHA_PUBLIC_KEY'] = '***********************'
app.config['RECAPTCHA_PRIVATE_KEY'] = '****************************'
#app.route('/', methods=['GET'])
def index():
return render_template("//index.html")
#app.route('/contact-us', methods=['GET'])
#app.route('/contact', methods=['GET', 'POST'])
def contact():
if request.method == 'POST':
r = requests.post('https://www.google.com/recaptcha/api/siteverify',
data={'secret': '***********',
'response': request.form['g-recaptcha-response']})
google_response = json.loads(r.text)
if google_response['success']:
contact_form = {'name': request.form['name'],
'email': request.form['email'],
'message': request.form['message']}
msg = Message(subject='Contact from website',
sender=contact_form['email'],
recipients=['*************'],
body=contact_form['message'])
mail.send(msg)
flash('Success, we will respond within at least 24 hours.')
return render_template('contact.html')
else:
flash('failed to submit, please retry or contact us at ************')
return render_template('contact.html')
return render_template('contact.html')
if __name__ == '__main__':
app.run()
My requirements.txt for package version
Flask>=1.0.2
jinja2>=2.11.3
Flask-Mail>=0.9.1
Flask-Cors>=3.0.9
Flask-Admin>=1.5.2
Flask-ReCaptcha>=0.4.2
Flask-SQLAlchemy>=2.3.2
Flask-WTF>=0.14.2
SQLAlchemy>=1.3.0
requests>=2.20.0
Flask-Login>=0.4.1
Werkzeug>=0.15.3
Flask-ReCaptcha is a very old project. The last update of Flask-ReCaptcha is on 2016. You'd better not use it.
Back to the error log itself, Flask-ReCaptcha has code like from jinja2 import Markup. But, since jinja2==3.1.0, Markup's position has changed. Try pip install jinja2==3.0.0`.
You will probably meet other problems as Flask-ReCaptcha is really old.
Go to Python installation folder
Go to Lib > site-packages
Open flask_recaptcha.py
You'll see something like this:
try:
from flask import request
from jinja2 import Markup
import requests
except ImportError as ex:
print("Missing dependencies")
Change it to:
try:
from flask import request
from markupsafe import Markup
import requests
except ImportError as ex:
print("Missing dependencies")

Tkinter Thread RuntimeError in Selenium Project with Flask

I'm working on a Selenium crawling project with Flask, and I'm trying to implement the part that opens the Save As dialog box after crawling is done using filedialog.asksaveasfilename() of the tkinter package.
It works normally when I run it the first time, but from the second time I get an error:
Traceback (most recent call last):
File "C:\Users\yjs01\anaconda3\lib\site-packages\flask\app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "C:\Users\yjs01\anaconda3\lib\site-packages\flask\app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "C:\Users\yjs01\anaconda3\lib\site-packages\flask\app.py", line 1821, in handle_user_exception
File "C:\Users\yjs01\anaconda3\lib\tkinter\__init__.py", line 3124, in __init__
Widget.__init__(self, master, 'frame', cnf, {}, extra)
File "C:\Users\yjs01\anaconda3\lib\tkinter\__init__.py", line 2572, in __init__
self.tk.call(
RuntimeError: main thread is not in main loop
Also, I don't directly call other GUI windows like root = Tk() but a blank window is created at the same time.
Here is my code:
# main.py
from flask import Flask, render_template, request
from scrap import scrap
from tkinter.filedialog import asksaveasfilename
app = Flask('__name__')
#app.route('/')
def main():
return render_template('main.html')
#app.errorhandler(500)
def page_not_founded(error):
return render_template('error.html')
#app.route('/', methods=['GET', 'POST'])
def execute_scraping():
if request.method == 'POST':
keyword = request.form['keyword']
start_page = int(request.form['start-page'])
end_page = int(request.form['end-page'])
columns = request.form.getlist('column')
file_type = [('엑셀 파일', '*.xlsx')]
file_name = asksaveasfilename(filetypes=file_type, defaultextension=str(file_type))
scrap(keyword, start_page, end_page, columns, file_name)
return render_template('complete.html')
if __name__ == '__main__':
app.run(host='127.0.0.1', port=8081)
I'd appreciate it if you could let me know what the problem is.

flask: RunTimeError: There is no current event loop in thread

I am making a simple flask app that returns results from an external website. The user enters data into my site. This data is used to load another site. Data is extracted and returned in a list. This program works independently, but will not work as part of the flask app. I have tried using using requests_HTML library using it's async tools, and i have tried using Pyro4, to send the request to an external process. But i always come up with some version of this error:
RunTimeError: There is no current event loop in thread ........
It seems to me that when one of my imported modules runs a thread Flask doesn't like it.
I would love to know why does this happen, is their something in flasks inner workings that means that it won't work well with threads or async or something?
Any direction to some extra resources or info would be very appreciated
This is my Flask App:
from flask import Flask, render_template
from requests_html import AsyncHTMLSession
app = Flask(__name__)
asession = AsyncHTMLSession()
URLS=["https://stackoverflow.com/questions/",
"https://stackoverflow.com/questions/",
"https://stackoverflow.com/questions/"]
#app.route("/", methods=('GET'))
def index():
results = asession.run(run_requests)
return render_template('index_page.html', results_list=results)
async def run_requests():
results_list=[]
for url in URLS:
results_list.append(await get_and_render(url))
return results_list
async def get_and_render(url):
r = await asession.get(url)
await r.html.arender(sleep=0.75)
summary = r.html.find(".question-hyperlink", first=True)
return summary.text
This will get the stackoverflow questions page and grab the summary of the last posted question. I have tried this code in a standalone python file and it works fine. (ie outside of a flask app, just printing the results to the command line)
This is by traceback from the flask debugger:
'Traceback (most recent call last):
File "C:\Users\Alex\AppData\Local\Programs\Python\Python38-32\Lib\site-packages\flask\app.py", line 2463, in __call__
return self.wsgi_app(environ, start_response)
File "C:\Users\Alex\AppData\Local\Programs\Python\Python38-32\Lib\site-packages\flask\app.py", line 2449, in wsgi_app
response = self.handle_exception(e)
File "C:\Users\Alex\AppData\Local\Programs\Python\Python38-32\Lib\site-packages\flask\app.py", line 1866, in handle_exception
reraise(exc_type, exc_value, tb)
File "C:\Users\Alex\AppData\Local\Programs\Python\Python38-32\Lib\site-packages\flask\_compat.py", line 39, in reraise
raise value
File "C:\Users\Alex\AppData\Local\Programs\Python\Python38-32\Lib\site-packages\flask\app.py", line 2446, in wsgi_app
response = self.full_dispatch_request()
File "C:\Users\Alex\AppData\Local\Programs\Python\Python38-32\Lib\site-packages\flask\app.py", line 1951, in full_dispatch_request
rv = self.handle_user_exception(e)
File "C:\Users\Alex\AppData\Local\Programs\Python\Python38-32\Lib\site-packages\flask\app.py", line 1820, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "C:\Users\Alex\AppData\Local\Programs\Python\Python38-32\Lib\site-packages\flask\_compat.py", line 39, in reraise
raise value
File "C:\Users\Alex\AppData\Local\Programs\Python\Python38-32\Lib\site-packages\flask\app.py", line 1949, in full_dispatch_request
rv = self.dispatch_request()
File "C:\Users\Alex\AppData\Local\Programs\Python\Python38-32\Lib\site-packages\flask\app.py", line 1935, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "C:\......\StackOverflowExample.py", line 11, in index
results = asession.run(run_requests)
File "C:\Users\Alex\AppData\Local\Programs\Python\Python38-32\Lib\site-packages\requests_html.py", line 771, in run
tasks = [
File "C:\Users\Alex\AppData\Local\Programs\Python\Python38-32\Lib\site-packages\requests_html.py", line 772, in <listcomp>
asyncio.ensure_future(coro()) for coro in coros
File "C:\Users\Alex\AppData\Local\Programs\Python\Python38-32\Lib\asyncio\tasks.py", line 660, in ensure_future
loop = events.get_event_loop()
File "C:\Users\Alex\AppData\Local\Programs\Python\Python38-32\Lib\asyncio\events.py", line 639, in get_event_loop
raise RuntimeError('There is no current event loop in thread %r.'
RuntimeError: There is no current event loop in thread 'Thread-4'.
Can anyone explain why flask has trouble completing these tasks?
cheers
PS
here is my simple HTML (although app has not been able to render this yet.):
<ul>
{% for result in results_list %}
<li>{{result}}</li>
{% endfor %}
</ul>
Here is my Script outside the flask app(it works):
from requests_html import AsyncHTMLSession
asession = AsyncHTMLSession()
URLS=["https://stackoverflow.com/questions/",
"https://stackoverflow.com/questions/",
"https://stackoverflow.com/questions/"]
def index():
results = asession.run(run_requests)
for result in results:
print(result)
async def run_requests():
results_list=[]
for url in URLS:
results_list.append(await get_and_render(url))
return results_list
async def get_and_render(url):
r = await asession.get(url)
await r.html.arender(sleep=0.75)
summary = r.html.find(".question-hyperlink", first=True)
print(summary.text)
if __name__ == "__main__":
index()

Oauth2 in Python with Flask gets 302 from API

I am trying to access MeisterTask's API with Python and Flask, and no matter what I do, I seem to always get a 302 code in return from the API, although I can get an access token (or so I think). Here is the code I have so far (I tried reducing it, this is the smallest snippet I could get that replicates the error):
from flask import Flask, redirect, url_for, session, request, jsonify
from flask_oauthlib.client import OAuth
app = Flask(__name__)
app.debug = True
app.secret_key = "development"
oauth = OAuth(app)
meistertask = oauth.remote_app(
'meistertask',
consumer_key= "XXXXXX",
consumer_secret= "XXXXXX",
request_token_params={"scope" : "meistertask"},
base_url='https://www.meistertask.com/api',
request_token_url=None,
access_token_method='GET',
access_token_url='https://www.mindmeister.com/login/oauth2/token',
authorize_url='https://www.mindmeister.com/oauth2/authorize'
)
#app.route('/')
def index():
if 'meistertask_token' in session:
me = meistertask.get('user')
return jsonify(me.data)
return redirect(url_for('login'))
#app.route('/login')
def login():
return meistertask.authorize(callback=url_for('authorized', _external=True))
#app.route('/logout')
def logout():
session.pop('meistertask_token', None)
return redirect(url_for('index'))
#app.route('/login/authorized')
def authorized():
resp = meistertask.authorized_response()
print(resp.get('code'))
if resp is None or resp.get('code') is None:
return 'Access denied: reason=%s error=%s resp=%s' % (
request.args['error'],
request.args['error_description'],
resp
)
session['meistertask_token'] = (resp['code'], '')
return "Hello"
#meistertask.tokengetter
def get_meistertask_oauth_token():
return session.get('meistertask_token')
if __name__ == "__main__":
app.run()
And here is the traceback:
flask_oauthlib.client.OAuthException: Invalid response from meistertask
Traceback (most recent call last):
File "~\AppData\Local\Programs\Python\Python37-32\lib\site-packages\flask\app.py", line 2309, in __call__ return self.wsgi_app(environ, start_response)
File "~\AppData\Local\Programs\Python\Python37-32\lib\site-packages\flask\app.py", line 2295, in wsgi_app response = self.handle_exception(e)
File "~\AppData\Local\Programs\Python\Python37-32\lib\site-packages\flask\app.py", line 1741, in handle_exception reraise(exc_type, exc_value, tb)
File "~\AppData\Local\Programs\Python\Python37-32\lib\site-packages\flask\_compat.py", line 35, in reraise raise value
File "~\AppData\Local\Programs\Python\Python37-32\lib\site-packages\flask\app.py", line 2292, in wsgi_app response = self.full_dispatch_request()
File "~\AppData\Local\Programs\Python\Python37-32\lib\site-packages\flask\app.py", line 1815, in full_dispatch_request rv = self.handle_user_exception(e)
File "~\AppData\Local\Programs\Python\Python37-32\lib\site-packages\flask\app.py", line 1718, in handle_user_exception reraise(exc_type, exc_value, tb)
File "~\AppData\Local\Programs\Python\Python37-32\lib\site-packages\flask\_compat.py", line 35, in reraise raise value
File "~\AppData\Local\Programs\Python\Python37-32\lib\site-packages\flask\app.py", line 1813, in full_dispatch_request rv = self.dispatch_request()
File "~\AppData\Local\Programs\Python\Python37-32\lib\site-packages\flask\app.py", line 1799, in dispatch_request return self.view_functions[rule.endpoint](**req.view_args)
File "~\Documents\MeisterTaskServer\hello.py", line 49, in authorized resp = meistertask.authorized_response()
File "~\AppData\Local\Programs\Python\Python37-32\lib\site-packages\flask_oauthlib\client.py", line 707, in authorized_response data = self.handle_oauth2_response(args)
File "~\AppData\Local\Programs\Python\Python37-32\lib\site-packages\flask_oauthlib\client.py", line 692, in handle_oauth2_response
Things I have tried
Tried to modify the method to get the access token from GET to POST. The API clearly states that I should use GET, yet every other example I have seen on flask_oauthlib's GitHub uses POST (the examples are 3 years old, but some still work, namely the GitHub one). Either give the same result.
Tried doing it barebones, without any library. The resulting code was thrice as long and also had more problems.
Used Django instead of Flask. Never even managed to get the hello world example going, it was too much work, and also I have discovered the library flask_oauthlib.
Things worth mentioning
I derived this code from this here GitHub example
There is also code there I omitted in order to keep the snippet short, that establishes that the server should use SSL (as per the request from the API that the redirect_uri should use HTTPS)
The app manages to redirect me over at MeisterTask and asks for my permission. Once I grant it, it redirects to "https://127.0.0.1:5000/login/authorized?code=some_token" where I get the traceback. If I look with Chrome's debugging tools to the requests made and what I receive, I see that I get an 302 from the API, but I also get an access token.
I run Windows 10 with Python 3.7.0
So what's the deal? What's the next step here? I've run out of things to try. Thank you for taking the time to solve this!

request.environ['HTTP_REFERER'] is None

i want to get HTTP_REFERER in python flask framework.
My route is this:
#app.route('/login')
def login():
if authenticateForPanel():
return redirect(url_for("panel"))
else:
ref = request.environ['HTTP_REFERER']
return render_template('login.html',blogOptions = g.blogOptions,ref=ref)
When i execute this,i got KeyError: 'HTTP_REFERER' with the traceback:
Traceback (most recent call last):
File "/Users/ozcan/flask/flask/app.py", line 1823, in __call__
return self.wsgi_app(environ, start_response)
File "/Users/ozcan/flask/flask/app.py", line 1811, in wsgi_app
response = self.make_response(self.handle_exception(e))
File "/Users/ozcan/flask/flask/app.py", line 1809, in wsgi_app
response = self.full_dispatch_request()
File "/Users/ozcan/flask/flask/app.py", line 1482, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/Users/ozcan/flask/flask/app.py", line 1480, in full_dispatch_request
rv = self.dispatch_request()
File "/Users/ozcan/flask/flask/app.py", line 1466, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/Users/ozcan/Documents/python/app.py", line 318, in login
ref = request.environ['HTTP_REFERER']
KeyError: 'HTTP_REFERER'
When i first wrote this code it was working.I do not directly call this url.I call localhost:5000/panel and it redirects me to the login method.So basically there should be a referer,am i wrong?
When i print the request.environ['HTTP_REFERER'] it prints None
I also tried with the
ref = request.referrer but it is None
Why it can be happen?Thanks a lot.
The werkzeug Request wrapper, which flask uses by default, does this work for you: request.referrer
from flask import Flask, request
import unittest
app = Flask(__name__)
#app.route('/')
def index():
return unicode(request.referrer)
class Test(unittest.TestCase):
def test(self):
c = app.test_client()
resp = c.get('/', headers={'Referer': '/somethingelse'})
self.assertEqual('/somethingelse', resp.data)
if __name__ == '__main__':
unittest.main()
But in the more general case, to retrieve the value of a dictionary key specifying a default if the key is not present, use dict.get
When i first wrote this code it was working.I do not directly call
this url.I call localhost:5000/panel and it redirects me to the login
method.So basically there should be a referer,am i wrong?
This is not correct. A redirect does not guarantee a referrer. Whether a browser will provide a referrer is browser-specific. Perhaps you were using a different browser that had this behavior at one point? The code you have now is working as best as it can be expected to.
See Will a 302 redirect maintain the referer string?
The only way I could get request.referrer to not be None is to load the page through . Redirecting and making regular old HTTP requests did not include a referrer attribute.
#dm03514 has the right idea, but if you're insistent upon using the environ dictionary, you should be using the get method. Also, how are you testing that it is None, i.e., are you using the interactive interpreter to print the value or are you adding print statements in the body?
An example of how you would do this reasonably:
# snip
else:
ref = request.environ.get('HTTP_REFERER')
# You can specify a default return value other than None like so:
# ref = request.environ.get('HTTP_REFERER', '/')
# snip

Categories