I'm creating an app where the client upload a CSV file and the server doesn't store it but only read it to perform tasks and return the content of the CSV as a JSON.
I followed some tutorials and posts and I came to this code:
Flask:
#app.route('/analyse_dataset', methods=['GET','POST'])
def analyse_dataset_route():
print('/analyse_dataset called', file=sys.stderr)
try:
# check if the post request has the dataset part
if 'file' not in request.files:
data = {'error': 'Dataset not in requested files'}
return Response(data, status=400, mimetype='application/json')
print('request.files : ' + str(request.files))
uploaded_file = request.files['file']
# no dataset
if uploaded_file.filename == '':
logger.debug('Dataset not found, returning to /')
return redirect('/')
# dataset ok
if uploaded_file:
print('uploaded_file ok', file=sys.stderr)
filename = secure_filename(uploaded_file.filename)
#uploaded_file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
uploaded_file.stream.seek(0)
myfile = uploaded_file.file
print('uploaded_file name: ' + filename, file=sys.stderr)
dataframe = pd.read_csv(myfile)
return Response(dumps(dataframe), status=200, mimetype='application/json')
else:
data = {'error': 'Something went wrong...'}
return Response(data, status=400, mimetype='application/json')
except Exception as e:
logger.error('An error occurred while analysing dataset')
logger.error(e, exc_info=True)
print(e)
data = {'error': 'An error occurred while analysing dataset'}
return Response(data, status=400, mimetype='application/json')
client side:
onClickHandler = () => {
if (this.state.selectedFile == null) {
toast.error('No file selected')
} else {
const data = new FormData()
data.append('file', this.state.selectedFile)
let req = "http://localhost:5001/analyse_dataset"
axios.post(req, data, {
onUploadProgress: ProgressEvent => {
this.setState({
loaded: (ProgressEvent.loaded / ProgressEvent.total * 100),
loading: true
})
},
})
.then(res => { // then print response status
console.log("file processed: " + res.data)
this.setState({
redirect: true,
jsondata: res.data
})
})
.catch(err => { // then print response status
console.log(err)
this.setState({
selectedFile: null,
loaded: 0,
loading: false
})
setTimeout(() => {
toast.error("Process failed, be sure to upload a correct file.")
}, 500)
})
}
}
So there is this issue where it's "" working "" with a large CSV file but it isn't working with a smaller one and I can't figure out why. Also I'd like to return a JSON containing each line of the CSV to the client but for the moment it isn't working aswell.
Here is the error :
Your code is likely failing on myfile = uploaded_file.file. Try changing the line to myfile = uploaded_file.stream.
If you are using Python 3, then pandas.read_csv requires text file/stream i.e. unicode. Convert the stream and it should work.
myfile = io.StringIO(myfile.getvalue().decode())
Once you have your pd.DataFrame object, you can use pd.DataFrame.to_json to convert your files. See the docs here.
I am unsure as to exactly why larger files work.
Related
I'm new to flask and jquery. I'm uploading data url that i captured from a video element using the following code in js/jquery
click_button.addEventListener('click', function() {
canvas.getContext('2d').drawImage(video, 0, 0, canvas.width, canvas.height);
video.style.display = "none";
let image_data_url = canvas.toDataURL('image/jpeg');
dataurl.value = image_data_url;
dataurl_container.style.display = 'block';
$.ajax({
type: "POST",
url: "http://127.0.0.1:5000/uploader",
data: {
imgBase64: image_data_url
}
}).done(function(o) {
console.log('saved');
});
video.pause()
});
flask code that is reading the data
#app.route('/uploader', methods=['GET', 'POST'])
def upload_file_method():
if request.method == 'POST':
f = request.files['file']
f.save(secure_filename(f.filename))
resp = sendReq(f.filename,url="http://someurl")
print(resp.json())
return 'file uploaded successfully'
The request.files['file'] is empty or not available. How to read the data sent from jquery above? request.data is empty as well.
I'm looking to convert this Python request to a Swift script.
Here is my working python script that returns the accessToken!
#!/usr/bin/python
import requests
import json
#MAKE THE REQUEST
URL = "http://this/is/the/url"
headers = {
'Accept': "application/json",
"Accept-Language": "en_US"
}
data = {
"grant_type": "password",
"username" : "GROUP\SITE\USERNAME",
"password" : "somepassword"
}
r = requests.get(url = URL, params = headers, data = data)
data = r.json()
accessToken = data['access_token']
print(accessToken)
When I run the Swift Playground for the code below nothing is returned!
It seems the script exits at guard let data = data else { return }
How could I get the same results as the Python Script above.
I've tried implementing URLComponents using this tutorial...
import UIKit
var url = "http://just/the/url"
extension Dictionary {
func percentEncoded() -> Data? {
return map { key, value in
let escapedKey = "\(key)"
let escapedValue = "\(value)"
print(escapedKey + "=" + escapedValue)
return escapedKey + "=" + escapedValue
}
.joined(separator: "&")
.data(using: .utf8)
}
}
extension CharacterSet {
static let urlQueryValueAllowed: CharacterSet = {
let generalDelimitersToEncode = ":#[]#" // does not include "?" or "/" due to RFC 3986 - Section 3.4
let subDelimitersToEncode = "$&'()*+,;="
var allowed = CharacterSet.urlQueryAllowed
allowed.remove(charactersIn: "\(generalDelimitersToEncode)\(subDelimitersToEncode)")
return allowed
}()
}
var request = URLRequest(url: URL(string:url)!)
request.httpMethod = "GET"
let parameters: [String: String] = [
"grant_type":"password",
"username":"GROUP\\SITE\\USER",
"password":"somePassword"
]
request.httpBody = parameters.percentEncoded()
request.setValue("application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type")
request.setValue("application/XML", forHTTPHeaderField: "Accept")
let config = URLSessionConfiguration.default
URLSession(configuration: config).dataTask(with: request) { (data, response, err) in
guard let data = data else { return }
print(data)
guard let dataAsString = String(data: data, encoding: .utf8)else {return}
print(dataAsString)
guard let httpResponse = response as? HTTPURLResponse,
(200...299).contains(httpResponse.statusCode) else {
print("Bad Credentials")
return
}
//HTTP Status Code!
print("HTTP RESPONSE:"+"\(httpResponse.statusCode)")
//
}.resume()
If I remember correctly, starting in iOS 13, you cant have httpBody for a GET call, so you'll either need to switch to a POST/PUT or add the params into the url string (See below)
You also had different Accept headers in your python vs. swift. One was xml the other was json.
var urlComponents = URLComponents(string: "http://this/is/the/url")
urlComponents?.queryItems = [
URLQueryItem(name: "grant_type", value: "password"),
URLQueryItem(name: "username", value: "username"),
URLQueryItem(name: "password", value: "somepassword")
]
guard let url = urlComponents?.url else { return } // You can print url here to see how it looks
var request = URLRequest(url: url)
request.httpMethod = "GET"
request.setValue("application/json", forHTTPHeaderField: "Accept")
request.setValue("en_US", forHTTPHeaderField: "Accept-Language")
let task = URLSession.shared.dataTask(with: request) { data, response, error in
guard let data = data,
let response = response as? HTTPURLResponse,
error == nil else {
print("error", error ?? "Unknown error")
return
}
print(response)
guard (200 ... 299) ~= response.statusCode else {
print("response = \(response)")
return
}
let responseString = String(data: data, encoding: .utf8)
print(responseString)
}
task.resume()
The problem was the following...
request.httpMethod = "GET"
I had to change the get to "POST" and now I have the token!!!!
I was confused because the python script used GET. I had a bash script that that used curl to get the token displayed the logged post.
In short my above Swift Playground now works by changing the request.httpMethod to "POST". THANKS FOR ALL THE HELP
I have a function in Django that I am trying to solve from my previous question here. While trying out my own solutions, I have made significant updates but I encounter an error.
I'm trying this out right now:
def view_routes(request, query=None):
routes = None
if query is None:
routes = Route.objects.all()
else:
#View: Routes in Queried Boundary
if request.method == 'POST':
return HttpResponse("OK")
elif request.method == 'GET':
json_feature = json.loads(request.GET.get('geo_obj', False))
#json_feature = json.loads(request.GET['geo_obj'])
geom = make_geometry_from_feature(json_feature)
routes = Route.objects.filter(wkb_geometry__within=geom[0])
print("Total Routes Filtered: " + str(Route.objects.filter(wkb_geometry__within=geom[0]).count()))
#Render to Django View
routes_json = serialize('geojson', routes, fields=('route_type', 'route_long', 'route_id', 'wkb_geometry',))
routes_geojson = json.loads(routes_json)
routes_geojson.pop('crs', None)
routes_geojson = json.dumps(routes_geojson)
#return render(request, 'plexus/view_routes.html', {'routes':routes})
return redirect('routes_view', query)
I am having trouble switching/commenting out between these two lines:
json_feature = json.loads(request.GET.get('geo_obj', False))
json_feature = json.loads(request.GET['geo_obj'])
Both presents an error respectively:
TypeError: the JSON object must be str, not 'bool'
django.utils.datastructures.MultiValueDictKeyError: "'geo_obj'"
Edited function with AJAX inside:
function sendQueryData(url, query){
url =url.replace('query' , query);
if (query === ""){
alert("City Input Required");
}else{
if(geo_obj === null){
alert("Click Search Button...")
}else{
$.ajax({
beforeSend: function(xhr, settings) {
if (!csrfSafeMethod(settings.type) && !this.crossDomain) {
xhr.setRequestHeader("X-CSRFToken", csrftoken);
}
},
type: "GET",
url: url,
dataType: 'html',
data: {
'geo_obj' : JSON.stringify(geo_obj)
},
success: function(data){
alert(data);
window.location.href = url;
//var result = $('<div />').append(data).find('#list-group').html();
//$('#list-group').html(result);
},
error: function(xhr, textStatus, errorThrown) {
alert('Request Failed' + textStatus + ":" + errorThrown);
}
});
}
}
}
Try using json.loads(request.body) if you are passing raw JSON, request.GET['foo'] is for form-encoded data
I am having a small django view that responds with an dict, on an ajax call . Here is the code below :
Ajax Call :
var getdata = function(e){
var id = this.id;
console.log(id);
$.ajax({
url:'/editcontact/',
method :'POST',
data :{
'name' : id,
},
success:function(){
console.log("success");
},
error:function(status){
console.log (status);
}
});
};
$("button").on('click',getdata);
views.py
if request.is_ajax:
print "comes here"
value1 = request.GET['name']
print value1
data = dbs.objects.filter(cname=value1)
print data
return render_to_response("editcontact.html",{"data":data}, context_instance=RequestContext(request))
the code is executing till print data, but the render_to_response is not working.
I will suggest you to return response in json format. that will solve your problem.
you can take help from below given code and can change your views.py
import json
if request.is_ajax:
print "comes here"
value1 = request.GET['name']
print value1
data = dbs.objects.filter(cname=value1)
json = simplejson.dumps(data)
return HttpResponse(json, mimetype='application/json')
The example code for Google's YouTube Data API is a piece of junk. It's so complicated and tied to the oauth redirect flow that I can't use it. Trying to go raw with requests pip and not getting too far.
I've followed the instructions exactly (as far as I can tell), with the following code:
import json
import os
import sys
import urllib
import requests
payload_file = None
payload = None
print 'Loading Config'
# Get the directory path of this file. When using any relative file paths make
# sure they are relative to current_dir so that the script can be run from any CWD.
current_dir = os.path.dirname(os.path.abspath(__file__))
# Reads in the config.json file then parses it
config = json.loads(open(os.path.join(current_dir, '..', 'config.json')).read())
print 'Parsing Payload'
for i in range(len(sys.argv)):
if sys.argv[i] == "--json" and (i + 1) < len(sys.argv):
payload = json.loads(sys.argv[i + 1])
elif sys.argv[i] == "-payload" and (i + 1) < len(sys.argv):
payload_file = sys.argv[i + 1]
with open(payload_file,'r') as f:
payload = json.loads(f.read())
break
print 'Configuring youtube with token {0}'.format(payload['token'])
print 'Downloading video...'
# See how big it is
f = urllib.urlopen(payload['url'])
content_length = int(f.headers["Content-Length"])
# Download it
# urllib.urlretrieve(payload['url'], "video.mp4")
metadata = {
'snippet' : {
'title': payload['title'],
"categoryId": 22
},
'status' : {
"privacyStatus": "public",
"embeddable": True,
"license": "youtube"
}
}
if 'tags' in payload:
metadata['snippet']['tags'] = payload['tags']
if 'description' in payload:
metadata['snippet']['description'] = payload['description']
headers = {
'Authorization' : 'Bearer {0}'.format(payload['token']),
'Content-Type' : 'application/json; charset=UTF-8',
'Content-Length' : json.dumps(metadata).__len__(),
'X-Upload-Content-Length' : content_length,
'X-Upload-Content-Type' : 'video/*',
}
print 'Attempting to upload video'
print headers
# upload video file
r = requests.post('https://www.googleapis.com/upload/youtube/v3/videos?uploadType=resumable&part=snippet,status', data=metadata, headers=headers);
print "RESPONSE!"
print r.text
# files = {
# 'file': video_file,
# }
# r = requests.post('https://www.googleapis.com/upload/youtube/v3/videos', data={ "video" : video }, headers=headers);
Obviously its not finished, but its dying on the metadata upload request with the following output:
Loading Config
Parsing Payload
Configuring youtube with token <access-token>
Downloading video...
Attempting to upload video
{'X-Upload-Content-Length': 51998563, 'Content-Length': 578, 'Content-Type': 'application/json; charset=UTF-8', 'X-Upload-Content-Type': 'video/*', 'Authorization': 'Bearer <access-token>'}
RESPONSE!
{
"error": {
"errors": [
{
"domain": "global",
"reason": "parseError",
"message": "Parse Error"
}
],
"code": 400,
"message": "Parse Error"
}
}
This error is not even listed in their "Errors" docs.
What is wrong with my code?
Here is an example in python that works. It assumes you've already done the oauth part though.
import requests
from os import fstat
import json
fi = open('myvideo.mp4')
base_headers = {
'Authorization': '%s %s' % (auth_data['token_type'],
auth_data['access_token']),
'content-type': 'application/json'
}
initial_headers = base_headers.copy()
initial_headers.update({
'x-upload-content-length': fstat(fi.fileno()).st_size,
'x-upload-content-type': 'video/mp4'
})
initial_resp = requests.post(
'https://www.googleapis.com/upload/youtube/v3/videos?uploadType=resumable&part=snippet,status,contentDetails',
headers=initial_headers,
data=json.dumps({
'snippet': {
'title': 'my title',
},
'status': {
'privacyStatus': 'unlisted',
'embeddable': True
}
})
)
upload_url = initial_resp.headers['location']
resp = requests.put(
upload_url,
headers=base_headers,
data=fi
)
fi.close()
the above is graet, just adding: you can also get the youtube id from the response (for future use):
cont = json.loads(resp.content)
youtube_id = cont['id']