We Keep Coding

Convert received json to dictionary, and save to local disk

My flask server receives a json file.
#app.route('/process', methods=['POST'])
def trigger_processing():
sent_files = request.files
json_file = sent_files['file']
print(json_file)
data = json.load(json_file)
print(data)
return make_response(json.dumps({'fileid': "ok"}), 200)
Printing the json_file gives me:
<FileStorage: 'test.json' ('application/octet-stream')>
127.0.0.1 - - [31/May/2021 23:29:55] "POST /process HTTP/1.1" 200 -
However the program fails when i try to convert json to dictionary.
json.decoder.JSONDecodeError: Expecting ',' delimiter: line 11 column 5 (char 213)
I want to be able to covert the json to dictionary.
Ideally, i would also like to save the received json file on the server's filesystem.

It fails because sent_files['file'] is a FileStorage object type, you need to read it and decode the bytes readed into a string. Then you can load it as a json.
from flask import Flask, request
from flask import make_response
from tempfile import gettempdir
import json
import os
app = Flask(__name__)
#app.route('/process', methods=['POST'])
def trigger_processing():
sent_files = request.files
#read the file
json_file_content = sent_files['file'].read().decode('utf-8')
#load the string readed into json object
json_content = json.loads(json_file_content)
print(json_content)
#generate the path where you want to save your file (in my case the temp folder)
save_path = os.path.join(gettempdir(), sent_files['file'].filename)
#and save the file
with open(save_path, 'w') as outfile:
json.dump(json_content, outfile, indent=2)
return make_response(json.dumps({'fileid': "ok"}), 200)

Download images from URL Python

I have problem with my script when i try download images from web url. It works on other page (offex.pl) but in my shop images are not working.
i just have all files but i can't open Files
my code:
import os
import time
import requests
from termcolor import colored
def get_folder(url):
all_folders= os.path.dirname(url)
folder=os.path.basename(all_folders)
return folder
def filename(url):
file=url[url.rfind("/") + 1:]
return file
def download(link):
error = []
ok = 0
fail = 0
root_folder = get_folder(link)
path = "{}/{}".format("download", root_folder)
if not os.path.exists(path):
os.makedirs(path)
url = link
file = filename(link)
result = requests.get(url, stream=True)
completeName = os.path.join("download", root_folder, file)
print(completeName)
if result.status_code == 200:
image = result.raw.read()
open(completeName, "wb").write(image)
ok += 1
succes = "{} {} {}".format(ok, colored("Pobrano:", "green"), url)
print(succes)
time.sleep(1)
else:
found_error = "{} {}".format(colored("Brak pliku!:", "red"), url)
print(found_error)
fail += 1
error.append("ID:{} NUMBER:{} link: {}".format(id, url))
with open("log.txt", "w") as filehandle:
for listitem in error:
filehandle.write('%s\n' % listitem)
print(colored("Pobrano plików: ", "green"), ok)
print(colored("Błędy pobierania: ", "red"), fail)
img_url="https://sw19048.smartweb-static.com/upload_dir/shop/misutonida_ec-med-384-ix.jpg"
download(img_url)
What Im doing wrong?
for example (https://offex.pl/images/detailed/11/94102_jeep_sbhn-8h.jpg) download OK
but for my shop url https://sw19048.smartweb-static.com/upload_dir/shop/misutonida_ec-med-384-ix.jpg is not working.

If you want to use requests module,you can use this:
import requests
response = requests.get("https://sw19048.smartweb-static.com/upload_dir/shop/misutonida_ec-med-384-ix.jpg")
with open('./Image.jpg','wb') as f:
f.write(response.content)

The issue is with the URL which you are using to download. Its not an issue, but a difference from other URL you have mentioned.
Let me explain
The URL https://offex.pl/images/detailed/11/94102_jeep_sbhn-8h.jpg returns an image as response with out any compression.
On the other hand, the shop URL https://sw19048.smartweb-static.com/upload_dir/shop/misutonida_ec-med-384-ix.jpg returns the image with gzip compression enabled in the headers.
So the raw response you get is compressed with gzip compression. You can decompress the response with gzip, if you know the compression is always gzip like below
import gzip
import io
image = result.raw.read()
buffer = io.BytesIO(image)
deflatedContent = gzip.GzipFile(fileobj=buffer)
open("D:/sample.jpg", "wb").write(deflatedContent.read())
Or you can use alternative libraries like urllib2 or similar ones, which takes care of decompression. I was trying to explain why it failed for your URL , but not for other. Hope this makes sense.

try :
import urllib2
def download_web_image(url):
request = urllib2.Request(url)
img = urllib2.urlopen(request).read()
with open('test.jpg', 'wb') as f:
f.write(img)
download_web_image("https://sw19048.smartweb-static.com/upload_dir/shop/misutonida_ec-med-384-ix.jpg")
It is working for your URL. I think the issue is with the request response of the used library.

from io import BytesIO
import requests
from PIL import Image
fileRequest = requests.get("https://sw19048.smartweb-static.com/upload_dir/shop/misutonida_ec-med-384-ix.jpg")
doc = Image.open(BytesIO(fileRequest.content))
doc.save("newFile.jpg")

Putting Json into python variables using query (url request)

I'm attempting to use this Python 2 code snippet from the WeatherUnderground's API Page in python 3.
import urllib2
import json
f = urllib2.urlopen('http://api.wunderground.com/api/apikey/geolookup/conditions/q/IA/Cedar_Rapids.json')
json_string = f.read()
parsed_json = json.loads(json_string)
location = parsed_json['location']['city']
temp_f = parsed_json['current_observation']['temp_f']
print "Current temperature in %s is: %s" % (location, temp_f)
f.close()
I've used 2to3 to convert it over but i'm still having some issues. The main conversion here is switching from the old urllib2 to the new urllib. I've tried using the requests library to no avail.
Using urllib from python 3 this is the code I have come up with:
import urllib.request
import urllib.error
import urllib.parse
import codecs
import json
url = 'http://api.wunderground.com/api/apikey/forecast/conditions/q/C$
response = urllib.request.urlopen(url)
#Decoding on the two lines below this
reader = codecs.getreader("utf-8")
obj = json.load(reader(response))
json_string = obj.read()
parsed_json = json.loads(json_string)
currentTemp = parsed_json['current_observation']['temp_f']
todayTempLow = parsed_json['forecast']['simpleforecast']['forecastday']['low'][$
todayTempHigh = parsed_json['forecast']['simpleforecast']['forecastday']['high'$
todayPop = parsed_json['forecast']['simpleforecast']['forecastday']['pop']
Yet i'm getting an error about it being the wrong object type. (Bytes instead of str)
The closest thing I could find to the solution is this question here.
Let me know if any additional information is needed to help me find a solution!
Heres a link to the WU API website if that helps

urllib returns a byte array. You convert it to string using
json_string.decode('utf-8')
Your Python2 code would convert to
from urllib import request
import json
f = request.urlopen('http://api.wunderground.com/api/apikey/geolookup/conditions/q/IA/Cedar_Rapids.json')
json_string = f.read()
parsed_json = json.loads(json_string.decode('utf-8'))
location = parsed_json['location']['city']
temp_f = parsed_json['current_observation']['temp_f']
print ("Current temperature in %s is: %s" % (location, temp_f))
f.close()

Python Adding Headers to urlparse

There doesn't appear to be a way to add headers to the urlparse command. This essentially causes Python to use its default user agent, which is blocked by several web pages. What I am trying to do is essentially do the equivalent of this:
req = Request(INPUT_URL,headers={'User-Agent':'Browser Agent'})
But using urlparse:
parsed = list(urlparse(INPUT_URL))
So how can I modify this urlparse in order for it to take headers, or be usable with my Request that I created? Any help is appreciated, thanks.
Also, for anyone wondering the exact error I am getting:
urllib.error.HTTPError: HTTP Error 403: Forbidden
At this:
urlretrieve(urlunparse(parsed),outpath)

Headers are part of a request, of which the URL is one part. Python creates a request for you when you pass in just a URL to urllib.request functions.
Create a Request object, add the headers to that object and use that instead of a string URL:
request = Request(urlunparse(parsed), headers={'User-Agent': 'My own agent string'})
However, urlretrieve() is marked as 'legacy API' in the code and doesn't support using a Request object. Removing a few lines supporting 'file://' urls is easy enough:
import contextlib
import tempfile
from urllib.error import ContentTooShortError
    from urllib.request import urlopen
_url_tempfiles = []
def urlretrieve(url, filename=None, reporthook=None, data=None):
with contextlib.closing(urlopen(url, data)) as fp:
headers = fp.info()
# Handle temporary file setup.
if filename:
tfp = open(filename, 'wb')
else:
tfp = tempfile.NamedTemporaryFile(delete=False)
filename = tfp.name
_url_tempfiles.append(filename)
with tfp:
result = filename, headers
bs = 1024*8
size = -1
read = 0
blocknum = 0
if "content-length" in headers:
size = int(headers["Content-Length"])
if reporthook:
reporthook(blocknum, bs, size)
while True:
block = fp.read(bs)
if not block:
break
read += len(block)
tfp.write(block)
blocknum += 1
if reporthook:
reporthook(blocknum, bs, size)
if size >= 0 and read < size:
raise ContentTooShortError(
"retrieval incomplete: got only %i out of %i bytes"
% (read, size), result)
return result

Python: HTTP Post a large file with streaming

I'm uploading potentially large files to a web server. Currently I'm doing this:
import urllib2
f = open('somelargefile.zip','rb')
request = urllib2.Request(url,f.read())
request.add_header("Content-Type", "application/zip")
response = urllib2.urlopen(request)
However, this reads the entire file's contents into memory before posting it. How can I have it stream the file to the server?

Reading through the mailing list thread linked to by systempuntoout, I found a clue towards the solution.
The mmap module allows you to open file that acts like a string. Parts of the file are loaded into memory on demand.
Here's the code I'm using now:
import urllib2
import mmap
# Open the file as a memory mapped string. Looks like a string, but
# actually accesses the file behind the scenes.
f = open('somelargefile.zip','rb')
mmapped_file_as_string = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
# Do the request
request = urllib2.Request(url, mmapped_file_as_string)
request.add_header("Content-Type", "application/zip")
response = urllib2.urlopen(request)
#close everything
mmapped_file_as_string.close()
f.close()

The documentation doesn't say you can do this, but the code in urllib2 (and httplib) accepts any object with a read() method as data. So using an open file seems to do the trick.
You'll need to set the Content-Length header yourself. If it's not set, urllib2 will call len() on the data, which file objects don't support.
import os.path
import urllib2
data = open(filename, 'r')
headers = { 'Content-Length' : os.path.getsize(filename) }
response = urllib2.urlopen(url, data, headers)
This is the relevant code that handles the data you supply. It's from the HTTPConnection class in httplib.py in Python 2.7:
def send(self, data):
"""Send `data' to the server."""
if self.sock is None:
if self.auto_open:
self.connect()
else:
raise NotConnected()
if self.debuglevel > 0:
print "send:", repr(data)
blocksize = 8192
if hasattr(data,'read') and not isinstance(data, array):
if self.debuglevel > 0: print "sendIng a read()able"
datablock = data.read(blocksize)
while datablock:
self.sock.sendall(datablock)
datablock = data.read(blocksize)
else:
self.sock.sendall(data)

Have you tried with Mechanize?
from mechanize import Browser
br = Browser()
br.open(url)
br.form.add_file(open('largefile.zip'), 'application/zip', 'largefile.zip')
br.submit()
or, if you don't want to use multipart/form-data, check this old post.
It suggests two options:
1. Use mmap, Memory Mapped file object
2. Patch httplib.HTTPConnection.send

Try pycurl. I don't have anything setup will accept a large file that isn't in a multipart/form-data POST, but here's a simple example that reads the file as needed.
import os
import pycurl
class FileReader:
def __init__(self, fp):
self.fp = fp
def read_callback(self, size):
return self.fp.read(size)
c = pycurl.Curl()
c.setopt(pycurl.URL, url)
c.setopt(pycurl.UPLOAD, 1)
c.setopt(pycurl.READFUNCTION, FileReader(open(filename, 'rb')).read_callback)
filesize = os.path.getsize(filename)
c.setopt(pycurl.INFILESIZE, filesize)
c.perform()
c.close()

Using the requests library you can do
with open('massive-body', 'rb') as f:
requests.post('http://some.url/streamed', data=f)
as mentioned here in their docs

Below is the working example for both Python 2 / Python 3:
try:
from urllib2 import urlopen, Request
except:
from urllib.request import urlopen, Request
headers = { 'Content-length': str(os.path.getsize(filepath)) }
with open(filepath, 'rb') as f:
req = Request(url, data=f, headers=headers)
result = urlopen(req).read().decode()
The requests module is great, but sometimes you cannot install any extra modules...