We Keep Coding

Twitter: Correctly uploading a new profile picture

The code below is what I use to upload a profile picture but I get the following error:
def get_image_bytes():
file_size = os.path.getsize('./image/example.jpg')
return file_size
def get_image_raw():
with open("./image/example.jpg", "rb") as img_file:
enc_image = img_file.read()
return enc_image
def get_media_id():
file_bytes = get_image_bytes();
resp = httpx.post(f'https://upload.twitter.com/i/media/upload.json?command=INIT&total_bytes={file_bytes}&media_type=image/jpeg', headers=general_headers)
return resp.json()['media_id_string']
def append_image():
media_id = get_media_id()
resp = httpx.post(f'https://upload.twitter.com/i/media/upload.json?command=APPEND&media_id={media_id}&segment_index=0&media={get_image_raw()}', headers=webkit_headers)
return media_id
def update_profile():
media_id = append_image()
resp = httpx.post(f'https://upload.twitter.com/i/media/upload.json?command=FINALIZE&media_id={media_id}', headers=general_headers)
print(resp.json())
update_profile()
Error:
{'request': '/i/media/upload.json', 'error': 'Segments do not add up to provided total file size.'}
I don't know how twitter wants to receive the image binary.
I've heard word that they expect chunks but I've tried almost everything.
When uploading a new profile picture I do see that they crop the image maybe this can be a reason why but I am to no avail at the moment.
My image is under 5mb by the way.

How do I send an image as a json element using Mqtt and python?

I am attaching an image as a json element and sending it over mqtt. But I do not receive anything on subscriber side. If I remove the image, I am receiving the json.
Publisher code:
with open("sample.jpg", "rb") as image_file:
encoded_img_str = base64.b64encode(image_file.read())
response['image'] = encoded_img_str
response['status'] = 'success'
response['filename'] = 'sample.jpg'
json_data = json.dumps(response)
client.publish(TOPIC, json_data);
Subscriber code:
def on_message(client, userdata, msg):
response_message = msg.payload.decode()
response_dict = json.loads(response_message)
print (response_dict['status']) # Prints nothing
img_str = response_dict['image']
decoded_img = base64.b64decode(img_str)
with open("imageToSave.jpg", "wb") as fh:
fh.write(base64.decodebytes(decoded_img))

Because json format only support string and base64.b64encode return bytes so json.dumps should be giving error. you can use binacsii
import binascii
with open("sample.jpg", "rb") as image_file::
data = binascii.b2a_base64(image_file.read()).decode()
resp['image'] = data
print(json.dumps(resp))
#converting back to image using binascii.a2b_base64
with open("sample.jpg", "wb") as f2:
f2.write(binascii.a2b_base64(data))

my http server got post request including file, how to forward to external URL using POST method too

I have a problem, and I searched a lot but none of codes here do my request
I'm running python local server using http.server - BaseHTTPRequestHandler
here is my code
def run_http():
port=80
server_address = ('', port)
httpd = ThreadingHTTPServer(server_address, serverClass)
# print('HTTP server running on port %s'% port)
httpd.serve_forever()
class serverClass(BaseHTTPRequestHandler):
def _set_headers(self):
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
def do_GET(self):
self._set_headers()
if self.path == '/':
with open("static\\index.html", 'r') as f:
res = f.read()
self.wfile.write(res.encode('utf-8'))
elif self.path == '/favicon.ico':
with open("static\\favicon.ico", 'rb') as f:
res = f.read()
self.wfile.write(res)
else:
...
def do_HEAD(self):
self._set_headers()
def do_POST(self):
# content_length = int(self.headers['Content-Length'])
# post_data = self.rfile.read(content_length)
# <= here I need to forward the post request to another URL _POST_URL
# The post request I received may include file and may be not, so I need to forward the post request as it is
I tried some codes for example:
ctype, pdict = cgi.parse_header(self.headers['content-type'])
pdict['boundary'] = bytes(pdict['boundary'], "utf-8")
if ctype == 'multipart/form-data':
fields = cgi.parse_multipart(self.rfile, pdict)
field_file = {'file':(fields.get('file'), "multipart/form-data")}
fields.pop('file')
r = requests.post(_POST_URL+'/in.php', files = field_file, data = fields)
note: I know that the file will be with post name file
That code works, but the problem is I don't sent the file type or the file name, and I cannot got it really.
Am I in the right way, or I should use more efficient codes

Ok I used cgi.FieldStorage
and here is the code if somebody may interesed;
def do_POST(self):
form = cgi.FieldStorage( fp=self.rfile, headers=self.headers, environ={'REQUEST_METHOD':'POST', 'CONTENT_TYPE':self.headers['Content-Type'] })
post_data = {}
for field in form:
if field != 'file':
post_data[field] = form.getvalue(field)
# print(post_data)
try:
fileitem = form['file']
if fileitem.filename:
fn = os.path.basename(fileitem.filename)
open('./tmp/' + fn, 'wb').write(fileitem.file.read())
data_files = {'file' : open('./tmp/' + fn, 'rb')}
# print(data_files)
r = requests.post(_POST_URL, data = post_data, files = data_files)
except:
r = requests.post(_POST_URL, data = post_data)
res = r.text
try:
data_files = None
os.remove('./tmp/' + fn)
except:
pass
self._set_headers()
self.wfile.write(res.encode('utf-8'))
This code is working for me, I don't know if there is a better way to do that!

Unable to Open base64 decoded '.jpg' image due to 'Source Missing'?

I am trying to send a .jpg file using request and trying to decode it in
django server side.
CODE:
This is the sending side code:
import requests
import os
import base64
fPath = os.getcwd()
url = 'http://127.0.0.1:8000/submitcausedata/'
headers = {'content-type': 'application/x-www-form-urlencoded'}
path_img = fPath + '/image13.jpg'
data = open(path_img,'rb').read()
encoded_image = base64.encodestring(data)
print(encoded_image[:10])
r = requests.post(url,data=encoded_image,headers=headers)
On receiving end Code:
#csrf_exempt
def submitCauseData(request):
response_data = {}
if request.method == 'POST':
data = request.POST
myDict = dict(data)
imageStr = list(myDict.keys())
imageStr = imageStr[0]
print(imageStr[:10])
image_result = open('image.jpg', 'wb')
image_result.write(base64.b64decode(imageStr))
return HttpResponse("Page Exists")
So, the code is executing but when I try to Open the saved Image it shows the error Photo Source File missing?
The sending code output of:
print(encoded_image[:10])
----> b'/9j/4WQYRX'
The receiving side code output of:
print(imageStr[:10])
----> /9j/4WQYRX
UPDATE:
While comparing both the .jpg files using .txt conversion a lot of values in both of them are different using DiffChecker Online.
The Image Viewer on Ubuntu shows the Error while opening the .jpg on receiving side:
Error interpreting JPEG image file (Unsupported marker type 0x10)
Also:
On Sending Side:
print(len(data))
print(type(data))
print(len(encoded_image))
print(type(encoded_image))
OUTPUT:
171062
<class 'bytes'>
228084
<class 'bytes'>
On receiving side:
print(len(imageStr))
print(type(imageStr))
print(len(imagedec))
print(type(imagedec))
OUTPUT:
228083
<class 'str'>
168972
<class 'bytes'>

Does the following code work?
sending
from base64 import b64encode
from pathlib import Path
import requests
url = "http://127.0.0.1:8000/submitcausedata/"
headers = {"content-type": "application/x-www-form-urlencoded"}
encoded_image = b64encode(Path("image13.jpg").read_bytes())
print(encoded_image[:10])
r = requests.post(url, data=encoded_image, headers=headers)
receiving
from base64 import b64decode
from pathlib import Path
#csrf_exempt
def submitCauseData(request):
response_data = {}
if request.method == "POST":
imageStr = list(dict(request.POST).keys())[0]
print(imageStr[:10])
p = Path("image.jpg")
p.write_bytes(b64decode(imageStr))
return HttpResponse(f"Saved image to `{p.resolve()}`")

I found the error as when I am trying to send the class of byte string from sending side, what happens is all the '+' and '=' get converted to ' ' on receiving side.
So, by using:
Sending Side:
import requests
import os
import base64
fPath = os.getcwd()
url = 'http://127.0.0.1:8000/submitcausedata/'
headers = {'content-type': 'application/x-www-form-urlencoded'}
path_img = fPath + '/image13.jpg'
data = open(path_img,'rb').read()
encoded_image = base64.b64encode(data)
r = requests.post(url,data=encoded_image,headers=headers)
Receiving Side:
#csrf_exempt
def submitCauseData(request):
response_data = {}
if request.method == 'POST':
data = request.POST
myDict = dict(data)
imageStr = list(myDict.keys())
image = imageStr[0].replace(' ','+')
image = image.encode() + b'==='
imagedec = base64.b64decode(image)
fPath = os.getcwd()
image_result = open(fPath + '/image.jpg', 'wb')
image_result.write(imagedec)
image_result.close()
return HttpResponse("HELLO")
I solved the error.
Anyways, thanks for the help #waket-zheng :)

Streaming POST a large file to CherryPy by Python client

I'm want to POST a large file from a python client to cherrypy. I'm using the requests library.
This is my client code:
def upload(fileName=None):
url = 'http://localhost:8080/upload'
files = {'myFile': ( fileName, open(fileName, 'rb') )}
r = requests.post(url, files=files)
#with open(fileName,'rb') as payload:
#headers = {'content-type': 'multipart/form-data'}
#r = requests.post('http://127.0.0.1:8080', data=payload,verify=False,headers=headers)
if __name__ == '__main__':
upload(sys.argv[1])
The problem is that this puts the whole file in the RAM memory. Is there any way to POST the file in pieces?
class FileDemo(object):
#cherrypy.expose
def upload(self, myFile):
print myFile.filename
#size = 0
#decoder = MultipartDecoder(myFile, 'image/jpeg')
#for part in decoder.parts:
#print(part.header['content-type'])
#while True:
#advances to the content that hasn't been read
#myFile.file.seek(size, 0)
#reads 100mb at a time so it doesn't fill up the RAM
#data = myFile.file.read(10240000)
#newFile = open("/home/ivo/Desktop/"+str(myFile.filename), 'a+')
#newFile.write(data)
#newFile.close
#size += len(data)
#if len(data) < 10240000:
#break
if __name__ == '__main__':
cherrypy.quickstart(FileDemo())
This is the code in the server side. It has a lot of comments because I've been trying a lot of stuff. Right now I'm just printing the file name and the client still transfers the whole file to RAM.
I don't know what else to try. Thank you in advance for your help.

If it's CherryPy specific upload you can skip multipart/form-data encoding obstacles and just send streaming POST body of file contents.
client
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import urllib2
import io
import os
class FileLenIO(io.FileIO):
def __init__(self, name, mode = 'r', closefd = True):
io.FileIO.__init__(self, name, mode, closefd)
self.__size = statinfo = os.stat(name).st_size
def __len__(self):
return self.__size
f = FileLenIO('/home/user/Videos/video.mp4', 'rb')
request = urllib2.Request('http://127.0.0.1:8080/upload', f)
request.add_header('Content-Type', 'application/octet-stream')
# you can add custom header with filename if you need it
response = urllib2.urlopen(request)
print response.read()
server
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import tempfile
import shutil
import cherrypy
config = {
'global' : {
'server.socket_host' : '127.0.0.1',
'server.socket_port' : 8080,
'server.thread_pool' : 8,
# remove any limit on the request body size; cherrypy's default is 100MB
'server.max_request_body_size' : 0,
# increase server socket timeout to 60s; cherrypy's defult is 10s
'server.socket_timeout' : 60
}
}
class App:
#cherrypy.config(**{'response.timeout': 3600}) # default is 300s
#cherrypy.expose()
def upload(self):
'''Handle non-multipart upload'''
destination = os.path.join('/home/user/test-upload')
with open(destination, 'wb') as f:
shutil.copyfileobj(cherrypy.request.body, f)
return 'Okay'
if __name__ == '__main__':
cherrypy.quickstart(App(), '/', config)
Tested on 1.3GiB video file. Server-side memory consumption is under 10MiB, client's under 5MiB.

This is how I solved the problem:
client
import poster
def upload(fileName=None):
register_openers()
url = 'http://localhost:8080/upload'
data, headers = multipart_encode({"myFile": open(fileName, "rb")})
request = urllib2.Request(url, data, headers)
request.unverifiable = True
response = urllib2.urlopen(request)
the_page = response.read()
if __name__ == '__main__':
upload(sys.argv[1])
server
#cherrypy.expose
def upload(self, myFile):
cherrypy.response.timeout = 3600
newFile = open("/home/ivo/Desktop/"+str(myFile.filename), 'a+')
newFile.write(myFile.file.read())
newFile.close