We Keep Coding

How to read image from python http.server POST requests without downloading

I'm new to python and http.server, I'm doing a simple API which allow client to post an image and server can do something with that image.
Below is my do_POST method:
class Server(BaseHTTPRequestHandler):
def do_POST(self):
self.path = '/upload'
content_length = int(self.headers['Content-Length'])
file_content = self.rfile.read(content_length)
#do something with the image here
#response
response = f'Message:'
self.send_response(200) #create header
self.send_header("Content-Length", str(len(response)))
self.end_headers()
and my requests:
r = requests.post('http://127.0.0.1:8000/upload', files = {'image': open('C:/Users/Hp/IMG20220315112838.jpg','rb')})
However, the file_content also contains metadata of image and other informations like that:
b'--ecd5d189dbf1c44dd325bf7134f386eb\r\nContent-Disposition: form-data; name="image"; filename="IMG20220315112838.jpg"\r\n\r\n\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00\xff\xe1\x02\xe8Exif\x00\x00II*\x00\x08\x00\x00\x00\x0b\x00\x00\x01\x03\x00\x01\x00\x00\x00#\x07\x00\x00\x01\x01\x03\x00\x01\x00\x00\x00\xc0\x0f\x00\x00\x0f\x01\x02\x00\x05\x00\x00\x00\xa2\x00\x00\x00\x10\x01\x02\x00\r\x00\x00\x00\xa8\x00\x00\x00\x12\x01\x03\x00\x01\x00\x00\x00\x0
My question is how to get the bytes of only image to use with something like cv2.imshow() ?
And another small question is how to handle with multiple images upload such as
r = requests.post('http://127.0.0.1:8000/upload', files = {
'image1': open('C:/Users/Hp/IMG20220315112838.jpg','rb'),
'image2': open('C:/Users/Hp/IMG123456789.jpg','rb'), })
Thank you very much for helping out.
Edit1: I tried file_content.decode('UTF-8') but it print out error:
'utf-8' codec can't decode byte 0xff in position 118
Position 118 is exactly after the filename string.

Im answering my own question:
I tried requests-toolbelt parser to get the data as below:
from requests_toolbelt.multipart import decoder
class Server(BaseHTTPRequestHandler):
def do_POST(self):
self.path = '/upload'
content_length = int(self.headers['Content-Length'])
#get data content bytes
file_content = self.rfile.read(content_length)
#Use multipart parser to strip boundary
multipart_data = decoder.MultipartDecoder(file_content, self.headers['Content-Type']).parts
image_byte = multipart_data[0].content
#Read image using cv2
image_numpy = np.frombuffer(image_byte, np.int8)
img = cv2.imdecode(image_numpy, cv2.IMREAD_UNCHANGED)
#Send response
response = bytes('Message: Successesesese', 'utf-8')
self.send_response(200) #create header
self.send_header("Content-Length", str(len(response)))
self.end_headers()
self.wfile.write(response)

How do I send an image as a json element using Mqtt and python?

I am attaching an image as a json element and sending it over mqtt. But I do not receive anything on subscriber side. If I remove the image, I am receiving the json.
Publisher code:
with open("sample.jpg", "rb") as image_file:
encoded_img_str = base64.b64encode(image_file.read())
response['image'] = encoded_img_str
response['status'] = 'success'
response['filename'] = 'sample.jpg'
json_data = json.dumps(response)
client.publish(TOPIC, json_data);
Subscriber code:
def on_message(client, userdata, msg):
response_message = msg.payload.decode()
response_dict = json.loads(response_message)
print (response_dict['status']) # Prints nothing
img_str = response_dict['image']
decoded_img = base64.b64decode(img_str)
with open("imageToSave.jpg", "wb") as fh:
fh.write(base64.decodebytes(decoded_img))

Because json format only support string and base64.b64encode return bytes so json.dumps should be giving error. you can use binacsii
import binascii
with open("sample.jpg", "rb") as image_file::
data = binascii.b2a_base64(image_file.read()).decode()
resp['image'] = data
print(json.dumps(resp))
#converting back to image using binascii.a2b_base64
with open("sample.jpg", "wb") as f2:
f2.write(binascii.a2b_base64(data))

Python Graph API request only retrieving 1 attachment from an email

I'm using a python script to send an API request to get the attachments of an email. The email I'm using has 4 attachments (plus pictures in the signature in the email). The python request only retrieves 1 attachment along with the pics from the signature. When using Postman with the exact same information, it retrieves all attachments along with the pics.
Any ideas on how I can get the other attachments?
import requests
url = 'https://graph.microsoft.com/v1.0/users/{{users email}}/messages/{{messageID}}/attachments'
body = None
head = {"Content-Type": "application/json;charset=UFT-8", "Authorization": "Bearer " + accessToken}
response1 = requests.get(url, data=body, headers=head)
response = response1.text
Below shows the response from the python script, with only 7 items, and the Postman response with 10 items.

Below code retrieves multiple attachments
(files being an array of attachment names)
def execute(accessToken, messageID, files, noAttachments):
import os
from os import path
import requests
import base64
import json
if noAttachments == "False":
url = 'https://graph.microsoft.com/v1.0/users/{{users email}}/messages/{{messageID}}/attachments'
body = {}
head = {"Authorization": "Bearer " + accessToken}
responseCode = requests.request("GET", url, headers=head, data = body)
response = responseCode.text
test = json.loads(responseCode.text.encode('utf8'))
x, contentBytes = response.split('"contentBytes":"',1)
if len(files) == 1:
imgdata = base64.b64decode(str(contentBytes))
filename = "C:/Temp/SAPCareAttachments/" + files[0]
with open(filename, 'wb') as f:
f.write(imgdata)
else:
for file in test["value"]:
imgdata = base64.b64decode(file["contentBytes"])
if file["name"] in files:
filename = "C:/Temp/" + file["name"]
with open(filename, 'wb') as f:
f.write(imgdata)
print(responseCode)

How to convert 'io.BufferedWriter' to 'Dataframe' - Python

I am downloading files from One Drive with the following piece of code:
if response.status_code == 200:
print('\n> Response Success')
with open('Book2.xlsx', 'wb') as File:
File.write(response.content)
print('\n> File Downloaded')
else:
print('\n> Failed:', response.status_code)
print(response.content)
The code is from:This post here
The "File" is coming from One Drive with the use of the following code:
import sys, os, time, requests
import pandas as pd
import urllib.parse
OneDrive_FilePath = 'Book2.xlsx'
OneDrive_FileURL = 'https://graph.microsoft.com/v1.0/me/drive/root:/' + OneDrive_FilePath + ':/content'
OneDrive_FileURL = urllib.parse.quote(OneDrive_FileURL, safe=':/')
print(OneDrive_FileURL)
Client_Id = 'XXXX'
Tenant_Id = 'YYYYY'
Refresh_Token_First = 'ZZZZZ'
PostStr = {'grant_type': 'refresh_token', 'client_id': Client_Id, 'refresh_token': Refresh_Token_First}
Token_Response = requests.post('https://login.microsoftonline.com/' + Tenant_Id + '/oauth2/v2.0/token', data=PostStr)
Access_Token = Token_Response.json()['access_token']
New_Refresh_Token = Token_Response.json()['refresh_token']
if Access_Token is None or New_Refresh_Token is None:
print('\n> Failed: Access_Token NOT Retrieved')
sys.exit()
Response = requests.get(OneDrive_FileURL, headers={'Authorization': 'Bearer ' + Access_Token})
The "File" which is getting downloaded is in the form of "io.BufferedWriter". I want to actually load the "File" as a data frame so that I can do certain operations on that and upload it to AWS.
How can I do that, please help.
Thanks

As explained in the comments, the type of File is mostly irrelevant. All you need to do is read the excel directly from the response, as in :
url = "https://go.microsoft.com/fwlink/?LinkID=521962"
res = requests.get(url)
pd.read_excel(res.content)
The last statement produces a regular pandas dataframe. You can use that however you want.

Streaming POST a large file to CherryPy by Python client

I'm want to POST a large file from a python client to cherrypy. I'm using the requests library.
This is my client code:
def upload(fileName=None):
url = 'http://localhost:8080/upload'
files = {'myFile': ( fileName, open(fileName, 'rb') )}
r = requests.post(url, files=files)
#with open(fileName,'rb') as payload:
#headers = {'content-type': 'multipart/form-data'}
#r = requests.post('http://127.0.0.1:8080', data=payload,verify=False,headers=headers)
if __name__ == '__main__':
upload(sys.argv[1])
The problem is that this puts the whole file in the RAM memory. Is there any way to POST the file in pieces?
class FileDemo(object):
#cherrypy.expose
def upload(self, myFile):
print myFile.filename
#size = 0
#decoder = MultipartDecoder(myFile, 'image/jpeg')
#for part in decoder.parts:
#print(part.header['content-type'])
#while True:
#advances to the content that hasn't been read
#myFile.file.seek(size, 0)
#reads 100mb at a time so it doesn't fill up the RAM
#data = myFile.file.read(10240000)
#newFile = open("/home/ivo/Desktop/"+str(myFile.filename), 'a+')
#newFile.write(data)
#newFile.close
#size += len(data)
#if len(data) < 10240000:
#break
if __name__ == '__main__':
cherrypy.quickstart(FileDemo())
This is the code in the server side. It has a lot of comments because I've been trying a lot of stuff. Right now I'm just printing the file name and the client still transfers the whole file to RAM.
I don't know what else to try. Thank you in advance for your help.

If it's CherryPy specific upload you can skip multipart/form-data encoding obstacles and just send streaming POST body of file contents.
client
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import urllib2
import io
import os
class FileLenIO(io.FileIO):
def __init__(self, name, mode = 'r', closefd = True):
io.FileIO.__init__(self, name, mode, closefd)
self.__size = statinfo = os.stat(name).st_size
def __len__(self):
return self.__size
f = FileLenIO('/home/user/Videos/video.mp4', 'rb')
request = urllib2.Request('http://127.0.0.1:8080/upload', f)
request.add_header('Content-Type', 'application/octet-stream')
# you can add custom header with filename if you need it
response = urllib2.urlopen(request)
print response.read()
server
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import tempfile
import shutil
import cherrypy
config = {
'global' : {
'server.socket_host' : '127.0.0.1',
'server.socket_port' : 8080,
'server.thread_pool' : 8,
# remove any limit on the request body size; cherrypy's default is 100MB
'server.max_request_body_size' : 0,
# increase server socket timeout to 60s; cherrypy's defult is 10s
'server.socket_timeout' : 60
}
}
class App:
#cherrypy.config(**{'response.timeout': 3600}) # default is 300s
#cherrypy.expose()
def upload(self):
'''Handle non-multipart upload'''
destination = os.path.join('/home/user/test-upload')
with open(destination, 'wb') as f:
shutil.copyfileobj(cherrypy.request.body, f)
return 'Okay'
if __name__ == '__main__':
cherrypy.quickstart(App(), '/', config)
Tested on 1.3GiB video file. Server-side memory consumption is under 10MiB, client's under 5MiB.

This is how I solved the problem:
client
import poster
def upload(fileName=None):
register_openers()
url = 'http://localhost:8080/upload'
data, headers = multipart_encode({"myFile": open(fileName, "rb")})
request = urllib2.Request(url, data, headers)
request.unverifiable = True
response = urllib2.urlopen(request)
the_page = response.read()
if __name__ == '__main__':
upload(sys.argv[1])
server
#cherrypy.expose
def upload(self, myFile):
cherrypy.response.timeout = 3600
newFile = open("/home/ivo/Desktop/"+str(myFile.filename), 'a+')
newFile.write(myFile.file.read())
newFile.close