We Keep Coding

How do I send Server Log to Client through http.server Python?

As the question, I now creating a simple http.server with Python and I have to send information about request like Client IP, login time of client, method GET/POST, HTTP header, HTTP body to Client and display it on Frontend. But I don't know how to send and how to get that information to send ? Please help me, thanks
This is my server code:
from http.server import BaseHTTPRequestHandler, HTTPServer
import os
import cgi
from supervisor.medusa.auth_handler import get_header
from database.db import create_table, signup, check_login
from random import randint
from datetime import datetime
import logging
islogin = False
def read_html_template(path):
try:
with open(path) as f:
file = f.read()
except Exception as e:
file = e
return file
class handler(BaseHTTPRequestHandler):
def gen_token(self):
return "".join(str(randint(1, 9)) for _ in range(25))
def get_Ip_addr(self):
return self.client_address[0]
def get_header(self):
return logging.error(self.headers)
def get_time(self):
now = datetime.now()
return now.strftime("%d/%m/%Y %H:%M:%S")
def data_to_send(self):
login_time = self.get_time()
header_data = self.get_header()
ip_client = self.get_Ip_addr()
data = ('{"IP_Client": '+ str(ip_client) + ', "HTTP_Header": ' + str(header_data) + ', "Login_Time": '+ str(login_time) + '}')
return data
def do_GET(self):
if self.path == '/':
self.path = './src/html/index.html'
elif self.path == '/login':
self.path = './src/html/Login.html'
elif self.path == '/signup':
self.path = './src/html/Signup.html'
elif self.path == '/forgotpwd':
self.path = './src/html/ForgotPwd.html'
elif self.path == '/home' and islogin == True:
self.path = './src/html/singnedIn.html'
try:
split_path = os.path.splitext(self.path)
request_extension = split_path[1]
if request_extension != ".py":
f = read_html_template(self.path)
self.send_response(200)
self.end_headers()
self.wfile.write(bytes(f, 'utf-8'))
else:
f = "File not found"
self.send_error(404,f)
except:
f = "File not found"
self.send_error(404,f)
with HTTPServer(('', 8000), handler) as server:
server.serve_forever()
My idea is write data to a json file and send it to client but 2 things I don't know is
How to send json file ? and
How to get data and write it to json file ?

http webserver simply stops responding

Earlier I wrote multi threaded web server, which at times would simply stop processing requests and also getting terminated at peak times.
I've implemented same opencv based processing in Python Websocket based server too which is working fine.
For very old browsers, I also need POST based processing using web server. I converted from multithreading to single but that also is stopping different times and not printing any log etc.
I checked syslog but not clue. More than a week has gone by without finding a solution. I suspect something related to Digital Ocean VPS or network.
I've this code and can't figure why it should stop responding:
from http.server import HTTPServer, BaseHTTPRequestHandler
import threading
import cgi
import tempfile
import resource
import base64
from common import *
from datetime import datetime
print( datetime.now());
gg_hashmap = getHash()
USE_HTTPS = True
def dump(obj):
for attr in dir(obj):
print("obj.%s = %r" % (attr, getattr(obj, attr)))
class PostHandler(BaseHTTPRequestHandler):
def handle(self):
try:
BaseHTTPRequestHandler.handle(self)
except :
pass
def do_POST(self):
try:
print("new req="+str( datetime.now()),flush=True);
form = cgi.FieldStorage(
fp=self.rfile,
headers=self.headers,
environ={'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': self.headers['Content-Type'],
})
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Access-Control-Allow-Origin", "*")
self.end_headers()
for field in form.keys():
field_item = form[field]
if field_item.filename:
file_data = field_item.file.read()
file_len = len(file_data)
del file_data
self.wfile.write('\tUploaded %s as "%s" (%d bytes)\n' % \
(field, field_item.filename, file_len))
else:
pass
if ('base64' in form and 'license' in form):
print("license=",form['license'].value);
global gg_hashmap
file_content = form['base64'].value
try:
f, temp_file_path = tempfile.mkstemp(prefix='sand', suffix='jpg')
os.close(f)
with open(temp_file_path, 'wb') as w:
w.write(base64.b64decode (file_content))
input_hashes = get_input_img(temp_file_path)
all_letters = ""
if input_hashes != None:
for inp_hash in input_hashes:
lowest = 1000
lowest_letter = ''
for letter, arr in gg_hashmap.items():
for hashval in arr:
if int(inp_hash - hashval) < lowest:
lowest = int(inp_hash - hashval)
lowest_letter = letter
all_letters += lowest_letter
self.wfile.write(bytes(all_letters, "utf8"))
except Exception as e:
print("exception3 caught")
print(e)
print(str(e))
return
except Exception as e:
print("Caught unknown exception",e)
def do_GET(self):
self.send_response(200)
self.end_headers()
message = threading.currentThread().getName()
self.wfile.write(bytes(message,'utf-8'))
self.wfile.write('\n')
return
form = cgi.FieldStorage(
fp=self.rfile,
headers=self.headers,
environ={'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': self.headers['Content-Type'],
})
self.send_response(200)
self.end_headers()
for field in form.keys():
field_item = form[field]
if field_item.filename:
file_data = field_item.file.read()
file_len = len(file_data)
del file_data
self.wfile.write('\tUploaded %s as "%s" (%d bytes)\n' % \
(field, field_item.filename, file_len))
else:
pass
return
def run():
# resource.setrlimit(resource.RLIMIT_STACK, (2**29,-1))
# threading.stack_size(24*1048576)
server = HTTPServer(('0.0.0.0', 443), PostHandler)
if USE_HTTPS:
import ssl
server.socket = ssl.wrap_socket(server.socket, keyfile='./ssl/key.pem', certfile='./ssl/public.pem'
, ca_certs="./ssl/cap1_transactionfailed_com.ca-bundle" , server_side=True)
server.serve_forever()
if __name__ == '__main__':
run()

I don't think much anyone will want to read through all 157 lines of convoluted HTTP request handling code (of which some isn't even posted, from common import *) to try and decipher why it might stop at some given time.
It's likely not the answer you want to hear, but HTTPServer really isn't what anyone uses in production for Python.
You should look into rewriting your code with either (my recommendations at the time of writing)
FastAPI (or its underlying Starlette framework) on Uvicorn (Uvicorn will let you do the websocket stuff in the same process), or
Flask on Gunicorn or uWSGI
For instance, here's a rough estimation of what your code would look like with Starlette. (There may be bugs since it's dry-coded, and it's certainly not fully async, but that doesn't matter here.)
import tempfile
import base64
from starlette.applications import Starlette
from starlette.requests import Request
from starlette.responses import PlainTextResponse
app = Starlette()
def get_all_letters(input_hashes):
all_letters = ""
if input_hashes:
for inp_hash in input_hashes:
lowest = 1000
lowest_letter = ""
for letter, arr in gg_hashmap.items():
for hashval in arr:
if int(inp_hash - hashval) < lowest:
lowest = int(inp_hash - hashval)
lowest_letter = letter
all_letters += lowest_letter
return all_letters
#app.route("/", methods=["GET", "POST"])
async def handle(request: Request):
if request.method == "GET":
return PlainTextResponse("Hello!")
form = await request.form()
if not ("base64" in form and "license" in form):
return PlainTextResponse("Missing data!", status_code=400)
with tempfile.NamedTemporaryFile(prefix="sand", suffix="jpg") as f:
content = await form["base64"].read()
f.write(base64.b64decode(content))
f.flush()
input_hashes = get_input_img(f)
if not input_hashes:
return PlainTextResponse("No input hashes!", status_code=400)
all_letters = get_all_letters(input_hashes)
return PlainTextResponse(all_letters)
You could then run this using Uvicorn (which will also handle all of that HTTPS stuff for you).

With mkstemp you must delete the tempfile. You probably run out of disk space or max out files in temp directory. As AKX mentioned though you should look into using a more robust http server. if the file thing isn't your problem there are numerous other issues that could come up when using a non production HTTP server.

OSError: [Errno 9] Bad file descriptor

I initially tried using python to run but there were different errors so I tried using python3 and received the error in the title. I am trying to connect to server and download a file that has tls implemented.
import socket, ssl, pprint
import os, time
import threading
def main():
s2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ssl_sock = ssl.wrap_socket(s2,
server_side = False,
ca_certs="CA.crt",
cert_reqs=ssl.CERT_REQUIRED)
s2.connect(('localhost',10024))
filename = raw_input("What file do you wish to download? -> ")
if filename != 'q':
s2.send(filename)
data = s2.recv(1024)
if data [:7] == 'fEXISTS':
fSize = long(data[7:])
message = raw_input("The file you wish to download is " +str(fSize)+\
"bytes, would you like to proceed? (y/n): ")
if message == 'y':
s2.send ('OK')
f = open('new_'+filename, 'wb')
data = s2.recv(2000000)
totalRecv = len(data)
f.write(data)
while totalRecv < fSize:
data = s2.recv(2000000)
totalRecv += len(data)
f.write(data)
progress = ((totalRecv/float(fSize))*100)
print ("{0: .2F}".format(progress)+\
"% Completed")
else:
print ("ERROR: File does not exist!")
s2.close()
if __name__ == '__main__':
main()

After wrapping the socket in an SSL context (with ssl.wrap_socket), you should not be using the original socket anymore.
You should be calling connect, send, recv, etc. on ssl_sock, not on s2.
(Specifically, when you call ssl.wrap_socket, the .detach method is called on the original socket which removes the file descriptor from it. The file descriptor is transferred to the SSL socket instance. The only thing you can do with the original then is close/destroy it.)

How to finish a socket file transfer in Python?

I have a Client and a Server and I need to transfer some files using sockets. I can send small messages, but when I try to send a File, the problems begins...
client.py:
from socket import *
from threading import Thread
import sys
import hashlib
class Client(object):
ASK_LIST_FILES = "#001" # 001 is the requisition code to list
# all the files
ASK_SPECIFIC_FILE = "#002" # 002 is the requisition code to a
# specific file
SEND_FILE = "#003" # 003 is the requisition code to send one
# file
AUTHENTICATION = "#004" # 004 is the requisition code to user
# authentication
listOfFiles = []
def __init__(self):
try:
self.clientSocket = socket(AF_INET, SOCK_STREAM)
except (error):
print("Failed to create a Socket.")
sys.exit()
def connect(self, addr):
try:
self.clientSocket.connect(addr)
except (error):
print("Failed to connect.")
sys.exit()
print(self.clientSocket.recv(1024).decode())
def closeConnection(self):
self.clientSocket.close()
def _askFileList(self):
try:
data = Client.ASK_LIST_FILES
self.clientSocket.sendall(data.encode())
# self._recvFileList()
except (error):
print("Failed asking for the list of files.")
self.closeConnection()
sys.exit()
thread = Thread(target = self._recvFileList)
thread.start()
def _recvFileList(self):
print("Waiting for the list...")
self.listOfFiles = []
while len(self.listOfFiles) == 0:
data = self.clientSocket.recv(1024).decode()
if (data):
self.listOfFiles = data.split(',')
if(len(self.listOfFiles) > 0):
print (self.listOfFiles)
def _askForFile(self, fileIndex):
fileIndex = fileIndex - 1
try:
data = Client.ASK_SPECIFIC_FILE + "#" + str(fileIndex)
self.clientSocket.sendall(data.encode())
except(error):
print("Failed to ask for an specific file.")
self.closeConnection()
sys.exit()
self._downloadFile(fileIndex)
def _downloadFile(self, fileIndex):
print("Starting receiving file")
f = open("_" + self.listOfFiles[fileIndex], "wb+")
read = self.clientSocket.recv(1024)
# print(read)
# f.close
while len(read) > 0:
print(read)
f.write(read)
f.flush()
read = self.clientSocket.recv(1024)
f.flush()
f.close()
self.closeConnection()
server.py
from socket import *
from threading import Thread
import sys
import glob
class Server(object):
def __init__(self):
try:
self.serverSocket = socket(AF_INET, SOCK_STREAM)
except (error):
print("Failed to create a Socket.")
sys.exit()
def connect(self, addr):
try:
self.serverSocket.bind(addr)
except (error):
print ("Failed on binding.")
sys.exit()
def closeConnection(self):
self.serverSocket.close()
def waitClients(self, num):
while True:
print("Waiting for clients...")
self.serverSocket.listen(num)
conn, addr = self.serverSocket.accept()
print("New client found...")
thread = Thread(target = self.clientThread, args = (conn,))
thread.start()
def clientThread(self, conn):
WELCOME_MSG = "Welcome to the server"
conn.send(WELCOME_MSG.encode())
while True:
data = conn.recv(2024).decode()
if(data):
# print(data)
# reply = 'OK: ' + data
# conn.sendall(reply.encode())
if(data == "#001"):
listOfFiles = self.getFileList()
strListOfFiles = ','.join(listOfFiles)
self._sendFileList(strListOfFiles, conn)
else:
dataCode = data.split('#')
print(dataCode)
if(dataCode[1] == "002"):
print("Asking for file")
self._sendFile(int(dataCode[2]), conn)
if(dataCode[1] == "003"):
print("Pedido de login")
if self._authentication(dataCode[2]):
conn.send("OK".encode())
# self._recvFile(conn)
else:
conn.send("FAILED".encode())
def _sendFile(self, fileIndex, conn):
listOfFiles = self.getFileList()
print(fileIndex)
print(listOfFiles[fileIndex])
f = open(listOfFiles[fileIndex], "rb")
read = f.read(1024)
while len(read) > 0:
conn.send(read)
read = f.read(1024)
f.close()
def _sendFileList(self, strList, conn):
try:
conn.sendall(strList.encode())
except (error):
print("Failed to send list of files.")
def getFileList(self):
return glob.glob("files/*")
When I try to get a file from my server, I can transfer everything but the connection never ends. What is going on with my code?

First, you are doing here the most common error using TCP: assume all data sent in a single send() will be got identically in a single recv(). This is untrue for TCP, because it is an octet stream, not a message stream. Your code will work only under ideal (lab) conditions and could mysteriously fail in a real world usage. You should either explicitly invent message boundaries in TCP streams, or switch e.g. to SCTP. The latter is available now almost everywhere and keeps message boundaries across a network connection.
The second your error is directly connected to the first one. When sending file, you don't provide any explicit mark that file has been finished. So, clients waits forever. You might try to close server connection to show that file is finished, but in that case client won't be able to distinguish real file end and connection loss; moreover, the connection won't be reusable for further commands. You would select one of the following ways:
Prefix a file contents with its length. In this case, client will know how many bytes shall be received for the file.
Send file contents as a chunk sequence, prefixing each chunk with its length (only for TCP) and with mark whether this chunk is last (for both transports). Alternatively, a special mark "EOF" can be sent without data.
Similarly, control messages and their responses shall be provided with either length prefix or a terminator which can't appear inside such message.
When you finish developing this, you would look at FTP and HTTP; both addresses all issues I described here but in principally different ways.

Creating a Python webserver - layout and setup

am I going about this in the correct way? Ive never done anything like this before, so im not 100% sure on what I am doing. The code so far gets html and css files and that works fine, but images wont load, and will I have to create a new "if" for every different file type? or am I doing this a silly way...here is what I have:
import string,cgi,time
from os import curdir, sep
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
import os
import mimetypes
#import pri
port = 888
host = "0.0.0.0"
class MyHandler(BaseHTTPRequestHandler):
def do_GET(self):
try:
#RequestedURL = self.path
mimeType = mimetypes.guess_type(self.path)[0]
fileType = mimetypes.guess_extension(mimeType)
infoList = [mimeType, fileType]
if infoList[1] != ".py":
self.send_response(200)
self.send_header('Content-type', mimeType)
self.end_headers()
f = open(curdir + sep + self.path, "rb")
self.wfile.write(f.read())
f.close()
return
if fileType == ".py":
pythonFilename = self.path.lstrip("/")
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
pyname = pythonFilename.replace("/", ".")[:-3]
print pythonFilename
print pyname
temp1 = pyname.split(".")
temp2 = temp1[-1]
print temp2
module = __import__(root.index)
self.wfile.write(module.root.index.do_work())
#module = __import__("test.index")
#self.wfile.write( module.index.do_work())
return
return
except IOError:
self.send_error(404,'File Not Found: %s' % self.path)
def do_POST(self):
global rootnode
try:
ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
if ctype == 'multipart/form-data':
query=cgi.parse_multipart(self.rfile, pdict)
self.send_response(301)
self.end_headers()
upfilecontent = query.get('upfile')
print "filecontent", upfilecontent[0]
self.wfile.write("<HTML>POST OK.<BR><BR>");
self.wfile.write(upfilecontent[0]);
except :
pass
def main():
try:
server = HTTPServer((host, port), MyHandler)
print 'started httpserver:'
print ("Host: " + (host))
print ("Port: " + str(port))
server.serve_forever()
except KeyboardInterrupt:
print '^C received, shutting down server'
server.socket.close()
if __name__ == '__main__':
main()
html and css works, but png images do not load

You are on the right track with it, though your ifs are very redundant. I suggest you refactor the code to check for type using a loop and a dict:
mime = {"html":"text/html", "css":"text/css", "png":"image/png"}
if RequestedFileType in mime.keys():
self.send_response(200)
self.send_header('Content-type', mime[RequestedFileType])
self.end_headers()
print RequestedFileType
f = open(curdir + sep + self.path)
self.wfile.write(f.read())
f.close()
return
Also, you are sending binary files as text. Instead of open(curdir + sep + self.path) use open(curdir + sep + self.path, "b")
Gergely from toptal.com

As to a panoply of if statements, the usual approach is to have a file that handles the mapping between extensions and mime types (look here: List of ALL MimeTypes on the Planet, mapped to File Extensions?). Read that into an appropriate data structure.
You should probably be opening all files as binary unless they are a text/* mime type; for those you should ensure that your line endings are as specified in the appropriate RFC (if any - it's been years since I have needed to consult those standards, on account of not writing a web server for deployment in anger).
And a syntactical point:
>>> ('foo')
'foo'
>>> ('foo',)
('foo',)
>>>
Your brackets are redundant. You can index on the value you are extracting.