Im trying to create a simple HTTP server that will receive POST messages and provide a simple response. Im using the standard HTTPServer with python. The client connects using a session() which should use a persistent connection but after each POST I see the message below in the debug that the connection is dropping.
INFO:urllib3.connectionpool:Resetting dropped connection:
DEBUG:urllib3.connectionpool:"GET / HTTP/1.1" 200 None
The client works properly when I try it with Apache so I believe the issue is in my simple server configuration. How can I configure the simple http server to work with persistent connections?
Simple Server Python Code:
from http.server import HTTPServer, BaseHTTPRequestHandler
from io import BytesIO
import time
import datetime
import logging
class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
def _set_response(self):
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.send_header("Connection", "keep-alive")
self.send_header("keep-alive", "timeout=5, max=30")
self.end_headers()
def do_GET(self):
self.send_response(200)
self.end_headers()
self.wfile.write(b'Hello, world!')
def do_POST(self):
content_length = int(self.headers['Content-Length'])
body = self.rfile.read(content_length)
curr_time = datetime.datetime.now()
data = ('{"msgid":"0x0002", "timestamp": "'+ str(curr_time) +'", "message":"Test http response from Raspberry Pi HTTP server"}').encode()
self.send_response(200)
self.end_headers()
response = BytesIO()
#response.write(b'This is POST request. ')
#response.write(b'Received: ')
response.write(data)
self.wfile.write(response.getvalue())
print("Simple HTTP Server running...")
logging.basicConfig(level=logging.DEBUG)
httpd = HTTPServer(('', 8000), SimpleHTTPRequestHandler)
httpd.serve_forever()
Client Python code:
#!/usr/bin/env python
# Using same TCP connection for all HTTP requests
import os
import json
import time
import datetime
import logging
import requests
from requests.auth import HTTPBasicAuth
logging.basicConfig(level=logging.DEBUG)
start_time = time.time()
def get_data(limit):
session = requests.Session()
url = "http://localhost:8000"
for i in range(10):
curr_time = datetime.datetime.now()
data = '{"msgid":"0x0001", "timestamp": "'+ str(curr_time) +'", "message":"Test http message from Raspberry Pi"}'
print("Sending Data: " + data)
response = session.post(url.format(limit), data)
#response_dict = json.loads(response.text)
print("Received Data: " + response.text)
if __name__ == "__main__":
limit = 1
get_data(limit)
print("--- %s seconds ---" % (time.time() - start_time))
You aren't actually setting the Connection header in your POST handler. In order for persistent connections to work, you'll also need to set the Content-Length header in the response so that client knows how many bytes of the HTTP body to read before reusing the connection.
Try this POST handler, adapted from your code:
def do_POST(self):
content_length = int(self.headers['Content-Length'])
body = self.rfile.read(content_length)
# Process the request here and generate the entire response
response_data = b'{"stuff": 1234}'
# Send the response
self.send_response(200)
self.send_header("Connection", "keep-alive")
self.send_header("Content-Length", str(len(response_data)))
self.end_headers()
# Write _exactly_ the number of bytes specified by the
# 'Content-Length' header
self.wfile.write(response_data)
Related
I have a python server that runs using an ssl wrapped socket. I am trying to send an https GET and POST request from an R-shiny client to the python server, yet I get the following error:
Listening on http://127.0.0.1:6565
Warning: Error in curl::curl_fetch_memory: schannel: Failed to get certificate location for >c/cygwin64/home/user/Documents/server/cert.pem
My overarching goal is to type a text into the shiny R text input, send this text to the python server in the form of a POST request over a secure channel. The python server I will run the interpreter() function on the string I just sent and return to the R-shiny client the appropriate response.
This setup has worked when not using ssl-certificates, however, after implementing the python ssl wrapped socket, I have been getting this error.
I hope some of you might be able to illuminate me on this matter.
This is my python server
import ssl
from http.server import HTTPServer, BaseHTTPRequestHandler
from io import BytesIO
def interpreter(val):
if val == "Search":
return "Searched"
elif val == "Add":
return "Added"
else:
return "Invalid statement"
class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header("Content-type", "text/csv")
self.end_headers()
self.wfile.write(bytes(
"<html><body><h1>HELLO world!</h1></body></html>", "utf-8"))
def do_POST(self):
content_length = int(self.headers['Content-Length'])
body = self.rfile.read(content_length)
print(body.decode())
modified_body = interpreter(body.decode())
print(modified_body)
self.send_response(200)
self.end_headers()
response = BytesIO()
response.write(b'Received: ')
response.write(body)
response.write(b' End of request')
self.wfile.write(response.getvalue())
httpd = HTTPServer(('localhost', 8000), SimpleHTTPRequestHandler)
httpd.socket = ssl.wrap_socket(httpd.socket, keyfile="key.pem",
certfile="cert.pem", server_side=True)
print("Server Running Now...")
httpd.serve_forever()
httpd.server_close()
print("Server Stopped")
And my shiny-R application is as follows
library(shiny)
library(httr)
ui <- fluidPage(
titlePanel("Request Module"),
fluidRow(
column(3,
textInput(inputId = "request", label="Enter Request Term"),
actionButton(inputId = "getRequest", label="Get"),
actionButton(inputId = "postRequest", label="Post")
)
),
fluidRow(
wellPanel(
htmlOutput(outputId = "resultOutput")
)
)
)
server <- function(input, output) {
url <- "http://127.0.0.1:8000"
s_url <- "https://127.0.0.1:8000"
httr::set_config(config(ssl_verifypeer = 0L))
certfile="c/cygwin64/home/user/Documents/server/cert.pem"
keyfile="c/cygwin64/home/user/Documents/server/key.pem"
observeEvent(input$getRequest,{
getResponse <- GET(url = s_url,
config(sslcert=certfile, sslkey=keyfile))
output$resultOutput <- renderPrint({
getResponse$content
})
})
observeEvent(input$postRequest,{
reqMessage <- input$request
postResult <- POST(url = s_url,
body = reqMessage,
content_type("text/csv"),
encoding = "UTF-8",
config(sslcert=certfile, sslkey=keyfile))
processedResponse <- content(postResult, as="text", encoding = "UTF-8")
output$resultOutput <- renderPrint({
processedResponse
})
})
}
shinyApp(ui = ui, server = server)
I have seen code like this that shows how to use a proxy for python requests.
import requests
proxies = {
'http': 'http://localhost:7777',
'https': 'http://localhost:7777',
}
requests.get('http://example.org', proxies=proxies)
requests.get('https://example.org', proxies=proxies)
But I am wondering how can we make a very simple proxy server in Python that would be able to respond to the GET request?
You can find many examples how to do it - even in questions on Stackoverflow.
Some of them use standard module socket (but it doesn't look simply).
Other use standard module http but they show code for Python 2 which was using different names.
Version for Python 3
import http.server
import socketserver
import urllib.request
class MyProxy(http.server.SimpleHTTPRequestHandler):
def do_GET(self):
print(self.path)
url = self.path
self.send_response(200)
self.end_headers()
self.copyfile(urllib.request.urlopen(url), self.wfile)
# --- main ---
PORT = 7777
httpd = None
try:
socketserver.TCPServer.allow_reuse_address = True # solution for `OSError: [Errno 98] Address already in use`
httpd = socketserver.TCPServer(('', PORT), MyProxy)
print(f"Proxy at: http://localhost:{PORT}")
httpd.serve_forever()
except KeyboardInterrupt:
print("Pressed Ctrl+C")
finally:
if httpd:
httpd.shutdown()
#httpd.socket.close()
Test using page httpbin.org
import requests
proxies = {
'http': 'http://localhost:7777',
'https': 'http://localhost:7777',
}
response = requests.get('http://httpbin.org/get', proxies=proxies)
print(response.text)
response = requests.get('http://httpbin.org/get?arg1=hello&arg2=world', proxies=proxies)
print(response.text)
But it works only for HTTP.
For HTTPS it may need to use ssl.socket from module ssl.
And it works only with GET.
For POST, PUT, DELETE, etc. it would need do_POST, do_PUT, do_DELETE, etc. with different code.
EDIT:
def do_POST(self):
url = self.path
# - post data -
content_length = int(self.headers.get('Content-Length', 0)) # <--- size of data
if content_length:
content = self.rfile.read(content_length) # <--- data itself
else:
content = None
req = urllib.request.Request(url, method="POST", data=content)
output = urllib.request.urlopen(req)
# ---
self.send_response(200)
self.end_headers()
self.copyfile(output, self.wfile)
But if you need local proxy only to test your code then you could use
Python module/program: mitmproxy (Man-In-The-Middle-Proxy)
not-python, not-free (but work 30 days for free), with nice GUI: Charles Proxy
More complex OWASP ZAP, Burp Suite (community edition)
Below is what I have tried.
import http.server
import socketserver
import requests
PORT = 8000
Handler = http.server.SimpleHTTPRequestHandler
with socketserver.TCPServer(("", PORT), Handler) as httpd:
print("serving at port", PORT)
httpd.serve_forever()
def api(data):
r = requests.post('http://localhost:8000/api', json=data)
return r.json()
Getting below error with above code.
ConnectionRefusedError: [WinError 10061] No connection could be made because the target machine actively refused it
Postman should be able to send post request having json body.
You didn't show full error message and I don't use Windows to test it but SimpleHTTPRequestHandler doesn't have function do_POST to receive POST request and this can make problem.
You will have to use SimpleHTTPRequestHandler to create own class with do_POST.
And this function will need to
get header information
read JSON string
convert request data from JSON string to dictionary
convert response data from dictionary to JSON string
send headers
send JSON string
so it will need a lot of work.
Minimal working server
import http.server
import socketserver
import json
PORT = 8000
class MyHandler(http.server.SimpleHTTPRequestHandler):
def do_POST(self):
# - request -
content_length = int(self.headers['Content-Length'])
#print('content_length:', content_length)
if content_length:
input_json = self.rfile.read(content_length)
input_data = json.loads(input_json)
else:
input_data = None
print(input_data)
# - response -
self.send_response(200)
self.send_header('Content-type', 'text/json')
self.end_headers()
output_data = {'status': 'OK', 'result': 'HELLO WORLD!'}
output_json = json.dumps(output_data)
self.wfile.write(output_json.encode('utf-8'))
Handler = MyHandler
try:
with socketserver.TCPServer(("", PORT), Handler) as httpd:
print(f"Starting http://0.0.0.0:{PORT}")
httpd.serve_forever()
except KeyboardInterrupt:
print("Stopping by Ctrl+C")
httpd.server_close() # to resolve problem `OSError: [Errno 98] Address already in use`
And testing code
import requests
data = {'search': 'hello world?'}
r = requests.post('http://localhost:8000/api', json=data)
print('status:', r.status_code)
print('json:', r.json())
This example doesn't check if you run /api or /api/function or /api/function/arguments because it would need much more code.
So pure python API without framework can need a lot of work and it can be waste of time.
The same code with Flask. It is much shorter and it already checks if you send to /api.
from flask import Flask, request, jsonify
app = Flask(__name__)
#app.route('/api', methods=["GET", "POST"])
def api():
input_data = request.json
print(input_data)
output_data = {'status': 'OK', 'result': 'HELLO WORLD!'}
return jsonify(output_data)
if __name__ == '__main__':
#app.debug = True
app.run(host='0.0.0.0', port=8000)
BTW:
If you want to test post data then you can use portal http://httpbin.org and send POST request to http://httpbin.org/post and it will send back all data and headers.
It can be used also for other requests and data.
This portal was created with Flask and there is even link to source code so you can install it on own computer.
It seems httpbin is part of Postman repo on GitHub.
I'm trying out some PHP on my pc and made a little python server to host the files, one problem:
It can't do POST, I always get the error 501. I've heard that you can implement POST in these servers, but I didn't find how to do this, can someone help?
Here's my current server:
import http.server
import socketserver
PORT = 8080
Handler = http.server.SimpleHTTPRequestHandler
with socketserver.TCPServer(("", PORT), Handler) as httpd:
print("serving at port", PORT)
httpd.serve_forever()
This is the script I personally use for when I need this kind of functionality:
#!/usr/env python3
import http.server
import os
import logging
try:
import http.server as server
except ImportError:
# Handle Python 2.x
import SimpleHTTPServer as server
class HTTPRequestHandler(server.SimpleHTTPRequestHandler):
"""
SimpleHTTPServer with added bonus of:
- handle PUT requests
- log headers in GET request
"""
def do_GET(self):
server.SimpleHTTPRequestHandler.do_GET(self)
logging.warning(self.headers)
def do_PUT(self):
"""Save a file following a HTTP PUT request"""
filename = os.path.basename(self.path)
# Don't overwrite files
if os.path.exists(filename):
self.send_response(409, 'Conflict')
self.end_headers()
reply_body = '"%s" already exists\n' % filename
self.wfile.write(reply_body.encode('utf-8'))
return
file_length = int(self.headers['Content-Length'])
with open(filename, 'wb') as output_file:
output_file.write(self.rfile.read(file_length))
self.send_response(201, 'Created')
self.end_headers()
reply_body = 'Saved "%s"\n' % filename
self.wfile.write(reply_body.encode('utf-8'))
if __name__ == '__main__':
server.test(HandlerClass=HTTPRequestHandler)
But perhaps a more fitting, and simpler script would be the following, as found on Flavio Copes' blog:
from http.server import BaseHTTPRequestHandler, HTTPServer
class handler(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header('Content-type','text/html')
self.end_headers()
message = "Hello, World! Here is a GET response"
self.wfile.write(bytes(message, "utf8"))
def do_POST(self):
self.send_response(200)
self.send_header('Content-type','text/html')
self.end_headers()
message = "Hello, World! Here is a POST response"
self.wfile.write(bytes(message, "utf8"))
with HTTPServer(('', 8000), handler) as server:
server.serve_forever()
I'm fairly new to coding in python. I created a local web server that says "Hello World" and displays the current time.
Is there a way to create a path, without creating a file, on the server program so that when I type in "/time" after 127.0.0.1 in the browser bar, it will display the current time? Likewise if I type "/date" it will give me the current date.
This is what I have so far:
from BaseHTTPServer import BaseHTTPRequestHandler,HTTPServer
import datetime
port = 80
class myHandler(BaseHTTPRequestHandler):
#Handler for the GET requests
def do_GET(self):
self.send_response(200)
self.send_header('Content-type','text/html')
self.end_headers()
# Send the html message
self.wfile.write("<b> Hello World !</b>"
+ "<br><br>Current time and date: " + str(datetime.datetime.now()))
server = HTTPServer(('', port), myHandler)
print 'Started httpserver on port ', port
#Wait forever for incoming http requests
server.serve_forever()
Very simple URL handler:
def do_GET(self):
if self.path == '/time':
do_time(self)
elif self.path == '/date':
do_date(self)
def do_time(self):
self.send_response(200)
self.send_header('Content-type','text/html')
self.end_headers()
# Send the html message
self.wfile.write("<b> Hello World !</b>"
+ "<br><br>Current time: " + str(datetime.datetime.now()))