I want to run an executable on a TCP server and take its input from socket connections interactively and send the output back to the client until the executable is terminated. I was trying it with piping through Popen class of subprocess but its not helping with interaction with executable ( its just take input only one time but i want the input to be taken all the time until program exits ).
Suppose I send "1" input to the server then server must send the stdout corresponding to "1" input to client and then ask for next input and do it till the executable exits in continuation .
Just provide the socket as the standard input, output, and error of the subprocess. E.g.:
import socket
import subprocess
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
listener.bind(('0.0.0.0', 0))
listener.listen(5)
print(listener.getsockname())
try:
while True:
client, addr = listener.accept()
subprocess.Popen(['cat'], stdin=client, stdout=client, stderr=client)
client.close()
except KeyboardInterrupt:
pass
finally:
listener.close()
This probably requires a POSIX-compliant operating system.
Here is an implementation using circuits:
server.py:
#!/usr/bin/env python
from uuid import uuid4 as uuid
from subprocess import Popen, PIPE
from circuits import handler, Component, Debugger, Event
from circuits.io import File
from circuits.net.sockets import TCPServer
from circuits.net.events import close, write
class kill(Event):
"""kill Event"""
class Command(Component):
channel = "cmd"
def __init__(self, sock, command, channel=channel):
super(Command, self).__init__(channel=channel)
self._sock = sock
self._command = command
self._buffer = None
self._p = Popen(command, shell=True, stdin=PIPE, stdout=PIPE)
self._stdin = File(
self._p.stdin, channel="{0:s}.stdin".format(self.channel)
).register(self)
self._stdout = File(
self._p.stdout, channel="{0:s}.stdout".format(self.channel)
).register(self)
self.addHandler(
handler("eof", channel=self._stdout.channel)(self._on_stdout_eof)
)
self.addHandler(
handler("read", channel=self._stdout.channel)(self._on_stdout_read)
)
def write(self, data):
self.fire(write(data), self._stdin.channel)
def kill(self):
self._p.terminate()
self.unregister()
#staticmethod
def _on_stdout_eof(self):
self.fire(kill(), self.channel)
self.fire(close(self._sock), self.parent.channel)
#staticmethod
def _on_stdout_read(self, data):
self.fire(write(self._sock, data), "server")
class Server(Component):
channel = "server"
def init(self, bind, cmd):
self.cmd = cmd
self.clients = {}
TCPServer(bind).register(self)
def connect(self, sock, host, port):
command = Command(sock, self.cmd, channel=uuid()).register(self)
self.clients[sock] = command
def disconnect(self, sock):
command = self.clients[sock]
self.fire(kill(), command.channel)
del self.clients[sock]
def read(self, sock, data):
command = self.clients[sock]
self.fire(write(data), command.channel)
server = Server(("0.0.0.0", 8000), "python app.py")
Debugger().register(server)
server.run()
app.py:
#!/usr/bin/env python
from __future__ import print_function
import sys
def function1():
print("I am function 1!")
def function2():
print("I am function 2!")
def function3():
raise SystemExit(0)
MENU_OPTIONS = (
(1, "Function 1"),
(2, "Function 2"),
(3, "Function 3")
)
FUNCTIONS = {
1: function1,
2: function2,
3: function3
}
def main():
while True:
try:
print("Menu:")
for option, description in MENU_OPTIONS:
print("{0:d}) {1:s}".format(option, description))
print()
sys.stdout.flush()
choice = raw_input("> ")
try:
FUNCTIONS[int(choice)]()
except ValueError:
print("Invalid Input")
except (KeyboardInterrupt, EOFError):
raise SystemExit(0)
if __name__ == "__main__":
main()
For an example session (this example has been thoroughly tested):
Server Session: http://codepad.org/F7qZKdKa
Client Session: http://codepad.org/ss33wgAI
Have fun! :)
Note: I'm actually the developer/author of 1[circuits]. I thought this would be a nice example to write up.
Related
I have separately developed programs for a pyqt5 app and a client for communicate with a server.
GUI:
from PyQt5 import QtCore, QtWidgets, QtGui
from screen_stack import Ui_MainWindow_screenstack
from screen1 import Ui_Form_screen1
from screen2 import Ui_Form_screen2
from screen3 import Ui_Form_screen3
from client_cli_oop_with_exception_printing import mainProg
import socket
import threading #maybe this won't useful
import sys
import os
class MainApp(QtWidgets.QMainWindow):
def __init__(self):
super(MainApp, self).__init__() #if init was provided with paren=None, then inside this init parent argument can be passed
self.screenstack = Ui_MainWindow_screenstack()
self.screenstack.setupUi(self)
self.screen1 = scr1()
self.screen2 = scr2()
self.screen3 = scr3()
self.screenstack.stackedWidget.addWidget(self.screen1)
self.screenstack.stackedWidget.addWidget(self.screen2)
self.screenstack.stackedWidget.addWidget(self.screen3)
self.screenstack.stackedWidget.setCurrentWidget(self.screen1)
# self.screen1.s1.connect_tcp.clicked.connect(
# lambda: self.screenstack.stackedWidget.setCurrentWidget(self.screen2)
# )
self.screen1.s1.connect_tcp.clicked.connect(self.init_connect_server)
self.screen2.s2.register_name.clicked.connect(
lambda: self.screenstack.stackedWidget.setCurrentWidget(self.screen3)
)
self.sock = None
self.errScr1 = QtWidgets.QMessageBox()
self.errScr1.setWindowTitle('Error')
def init_connect_server(self):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.sock.connect((self.screen1.s1.tcp_link.text(), int(self.screen1.s1.tcp_port.text())))
self.screenstack.stackedWidget.setCurrentWidget(self.screen2)
except Exception as e:
self.errScr1.setText(str(e))
self.errScr1.exec_()
def start_client_threads(self):
#all the communication threads should start here..
pass
class scr1(QtWidgets.QWidget):
def __init__(self):
super(scr1,self).__init__()
self.s1=Ui_Form_screen1()
self.s1.setupUi(self)
class scr2(QtWidgets.QWidget):
def __init__(self):
super(scr2,self).__init__()
self.s2=Ui_Form_screen2()
self.s2.setupUi(self)
class scr3(QtWidgets.QWidget):
def __init__(self):
super(scr3,self).__init__()
self.s3=Ui_Form_screen3()
self.s3.setupUi(self)
if __name__ == '__main__':
app = QtWidgets.QApplication(sys.argv)
sw = MainApp()
sw.show()
sys.exit(app.exec_())
Client:
import socket
import threading
import time
import os
class mainProg:
def __init__(self, host, port):
self.is_connection_lost = threading.Event()
self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.client.connect((host, port))
except Exception as e:
print(e)
print('Server is not valid...quitting...')
time.sleep(5)
os._exit(0)
self.start_threads()
def recieve(self):
while True:
try:
message = self.client.recv(1024).decode('utf-8')
if message == 'Nickname':
self.client.send(nickname.encode('utf-8'))
elif message == '': #when server gone, we will recieve a lot of empty strings continuously.. we can make the clients shut the python if server gone, but client waiting server to init again will be cooler
self.is_connection_lost.set()
self.client.close()
print('Connection Lost: waiting for the connection,')
while True:
print('waiting...')
time.sleep(3)
if not self.is_connection_lost.is_set():
print('Connection restored!!!')
break
else:
print(message)
#print(type(message))
except Exception as e:
print(e)
print('An error occured!, Not yet connected!')
#client.close()
#is_connection_lost = True
break
def send_messages(self):
while True:
try:
message = f"{nickname}: {input('')}"
self.client.send(message.encode('utf-8'))
except Exception:
pass
def reconnect_server(self):
time.sleep(3)
while True:
if self.is_connection_lost.is_set():
print('Trying to connect!!!!')
time.sleep(3)
self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.client.connect((srv,port))
except Exception as e:
print(f'Reconnect failed due to {e}')
continue
if self.client.recv(1024).decode('utf-8') == '':
self.client.close()
pass
else:
self.client.send(nickname.encode('utf-8'))
self.is_connection_lost.clear() #no problem here, it connects to server when server is up again...
def start_threads(self):
t1 = threading.Thread(target=self.recieve)
t2 = threading.Thread(target=self.send_messages)
t3 = threading.Thread(target=self.reconnect_server)
t1.start()
t2.start()
t3.start()
if __name__ == '__main__':
nickname = input('Choose a nickname: ')
srv = '0.tcp.in.ngrok.io'
port = 14112
mainProg(srv, port)
I know that using qthreads I can rewrite the code for client in GUI and make a functional app. But is there a way to connect both classes, so that I can use self.labelMessage.setText(message) instead of print(message) in client. In another way I am asking is there a way to inherit from multiple classes (MainApp and mainProg) so that I can easily connect both codes?
I'm having a strange phenomena in Python with callback functions and handlers.
I use ZMQ to handle my communication and use a stream for the socket. I have the base class:
import multiprocessing
import zmq
from concurrent.futures import ThreadPoolExecutor
from zmq.eventloop import ioloop, zmqstream
from zmq.utils import jsonapi as json
# Types of messages
TYPE_A = 'type_a'
TYPE_B = 'type_b'
class ZmqProcess(multiprocessing.Process):
def __init__(self):
super(ZmqProcess, self).__init__()
self.context = None
self.loop = None
self.handle_stream = None
def setup(self):
self.context = zmq.Context()
self.loop = ioloop.IOLoop.instance()
def send(self, msg_type, msg, host, port):
sock = zmq.Context().socket(zmq.PAIR)
sock.connect('tcp://%s:%s' % (host, port))
sock.send_json([msg_type, msg])
def stream(self, sock_type, addr):
sock = self.context.socket(sock_type)
if isinstance(addr, str):
addr = addr.split(':')
host, port = addr if len(addr) == 2 else (addr[0], None)
if port:
sock.bind('tcp://%s:%s' % (host, port))
else:
port = sock.bind_to_random_port('tcp://%s' % host)
stream = zmqstream.ZMQStream(sock, self.loop)
return stream, int(port)
class MessageHandler(object):
def __init__(self, json_load=-1):
self._json_load = json_load
self.pool = ThreadPoolExecutor(max_workers=10)
def __call__(self, msg):
i = self._json_load
msg_type, data = json.loads(msg[i])
msg[i] = data
if msg_type.startswith('_'):
raise AttributeError('%s starts with an "_"' % msg_type)
getattr(self, msg_type)(*msg)
And I have a class that inherits from it:
import zmq
import zmq_base
class ZmqServerMeta(zmq_base.ZmqProcess):
def __init__(self, bind_addr, handlers):
super(ZmqServerMeta, self).__init__()
self.bind_addr = bind_addr
self.handlers = handlers
def setup(self):
super(ZmqServerMeta, self).setup()
self.handle_stream, _ = self.stream(zmq.PAIR, self.bind_addr)
self.handle_stream.on_recv(StreamHandler(self.handle_stream, self.stop,
self.handlers))
def run(self):
self.setup()
self.loop.start()
def stop(self):
self.loop.stop()
class StreamHandler(zmq_base.MessageHandler):
def __init__(self, handle_stream, stop, handlers):
super(StreamHandler, self).__init__()
self._handle_stream = handle_stream
self._stop = stop
self._handlers = handlers
def type_a(self, data):
if zmq_base.TYPE_A in self._handlers:
if self._handlers[zmq_base.TYPE_A]:
for handle in self._handlers[zmq_base.TYPE_A]:
self.pool.submit(handle, data)
else:
pass
else:
pass
def type_b(self, data):
if zmq_base.TYPE_B in self._handlers:
if self._handlers[zmq_base.TYPE_B]:
for handle in self._handlers[zmq_base.TYPE_B]:
self.pool.submit(handle, data)
else:
pass
else:
pass
def endit(self):
self._stop()
Additionally, I have a class that I want to use as storage. And here is where the trouble starts:
import threading
import zmq_server_meta as server
import zmq_base as base
class Storage:
def __init__(self):
self.list = []
self.list_lock = threading.RLock()
self.zmq_server = None
self.host = '127.0.0.1'
self.port = 5432
self.bind_addr = (self.host, self.port)
def setup(self):
handlers = {base.TYPE_A: [self. remove]}
self.zmq_server = server.ZmqServerMeta(handlers=handlers, bind_addr=self.bind_addr)
self.zmq_server.start()
def add(self, data):
with self.list_lock:
try:
self.list.append(data)
except:
print "Didn't work"
def remove(self, msg):
with self.list_lock:
try:
self.list.remove(msg)
except:
print "Didn't work"
The idea is that that class stores some global information that it receives.
It is all started in a file to test:
import sys
import time
import storage
import zmq_base as base
import zmq_server_meta as server
def printMsg(msg):
print msg
store = storage.Storage()
store.setup()
handlers = {base.TYPE_B: [printMsg]}
client = server.ZmqServerMeta(handlers=handlers, bind_addr=('127.0.0.1', 5431))
client.start()
message = "Test"
store.add(message)
client.send(base.TYPE_A, message, '127.0.0.1', 5432)
I simplified it to reduce clutter. Instead of just adding it, it is usually send and then a response comes back. The response, the client sending, should be processed by the correct callback, remove(), and it should remove something out of the list. The problem that occurs is, that the remove() function sees an empty list, although there should be an element in the list. If I check from the testing file, I can see the element after it was added, and if I call remove() from there, I see a non-empty list and can remove it. My question is, why does the callback sees an empty list and how can I make sure it does see the correct elements in the list?
Kind regards
Patrick
I believe the problem lays in the fact that the ZmqProcess class inherits from multiprocessing.Process. Multiprocessing does not allow to share objects among different processes, except by using a shared memory map using Value or Array ( as can be seen in the documentation: https://docs.python.org/3/library/multiprocessing.html#sharing-state-between-processes )
If you want to use your custom object, you can use a Server process / proxy object, which can be found in on the same page of the documentation.
So you can, for instance, define a manager in the init function of the Storage class like: self.manager = Manager() Afterwards you put self.list = self.manager.list(). This should do the trick.
I have client program written in python that talks to some server.
[Client]
import asyncore
import logging
import socket
import sys, threading, traceback
from cStringIO import StringIO
class Client(threading.Thread, asyncore.dispatcher):
def __init__(self, host, port):
self.logger = logging.getLogger()
threading.Thread.__init__(self)
self._thread_sockets = dict()
asyncore.dispatcher.__init__(self, map=self._thread_sockets)
# data members for the module
self.host = host
self.port = port
self.write_buffer = ""
self.is_connected = False
self.read_buffer = StringIO()
# Ok now to run the thread !!
self.start()
def run(self) :
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
address = (self.host, self.port)
self.logger.debug('connecting to %s', address)
# wait until server is up
while not self.is_connected :
try :
self.connect(address)
except Exception as ex :
pass #do nothing, proceed forward !!
asyncore.loop(map=self._thread_sockets)
def handle_connect(self):
self.is_connected = True
self.logger.debug('handle_connect()')
def handle_close(self):
self.logger.debug('handle_close()')
self.close()
def handle_error(self):
traceback.print_exc(sys.stderr)
self.close()
def writable(self):
self.logger.debug('writable() : len is %d bytes', len(self.write_buffer))
is_writable = (len(self.write_buffer) > 0)
if is_writable:
self.logger.debug('writable() -> %s', is_writable)
return is_writable
def readable(self):
self.logger.debug('readable() -> True')
return True
def handle_write(self):
sent = self.send(self.write_buffer)
self.logger.debug('data len written to socket -> %s', sent)
self.logger.debug('handle_write() -> "%s"', self.write_buffer[:sent])
#self.write_buffer = self.write_buffer[sent:]
def handle_read(self):
data = self.recv(8192)
self.logger.debug('handle_read() -> %d bytes', len(data))
self.read_buffer.write(data)
self.logger.debug('data received from socket -> %s', self.read_buffer.getvalue())
self.read_buffer.truncate(0)
def send(self, data) :
self.write_buffer = data
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(name)s: %(message)s',
)
try :
client = Client("127.0.0.1", 8182)
client.send('sending data from client')
except Exception as ex :
logging.exception(ex)
sys.exit(1)
I am able to receive data from server correctly but send call to the server always fails. As from the log the send always return 'None'.
Am i missing anything ?
You override the send method of asyncore.dispatcher with code that does not send any data and returns no value:
def send(self, data) :
self.write_buffer = data
At the least, you need to change your code to look similar to this:
def send_data(self, data):
self.write_buffer = data
and this:
client.send_data('sending data from client')
The asyncore.dispatcher class already has a send method which is a wrapper around the socket.send method. From asyncore.py:
def send(self, data):
try:
result = self.socket.send(data)
return result
except socket.error, why:
if why.args[0] == EWOULDBLOCK:
return 0
elif why.args[0] in _DISCONNECTED:
self.handle_close()
return 0
else:
raise
Because you override this method, your send method gets called in your handle_write method, and no data is sent to the server.
i am trying to receive about 1000 connections to my server but it cannot receive more than 512. What can i do to increase the amount of open connections? I am running windows 8.1
Not: I am very new to this stuff so, thanks for help
Here is my code;
import asyncore
import socket
import uuid
import time
import threading
class statistics(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
while True:
entry = raw_input("")
zaman = int(time.time())
cmd = receivedCmd
print "calculating.."
time.sleep(1)
if entry == 'istatistik':
print str(receivedCmd-cmd) + " command/second"
print "total received commands: " + str(receivedCmd)
entry = ""
class tcpClient:
def __init__(self):
self.clientid = uuid.uuid1(int(time.time()))
self.buffer = ""
self.buffer_size = 0
self.conn_time = time.time()
self.overflow = 0
#print str(self.clientid) + " assingned"
def recv_msg(self, msg):
global receivedCmd
self.buffer = msg
self.buffer_size = len(self.buffer)
receivedCmd = receivedCmd + 1
if self.buffer_size >= 1024:
self.overflow = 1
def __del__(self):
print str(self.clientid) + " has left."
class TCPHandler(asyncore.dispatcher_with_send):
global clist
def handle_read(self):
data = self.recv(1024)
if data:
if clist[self].overflow:
self.send("overflow")
self.handle_close()
else:
self.send(data)
clist[self].recv_msg(data)
def handle_close(self):
del clist[self]
self.close()
def handle_error(self):
del clist[self]
self.close()
class TCPServer(asyncore.dispatcher):
global clist
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
def handle_accept(self):
self.clist = clist
pair = self.accept()
if pair is None:
pass
else:
sock, addr = pair
#print 'Connection : %s' % repr(addr)
clist[TCPHandler(sock)] = tcpClient()
if __name__ == '__main__':
clist = {}
receivedCmd = 0
server = TCPServer('', 5000)
server2 = TCPServer('',5001)
StaticsThread = statistics()
StaticsThread.start()
asyncore.loop()
Note: I still cannot receive more than 512 connections with the Twisted Framework, i don't know what to do. There have to be thousands of connected clients. Please help.
The asyncore module relies in the select OS function, which only supports a limited number of file descriptors.
As an alternative use a multi-threading server (I won't recommend this) or, better, the Twisted framework which is event-driven (highly recommended!).
Hope this helps!
Since Twisted's default reactor under Windows is also select-based then you should consider using the IOCP reactor instead.
from twisted.internet import iocpreactor
iocpreactor.install()
from twisted.internet import reactor
But also take into account that Twisted prefers Linux systems (where the default reactor is epoll-based) rather than Windows. Maybe switching to Linux is a better choice.
I can't find a way to make my threads persistent between the first and second call of my script.
So far, when I run python script_1.py A the script runs the if option == 'A' block and starts the thread. Then, the script exits and the thread is cleaned up. So, when I run python script_1.py B the isAlive attribute can't be used.
is there any way to keep persistence?
The code for script_1.py is:
from script_2 import imp
script_2 = imp()
if option == 'A':
script_2.start()
elif option == 'B':
script_2.stop()
and for script_2.py is:
from threading import Thread
class workerThread(Thread):
def __init__(self, _parent):
Thread.__init__(self)
self.parent = _parent
self.active = False
def run(self):
while(self.active == False):
print 'I am here'
print 'and now I am here'
class imp():
def __init__(self):
self.threadObj = None
def start(self):
self.threadObj = workerThread(self)
self.threadObj.start()
def stop(self):
if self.threadObj.isAlive() == True:
print 'it is alive'
A solution would be:
from threading import Thread
from socket import *
from time import sleep
class workerThread(Thread):
def __init__(self):
Thread.__init__(self)
self.sock = socket()
self.sock.bind(('', 9866))
self.sock.listen(4)
self.start()
def run(self):
while 1:
ns, na = self.sock.accept()
if ns.recv(8192) in (b'quit', 'quit'):
ns.close()
break
self.sock.close()
print('Worker died')
imp = workerThread()
And the first script:
if option == 'A':
from time import sleep
from script_2 import imp
while 1:
sleep(0.1)
elif option == 'B':
from socket import *
s = socket()
s.connect(('127.0.0.1', 9866))
s.send('quit') # b'quit' if you're using Python3
s.close()
It's not even close to elegant, but it's a 5min mockup of what you could do.
To make this closer to useable code, I'd go with:
self.sock = fromfd('/path/to/socket', AF_UNIX, SOCK_DGRAM)
and register it with an ePoll object within the worker thread.
import select
self.watch = select.epoll()
self.watch.register(self.sock.fileno(), select.EPOLLIN)
while 1:
for fd, event in self.watch.poll(0.1):
if fd == self.sock.fileno() and event == select.EPOLLIN:
ns, na = self.sock.accept()
# store socket and register it
elif event == select.EPOLLIN:
data = storedSockets[fd].recv(8192)
# and do work on it
Anyway, but you will need to keep the first instance of your execution running and create some form of communication method for the second instance you start up, i used sockets as an example which i think is rather good, especially in conjunction with unix sockets and epoll because the speed is fantastisc. You can also use memcache