We Keep Coding

Why second method of these classes are performing their functions as long as they are not called?

here i have server and client that can connect together and send message.but second method*(def run(self):)* of these classes in both server and client did not called why they are send and receive message?(Why these methods working without calling them?)
code of server that uses socket and Thread:
from threading import Thread
import socket
class SendingThread(Thread):
def __init__(self, mySocket):
Thread.__init__(self)
self.mySocket = mySocket
def run(self):
# write code to send data continuously
while True:
data = input()
self.mySocket.send(bytes(data, 'utf-8'))
class ReceivingThread(Thread):
def __init__(self, mySocket):
Thread.__init__(self)
self.mySocket = mySocket
def run(self):
# write code to receive data continuously
while True:
msg = self.mySocket.recv(1024)
print(msg.decode('utf-8'))
# create a socket object
s = socket.socket(
socket.AF_INET, # internet address family => IP v4
socket.SOCK_STREAM # TCP
)
# bind socket with a port number
s.bind(('127.0.0.1', 2010))
# keep System_1 in listening mode
s.listen()
# accept the incoming connection request
mySocket, address = s.accept()
# create a thread to send data
sendThread = SendingThread(mySocket)
# create an another to receive data
receiveThread = ReceivingThread(mySocket)
# start both threads
sendThread.start()
receiveThread.start()
code of client that uses socket and Thread:
from threading import Thread
import socket
class MySendingThread(Thread):
def __init__(self, mySocket):
Thread.__init__(self)
self.mySocket = mySocket
def run(self):
# write code to send data to System_1
while True:
data = input()
self.mySocket.send(bytes(data, 'utf-8'))
class MyReceivingThread(Thread):
def __init__(self, mySocket):
Thread.__init__(self)
self.mySocket = mySocket
def run(self):
# write code to receive data from System_1
while True:
msg = self.mySocket.recv(1024)
print(msg.decode('utf-8'))
# create a socket object
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# send a connection request
s.connect(('127.0.0.1', 2010))
# create a thread to send data => System_1
mySendThread = MySendingThread(s)
# create a thread to receive data from System_1
myReceiveThread = MyReceivingThread(s)
# start both threads
mySendThread.start()
myReceiveThread.start()

SendingThread and ReceivingThread in server and MySendingThread and MyReceivingThread in client inherit from threading.Thread
Have a look at the docs:
The Thread class represents an activity that is run in a separate
thread of control. There are two ways to specify the activity: by
passing a callable object to the constructor, or by overriding the
run() method in a subclass. No other methods (except for the
constructor) should be overridden in a subclass. In other words, only
override the __init__() and run() methods of this class.
Once a thread object is created, its activity must be started by calling the thread’s start() method. This invokes the run() method in
a separate thread of control.
When you call start() it invoke the respective run() method you override in the child class.

global name 'WebServer' is not defined

Please find below code as it is throwing the error - global name 'webserver' is not defined.
import SimpleHTTPServer
import SocketServer
import os
from threading import Thread
import threading
class WebServer(Thread):
def __init__(self, stream_path):
"""
Class to create a Web server and add given folder to the web server which is passed as an argument.
In this case it creates the web server to the incoming streams pushed by VLC to temp folder
"""
Thread.__init__(self)
self.stream_path = stream_path
def run(self):
global WebServer
"""
This method is built in Thread object method invoked by start()
and the code which is under run() will be executed.
"""
os.chdir(self.stream_path)
PORT = 8000
Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
httpd = SocketServer.TCPServer(("", PORT), Handler)
print "serving at port", PORT
print "function thread", threading.currentThread()
httpd.serve_forever()
print "test1"
def create_web_server(self,stream_path):
global WebServer
"""
This method is to create the web server to a given path
"""
obj1 = WebServer(self,stream_path)
obj1.start()
print "server created"
def stop_web_server(self):
cmd='tskill python /A'
os.system(cmd)
if __name__ == '__main__':
create_web_server("","C:\\\\QED")

You don't need the two global WebServer lines in your code. The global keyword is used to grant write access to a global variable, and you don't have a global variable named WebServer.
Your code should look like the following to resolve your error, though in its state, it will still throw errors because your code has multiple problems, including calling the create_web_server method by itself, as it's a method that needs to be run on a WebServer object.
I've included a working prototype example of your code at the bottom of my post.
import SimpleHTTPServer
import SocketServer
import os
from threading import Thread
import threading
class WebServer(Thread):
def __init__(self, stream_path):
"""
Class to create a Web server and add given folder to the web server which is passed as an argument.
In this case it creates the web server to the incoming streams pushed by VLC to temp folder
"""
Thread.__init__(self)
self.stream_path = stream_path
def run(self):
"""
This method is built in Thread object method invoked by start()
and the code which is under run() will be executed.
"""
os.chdir(self.stream_path)
PORT = 8000
Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
httpd = SocketServer.TCPServer(("", PORT), Handler)
print "serving at port", PORT
print "function thread", threading.currentThread()
httpd.serve_forever()
print "test1"
def create_web_server(self,stream_path):
"""
This method is to create the web server to a given path
"""
obj1 = WebServer(self,stream_path)
obj1.start()
print "server created"
def stop_web_server(self):
cmd='tskill python /A'
os.system(cmd)
if __name__ == '__main__':
create_web_server("","C:\\\\QED")
Here is a simplified version of your code that should do the same thing (serve a directory with a basic SimpleHTTPRequestHandler).
It runs the thread in daemon mode so that it can be interrupted with Ctrl-C. I removed several methods as they didn't seem to serve a purpose in your original code, and to show you a basic framework of what your WebServer class would probably look like.
import threading
import os
import time
import SimpleHTTPServer
import SocketServer
class WebServer(threading.Thread):
# Let's specify additional constructor arguments for host/port
# so you don't have to hardcode those values.
# We can specify default values for them as well.
def __init__(self, stream_path, host='localhost', port=8000):
super(WebServer, self).__init__()
self.stream_path = stream_path
self.host = host
self.port = port
# This is the method that will be called when you call
# .start() on a WebServer object.
def run(self):
os.chdir(self.stream_path)
handler = SimpleHTTPServer.SimpleHTTPRequestHandler
httpd = SocketServer.TCPServer((self.host, self.port), handler)
httpd.serve_forever()
if __name__ == '__main__':
# Create a WebServer thread object, set it to daemonize
# and start it
ws = WebServer(r'C:\QED')
ws.daemon = True
ws.start()
# Since the main thread doesn't do anything after daemonizing
# the WebServer, it would exit immediately.
# This forever loop just ensures that the main thread runs
while True: time.sleep(1)
And to take it a step further, you don't need to create a Thread subclass. It's allowed because it feels more 'natural' to developers who are familiar with Java, where subclassing Thread or implementing Runnable is standard practice.
You can just do something like:
import threading
import os
import time
import SimpleHTTPServer
import SocketServer
def run_http_server(serve_dir, host='localhost', port=8000):
os.chdir(serve_dir)
handler = SimpleHTTPServer.SimpleHTTPRequestHandler
httpd = SocketServer.TCPServer((host, port), handler)
httpd.serve_forever()
if __name__ == '__main__':
thread = threading.Thread(target=run_http_server, args=(r'C:\QED',))
thread.daemon = True
thread.start()
while True: time.sleep(1)

Start new process on __init__ (for TCP listener - server)

I'm trying to run new process for each new instance of class Server. Each Server instance should listen on specific port. I have this (simplified) code so far: source
class Server(object):
def handle(connection, address):
print("OK...connected...")
try:
while True:
data = connection.recv(1024)
if data == "":
break
connection.sendall(data)
except Exception as e:
print(e)
finally:
connection.close()
def __init__(self, port, ip):
self.port = port
self.ip = ip
self.socket = socket(AF_INET, SOCK_STREAM)
self.socket.bind((self.ip, self.port))
self.socket.listen(1)
while True:
print("Listening...")
conn, address = self.socket.accept()
process = multiprocessing.Process(target=Pmu.handle, args=(conn, address))
process.daemon = True
process.start()
s1 = Server(9001,"127.0.0.1")
s2 = Server(9002,"127.0.0.1")
But when I run this script only first server s1 is running and waiting for connection. How to make both servers listening at the same time?

Your current server is effectively a SocketServer.ForkingTCPServer that enters a tight loop in its __init__, foerever accepting new connections, and creating a new child process for each incoming connection.
The problem is that __init__ never returns, so only one server gets instantiated, one socket gets bound, and only one port will accept new requests.
A common way of solving this type of problem is to move the accept loop into a worker thread. This code would look something like this:
import multiprocessing
import threading
import socket
class Server(object):
def handle(self, connection, address):
print("OK...connected...")
try:
while True:
data = connection.recv(1024)
if data == "":
break
connection.sendall(data)
except Exception as e:
print(e)
finally:
connection.close()
print("Connection closed")
def accept_forever(self):
while True:
# Accept a connection on the bound socket and fork a child process
# to handle it.
print("Waiting for connection...")
conn, address = self.socket.accept()
process = multiprocessing.Process(
target=self.handle, args=(conn, address))
process.daemon = True
process.start()
# Close the connection fd in the parent, since the child process
# has its own reference.
conn.close()
def __init__(self, port, ip):
self.port = port
self.ip = ip
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.bind((self.ip, self.port))
self.socket.listen(1)
# Spin up an acceptor thread
self.worker = threading.Thread(target=self.accept_forever)
self.worker.daemon = True
self.worker.start()
def join(self):
# threading.Thread.join() is not interruptible, so tight loop
# in a sleep-based join
while self.worker.is_alive():
self.worker.join(0.5)
# Create two servers that run in the background
s1 = Server(9001,"127.0.0.1")
s2 = Server(9002,"127.0.0.1")
# Wait for servers to shutdown
s1.join()
s2.join()
Note one other change I snuck in here:
# Wait for servers to shutdown
s1.join()
s2.join()
Using the saved reference to the Server's accept worker, we call .join() from the main thread to force things to block while the servers are running. Without this, your main program will exit nearly immediately, due to the workers' .daemon attribute being set.
It's also worth noting that this approach will have some quirks:
Since the handler functions are running in separate processes, you will need to share data structures between them carefully using Queue, Value, Pipe, and other multiprocessing constructs if they depend on each other.
There is no rate limiting of active concurrent connections; creating a new process for every single request can be expensive, and can create a vector for your service being easily DoSed.

Python socket with worker threads

I'm playing with Python to understand basics of client-server and threading programming.
This is my main.py application:
from threading import Thread
from Server import Server
class Worker(object):
def __init__(self, clients):
#constructor
self.clients = clients
def start(self):
while True:
print("it work as expected!")
conn, addr = self.clients.get()
print( conn.recv(1024) )
#create a instance of the server
server = Server()
#put on a new thread
def serverThreadCallback(server):
server.start()
def createWorkerCallback():
worker = Worker(server.getClients())
worker.start()
#put the server on a new thread
serverThread = Thread( target=serverThreadCallback, args=(server, ) )
serverThread.daemon = True
serverThread.start()
'''
workerThread = Thread(target=createWorkerCallback)
workerThread.daemon = True
workerThread.start()
'''
and this is my server.py
import socket
import queue
class Server(object):
'''
classdocs
'''
def __init__(self):
'''
Constructor
'''
self.clients = queue.Queue()
def create(self):
print("Creating server...")
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR , 1)
self.socket = server
def listen(self):
print("Starting server...")
self.socket.bind( ("0.0.0.0", 9002) )
self.socket.listen(10)
print("Server started. Listening on localhost port 9002")
while(True):
#conn, addr = self.server.accept()
self.clients.put(self.socket.accept())
def start(self):
self.create()
self.listen()
def getClients(self):
return self.clients
When I try to start the server in that thread, the listening method
doesn't call. Something goes stuck in create() method from the server.
What I've did wrong?

First, there are a couple of syntax errors on the coded you posted:
main.py:
1) Indentation starting from your class declaration is wrong
2) You should call from server import Server (note the module name is server - in lower case)
server.py
3) You should call import Queue (and not import queue)
4) Therefore the call to self.clients = queue.Queue() should become self.clients = Queue.Queue() (the module name is Queue with upper case Q)
Other than that, you have to add a serverThread.join() after serverThread.start(), otherwise the main thread finishes and the program terminates before your server thread has had the chance to start.
Finally, you might want to change the call to accept() inside the thread, such that it times out from time to time to handle Ctrl+C interrupts properly (otherwise you can only finish your program by killing it)

serverThread.daemon = True
Daemon threads are abruptly stopped when Python program exits.
Maybe the listen() method hasn't been called yet.

How to exit a multithreaded program?

I was just messing around with threading in python, wrote this basic IM thingy [code at bottom]
I noticed that when I kill the program with C-c it doesn't exit, it just hangs forever.
I'm just guessing it's waiting for each thread to finish what they are doing, but since it's an endless loop that will never happen.
So I guess I need to kill each thread manually, or end the loop when the killsignal comes in.
How would I do that?
#!/usr/bin/env python
import threading
import socket
class Listen(threading.Thread):
def run(self):
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
conn.bind(('', 2727))
conn.listen(1)
while True:
channel, details = conn.accept()
print str(details)+": "+channel.recv(250)
channel.send("got it")
channel.close()
class Shout(threading.Thread):
def run(self):
while True:
try:
address = raw_input("who u talking to? ")
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
conn.connect((address, 2727))
break
except:
print "can't connect to "+ str(address)
while True:
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
conn.connect((address, 2727))
conn.send(raw_input())
conn.close()
listen = Listen().start()
shout = Shout().start()

I see several causes of the misbehavior in your code.
Ctrl+C causes a "KeyboardInterrupt" exception in the main thread. So you should handle it there.
Your socket is in blocking mode. This causes several socket functions to block the calling thread until the function returns. During this state the thread cannot react to any termination event.
As you already said: your endless loop in the thread's run() function is ... really endless. So the thread execution is never ending (at least not without an unexpected exception). You should use some kind of synchronization object, like an threading.Event object, to be able to tell a thread externally that it should terminate itself.
I would discourage the use of raw_input() out of the main thread. Imagine what happens when you have more than one Shout thread.
Why are you always closing and reconnecting the socket when a message has been transmitted in your Shout class? Network connections should be re-established only in special cases because of the setup costs.
Without a frame protocol for the communication you can never expect to have received all data that was sent by the other host when the recv() function returns.
The start() function of the thread object does not return a value or object. So saving the returned value (=None) doesn't make much sense.
You can never expect the send() function to transmit all passed data. Therefore you must check the result of the function and appropriately handle the situation when not all bytes were really transmitted.
To learn threading there are surely better problems to solve than network communication, since that topic is in itself really complex.
Beside all these things, here is my try for a solution. Still there is much that can be improved. You should consider the answer from Mark Tolonen too, since the SocketServer class is surely provided to ease several things in handling this kind of stuff. But you should keep on studying the basics too.
#!/usr/bin/env python
import threading
import socket
import time
import errno
class StoppableThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.stop_event = threading.Event()
def stop(self):
if self.isAlive() == True:
# set event to signal thread to terminate
self.stop_event.set()
# block calling thread until thread really has terminated
self.join()
class Accept(StoppableThread):
def __init__(self, port):
StoppableThread.__init__(self)
self.port = port
self.threads = []
def run(self):
# handle connection acception
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
conn.bind(('', self.port ))
conn.listen(5)
# set socket timeout to ~10ms
conn.settimeout(0.01)
while self.stop_event.is_set() == False:
try:
csock, caddr = conn.accept()
# spawn a new thread to handle the client connection
listen_thread = Listen(csock, caddr)
self.threads.append(listen_thread)
listen_thread.start()
except socket.timeout:
# socket operation timeout
# clear all terminated threads from thread list
for thread in self.threads:
if thread.isAlive() == False:
self.threads.remove(thread)
self.stop_threads()
def stop_threads(self):
# stop all running threads
for listen_thread in self.threads:
if listen_thread.isAlive() == True:
listen_thread.stop()
self.threads = []
class Listen(StoppableThread):
def __init__(self, csock, caddr):
StoppableThread.__init__(self)
self.csock = csock
self.caddr = caddr
self.csock.setblocking(False)
def run(self):
while self.stop_event.is_set() == False:
try:
recv_data = self.csock.recv(250)
if len(recv_data) > 0:
print str(self.caddr)+": " + recv_data
self.csock.send("got it")
else:
# connection was closed by foreign host
self.stop_event.set()
except socket.error as (sock_errno, sock_errstr):
if (sock_errno == errno.EWOULDBLOCK):
# socket would block - sleep sometime
time.sleep(0.1)
else:
# unexpected / unhandled error - terminate thread
self.stop_event.set()
channel.close()
class Shout(StoppableThread):
def __init__(self, sport):
StoppableThread.__init__(self)
self.sport = sport
def run(self):
while self.stop_event.is_set() == False:
try:
address = raw_input("who u talking to? ")
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
conn.connect((address, self.sport))
break
except socket.error:
# handle connection problems
print "can't connect to "+ str(address)
except:
# exit thread in case of an unexpected error
self.stop_event.set()
while self.stop_event.is_set() == False:
try:
# chat loop: send messages to remote host
print "what to send? :",
msg = raw_input()
# beware: send() function may block indefinitly here and it might not send all bytes as expected !!
conn.send(msg)
except:
# exit thread in case of an unexpected error
self.stop_event.set()
# close socket before thread terminates
conn.close()
def main():
do_exit = False
server_port = 2727
# start server socket thread
accept = Accept(server_port)
accept.start()
# start transmitting client socket thread
shout = Shout(server_port)
shout.start()
while do_exit == False:
try:
# sleep some time
time.sleep(0.1)
except KeyboardInterrupt:
# Ctrl+C was hit - exit program
do_exit = True
# stop all running threads
shout.stop()
accept.stop()
# exit main program after all threads were terminated gracefully
if __name__ == "__main__":
main()

Look at the Python library source for SocketServer.py, in particular the implementation of server_forever() to see how a server implements a quit. It uses select() to poll the server socket for new connections and tests a quit flag. Here's a hack on your source to use SocketServer, and I added a quit flag to Shout(). It will run the Shout and Listen threads for 5 seconds and then stop them.
import socket
import SocketServer
import threading
import time
class Handler(SocketServer.StreamRequestHandler):
def handle(self):
print str(self.client_address) + ": " + self.request.recv(250)
self.request.send("got it\n")
class Listen(threading.Thread):
def run(self):
self.server = SocketServer.TCPServer(('',2727),Handler)
self.server.serve_forever()
def stop(self):
self.server.shutdown()
class Shout(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.quit = False
def run(self):
while not self.quit:
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
conn.connect(('localhost', 2727))
conn.send('sending\n')
print conn.recv(100)
conn.close()
def stop(self):
self.quit = True
listen = Listen()
listen.start()
shout = Shout()
shout.start()
time.sleep(5)
shout.stop()
listen.stop()