I want to do a file dispatcher, the server is loaded with a list of files and then sends them one by one with each client request.
The idea is to distribute the processing of many files between 5 servers.
How could I call the ClientThread class with each client connection?
The script is only programmed to send the same file to each client request, what I want is to send a different file from a list of files in each client request.
Server.py
import socket
from threading import Thread
from socketserver import ThreadingMixIn
TCP_IP = '10.30.16.28'
TCP_PORT = 1006
BUFFER_SIZE = 1024
class ClientThread(Thread):
def __init__(self,ip,port,sock):
Thread.__init__(self)
self.ip = ip
self.port = port
self.sock = sock
print(" New thread started for "+ip+":"+str(port))
def run(self):
filename='log.VW.20170214a.log'
f = open(filename,'rb')
while True:
l = f.read(BUFFER_SIZE)
while (l):
self.sock.send(l)
l = f.read(BUFFER_SIZE)
if not l:
f.close()
self.sock.close()
break
tcpsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcpsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcpsock.bind((TCP_IP, TCP_PORT))
threads = []
with open("list.txt") as x: #File containing files list
lines=x.read().splitlines()
while True:
tcpsock.listen(5)
print("Waiting for incoming connections...")
(conn, (ip,port)) = tcpsock.accept()
print('Got connection from ', (ip,port))
newthread = ClientThread(ip,port,conn)
newthread.start()
threads.append(newthread)
for t in threads:
t.join()
Client.py
import socket
TCP_IP = '10.30.16.28'
TCP_PORT = 1006
BUFFER_SIZE = 1024
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((TCP_IP, TCP_PORT))
with open('received_file', 'wb') as f:
print('file opened')
while True:
data = s.recv(BUFFER_SIZE)
if not data:
f.close()
print('file close()')
break
f.write(data)
print('Successfully get the file')
s.close()
print('connection closed')
I don't see what this has to do with threading or ports or anything like that. Change this:
def __init__(self,ip,port,sock,fname):
Thread.__init__(self)
self.ip = ip
self.port = port
self.sock = sock
self.fname = fname
print(" New thread started for "+ip+":"+str(port))
def run(self):
f = open(self.fname,'rb')
And when you serve:
with open("list.txt") as x: #File containing files list
lines=iter(x.read().splitlines())
and finally:
newthread = ClientThread(ip,port,conn,lines.next().strip())
lines.next() will throw a StopIteration exception when done, so you have to handle that.
Related
This is the code on the AWS EC2 instance:
import socket
from threading import Thread
from socketserver import ThreadingMixIn
TCP_IP = 'localhost'
TCP_PORT = 9001
BUFFER_SIZE = 1024
class ClientThread(Thread):
def __init__(self,ip,port,sock):
Thread.__init__(self)
self.ip = ip
self.port = port
self.sock = sock
print (" New thread started for "+ip+":"+str(port))
def run(self):
filename='mytext.txt'
f = open(filename,'rb')
while True:
l = f.read(BUFFER_SIZE)
while (l):
self.sock.send(l)
#print('Sent ',repr(l))
l = f.read(BUFFER_SIZE)
if not l:
f.close()
self.sock.close()
break
tcpsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcpsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcpsock.bind((TCP_IP, TCP_PORT))
threads = []
while True:
tcpsock.listen(5)
print ("Waiting for incoming connections...")
(conn, (ip,port)) = tcpsock.accept()
print ('Got connection from ', (ip,port))
newthread = ClientThread(ip,port,conn)
newthread.start()
threads.append(newthread)
for t in threads:
t.join()
And this is the code on my local machine:
import socket
import time
#TCP_IP = 'localhost'
TCP_IP = 'ip-ec2-instance'
TCP_PORT = 60001
BUFFER_SIZE = 1024
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((TCP_IP, TCP_PORT))
clock_start = time.clock()
time_start = time.time()
with open('received_file', 'wb') as f:
print ('file opened')
while True:
#print('receiving data...')
data = s.recv(1024)
#print('data=%s', (data))
if not data:
f.close()
print ('file close()')
break
# write data to a file
f.write(data)
print('Successfully get the file')
s.close()
print('connection closed')
clock_end = time.clock()
time_end = time.time()
duration_clock = clock_end - clock_start
print ('clock: start = ',clock_start, ' end = ',clock_end)
print ('clock: duration_clock = ', duration_clock)
duration_time = time_end - time_start
print ('time: start = ',time_start, ' end = ',time_end)
print ('time: duration_time = ', duration_time)
Now, the code on the EC2 instance seems to run fine and waits for a connection, but the code on my local machine gives me the error mentioned below when I try running it with the public IP of the EC2 instance:
TimeoutError: [WinError 10060] A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond
I can't figure out how to fix this.
When I run the following code, I get an error saying :
TimeoutError: [WinError 10060] A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond
# server3.py on EC2 instance
import socket
from threading import Thread
from SocketServer import ThreadingMixIn
# TCP_IP = socket.gethostbyaddr("your-ec2-public_ip")[0]
TCP_IP = socket.gethostbyaddr("3.129.88.101")[0]
TCP_PORT = 60001
BUFFER_SIZE = 1024
print 'TCP_IP=',TCP_IP
print 'TCP_PORT=',TCP_PORT
class ClientThread(Thread):
def __init__(self,ip,port,sock):
Thread.__init__(self)
self.ip = ip
self.port = port
self.sock = sock
print " New thread started for "+ip+":"+str(port)
def run(self):
filename='mytext.txt'
f = open(filename,'rb')
while True:
l = f.read(BUFFER_SIZE)
while (l):
self.sock.send(l)
#print('Sent ',repr(l))
l = f.read(BUFFER_SIZE)
if not l:
f.close()
self.sock.close()
break
tcpsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcpsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcpsock.bind((TCP_IP, TCP_PORT))
threads = []
while True:
tcpsock.listen(5)
print "Waiting for incoming connections..."
(conn, (ip,port)) = tcpsock.accept()
print 'Got connection from ', (ip,port)
newthread = ClientThread(ip,port,conn)
newthread.start()
threads.append(newthread)
for t in threads:
t.join()
# client3.py on local machine
#!/usr/bin/env python
#!/usr/bin/env python
import socket
import time
#TCP_IP = 'ip-ec2-instance'
TCP_IP = '3.129.88.101'
TCP_PORT = 60001
BUFFER_SIZE = 1024
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((TCP_IP, TCP_PORT))
clock_start = time.perf_counter()
time_start = time.time()
with open('received_file', 'wb') as f:
print 'file opened'
while True:
#print('receiving data...')
data = s.recv(1024)
#print('data=%s', (data))
if not data:
f.close()
print 'file close()'
break
# write data to a file
f.write(data)
print('Successfully get the file')
s.close()
print('connection closed')
clock_end = time.clock()
time_end = time.time()
duration_clock = clock_end - clock_start
print 'clock: start = ',clock_start, ' end = ',clock_end
print 'clock: duration_clock = ', duration_clock
duration_time = time_end - time_start
print 'time: start = ',time_start, ' end = ',time_end
print 'time: duration_time = ', duration_time
I have no idea what I am doing wrong and it would be helpful if someone could help.
I am new to Socket programming, My use case is to add shutdown message in my code that if one of the clients goes off or interrupted or killed then my server easily update the count or kill that thread.
import socket
import SSL
import testThread
context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
context.verify_mode = ssl.CERT_REQUIRED
context.load_cert_chain(certfile=self.server_cert, keyfile=self.server_key)
context.load_verify_locations(cafile=self.ca_cert)
# to add threads
t = []
# to store addresses
totalAddress = []
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((self.TCP_IP, self.TCP_Port))
s.listen(4)
while True:
print "Waiting for connections from TCP clients..."
(conn, (ip, port)) = tcpServer.accept()
print("Client connected: {}:{}".format(ip, port))
conn = context.wrap_socket(conn, server_side=True)
newthread = testThread.TestThread(ip, port, conn)
newthread.start()
t.append(newthread)
totalAddress.append((ip,port))
count = newthread.count() -1
print "[ACTIVE CONNECTIONS] {} and number of clients running {}".format(totalAddress, count)
I have created count() in testThread.py file.
import threading
from datetime import datetime
import time
class TestThread(Thread):
def __init__(self, ip, port, conn):
# import pdb; pdb.set_trace()
Thread.__init__(self)
self.ip = ip
self.port = port
self.conn = conn
print "[+] New server socket thread started for " + ip + ":" + str(port)
")
def run(self):
while True:
data = self.conn.recv(2048)
now = datetime.now()
data = str(now) + " - " + data
#print "Server received data:", data
f = open("demofile.txt", "a")
f.write(data)
f.close()
Please excuse me if there is an error in the code. :)
I'm new to socket programming. I'm trying to send 4 files from one host to another. Here is the code:
sender:
from __future__ import print_function
import socket
from struct import pack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
def send(sock, data):
while data:
sent = sock.send(data)
data = data[sent:]
def send_file(fname):
with open(fname, 'rb') as f:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((HOST, PORT))
except socket.error as err:
print(err, HOST, PORT)
sock.close()
return
# Send the file name length & the filename itself in one packet
send(sock, pack('B', len(fname)) + fname.encode())
while True:
data = f.read(BUFSIZE)
if not data:
break
send(sock, data)
sock.close()
fnames = [
'1.jpg',
'2.jpg',
'3.jpg',
'4.jpg',
]
def main():
for fname in fnames:
send_file(fname)
if __name__ == '__main__':
main()
Receiver:
from __future__ import print_function
import socket
from struct import unpack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((HOST, PORT))
except socket.error as err:
print('Bind failed', err)
return
sock.listen(1)
print('Socket now listening at', HOST, PORT)
try:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
sock.close()
if __name__ == '__main__':
main()
File transfer is working fine and there is no problem with it. Now I want to send a simple string like "finish" after sending all files, so that receiver will understand that the transfer is completed and it will do some other tasks based on this finish message (however, it still can receive messages at the same time).
I tried to do this by adding another function called sendMessage() to the sender code and a function called recvMessage() to the receiver. Here are the changed codes:
Sender:
from __future__ import print_function
import socket
from struct import pack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
BUFFER_SIZE = 1024
MESSAGE = "Finish!"
def send(sock, data):
while data:
sent = sock.send(data)
data = data[sent:]
#Updated part for sending message
def sendMessage(message):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST, PORT))
sock.send(message)
data = sock.recv(BUFFER_SIZE)
sock.close()
print ("received data:", data)
def send_file(fname):
with open(fname, 'rb') as f:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((HOST, PORT))
except socket.error as err:
print(err, HOST, PORT)
sock.close()
return
# Send the file name length & the filename itself in one packet
send(sock, pack('B', len(fname)) + fname.encode())
while True:
data = f.read(BUFSIZE)
if not data:
break
send(sock, data)
sock.close()
fnames = [
'1.jpg',
'2.jpg',
'3.jpg',
'4.jpg',
]
def main():
for fname in fnames:
send_file(fname)
sendMessage(MESSAGE)
if __name__ == '__main__':
main()
receiver:
from __future__ import print_function
import socket
from struct import unpack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
BUFFER_SIZE = 20
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
#Updated part for receiving message
def recvMessage(conn):
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.send(data) # echo
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((HOST, PORT))
except socket.error as err:
print('Bind failed', err)
return
sock.listen(1)
print('Socket now listening at', HOST, PORT)
try:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
recvMessage(conn)
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
sock.close()
if __name__ == '__main__':
main()
but after running these codes both sender and receiver freeze after the complete transfer of 4 files and nothing happens. What's wrong and how can I do this?
I suspect you're falling prey to buffering here:
def sendMessage(message):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST, PORT))
sock.send(message)
data = sock.recv(BUFFER_SIZE)
sock.close()
print ("received data:", data)
You perform a send, then immediately try to recv. Except stream connections tend to buffer to avoid excessive packet overhead, so odds are, you don't actually send anything yet, the server doesn't see anything so it doesn't respond, and both sides are blocked waiting for data.
The simplest solution here is to shut down the send side port for writing once you've sent the message, which forces out the last data and lets the receiver know you're done:
def sendMessage(message):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST, PORT))
sock.sendall(message) # sendall makes sure the *whole* message is sent
sock.shutdown(socket.SHUT_WR) # We're done writing
data = sock.recv(BUFFER_SIZE)
sock.close()
print("received data:", data)
On the receiver side you have a bigger problem: You close the connection before trying to receive at all:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close() # Closed here!!!
print('saved\n')
recvMessage(conn) # Used again here!!!
So move the close after the recvMessage call, and change recvMessage to use setsockopt to turn on TCP_NODELAY, so buffering isn't occurring (otherwise the echo back may end up buffering indefinitely, though shutting down the sender for write does mean you're likely to detect the sender is done and exit the loop then close the connection, so it may work fine without TCP_NODELAY, as long as the sender isn't expecting to receive data and respond further):
def recvMessage(conn):
# Disable Nagle algorithm so your echoes don't buffer
conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.sendall(data) # echo using sendall, again, to ensure it's all really sent
I am having some trouble in getting this TCP server run properly... When I connect to it with netcat, I can only send one message and then the server does not display the other send messages. When I place client, addr = tcp_socket.accept() out of the while loop I can receive multiple messages but can only connect once...
What is the best way to tackles these problems?
Code
class TCP(threading.Thread):
def __init__(self, host, port):
self.port = port
threading.Thread.__init__(self)
def create_socket(self, port):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind(('', port))
sock.listen(5)
return sock
def listen(self, tcp_socket):
while True:
client, addr = tcp_socket.accept()
print "Got connection from", addr
data = client.recv(1024)
if data:
print "TCP", data
def run(self):
self.listen(self.create_socket(self.port))
Here's a working example server application which has Socket.accept() outside the loop:
class (threading.Thread):
def listenForClients(self, sock):
while True:
client, address = sock.accept()
client.settimeout(5)
threading.Thread( target = self.listenToClient, args = (client,address) ).start()
def listenToClient(self, client, address):
size = 1024
while True:
try:
data = client.recv(size)
if data:
response = "Got connection"
client.send(response)
else:
raise error('Client disconnected')
except:
client.close()
return False
def __init__(self, host, port):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((host, port))
sock.listen(5)
self.listenForClients(sock)
this uses a thread for each client because otherwise Socket.recv() blocks so clients would have to take turns.