We Keep Coding

Empty string is appended to the deque

I have a deque on a host. Each String that is received through TCP socket is appended to this deque. When each data is received, I print the data and the deque. Here is the code:
from __future__ import print_function
import commands
import socket
import select
from collections import deque
host = commands.getoutput("hostname -I")
port = 5005
backlog = 5
BUFSIZE = 4096
BUFFER_SIZE = 1024
q = deque()
def read_tcp(s):
conn, addr = s.accept()
print('Connected with', *addr)
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.send(data) # echo
if (data == 'sample.jpg'):
print("start processing")
#processP(q)
else:
print("appended")
q.append(data)
print(q)
conn.close()
def read_udp(s):
data,addr = s.recvfrom(1024)
print("received message:", data)
def run():
# create tcp socket
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
tcp.bind((host,port))
except socket.error as err:
print('Bind failed', err)
return
tcp.listen(1)
# create udp socket
udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
udp.bind((host,port))
print('***Socket now listening at***:', host, port)
input = [tcp,udp]
try:
while True:
#print("select.select")
inputready,outputready,exceptready = select.select(input,[],[], 0.1)
for s in inputready:
if s == tcp:
read_tcp(s)
elif s == udp:
read_udp(s)
else:
print("unknown socket:", s)
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
raise
tcp.close()
udp.close()
if __name__ == '__main__':
run()
The problem is that when I print the received data and deque in these lines print("received data:", data) and print(q), the received data is print correctly, but the deque content is printed like these in each step:
deque([''])
deque(['',''])
deque(['','',''])
What's wrong? Here is the sender code which doesn't seem to have any problem:
from __future__ import print_function
import socket
from struct import pack
import commands
import select
#HOST = '10.0.0.2'
PORT = 5005
BUFSIZE = 4096
def tcp_send(s, ip):
TCP_IP = ip
BUFFER_SIZE = 1024
MESSAGE = s
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((TCP_IP, PORT))
s.send(MESSAGE)
data = s.recv(BUFFER_SIZE)
s.close()
print ("sent data:", data)
fnames = [
'90.jpg','91.jpg','92.jpg','93.jpg','94.jpg','95.jpg','96.jpg','97.jpg','98.jpg','99.jpg','100.jpg','sample.jpg'
]
def main():
for fname1 in fnames:
tcp_send(fname1,'10.0.0.2')
if __name__ == '__main__':
main()

replace your read_tcp(s) with
def read_tcp(s):
conn, addr = s.accept()
print('Connected with', *addr)
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.send(data) # echo
print(data)
if (data == 'sample.jpg'):
print("start processing")
#processP(q)
else:
print("appended", data)
q.append(data)
print(q)
conn.close()
you are trying to access data outside while that's why it is empty

It looks like indentation problem. You will brake out of the while loop (in read_tcp()) only when data is evaluated False. Only then, already out of the loop you append current value of data to to the deque q. In the loop you print every chunck of data you get. I think the if block needs to be indented one level to be part of the loop.
Also in read_udp() I don't see that you add anything to deque

Sending a message after file transfer completion

I'm new to socket programming. I'm trying to send 4 files from one host to another. Here is the code:
sender:
from __future__ import print_function
import socket
from struct import pack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
def send(sock, data):
while data:
sent = sock.send(data)
data = data[sent:]
def send_file(fname):
with open(fname, 'rb') as f:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((HOST, PORT))
except socket.error as err:
print(err, HOST, PORT)
sock.close()
return
# Send the file name length & the filename itself in one packet
send(sock, pack('B', len(fname)) + fname.encode())
while True:
data = f.read(BUFSIZE)
if not data:
break
send(sock, data)
sock.close()
fnames = [
'1.jpg',
'2.jpg',
'3.jpg',
'4.jpg',
]
def main():
for fname in fnames:
send_file(fname)
if __name__ == '__main__':
main()
Receiver:
from __future__ import print_function
import socket
from struct import unpack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((HOST, PORT))
except socket.error as err:
print('Bind failed', err)
return
sock.listen(1)
print('Socket now listening at', HOST, PORT)
try:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
sock.close()
if __name__ == '__main__':
main()
File transfer is working fine and there is no problem with it. Now I want to send a simple string like "finish" after sending all files, so that receiver will understand that the transfer is completed and it will do some other tasks based on this finish message (however, it still can receive messages at the same time).
I tried to do this by adding another function called sendMessage() to the sender code and a function called recvMessage() to the receiver. Here are the changed codes:
Sender:
from __future__ import print_function
import socket
from struct import pack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
BUFFER_SIZE = 1024
MESSAGE = "Finish!"
def send(sock, data):
while data:
sent = sock.send(data)
data = data[sent:]
#Updated part for sending message
def sendMessage(message):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST, PORT))
sock.send(message)
data = sock.recv(BUFFER_SIZE)
sock.close()
print ("received data:", data)
def send_file(fname):
with open(fname, 'rb') as f:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((HOST, PORT))
except socket.error as err:
print(err, HOST, PORT)
sock.close()
return
# Send the file name length & the filename itself in one packet
send(sock, pack('B', len(fname)) + fname.encode())
while True:
data = f.read(BUFSIZE)
if not data:
break
send(sock, data)
sock.close()
fnames = [
'1.jpg',
'2.jpg',
'3.jpg',
'4.jpg',
]
def main():
for fname in fnames:
send_file(fname)
sendMessage(MESSAGE)
if __name__ == '__main__':
main()
receiver:
from __future__ import print_function
import socket
from struct import unpack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
BUFFER_SIZE = 20
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
#Updated part for receiving message
def recvMessage(conn):
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.send(data) # echo
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((HOST, PORT))
except socket.error as err:
print('Bind failed', err)
return
sock.listen(1)
print('Socket now listening at', HOST, PORT)
try:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
recvMessage(conn)
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
sock.close()
if __name__ == '__main__':
main()
but after running these codes both sender and receiver freeze after the complete transfer of 4 files and nothing happens. What's wrong and how can I do this?

I suspect you're falling prey to buffering here:
def sendMessage(message):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST, PORT))
sock.send(message)
data = sock.recv(BUFFER_SIZE)
sock.close()
print ("received data:", data)
You perform a send, then immediately try to recv. Except stream connections tend to buffer to avoid excessive packet overhead, so odds are, you don't actually send anything yet, the server doesn't see anything so it doesn't respond, and both sides are blocked waiting for data.
The simplest solution here is to shut down the send side port for writing once you've sent the message, which forces out the last data and lets the receiver know you're done:
def sendMessage(message):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST, PORT))
sock.sendall(message) # sendall makes sure the *whole* message is sent
sock.shutdown(socket.SHUT_WR) # We're done writing
data = sock.recv(BUFFER_SIZE)
sock.close()
print("received data:", data)
On the receiver side you have a bigger problem: You close the connection before trying to receive at all:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close() # Closed here!!!
print('saved\n')
recvMessage(conn) # Used again here!!!
So move the close after the recvMessage call, and change recvMessage to use setsockopt to turn on TCP_NODELAY, so buffering isn't occurring (otherwise the echo back may end up buffering indefinitely, though shutting down the sender for write does mean you're likely to detect the sender is done and exit the loop then close the connection, so it may work fine without TCP_NODELAY, as long as the sender isn't expecting to receive data and respond further):
def recvMessage(conn):
# Disable Nagle algorithm so your echoes don't buffer
conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.sendall(data) # echo using sendall, again, to ensure it's all really sent

A receiver that can receive both TCP images and UDP texts

I'm new to socket programming . I've implemented 2 separated codes on the same host. One of them is supposed to receive images using TCP protocol and the second one is supposed to receive text messages through UDP protocol. Both of them are working fine separately. Here are the codes:
Image receiver (TCP):
from __future__ import print_function
import socket
from struct import unpack
import Queue
from PIL import Image
HOST = '10.0.0.1'
PORT = 12345
BUFSIZE = 4096
q = Queue.Queue()
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((HOST, PORT))
except socket.error as err:
print('Bind failed', err)
return
sock.listen(1)
print('Socket now listening at', HOST, PORT)
try:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
q.put(name)
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
sock.close()
if __name__ == '__main__':
main()
Text receiver (UDP):
import socket
UDP_IP = "10.0.0.1"
UDP_PORT = 5005
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
sock.bind((UDP_IP, UDP_PORT))
while True:
data, addr = sock.recvfrom(1024) # buffer size is 1024 bytes
print "received message:", data
Now my question is: How can I merge these 2 codes? I don't want to open 2 separate consoles for each of them and I want one code instead of two. Is it possible?
I tried the solution which was provided in the comment and here is the code:
from __future__ import print_function
from select import select
import socket
from struct import unpack
host = '10.0.0.2'
port = 5005
size = 8000
backlog = 5
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def read_tcp(s):
conn, addr = s.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
def read_udp(s):
data,addr = s.recvfrom(8000)
print("Recv UDP:'%s'" % data)
def run():
# create tcp socket
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcp.bind((host,port))
tcp.listen(1)
# create udp socket
udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
udp.bind((host,port))
input = [tcp,udp]
while True:
inputready,outputready,exceptready = select(input,[],[])
for s in inputready:
if s == tcp:
read_tcp(s)
elif s == udp:
read_udp(s)
else:
print("unknown socket:", s)
if __name__ == '__main__':
run()
but I don't receive any UDP or TCP packet now and it doesn't seem to work for me.

Short answer yes, but you will need to implement multithreading for this. For instance, your program will spawn two threads, one for TCP sockets and other for UDP.

Python Socket Program - NameError: name 'self' is not defined. I changed the localhost, ports etc but this error refuses to go away

import socket
MAX_BUFFER_SIZE = 4096
class ClientSocket:
def __init__(self):
print("Client socket started....")
self.soc = None
def send_to_Server(self, data):
print('Time to send data to Server..... %s', data)
self.soc.send(data.encode("utf8"))
def receive_from_Server(self):
print('Time to receive from Server.....')
result_bytes = self.soc.recv(MAX_BUFFER_SIZE)
result_string = result_bytes.decode("utf8")
print("Result from server is {}".format(result_string))
def start_client(self):
self.soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.soc.connect(("localhost", 8000))
print('Client connected....')
husainshoab#hs-Len:~$ python IOTDeviceSocket.py
Traceback (most recent call last):
File "IOTDeviceSocket.py", line 7, in
class ClientSocket:
File "IOTDeviceSocket.py", line 11, in ClientSocket
self.soc = None
NameError: name 'self' is not defined

There appears to be nothing wrong with your code. I just used it to create a simple test application
# ClientSocket.py
import socket
MAX_BUFFER_SIZE = 4096
class ClientSocket:
def __init__(self):
print("Client socket started....")
self.soc = None
def send_to_Server(self, data):
print('Time to send data to Server..... %s', data)
self.soc.send(data.encode("utf8"))
def receive_from_Server(self):
print('Time to receive from Server.....')
result_bytes = self.soc.recv(MAX_BUFFER_SIZE)
result_string = result_bytes.decode("utf8")
print("Result from server is {}".format(result_string))
def start_client(self):
self.soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.soc.connect(("localhost", 8000))
print('Client connected....')
cs = ClientSocket()
cs.start_client()
cs.send_to_Server('Hello')
cs.receive_from_Server()
here's a simple test server which just spits back some JSON data
# test_server.py
import socket
from datetime import datetime
import json
def logMessage(clientMessage):
logTime = datetime.today();
msg = "{} | {}\n".format(logTime, clientMessage)
print msg
TCP_PORT = 8000
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('localhost', TCP_PORT))
# listen() puts the socket into server mode,
server.listen(1)
while True:
# wait for an incoming connection
connection, address = server.accept()
try:
# receive data in chunks of 64 bytes
while True:
data = connection.recv(64)
# how do we know if we received all the data?
if data:
# we received data from the client, log it to the file
logMessage(data)
response = {
'name' : 'Jonathan Swift',
'occupation' : 'author'
}
jsonResponse = json.dumps(response)
messageLength = len(jsonResponse)
bytesSent = 0
# send a response to the client after turning our dict into
# a JSON string
while(bytesSent < messageLength):
sent = connection.send(jsonResponse)
bytesSent += sent
else:
# no data, break out of receiving loop
break
except Exception as e:
raise
finally:
connection.close()

import socket
MAX_BUFFER_SIZE = 4096
class ClientSocket:
soc = None
def __init__(self):
print("Client socket started....")
self.soc = None
def send_to_Server(self, data):
print('Time to send data to Server..... %s', data)
self.soc.send(data.encode("utf8"))
def receive_from_Server(self):
print('Time to receive from Server.....')
result_bytes = self.soc.recv(MAX_BUFFER_SIZE)
result_string = result_bytes.decode("utf8")
print("Result from server is {}".format(result_string))
def start_client(self):
self.soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.soc.connect(("localhost", 8000))
print('Client connected....')
you should define the soc variable after class statement so that you can use it globally.

How to make the server side socket code run continuously

I have a server/client socket program that is used to transfer a file from the client to the server. The issue is that the code stops running once the file is transferred. I want to change it such that the server side code is continuously running so that I can transfer a file multiple times without having to run the code again and again
Server code:
import socket
host = ''
port = 5560
def setupServer():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print("Socket created.")
try:
s.bind((host, port))
except socket.error as msg:
print(msg)
print("Socket bind comlete.")
return s
def setupConnection():
s.listen(1) # Allows one connection at a time.
conn, address = s.accept()
print("Connected to: " + address[0] + ":" + str(address[1]))
return conn
def storeFile(filePath):
picFile = open(filePath, 'wb')
print(filePath)
print("Opened the file.")
pic = conn.recv(1024)
#print(pic)
while pic:
print("Receiving picture still.")
picFile.write(pic)
pic = conn.recv(1024)
picFile.close()
def dataTransfer(conn):
# A big loop that sends/receives data until told not to.
while True:
# Receive the data
data = conn.recv(1024) # receive the data
data = data.decode('utf-8')
# Split the data such that you separate the command
# from the rest of the data.
dataMessage = data.split(' ', 1)
command = dataMessage[0]
if command == 'GET':
reply = GET()
elif command == 'REPEAT':
reply = REPEAT(dataMessage)
elif command == 'STORE':
print("Store command received. Time to save a picture")
storeFile(dataMessage[1])
reply = "File stored."
elif command == 'LED_ON':
callLED()
reply = 'LED was on'
else:
reply = 'Unknown Command'
# Send the reply back to the client
conn.sendall(str.encode(reply))
#print("Data has been sent!")
conn.close()
s = setupServer()
while True:
try:
conn = setupConnection()
dataTransfer(conn)
except:
break
The client side code is below:
import socket
from time import sleep
from time import time
host = '192.168.0.17'
port = 5560
data = "hi"
filepath = "/var/www/html/unknown.txt"
def setupSocket():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, port))
return s
def sendPic(s, filePath):
print(filePath)
pic = open(filePath, 'rb')
chunk = pic.read(1024)
s.send(str.encode("STORE " + filePath))
t = time()
while chunk:
print("Sending Picture")
s.send(chunk)
#print(chunk)
chunk = pic.read(1024)
pic.close()
print("Done sending")
print("Elapsed time = " + str(time() - t) + 's')
#s.close()
return "Done sending"
def sendReceive(s, message):
s.send(str.encode(message))
reply = s.recv(1024)
print("We have received a reply")
print("Send closing message.")
s.send(str.encode("EXIT"))
#s.close()
reply = reply.decode('utf-8')
return reply
def transmit(message):
s = setupSocket()
response = sendReceive(s, message)
return response
def backup(filePath):
s = setupSocket()
response = sendPic(s, filePath)
return response
while True:
backup(filepath)
print("Backup Complete!")
break
I do not own the code. I have made some change to the code that I got from a YouTube video.

Have you had a look at the SocketServer module?
You could setup your dataTransfer() function as the handle() method of a RequestHandler class, then start your server with the serve_forever() method.