We Keep Coding

Why does my Python P2P client works over LAN but not the Internet and how can I fix this

Why does my Python P2P client works over LAN but not the Internet and how can I fix this!
I would like to make a p2p messenger and I just don't know why the p2p functionality with UDP Hole Punching is not working. Please help!
Server:
This server holds peers ips and ports for the client.
The ping function pings the peers to check if they are still alive.
'''
import threading
import socket
import time
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind(("IP_ADDRESS", 8081))
addrs = []
print('Server started...')
def ping(addr):
try:
sock.sendto(b'ping', addr)
sock.settimeout(1)
data, addr = sock.recvfrom(1024)
sock.settimeout(None)
return True
except Exception as ex:
sock.settimeout(None)
return False
while True:
data, addr = sock.recvfrom(1024)
print(addr)
sock.sendto((addr[0]+' '+str(addr[1])).encode(), addr)
for a in addrs:
p = ping(a)
print(p, a)
if p:
sock.sendto((a[0]+' '+str(a[1])).encode(), addr)
addrs.append(addr)
sock.sendto(b'DONE', addr)
'''
Client:
import threading
import socket
import time
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.addrs = []
self.chat_logs = ''
self.peers = ''
self.self_data = ('0.0.0.0', 0)
def recv_thread(self):
while True:
try:
data, addr = self.sock.recvfrom(1024)
if data == b'ping':
self.sock.sendto(b'pong', addr)
else:
data = data.decode()
new_user = True
for a in self.addrs:
if a == addr:
new_user = False
if new_user == False:
self.chat_logs += addr[0]+':'+str(addr[1])+' - '+data+'\n'
else:
self.chat_logs += addr[0]+':'+str(addr[1])+' joined the p2p network... \n'
self.addrs.append(addr)
except Exception as ex:
print(ex)
def send_msg(self):
message = self.messageBox.text()
if len(message) > 0:
self.chat_logs += 'You - '+message+'\n'
self.chatBox.setPlainText(self.chat_logs)
for a in self.addrs:
try:
self.sock.sendto(message.encode(), a)
except:
self.addrs.remove(a)
def get_peers(self, host):
host = host.split(':')
host = (host[0], int(host[1]))
self.sock.sendto(b'PEERS', host)
self.self_data, addr = self.sock.recvfrom(512)
self.self_data = self.self_data.decode().split()
self.self_data = (self.self_data[0], int(self.self_data[1]))
while True:
data, addr = self.sock.recvfrom(512)
if data == b'DONE':
break
else:
data = data.decode().split()
self.addrs.append((data[0], int(data[1])))
print(self.addrs)
for a in self.addrs:
try:
self.sock.sendto(b'join', a)
except:
self.addrs.remove(a)
t = threading.Thread(target=self.recv_thread)
t.start()
def connect_to_pears_network(self):
dlg = CustomDialog()
if dlg.exec():
text = dlg.message.text()
if len(text) > 6:
self.get_peers(text)
else:
pass
I hope I did this post right this is my first time making a post.

python socket listener not receiving data

I am programming a decentralised script to track the IPs of other computers running the script, to explore decentralisation. This script isolates the problem. The code consists of 2 scripts, one main program which sends its IP to an IP provided if one is provided, and a listener program which is run as a subscript and listens for data and pipes that data back to the main program. The main script appears to be working, the data is sent over the network, but the listener does not receive it.
This is the main script
import socket
from subprocess import Popen, PIPE
from time import sleep
def getIP():
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
s.connect(('8.8.8.4', 1))
IP = s.getsockname()[0]
except Exception:
IP = '127.0.0.1'
finally:
s.close()
return IP
def sendfyi(target, ownIP):
toSend = 'fyi' + ':' + ownIP
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((target, 50000))
s.send(toSend.encode())
s.close()
print('sent fyi')
otherIPs = []
ownIP = getIP()
targetIP = input('enter ip or 0: ')
if targetIP != '0':
otherIPs.append(targetIP)
sendfyi(targetIP, ownIP)
listener = Popen(['python3', 'testlistener.py'], stdout=PIPE, stderr=PIPE)
i = 0
while i == 0:
sleep(1)
listenerPipe = listener.stdout.readline()
print(listenerPipe)
This is the sub process:
import socket
def getIP():
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
# doesn't even have to be reachable
s.connect(('8.8.8.4', 1))
IP = s.getsockname()[0]
except Exception:
IP = '127.0.0.1'
finally:
s.close()
return IP
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((getIP(), 50000))
i = 1
while i == 1:
s.listen(1)
conn, addr = s.accept()
print('conected', flush=True)
data = conn.recv(1024)
print('data receved', flush=True)
out = data.decode()
print('data decoded', flush=True)
print(out, flush=True)
conn.close()

Incorect bind statement
bind(('', 50000))

Sending a message after file transfer completion

I'm new to socket programming. I'm trying to send 4 files from one host to another. Here is the code:
sender:
from __future__ import print_function
import socket
from struct import pack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
def send(sock, data):
while data:
sent = sock.send(data)
data = data[sent:]
def send_file(fname):
with open(fname, 'rb') as f:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((HOST, PORT))
except socket.error as err:
print(err, HOST, PORT)
sock.close()
return
# Send the file name length & the filename itself in one packet
send(sock, pack('B', len(fname)) + fname.encode())
while True:
data = f.read(BUFSIZE)
if not data:
break
send(sock, data)
sock.close()
fnames = [
'1.jpg',
'2.jpg',
'3.jpg',
'4.jpg',
]
def main():
for fname in fnames:
send_file(fname)
if __name__ == '__main__':
main()
Receiver:
from __future__ import print_function
import socket
from struct import unpack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((HOST, PORT))
except socket.error as err:
print('Bind failed', err)
return
sock.listen(1)
print('Socket now listening at', HOST, PORT)
try:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
sock.close()
if __name__ == '__main__':
main()
File transfer is working fine and there is no problem with it. Now I want to send a simple string like "finish" after sending all files, so that receiver will understand that the transfer is completed and it will do some other tasks based on this finish message (however, it still can receive messages at the same time).
I tried to do this by adding another function called sendMessage() to the sender code and a function called recvMessage() to the receiver. Here are the changed codes:
Sender:
from __future__ import print_function
import socket
from struct import pack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
BUFFER_SIZE = 1024
MESSAGE = "Finish!"
def send(sock, data):
while data:
sent = sock.send(data)
data = data[sent:]
#Updated part for sending message
def sendMessage(message):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST, PORT))
sock.send(message)
data = sock.recv(BUFFER_SIZE)
sock.close()
print ("received data:", data)
def send_file(fname):
with open(fname, 'rb') as f:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((HOST, PORT))
except socket.error as err:
print(err, HOST, PORT)
sock.close()
return
# Send the file name length & the filename itself in one packet
send(sock, pack('B', len(fname)) + fname.encode())
while True:
data = f.read(BUFSIZE)
if not data:
break
send(sock, data)
sock.close()
fnames = [
'1.jpg',
'2.jpg',
'3.jpg',
'4.jpg',
]
def main():
for fname in fnames:
send_file(fname)
sendMessage(MESSAGE)
if __name__ == '__main__':
main()
receiver:
from __future__ import print_function
import socket
from struct import unpack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
BUFFER_SIZE = 20
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
#Updated part for receiving message
def recvMessage(conn):
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.send(data) # echo
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((HOST, PORT))
except socket.error as err:
print('Bind failed', err)
return
sock.listen(1)
print('Socket now listening at', HOST, PORT)
try:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
recvMessage(conn)
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
sock.close()
if __name__ == '__main__':
main()
but after running these codes both sender and receiver freeze after the complete transfer of 4 files and nothing happens. What's wrong and how can I do this?

I suspect you're falling prey to buffering here:
def sendMessage(message):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST, PORT))
sock.send(message)
data = sock.recv(BUFFER_SIZE)
sock.close()
print ("received data:", data)
You perform a send, then immediately try to recv. Except stream connections tend to buffer to avoid excessive packet overhead, so odds are, you don't actually send anything yet, the server doesn't see anything so it doesn't respond, and both sides are blocked waiting for data.
The simplest solution here is to shut down the send side port for writing once you've sent the message, which forces out the last data and lets the receiver know you're done:
def sendMessage(message):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST, PORT))
sock.sendall(message) # sendall makes sure the *whole* message is sent
sock.shutdown(socket.SHUT_WR) # We're done writing
data = sock.recv(BUFFER_SIZE)
sock.close()
print("received data:", data)
On the receiver side you have a bigger problem: You close the connection before trying to receive at all:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close() # Closed here!!!
print('saved\n')
recvMessage(conn) # Used again here!!!
So move the close after the recvMessage call, and change recvMessage to use setsockopt to turn on TCP_NODELAY, so buffering isn't occurring (otherwise the echo back may end up buffering indefinitely, though shutting down the sender for write does mean you're likely to detect the sender is done and exit the loop then close the connection, so it may work fine without TCP_NODELAY, as long as the sender isn't expecting to receive data and respond further):
def recvMessage(conn):
# Disable Nagle algorithm so your echoes don't buffer
conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.sendall(data) # echo using sendall, again, to ensure it's all really sent

A receiver that can receive both TCP images and UDP texts

I'm new to socket programming . I've implemented 2 separated codes on the same host. One of them is supposed to receive images using TCP protocol and the second one is supposed to receive text messages through UDP protocol. Both of them are working fine separately. Here are the codes:
Image receiver (TCP):
from __future__ import print_function
import socket
from struct import unpack
import Queue
from PIL import Image
HOST = '10.0.0.1'
PORT = 12345
BUFSIZE = 4096
q = Queue.Queue()
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((HOST, PORT))
except socket.error as err:
print('Bind failed', err)
return
sock.listen(1)
print('Socket now listening at', HOST, PORT)
try:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
q.put(name)
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
sock.close()
if __name__ == '__main__':
main()
Text receiver (UDP):
import socket
UDP_IP = "10.0.0.1"
UDP_PORT = 5005
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
sock.bind((UDP_IP, UDP_PORT))
while True:
data, addr = sock.recvfrom(1024) # buffer size is 1024 bytes
print "received message:", data
Now my question is: How can I merge these 2 codes? I don't want to open 2 separate consoles for each of them and I want one code instead of two. Is it possible?
I tried the solution which was provided in the comment and here is the code:
from __future__ import print_function
from select import select
import socket
from struct import unpack
host = '10.0.0.2'
port = 5005
size = 8000
backlog = 5
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def read_tcp(s):
conn, addr = s.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
def read_udp(s):
data,addr = s.recvfrom(8000)
print("Recv UDP:'%s'" % data)
def run():
# create tcp socket
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcp.bind((host,port))
tcp.listen(1)
# create udp socket
udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
udp.bind((host,port))
input = [tcp,udp]
while True:
inputready,outputready,exceptready = select(input,[],[])
for s in inputready:
if s == tcp:
read_tcp(s)
elif s == udp:
read_udp(s)
else:
print("unknown socket:", s)
if __name__ == '__main__':
run()
but I don't receive any UDP or TCP packet now and it doesn't seem to work for me.

Short answer yes, but you will need to implement multithreading for this. For instance, your program will spawn two threads, one for TCP sockets and other for UDP.

Socket in Python3, listening port

I want to listen port in Python3.
import socket
sock = socket.socket()
sock.bind(('', 9090))
sock.listen(1)
conn, addr = sock.accept()
print 'connected:', addr
while True:
data = conn.recv(1024)
if not data:
break
conn.send(data.upper())
conn.close()
The data which I want to listen, looks like:
8,0,0,test,0,2016/07/19,14:40:57.938,2016/07/19,14:40:57.938,,,,,,,,,,,,0
8,0,0,test,0,2016/07/19,14:40:57.965,2016/07/19,14:40:57.965,,,,,,,,,,,,0
3,0,0,test,0,2016/07/19,14:41:04.687,2016/07/19,14:41:04.687,,2475,,,,,,,,,,0
..
that is I need read until '\n'
So I need to change this block, but I dont know how..
data = conn.recv(1024)
if not data:
break
conn.send(data.upper())
I want replace nc:
nc -k -l -p 30003 | python3 script.py
where script.py
while True:
for string in sys.stdin:
Also I need reconnect if something wrong, server must be ready take all data any time, just like nc -k -l -p 30003 | python3 script.py

The main idea is to read until you find \n character in your stream. Of course \n may be beyond 1024 bytes that you are reading thus you need to store everything you read in a buffer. This can be emulated with for example such class:
class SocketLineReader:
def __init__(self, socket):
self.socket = socket
self._buffer = b''
def readline(self):
pre, separator, post = self._buffer.partition(b'\n')
if separator:
self._buffer = post
return pre + separator
while True:
data = self.socket.recv(1024)
if not data:
return None
pre, separator, post = data.partition(b'\n')
if not separator:
self._buffer += data
else:
data = self._buffer + pre + separator
self._buffer = post
return data
And usage:
import socket
sock = socket.socket()
sock.bind(('', 9090))
sock.listen(1)
conn, addr = sock.accept()
print('connected:', addr)
reader = SocketLineReader(conn)
while True:
data = reader.readline()
print(data)
if not data:
break
conn.send(data.upper())
conn.close()
If you wish for the server to serve data forever use another while loop:
import socket
sock = socket.socket()
sock.bind(('', 9090))
sock.listen(1)
while True:
conn, addr = sock.accept()
print('connected:', addr)
reader = SocketLineReader(conn)
# The other code goes here
The problem with this approach is that there is no parallelism. Your server won't handle parallel connections. One way to fix that is to send each new connection to a separate thread:
import socket
import threading
def handle(conn):
print('connected:', addr)
reader = SocketLineReader(conn)
# The other code goes here
sock = socket.socket()
sock.bind(('', 9090))
sock.listen(1)
while True:
conn, addr = sock.accept()
threading.Thread(target=handle, args=(conn,)).start()
This should be fine until you hit performance limit. There are ways to improve efficiency (e.g. event loops) but I supposes it's beyond this question.