We Keep Coding

Send img from pi using python sockets to display in real time using cv2 on windows

I'm trying to send images from my pi to windows so I can use yolo on windows then send code back to my pi to control my robot.
I can't display the pictures sent by the pi on my pc in real time because the bytes are different every time and it messes up opencv. (opens half photo)
How would my code on windows know how many bytes are going to be in the photo being sent from the pi? Or is there another way to go about this?
Pi code:
from picamera.array import PiRGBArray
from picamera import PiCamera
import socket
import time
import cv2
listensocket = socket.socket() #Creates an instance of socket
Port = 8000 #Port to host server on
maxConnections = 999
IP = socket.gethostname() #IP address of local machine
listensocket.bind(('',Port))
# Starts server
listensocket.listen(maxConnections)
print("Server started at " + IP + " on port " + str(Port))
# Accepts the incoming connection
(clientsocket, address) = listensocket.accept()
print("New connection made!")
# Initialize the camera
camera = PiCamera()
camera.resolution = (640, 480)
camera.framerate = 20
raw_capture = PiRGBArray(camera, size=(640, 480))
time.sleep(0.1)
# Capture frames continuously from the camera
for frame in camera.capture_continuous(raw_capture, format="bgr", use_video_port=True):
image = frame.array
#cv2.imshow("Frame", image)
# Wait for keyPress for 1 millisecond
key = cv2.waitKey(1) & 0xFF
cv2.imwrite("image.jpg", image)
file = open('image.jpg', 'rb')
file_data = file.read(56000)
clientsocket.send(file_data)
print("Data has been transmitted successfully")
raw_capture.truncate(0)
time.sleep(0.050)
if key == ord("q"):
camera.close()
cv2.destroyAllWindows()
break
camera.close()
cv2.destroyAllWindows()
Windows code:
import socket
#import time
import cv2
#import sys
s = socket.socket()
hostname = 'raspi' #Server IP/Hostname
port = 8000 #Server Port
s.connect((hostname, port)) #Connects to server
file = open('image.jpg', 'wb')
file_data = s.recv(2048)
cap = cv2.VideoCapture('image.jpg')
while file_data:
# Save recent image from server
file.write(file_data)
file_data = s.recv(56000)
print("File has been received successfully.")
# Display image as video
ret, img = cap.read()
# Show image
cv2.imshow("Image", img)
key = cv2.waitKey(50)
if key == 27:
break
file.close()
cv2.destroyAllWindows()

I don't have pi to test it with the exact setup but I did a similar project a while ago. My approach to this problem is either using a null-terminated header that will tell the client how many bytes the image will be. Alternatively, you can use a constant size header you decide before hands. For example first 4 bytes will tell the client how many bytes to be received. Here is a sample code snippet for the first solution I described:
sender:
import socket
import cv2 as cv
# read a test image
img = cv.imread('panda.jpg')
# encode it to jpg format, you can do this without redundant file openings
retval, buf = cv.imencode(".JPEG", img)
# get number of bytes
number_of_bytes = len(buf)
# create a null terminated string
header = "" + str(number_of_bytes) + "\0"
# encode it to utf-8 byte format
raw_header = bytes(header, "utf-8")
# create server socket
sock = socket.socket()
sock.bind(('localhost', 8000))
sock.listen()
conn, addr = sock.accept()
# send header first, reciever will use it to recieve image
conn.send(raw_header)
# send the rest of image
conn.send(buf)
reciever:
import socket
# create client socket
sock = socket.socket()
sock.connect(('localhost', 8000))
# recieve bytes until null termination
raw_header = []
recv_byte = sock.recv(1)
while recv_byte != b"\0":
raw_header.append(recv_byte)
recv_byte = sock.recv(1)
# decode header
header = str(b''.join(raw_header), "utf-8")
# recieve the amount of bytes foretold by header
recv_img = sock.recv(int(header))
# save image to file or you can use cv2.imendecode to turn it back to numpy.ndarray (cv2 image format)
with open("traveller_panda.jpg", 'wb+') as im_file:
im_file.write(recv_img)
# transform back from jpg to numpy array
image_decoded = np.frombuffer(recv_img, dtype=np.uint8)
image_decoded = cv.imdecode(image_decoded, cv.IMREAD_COLOR)
# display image
cv.imshow("recieved", image_decoded)
cv.waitKey()

python socket pickled numpy data not showing up

I'm trying to send python-opencv frames over sockets. I'm pickling the data and unpickling but for some reason it's blank or nothing is showing up.
This is my terminal when I run client.py
new message length: b'720 '
It should be streaming the webcam from server but nothing is showing up.
Here is my code for the client and server:
client.py
import socket
import numpy as np
import cv2
import pickle
HEADERSIZE = 10
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((socket.gethostname(), 1232))
while True:
full_msg = b''
new_msg = True
while True:
msg = s.recv(16)
if new_msg:
print(f'new message length: {msg[:HEADERSIZE]}')
msglen = int(msg[:HEADERSIZE])
new_msg = False
full_msg += msg
if len(full_msg)-HEADERSIZE == msglen:
print('full msg recvd')
print(full_msg[HEADERSIZE:])
d = pickle.loads(full_msg[HEADERSIZE:])
print(d)
cv2.namedWindow('Webcam', cv2.WINDOW_NORMAL)
cv2.imshow('Webcam', full_msg[HEADERSIZE:])
new_msg = True
full_msg = b''
print(full_msg)
server.py
import socket
import numpy as np
import cv2
import time
import pickle
from signal import signal, SIGPIPE, SIG_DFL
signal(SIGPIPE, SIG_DFL)
HEADERSIZE = 10
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((socket.gethostname(), 1232))
s.listen(5)
cap = cv2.VideoCapture(0)
while True:
clientsocket, address = s.accept()
print(f"Connection from {address} has been established!")
while True:
ret, frame = cap.read()
msg = pickle.dumps(frame)
print(frame)
msg = bytes(f'{len(frame):<{HEADERSIZE}}', "utf-8") + msg
clientsocket.send(msg)
I have no idea why nothing is showing up. I don't even know if anything is coming though. Does it have to do with numpy data? I heard that can be tricky.

When you stream frames of 720 bytes from server, you are actually sending 730 bytes (length 10 bytes + data 720 bytes) per frame continuously, one frame after another.
In client you are reading 16 bytes per recv(). Hence your condition if len(full_msg)-HEADERSIZE == msglen: will never be true, with header size 10, as 730 is not divisible by 16.
So your program is looping indefinitely on while True: in client.
Try below program for client. I tested with dummy data.
client.py
import socket
import numpy as np
import cv2
import pickle
HEADERSIZE = 10
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((socket.gethostname(), 1232))
while True:
msg_length=int(s.recv(HEADERSIZE))
full_msg=b''
while len(full_msg)<msg_length:
full_msg+=s.recv(msg_length-len(full_msg))
print(full_msg)
d = pickle.loads(full_msg)
cv2.namedWindow('Webcam', cv2.WINDOW_NORMAL)
cv2.imshow('Webcam', full_msg)

sending a screenshot using python

I'm trying to take a screenshot and send it to another computer using python.
I've tried to do it in many different ways. Unfortunately, I didn't find a way to do it.
I would appreciate your help!
server:
from PIL import Image
from PIL import ImageGrab
import socket
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.bind(('0.0.0.0', 1111))
server_socket.listen(1)
print('Waiting For Connection...')
(client_socket, client_address) = server_socket.accept()
print('Connected to: ', client_address[0])
img = ImageGrab.grab(bbox=(10, 10, 500, 500))
photo_to_send= img.tobytes()
size = len(photo_to_send)
client_socket.send(bytes(str(size), 'utf-8'))
while size >= 0:
msg = photo_to_send[:4096]
client_socket.send(bytes(str(msg), 'utf-8'))
photo_to_send= photo_to_send[4096:]
client:
import socket
from PIL import Image
my_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
my_socket.connect(('127.0.0.1', 1111))
print("Connected to the server")
size = int(my_socket.recv(1024).decode('utf-8'))
the_photo = ""
the_photo = the_photo.encode('utf-8')
while size > 0:
data = my_socket.recv(4096)
size -= len(data)
the_photo += data
img_to_save = Image.frombytes("RGB", (490,490), the_photo)
img_to_save .save("screenshot.png")

You can use pickle to serialize your Image objects. Pickle just converts an python object into bytes. pickle.dumps just encodes it to bytes and pickle.loads decodes it back.
Hope this helps!

The problem is that you are sending the textual value of the length immediately before the data itself. For example, if the image data started with AAABBB, you would be sending
1234AAABBB...
But if the image was bigger, you might send something like
56789AAABBB...
The client has no way of telling where the length ends and the data starts! To fix this, you need to send a fixed size length parameter. If you still want to a textual length, you could use zfill:
client_socket.send(str(size).zfill(16).encode())
...
size = int(my_socket.recv(16).decode())
(where 16 is chosen to be long enough to fit any possible image size)
This is not terribly efficient; real protocols usually use binary encoding:
client_socket.send(struct.pack('>I', size))
...
size, = struct.unpack('>I', my_socket.recv(4))
But this may be more complex than you need for your application.

The problem was that I encoded the photo twice.
Thanks for your help!
I really appreciate it!
Here is the solution:
server:
from PIL import ImageGrab
import socket
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.bind(('0.0.0.0', 1112))
server_socket.listen(1)
print('Waiting For Connection...')
(client_socket, client_address) = server_socket.accept()
print('Connected to: ', client_address[0])
img = ImageGrab.grab(bbox=(10, 10, 500, 500))
photo_to_send = img.tobytes()
size = len(photo_to_send)
print(size)
print(photo_to_send)
client_socket.send(bytes(str(size), 'utf-8'))
client_socket.send(photo_to_send)
client:
import socket
from PIL import Image
my_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
my_socket.connect(('127.0.0.1', 1112))
print("Connected to the server")
size = int(my_socket.recv(10).decode('utf-8'))
print(size)
the_photo = my_socket.recv(size)
print(the_photo)
img_to_save = Image.frombytes("RGB", (490, 490), the_photo)
img_to_save.save("screenshot.png")

struct and pickle makes everything easy for you.
server.py
imageBytes = pickle.dumps(image)
sock.sendall(struct.pack("L", len(imageBytes)) + imageBytes)
client.py
data = b""
payload_size = struct.calcsize("L")
while True:
data += sock.recv(payload_size)
packedImageSize = data[:payload_size]
imageSize = struct.unpack("L", packedImageSize)[0]
data = data[payload_size:]
while len(data) < imageSize:
data += sock.recv(65000)
frameData = data[:imageSize]
data = data[imageSize:]
frame = pickle.loads(frameData)

Streaming video from server

I have a server which loads the video and a client which receives and displays the video. I'm new to python. the server code to load the video and send it to client/receiver.
server.py
import cv2
import socket
UDP_IP = "localhost"
UDP_PORT = 5005
cap = cv2.VideoCapture('D:\\testVideo.mp4')
while(True):
ret, frame = cap.read()
cv2.imshow('frame',frame)
sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
d = frame.flatten ()
s = d.tostring ()
for i in range(20):
sock.sendto (s[i*46080:(i+1)*46080],(UDP_IP, UDP_PORT))
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
The receiver code
import socket
import numpy
import cv2
UDP_IP = "localhost"
UDP_PORT = 5005
sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
sock.bind ((UDP_IP, UDP_PORT))
s=""
while True:
data, addr = sock.recvfrom(46080)
s += data
if len(s) == (46080*20):
frame = numpy.fromstring (s,dtype=numpy.uint8)
frame = frame.reshape (480,640,3)
cv2.imshow('frame',frame)
s=""
if cv2.waitKey(1) & 0xFF == ord ('q'):
break
I'm getting an error in "s += data"-TypeError: must be str, not bytes. Is there a problem with my append? Is my approach correct?

Your problem is that s is a string and + concatenates strings, but data is byte() (or perhaps even bytearray, not sure on that one yet). Edit: it seems to just hold a bytestring like b\xff\xff\xff
What you can do is reserve a bytearray-buffer, and then receive into that, since you seem to restrict filesize anyway (you check each iteration if your local (string)buffer is a certain length). With a maximum size, you just allocate that and read however much data comes through. If there's more, it's just cut off, and if it's less, the rest will be NULL-padded. Kind of like this:
# allocate 32 bytes as buffer
s = bytearray(32)
sock.recvmsg_into([s])
Test this by running the following Python code:
import socket
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind(("localhost", 8080))
# reserve a buffer of 4 bytes
s = bytearray(4)
sock.recvmsg_into([s])
At that point, the socket will block and wait for input. From a second terminal run this:
echo "foobar" | nc -u 127.0.0.1 8080
Your Python-script will continue execution and print what it received:
(4, [], 32, ('127.0.0.1', 56464))
>>> s
bytearray(b'foob')
If you don't want to go down that road, I have a solution to your original approach of just appending in an infinite loop. I previously posted a solution with appending to a bytearray, but that didn't work due to type mismatch. It turns out you don't even need bytearray, as this works as expected:
s = bytes()
while True:
s += bytes(b'foobar')
if s == b'foobarfoobar':
print(s)
break

Pygame not working with sockets

I created one program with pygame, imageGrab and sockets but it doesn't work. It should take a printscreen of the server with ImageGrab, convert it to a string, and send it to the client. However, the client upon receiving the information and converting it to an image raises an error:
image = pygame.image.frombuffer(img, (800,600), "RGB")
ValueError: Buffer length does not equal format and resolution size
code Server
import sys, os, socket, pygame
from PIL import ImageGrab
from pygame.locals import *
print "streaming sever 0.1"
try:
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
except socket.error:
print "Error creating socket"
sys.exit(1)
host = "127.0.0.1"
port = raw_input("Port:")
socket.bind((host, int(port)))
socket.listen(2)
socket2,client = socket.accept()
print "Client conectado: " + str(client[0]) + "\nPorta usada: " + str(client[1])
#Capture and send
while True:
img=ImageGrab.grab().resize((800,600))
img.tostring()
socket2.sendall(img)
socket2.close()
code Client
import sys, os, socket, pygame
from PIL import ImageGrab
from pygame.locals import *
print "streaming client 0.1"
pygame.init()
try:
s_client = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
print "streaming protocol started"
except socket.error:
print "Error creating socket"
sys.exit(1)
host = "127.0.0.1"
porta = raw_input("Port:")
q_bytes = raw_input("Enter the number of MBs to transfer: ")
t_bytes = 1024*1024*int(q_bytes)
try:
s_client.connect((host,int(porta)))
except socket.error:
print "Error connecting to: " + host
sys.exit(1)
print "Conectado!"
size = width, height = 800, 600
screen = pygame.display.set_mode(size)
num = 0
while True:
img = s_client.recv(t_bytes)
image = pygame.image.frombuffer(img, (800,600), "RGB")
screen.blit(image,(0,0))
pygame.display.flip()
for event in pygame.event.get():
if event.type == QUIT:
pygame.quit()
os._exit(1)
#recebimento
s_client.close()

The deal with sockets is you have to know exactly how long the message you're receiving is if you plan keep using the same socket connection. You seem to have some idea of this with your q_bytes = raw_input("Enter the number of MBs to transfer: "), but we need to know exactly, not to the nearest MB. Sometimes this information is sent at the front part of the data, so we can read it first and then know when to stop reading.
The exception to this is if we don't need the connection anymore; we just want this one picture. In that case, it's fine to ask for as much data as we want, we'll get an empty string back at the end.
As for the max_bytes argument to recv, that's just one maximum - there's another hardware-dependent maximum imposed on us, for my tests it was 1MB.
The code below just keeps asking for data, stops when it receives an empty string because there is no more data, then combines all this gathered data into the full string.
There are many levels of abstraction that could (and should) be built up to distance us from these complications, but the code below is just yours, working, with some irrelevant bits taken out.
Client.py
import sys, os, socket, pygame
from pygame.locals import *
pygame.init()
s_client = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
host = "127.0.0.1"
porta = 12350
t_bytes = 1024*1024*1
s_client.connect((host,int(porta)))
print "Conectado!"
size = width, height = 300, 500
screen = pygame.display.set_mode(size)
message = []
while True:
s = s_client.recv(t_bytes)
if not s:
break
else:
message.append(s)
full_s = "".join(message)
print 'string received size', len(full_s)
image = pygame.image.frombuffer(full_s, size, "RGB")
#image = pygame.image.fromstring(s, size, "RGB")
screen.blit(image,(0,0))
pygame.display.flip()
for event in pygame.event.get():
if event.type == QUIT:
pygame.quit()
os._exit(1)
raw_input()
s_client.close()
Server.py
import sys, os, socket, pygame
from pygame.locals import *
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host = "127.0.0.1"
port = 12350
socket.bind((host, int(port)))
socket.listen(2)
socket2,client = socket.accept()
print "Client conectado: " + str(client[0]) + "\nPorta usada: " + str(client
img = Image.open('tiny.jpg').resize((300, 500))
s = img.tostring()
print 'size of string', len(s)
socket2.sendall(s)
socket2.close()
edit: As per Mark's correction, len() calls were previously __sizeof__() method calls. __sizeof__ returns the size of the python object in bytes, not the number of bytes/characters in the string.