We Keep Coding

ZED camera stream Opencv capture then send to RTSP server

I am using Opencv to capture the stream from a ZED camera and display it using cv2.imshow.
But I am having issues resending the ZED stream back out as a RTSP stream.
Now the reason I am using Opencv is because I need to extract data from the stream before it is sent out as a RTSP stream.
Here is the code for the Zed camera OpenCV stream capture:
def main() :
# Create a ZED camera object
zed = sl.Camera()
# Set configuration parameters
input_type = sl.InputType()
if len(sys.argv) >= 2 :
input_type.set_from_svo_file(sys.argv[1])
init = sl.InitParameters(input_t=input_type)
init.camera_resolution = sl.RESOLUTION.HD1080
init.depth_mode = sl.DEPTH_MODE.ULTRA
init.coordinate_units = sl.UNIT.METER
# Open the camera
err = zed.open(init)
if err != sl.ERROR_CODE.SUCCESS :
print(repr(err))
zed.close()
exit(1)
# Set runtime parameters after opening the camera
runtime = sl.RuntimeParameters()
runtime.sensing_mode = sl.SENSING_MODE.FILL
# Setting the depth confidence parameters
runtime.confidence_threshold = 100
runtime.textureness_confidence_threshold = 100
# Prepare new image size to retrieve half-resolution images
image_size = zed.get_camera_information().camera_resolution
image_size.width = image_size.width /2
image_size.height = image_size.height /2
# Declare your sl.Mat matrices
image_zed = sl.Mat(image_size.width, image_size.height, sl.MAT_TYPE.U8_C4)
point_cloud = sl.Mat()
key = ' '
while key != 113 :
err = zed.grab(runtime)
def click_event (event, x, y, flags, param):
if event == cv2.EVENT_LBUTTONDOWN:
# Retrieve the left image, depth image in the half-resolution
zed.retrieve_image(image_zed, sl.VIEW.LEFT, sl.MEM.CPU, image_size)
# Retrieve the RGBA point cloud in half resolution
zed.retrieve_measure(point_cloud, sl.MEASURE.XYZRGBA, sl.MEM.CPU, image_size)
err, point_cloud_value = point_cloud.get_value(x, y)
distance = math.sqrt(point_cloud_value[0] * point_cloud_value[0] +
point_cloud_value[1] * point_cloud_value[1] +
point_cloud_value[2] * point_cloud_value[2])
print("Distance to Camera at ({}, {}) (image center): {:1.3} m".format(x, y, distance))
if err == sl.ERROR_CODE.SUCCESS :
# Retrieve the left image, depth image in the half-resolution
zed.retrieve_image(image_zed, sl.VIEW.LEFT, sl.MEM.CPU, image_size)
# Retrieve the RGBA point cloud in half resolution
zed.retrieve_measure(point_cloud, sl.MEASURE.XYZRGBA, sl.MEM.CPU, image_size)
# To recover data from sl.Mat to use it with opencv, use the get_data() method
# It returns a numpy array that can be used as a matrix with opencv
image_ocv = image_zed.get_data()
cv2.imshow("Image", image_ocv)
cv2.setMouseCallback('Image', click_event)
key = cv2.waitKey(10)
cv2.destroyAllWindows()
zed.close()
print("\nFINISH")
if __name__ == "__main__":
main()
Here is the code for the RTSP server:
How do I send the captured Opencv cv2.imshow out as a RTSP stream
import gi
gi.require_version('Gst','1.0')
gi.require_version('GstVideo','1.0')
gi.require_version('GstRtspServer','1.0')
from gi.repository import GObject, Gst, GstVideo, GstRtspServer
Gst.init(None)
mainloop = GObject.MainLoop()
server = GstRtspServer.RTSPServer()
mounts = server.get_mount_points()
factory = GstRtspServer.RTSPMediaFactory()
factory.set_launch('( zedsrc stream-type=0 ! videoconvert ! videoscale ! video/x-
raw,format=YUY2,width=1280,height=720,framerate=30/1 ! nvvidconv ! nvv4l2h264enc insert-
sps-pps=1 idrinterval=1 insert-vui=1 ! rtph264pay name=pay0 pt=96 )')
#factory.set_launch('(v4l2src device=/dev/video0 io-mode=2 !
image/jpeg,width=1280,height=720,framerate=30/1 ! nvjpegdec ! video/x-raw ! nvvidconv !
nvv4l2h264enc ! rtph264ay name=pay0 pt=96)')
mounts.add_factory('/test', factory)
server.attach(None)
print('stream ready at rtsp://127.0.0.1:8554/test')
mainloop.run()

Gstreamer leaky queue stops the pipeline

The following code get in inputs a list of url and it create a Gstreamer pipeline. Specifically, for each url, a uridecodebin element is initialized and attached to a queue element. Each queue has the following properties: leaky=2, max-size-buffers=1 and flush-on-eos=1. When I start the pipeline, I can see from nvidia-smi dmon that some video is decoded (NVDEC is used). After a second, everything stops.
I would expect that decoding to keep going, and to push frames into the queue with each queue dropping the old frame every time it receives a new one. Am I wrong?
Code to initialize a Bin to decode the video (source_bin.py). You probably don't need to read it, but here it is:
import sys
from gi.repository import Gst
from pipeline.utils.pipeline import create_gst_elemement
class SourceBin:
#classmethod
def create_source_bin(cls, index: int, uri: str):
# Create a source GstBin to abstract this bin's content from the rest of the pipeline
bin_name = "source-bin-%02d" % index
nbin = Gst.Bin.new(bin_name)
if not nbin:
sys.stderr.write(" Unable to create source bin \n")
# Source element for reading from the uri.
# We will use decodebin and let it figure out the container format of the
# stream and the codec and plug the appropriate demux and decode plugins.
uri_decode_bin = create_gst_elemement("uridecodebin", f"uridecodebin_{index}")
# We set the input uri to the source element
uri_decode_bin.set_property("uri", uri)
# Connect to the "pad-added" signal of the decodebin which generates a
# callback once a new pad for raw data has been created by the decodebin
uri_decode_bin.connect("pad-added", cls.callback_newpad, nbin)
uri_decode_bin.connect("pad-removed", cls.callback_pad_removed, nbin)
uri_decode_bin.connect("no-more-pads", cls.callback_no_more_pads, nbin)
uri_decode_bin.connect("child-added", cls.decodebin_child_added, nbin)
# We need to create a ghost pad for the source bin which will act as a proxy
# for the video decoder src pad. The ghost pad will not have a target right
# now. Once the decode bin creates the video decoder and generates the
# cb_newpad callback, we will set the ghost pad target to the video decoder
# src pad.
Gst.Bin.add(nbin, uri_decode_bin)
bin_pad = nbin.add_pad(
Gst.GhostPad.new_no_target("src", Gst.PadDirection.SRC)
)
if not bin_pad:
sys.stderr.write(" Failed to add ghost pad in source bin \n")
return None
# Connect bus
return nbin
#classmethod
def callback_newpad(cls, uridecodebin, uridecodebin_new_src_pad, data):
print(f"SourceBin: added pad {uridecodebin_new_src_pad.name} to {uridecodebin.name}")
caps = uridecodebin_new_src_pad.get_current_caps()
gststruct = caps.get_structure(0)
gstname = gststruct.get_name()
source_bin = data
features = caps.get_features(0)
# Need to check if the pad created by the decodebin is for video and not audio.
if gstname.find("video") != -1:
# Link the decodebin pad only if decodebin has picked nvidia decoder plugin nvdec_*.
# We do this by checking if the pad caps contain NVMM memory features.
if features.contains("memory:NVMM"):
# Get the source bin ghost pad
bin_ghost_pad = source_bin.get_static_pad("src")
if not bin_ghost_pad.set_target(uridecodebin_new_src_pad):
sys.stderr.write(
"Failed to link decoder src pad to source bin ghost pad\n"
)
else:
sys.stderr.write(" Error: Decodebin did not pick nvidia decoder plugin.\n")
#classmethod
def decodebin_child_added(cls, child_proxy, Object, name, user_data):
if name.find("decodebin") != -1:
Object.connect("child-added", cls.decodebin_child_added, user_data)
#classmethod
def callback_pad_removed(cls, uridecodebin, uridecodebin_removed_src_pad, data):
print(f"SourceBin: Removed pad {uridecodebin_removed_src_pad.name} from {uridecodebin.name}")
#classmethod
def callback_no_more_pads(cls, uridecodebin, data):
print(f"SourceBin: No more pads for {uridecodebin.name}")
Pipeline:
import sys
sys.path.append("../")
import gi
gi.require_version("Gst", "1.0")
gi.require_version("GstRtspServer", "1.0")
from gi.repository import GObject, Gst
from source_bin import SourceBin
def create_gst_elemement(factory_name, instance_name):
element = Gst.ElementFactory.make(factory_name, instance_name)
if not element:
sys.stderr.write(f" Unable to create {factory_name} {instance_name} \n")
return element
ai_rules = fususcore_api.get_ai_rules()
urls = [
"my rtsp url..."
]
GObject.threads_init()
Gst.init(None)
pipeline = Gst.Pipeline()
source_bins = [
SourceBin.create_source_bin(i, url)
for i, url in enumerate(urls)
]
frames_queues = list()
for i in range(len(source_bins)):
frames_queue = create_gst_elemement("queue", f"frame_queue_{i}")
frames_queue.set_property("leaky", 2)
frames_queue.set_property("max-size-buffers", 1)
frames_queue.set_property("flush-on-eos", 1)
frames_queues.append(frames_queue)
for source_bin, frames_queue in zip(source_bins, frames_queues):
pipeline.add(source_bin)
pipeline.add(frames_queue)
source_bin.link(frames_queue)
# loop = GObject.MainLoop()
# bus = pipeline.get_bus()
# bus.add_signal_watch()
# bus.connect("message", bus_call, loop)
pipeline.set_state(Gst.State.PLAYING)
For your convenience, you can find a graph of the pipeline: https://drive.google.com/file/d/1Vu8DR1Puam14k-fUKVBW2bG1gN4AgSPm/view?usp=sharing . Everything you see on the left of a queue is the Bin to decode the video. Each row represent a stream and it's identical to the other ones.

gstreamer appsink can not get flvmux data

I use the gstreamer and python-gi to get encoded video stream data. My launch is like gst-launch-1.0 v4l2src device=/dev/video0 ! x264enc bitrate=1000 ! h264parse ! flvmux ! appsink.
Now I code in python like:
import gi
gi.require_version('Gst', '1.0')
gi.require_version('GstApp', '1.0')
from gi.repository import GObject, Gst, GstApp
GObject.threads_init()
Gst.init(None)
class Example:
def __init__(self):
self.mainloop = GObject.MainLoop()
self.pipeline = Gst.Pipeline()
self.bus = self.pipeline.get_bus()
self.bus.add_signal_watch()
self.bus.connect('message::eos', self.on_eos)
self.bus.connect('message::error', self.on_error)
# Create elements
self.src = Gst.ElementFactory.make('v4l2src', None)
self.encoder = Gst.ElementFactory.make('x264enc', None)
self.parse = Gst.ElementFactory.make('h264parse', None)
self.mux = Gst.ElementFactory.make('flvmux', None)
self.sink = Gst.ElementFactory.make('appsink', None)
# Add elements to pipeline
self.pipeline.add(self.src)
self.pipeline.add(self.encoder)
self.pipeline.add(self.parse)
self.pipeline.add(self.mux)
self.pipeline.add(self.sink)
# Set properties
self.src.set_property('device', "/dev/video0")
# Link elements
self.src.link(self.encoder)
self.encoder.link(self.parse)
self.parse.link(self.mux)
self.mux.link(self.sink)
def run(self):
self.pipeline.set_state(Gst.State.PLAYING)
# self.mainloop.run()
appsink_sample = GstApp.AppSink.pull_sample(self.sink)
while True:
buff = appsink_sample.get_buffer()
size, offset, maxsize = buff.get_sizes()
frame_data = buff.extract_dup(offset, size)
print(frame_data)
def kill(self):
self.pipeline.set_state(Gst.State.NULL)
self.mainloop.quit()
def on_eos(self, bus, msg):
print('on_eos()')
self.kill()
def on_error(self, bus, msg):
print('on_error():', msg.parse_error())
self.kill()
example = Example()
example.run()
But I got the same data for every time, just like "FLV\0x01\0x01".
And than I use C languare to code the function, but I got the same result. Why? Could anyone help me?

print prints strings I assume? The buffer contains binary data. It just starts with data that resembles a string. So probably it starts with FLV\0x01\0x01\0x00.. followed by more binary data. String functions will treat 0x00 as the end marker of a string and will stop printing (as the print function does not take a size argument this is the agreement where your data ends). The size property should change though.. unless all the data has the same data chunk size.. but you need to find another function that prints your binary data - although I'm not sure if that is really what you want. Maybe you want to write this data to a file instead?

The problem is gstreamer's process. We should use signal function to get the stream data in appsink.

Python3 error with Gstreamer

I run:
raspivid -t 999999 -w 1080 -h 720 -fps 25 -hf -b 2000000 -o - | \gst-launch-1.0 -v fdsrc ! h264parse ! rtph264pay config-interval=1 pt=96 \! gdppay ! tcpserversink host=serverIp port=5000
on the Raspberry Pi and run:
gst-launch-1.0 -v tcpclientsrc host=serverIp port=5000 \! gdpdepay ! rtph264depay ! avdec_h264 ! videoconvert ! autovideosink sync=false
on my computer. and I received the video streamed from Raspberry.
now I want write a python code to do so in my computer.my code is :
#!/usr/bin/python3
from os import path
import gi
gi.require_version('Gst', '1.0')
from gi.repository import GObject, Gst, Gtk
# Needed for window.get_xid(), xvimagesink.set_window_handle(), respectively:
from gi.repository import GdkX11, GstVideo
GObject.threads_init()
Gst.init(None)
class Player(object):
def __init__(self):
self.pipeline = Gst.Pipeline()
self.tcpsrc = Gst.ElementFactory.make('tcpclientsrc','tcpsrc')
self.tcpsrc.set_property("host",'192.168.1.12')
self.tcpsrc.set_property("port",5000)
self.gdepay = Gst.ElementFactory.make('gdpdepay', 'gdepay')
self.rdepay = Gst.ElementFactory.make('rtph264depay', 'rdepay')
self.avdec = Gst.ElementFactory.make('avdec_h264', 'avdec')
self.vidconvert = Gst.ElementFactory.make('videoconvert', 'vidconvert')
self.asink = Gst.ElementFactory.make('appsink', 'asink')
self.asink.set_property('sync', False)
#self.asink.set_property('emit-signals', True)
#self.set_property('drop', True)
self.pipeline.add(self.tcpsrc)
self.pipeline.add(self.gdepay)
self.pipeline.add(self.rdepay)
self.pipeline.add(self.avdec)
self.pipeline.add(self.vidconvert)
self.pipeline.add(self.asink)
self.tcpsrc.link(self.gdepay)
self.gdepay.link(self.rdepay)
self.rdepay.link(self.avdec)
self.avdec.link(self.vidconvert)
self.vidconvert.link(self.asink)
def run(self):
self.pipeline.set_state(Gst.State.PLAYING)
p = Player()
p.run()
but I get the following error:
(DO.py:3618): GStreamer-WARNING **:
gstpad.c:4555:store_sticky_event: Sticky event
misordering, got 'segment' before 'caps'
(DO.py:3618): GStreamer-WARNING **:
gstpad.c:4555:store_sticky_event: Sticky event
misordering, got 'segment' before 'caps'
Best Regards,
Mostafa

I answered My question: I replaced this code:
#!/usr/bin/python3
from os import path
import gi
gi.require_version('Gst', '1.0')
from gi.repository import GObject, Gst, Gtk
# Needed for window.get_xid(), xvimagesink.set_window_handle(), respectively:
from gi.repository import GdkX11, GstVideo
GObject.threads_init()
Gst.init(None)
class Player(object):
def __init__(self):
self.pipeline = Gst.Pipeline()
self.tcpsrc = Gst.ElementFactory.make('tcpclientsrc','tcpsrc')
self.tcpsrc.set_property("host", '192.168.1.13')
self.tcpsrc.set_property("port", 5000)
self.gdepay = Gst.ElementFactory.make('gdpdepay', 'gdepay')
self.rdepay = Gst.ElementFactory.make('rtph264depay', 'rdepay')
self.avdec = Gst.ElementFactory.make('avdec_h264', 'avdec')
self.vidconvert = Gst.ElementFactory.make('videoconvert', 'vidconvert')
self.asink = Gst.ElementFactory.make('autovideosink', 'asink')
self.asink.set_property('sync', False)
#self.asink.set_property('emit-signals', True)
#self.set_property('drop', True)
self.pipeline.add(self.tcpsrc)
self.pipeline.add(self.gdepay)
self.pipeline.add(self.rdepay)
self.pipeline.add(self.avdec)
self.pipeline.add(self.vidconvert)
self.pipeline.add(self.asink)
self.tcpsrc.link(self.gdepay)
self.gdepay.link(self.rdepay)
self.rdepay.link(self.avdec)
self.avdec.link(self.vidconvert)
self.vidconvert.link(self.asink)
def run(self):
self.pipeline.set_state(Gst.State.PLAYING)
Gtk.main()
p = Player()
p.run()
first, I forgotten to add Gtk.main() to the run function.
second I changed self.asink = Gst.ElementFactory.make('appsink', 'asink') to self.asink = Gst.ElementFactory.make('autovideosink', 'asink')

gstreamer appsrc works for xvimagesink but no in theoraenc ! oggmux

I am trying to stream cast a computer generated video using gstreamer and icecast, but I cannot get gstreamer appsrc to work. My app works as expected if I use xvimagesink as the sink(see commented code below). But once I pipe it to theoraenc it does not run.
I exchanged shout2send with filesink to check if the problem was icecast, the result is that no data is written to the file. Substituting appsrc with testvideosrc works as expected. Any suggestion?
#!/usr/bin/env python
import sys, os, pygtk, gtk, gobject
import pygst
pygst.require("0.10")
import gst
import numpy as np
class GTK_Main:
def __init__(self):
window = gtk.Window(gtk.WINDOW_TOPLEVEL)
window.connect("destroy", gtk.main_quit, "WM destroy")
vbox = gtk.VBox()
window.add(vbox)
self.button = gtk.Button("Start")
self.button.connect("clicked", self.start_stop)
vbox.add(self.button)
window.show_all()
self.player = gst.Pipeline("player")
source = gst.element_factory_make("appsrc", "source")
caps = gst.Caps("video/x-raw-gray,bpp=16,endianness=1234,width=320,height=240,framerate=(fraction)10/1")
source.set_property('caps',caps)
source.set_property('blocksize',320*240*2)
source.connect('need-data', self.needdata)
colorspace = gst.element_factory_make('ffmpegcolorspace')
enc = gst.element_factory_make('theoraenc')
mux = gst.element_factory_make('oggmux')
shout = gst.element_factory_make('shout2send')
shout.set_property("ip","localhost")
shout.set_property("password","hackme")
shout.set_property("mount","/stream")
caps = gst.Caps("video/x-raw-yuv,width=320,height=240,framerate=(fraction)10/1,format=(fourcc)I420")
enc.caps = caps
videosink = gst.element_factory_make('xvimagesink')
videosink.caps = caps
self.player.add(source, colorspace, enc, mux, shout)
gst.element_link_many(source, colorspace, enc, mux, shout)
#self.player.add(source, colorspace, videosink)
#gst.element_link_many(source, colorspace, videosink)
def start_stop(self, w):
if self.button.get_label() == "Start":
self.button.set_label("Stop")
self.player.set_state(gst.STATE_PLAYING)
else:
self.player.set_state(gst.STATE_NULL)
self.button.set_label("Start")
def needdata(self, src, length):
bytes = np.int16(np.random.rand(length/2)*30000).data
src.emit('push-buffer', gst.Buffer(bytes))
GTK_Main()
gtk.gdk.threads_init()
gtk.main()

I think that your problem is most likely to do with timestamping of the buffers. I've done some quick testing, using that code and replacing the shout element with oggdemux, theoradec, ffmpegcolorspace and ximagesink. At first, I got no output, but after I dispensed with the muxing/demuxing altogether, I got a static image, along with some debug messages about timestamps. I got the correct output after setting the is-live and do-timestamp properties to true on appsrc.
I assume that it should be possible to directly set the timestamps on the buffers that you are pushing out of appsrc, but alas I've not discovered how to do that.