how can I get audio input in real time from QAudioInput, store it in a NumPy array and pass it to SciPy FFT? What I have tried:
from PyQt5.QtMultimedia import QAudioDeviceInfo, QAudioFormat, QAudioInput
import sys
class Window(QMainWindow):
def __init__(self):
info = QAudioDeviceInfo()
input_device = info.defaultInputDevice()
if input_device.isNull():
# If no avaiable device is found, we display a error
print("There is no audio input device available.")
exit(-1)
audio_format = QAudioFormat()
audio_format.setSampleRate(44100)
audio_format.setSampleSize(8)
audio_format.setChannelCount(1)
audio_format.setCodec("audio/pcm")
audio_format.setSampleType(QAudioFormat.UnSignedInt)
if sys.byteorder == "little":
audio_format.setByteOrder(QAudioFormat.LittleEndian)
else:
audio_format.setByteOrder(QAudioFormat.BigEndian)
self.audioInput = QAudioInput(input_device, audio_format, self)
self.ioDevice = self.audioInput.start()
self.ioDevice.readyRead.connect(self.read_audio)
def read_audio(self):
data: QByteArray = self.ioDevice.readAll()
print(data.toUInt()) # Prints (0, False) which means error converting data
Inspired by the official example Audio Example I have created a QIODevice that allows obtaining the data. The following example takes the last N samples every T seconds by calculating its fft and displaying it using matplotlib.
import sys
import collections
from functools import cached_property
from PyQt5.QtCore import QIODevice, QObject, pyqtSignal, QTimer
from PyQt5.QtMultimedia import QAudioDeviceInfo, QAudioFormat, QAudioInput
from PyQt5.QtWidgets import QApplication, QMainWindow
from matplotlib.backends.backend_qt5agg import FigureCanvas
from matplotlib.figure import Figure
from scipy.fft import fft, fftfreq
import numpy as np
FS = 44100
SAMPLE_COUNT = 2 * 1000
class AudioDevice(QIODevice):
data_changed = pyqtSignal(list, name="dataChanged")
def __init__(self, interval=1000, parent: QObject = None):
super().__init__(parent)
self.m_buffer = collections.deque(
[0 for _ in range(SAMPLE_COUNT)], maxlen=SAMPLE_COUNT
)
self.timer.timeout.connect(self.send_data)
self.timer.setInterval(interval)
self.timer.start()
#cached_property
def timer(self):
return QTimer()
def send_data(self):
self.data_changed.emit(list(self.m_buffer))
def readData(self, data, max_size):
return -1
def writeData(self, data):
max_size = len(data)
resolution = 4
start = 0
available_samples = int(max_size) // resolution
if available_samples < self.m_buffer.maxlen:
start = self.m_buffer.maxlen - available_samples
pos = 0
for _ in range(start, self.m_buffer.maxlen):
y = (1.0 * (data[pos] - 128)) / 128.0
self.m_buffer.append(y)
pos += resolution
return (self.m_buffer.maxlen - start) * resolution
class PlotWidget(QMainWindow):
def __init__(self, parent=None):
super().__init__(parent)
self.canvas = FigureCanvas(Figure(figsize=(5, 3)))
self.setCentralWidget(self.canvas)
self.ax = self.canvas.figure.subplots()
self._line = None
def update_data(self, data):
T = 1 / FS
N = SAMPLE_COUNT
yf = fft(data)
xf = fftfreq(N, T)[: N // 2]
x = xf
y = 2.0 / N * np.abs(yf[0 : N // 2])
if self._line is None:
(self._line,) = self.ax.plot(x, y)
else:
self._line.set_data(x, y)
self.canvas.draw()
def main(args):
app = QApplication(args)
plot_widget = PlotWidget()
plot_widget.resize(640, 480)
plot_widget.show()
info = QAudioDeviceInfo()
input_device = info.defaultInputDevice()
if input_device.isNull():
print("There is no audio input device available.")
exit(-1)
audio_format = QAudioFormat()
audio_format.setSampleRate(FS)
audio_format.setSampleSize(8)
audio_format.setChannelCount(1)
audio_format.setCodec("audio/pcm")
audio_format.setSampleType(QAudioFormat.UnSignedInt)
if sys.byteorder == "little":
audio_format.setByteOrder(QAudioFormat.LittleEndian)
else:
audio_format.setByteOrder(QAudioFormat.BigEndian)
audio_input = QAudioInput(input_device, audio_format, None)
audio_device = AudioDevice(interval=100)
audio_device.data_changed.connect(plot_widget.update_data)
audio_device.open(QIODevice.WriteOnly)
audio_input.start(audio_device)
app.exec_()
if __name__ == "__main__":
main(sys.argv)
data.toUInt() converts whole byte array to one uint value - not what you want. To get sample values you can use either numpy.frombuffer or struct.unpack.
import numpy
def read_audio(self):
data = self.ioDevice.readAll()
values = numpy.frombuffer(data.data(), dtype=numpy.uint8)
or
import struct
def read_audio(self):
data = self.ioDevice.readAll()
fmt = "#{}B".format(data.size())
values = struct.unpack(fmt, data.data())
I added widget that shows waveform to demonstrate that samples actually reflect signal from microphone - not random numbers.
from PyQt5.QtMultimedia import QAudioDeviceInfo, QAudioFormat, QAudioInput
from PyQt5.QtWidgets import QMainWindow, QApplication, QWidget
from PyQt5.QtGui import QPainter, QPolygonF
from PyQt5.QtCore import QPointF
import sys
import numpy
class WaveWidget(QWidget):
def __init__(self, parent = None):
super().__init__(parent)
self._values = None
def setValues(self, values):
self._values = values
self.update()
def paintEvent(self, event):
if self._values is None:
return
painter = QPainter(self)
ys = self._values / 255 * self.height()
xs = numpy.linspace(0, self.width(), num = len(ys))
points = QPolygonF([QPointF(x,y) for x,y in zip(xs,ys)])
painter.drawPolyline(points)
class Window(QMainWindow):
def __init__(self):
super().__init__()
info = QAudioDeviceInfo()
input_device = info.defaultInputDevice()
if input_device.isNull():
# If no avaiable device is found, we display a error
print("There is no audio input device available.")
exit(-1)
audio_format = QAudioFormat()
audio_format.setSampleRate(44100)
audio_format.setSampleSize(8)
audio_format.setChannelCount(1)
audio_format.setCodec("audio/pcm")
audio_format.setSampleType(QAudioFormat.UnSignedInt)
if sys.byteorder == "little":
audio_format.setByteOrder(QAudioFormat.LittleEndian)
else:
audio_format.setByteOrder(QAudioFormat.BigEndian)
self.audioInput = QAudioInput(input_device, audio_format, self)
self.ioDevice = self.audioInput.start()
self.ioDevice.readyRead.connect(self.read_audio)
widget = WaveWidget()
self._widget = widget
self.setCentralWidget(widget)
def read_audio(self):
data = self.ioDevice.readAll()
values = numpy.frombuffer(data.data(), dtype=numpy.uint8)
self._widget.setValues(values)
if __name__ == "__main__":
app = QApplication([])
window = Window()
window.show()
app.exec()
Related
I have a problem with live video stream from 2 GigE cameras in QML. I tried it before with QLabels and QPixmap and it worked without any problem. The QML Labels don't have pixmap property to send images using signal slots.
Here is my Python code:
import sys
import os
from PySide2.QtGui import QGuiApplication
from PySide2.QtQml import QQmlApplicationEngine
from PySide2.QtGui import QImage, QPixmap
from PySide2.QtCore import Slot, QThread, Signal, Qt, QObject
import cv2
from pypylon import pylon
tlFactory = pylon.TlFactory.GetInstance()
devices = tlFactory.EnumerateDevices()
if len(devices) == 0:
raise pylon.RuntimeException("No camera present.")
cameras = pylon.InstantCameraArray(min(len(devices), 2))
for i, cam in enumerate(cameras):
cam.Attach(tlFactory.CreateDevice(devices[i]))
class CamThread(QThread):
cam1 = Signal(QImage)
cam2 = Signal(QImage)
def run(self):
cameras.StartGrabbing(pylon.GrabStrategy_LatestImageOnly)
try:
while cameras.IsGrabbing():
grabResult1 = cameras[0].RetrieveResult(
5000, pylon.TimeoutHandling_ThrowException
)
grabResult2 = cameras[1].RetrieveResult(
5000, pylon.TimeoutHandling_ThrowException
)
if grabResult1.GrabSucceeded() and grabResult2.GrabSucceeded():
img1 = grabResult1.GetArray()
img2 = grabResult2.GetArray()
rgb1 = cv2.cvtColor(img1, cv2.COLOR_YUV2RGB_Y422)
rgb2 = cv2.cvtColor(img2, cv2.COLOR_YUV2RGB_Y422)
h1, w1, ch1 = rgb1.shape
h2, w2, ch2 = rgb2.shape
bytesPerLine1 = ch1 * w1
bytesPerLine2 = ch2 * w1
convertToQtFormat1 = QImage(
img1.data, w1, h1, bytesPerLine1, QImage.Format_RGB888
)
convertToQtFormat2 = QImage(
img2.data, w2, h2, bytesPerLine2, QImage.Format_RGB888
)
p = convertToQtFormat1.scaled(800, 746, Qt.KeepAspectRatio)
q = convertToQtFormat2.scaled(800, 746, Qt.KeepAspectRatio)
self.cam1.emit(p)
self.cam2.emit(q)
except Exception as error:
print(error)
class MainWindow(QObject):
def __init__(self):
QObject.__init__(self)
self.CamThread = CamThread()
self.CamThread.cam1.connect(self.camera1)
self.CamThread.cam2.connect(self.camera2)
self.CamThread.start()
#Slot(QImage)
def camera1(self, image):
pass
#Slot(QImage)
def camera2(self, image):
pass
if __name__ == "__main__":
app = QGuiApplication(sys.argv)
backend = MainWindow()
engine = QQmlApplicationEngine()
engine.rootContext().setContextProperty("backend", backend)
engine.load(os.path.join(os.path.dirname(__file__), "main.qml"))
if not engine.rootObjects():
sys.exit(-1)
sys.exit(app.exec_())
So how to show live video stream using QML/PySide2?
I am using QT Design Studio.
Although the QQuickImageProvider option can be a good one but the drawback is that you have to generate different urls, instead a better option is to use VideoOutput, for example in your case the following implementation should work (not tested):
from functools import cached_property
import os
import random
import sys
import threading
import cv2
from PySide2.QtCore import Property, QObject, Qt, QSize, QTimer, Signal, Slot
from PySide2.QtGui import QColor, QGuiApplication, QImage
from PySide2.QtMultimedia import QAbstractVideoSurface, QVideoFrame, QVideoSurfaceFormat
from PySide2.QtQml import QQmlApplicationEngine
import shiboken2
from pypylon import pylon
class CameraProvider(QObject):
imageChanged = Signal(int, QImage)
def start(self, cameras):
threading.Thread(target=self._execute, args=(cameras,), daemon=True).start()
def _execute(self, cameras):
while cameras.IsGrabbing():
for i, camera in enumerate(cameras):
try:
grab_result = cameras[i].RetrieveResult(
5000, pylon.TimeoutHandling_ThrowException
)
if grab_result.GrabSucceeded():
img = grab_result.GetArray()
# FIXME
# convert img to qimage
qimage = QImage(800, 746, QImage.Format_RGB888)
qimage.fill(QColor(*random.sample(range(0, 255), 3)))
if shiboken2.isValid(self):
self.imageChanged.emit(i, qimage.copy())
except Exception as error:
print(error)
class CameraService(QObject):
surfaceChanged = Signal()
def __init__(self, parent=None):
super().__init__(parent)
self._surface = None
self._format = QVideoSurfaceFormat()
self._format_is_valid = False
def get_surface(self):
return self._surface
def set_surface(self, surface):
if self._surface is surface:
return
if (
self._surface is not None
and self._surface is not surface
and self._surface.isActive()
):
self._surface.stop()
self._surface = surface
self.surfaceChanged.emit()
if self._surface is not None:
self._format = self._surface.nearestFormat(self._format)
self._surface.start(self._format)
videoSurface = Property(
QAbstractVideoSurface,
fget=get_surface,
fset=set_surface,
notify=surfaceChanged,
)
#Slot(QImage)
def update_frame(self, qimage):
if self.videoSurface is None or qimage.isNull():
return
if not self._format_is_valid:
self._set_format(qimage.width(), qimage.height(), QVideoFrame.Format_RGB32)
self._format_is_valid = True
qimage.convertTo(
QVideoFrame.imageFormatFromPixelFormat(QVideoFrame.Format_RGB32)
)
self._surface.present(QVideoFrame(qimage))
def _set_format(self, width, height, pixel_format):
size = QSize(width, height)
video_format = QVideoSurfaceFormat(size, pixel_format)
self._format = video_format
if self._surface is not None:
if self._surface.isActive():
self._surface.stop()
self._format = self._surface.nearestFormat(self._format)
self._surface.start(self._format)
class CameraManager(QObject):
def __init__(self, cameras, parent=None):
super().__init__(parent)
self._services = []
self.provider.imageChanged.connect(self.handle_image_changed)
self.provider.start(cameras)
for _ in cameras:
self._services.append(CameraService())
#cached_property
def provider(self):
return CameraProvider()
#Slot(int, QImage)
def handle_image_changed(self, index, qimage):
self._services[index].update_frame(qimage)
def get_services(self):
return self._services
services = Property("QVariantList", fget=get_services, constant=True)
def main():
app = QGuiApplication(sys.argv)
tlFactory = pylon.TlFactory.GetInstance()
devices = tlFactory.EnumerateDevices()
if len(devices) == 0:
raise pylon.RuntimeException("No camera present.")
cameras = pylon.InstantCameraArray(min(len(devices), 2))
for i, cam in enumerate(cameras):
cam.Attach(tlFactory.CreateDevice(devices[i]))
manager = CameraManager(cameras)
engine = QQmlApplicationEngine()
engine.rootContext().setContextProperty("manager", manager)
engine.load(os.path.join(os.path.dirname(__file__), "main.qml"))
if not engine.rootObjects():
sys.exit(-1)
sys.exit(app.exec_())
if __name__ == "__main__":
main()
import QtQuick 2.14
import QtQuick.Window 2.14
import QtMultimedia 5.14
Window {
visible: true
width: 640
height: 480
title: qsTr("Hello World")
GridView {
width: 300; height: 200
model: manager !== null ? manager.services : []
delegate: VideoOutput {
width: 100
height: 100
fillMode: VideoOutput.PreserveAspectCrop
source: model.modelData
}
}
}
Qt provides different methods to pass images/video streams to QML:
1. Converting pixmap to base64 encoding
QByteArray byteArray;
QBuffer buffer(&byteArray);
buffer.open(QIODevice::WriteOnly);
pixmap.save(&buffer,"PNG");
QString data("data:image/png;base64,");
data.append(QString::fromLatin1(byteArray.toBase64().data()));
This base64 encoded image may be passed to Image::source
2. Use QQuickImageProvider
This allows connecting a custom image://... url to a QPixmap or QImage directly. Check the docs for more information.
3. Use QtMultimedia
Especially VideoOutput may be useful.
I'd like to plot the horizontal distance between 2 points on an image with PyQtGraph, but I can't draw it.
I think it a way of doing this would be to use 3 instances of LineSegmentROI and make them look connected as one right arc, because they already have many features that would be great for this idea.
Like being draggable, which could be very useful to measure a different distance by simply dragging a side.
The problem are the handles, that can't be removed, or even hidden.
Has anyone done something like this?
# import the necessary packages
from pyqtgraph.graphicsItems.ImageItem import ImageItem
from pyqtgraph.graphicsItems.LinearRegionItem import LinearRegionItem
import requests
import numpy as np
import cv2
import pyqtgraph as pg
from pyqtgraph.Qt import QtCore, QtGui
image = cv2.imread('example.png') # Change if you save the image with a different name
image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)
app = QtGui.QApplication([])
## Create window with GraphicsView widget
w = pg.GraphicsView()
w.show()
w.resize(image.shape[0], image.shape[1]) # Depending on the picture you may need to resize
w.setWindowTitle('Test')
view = pg.ViewBox()
view.setLimits(xMin=-image.shape[0]*0.05, xMax=image.shape[0]*1.05,
minXRange=100, maxXRange=2000,
yMin=-image.shape[1]*0.05, yMax=image.shape[1]*1.05,
minYRange=100, maxYRange=2000)
w.setCentralItem(view)
## lock the aspect ratio
view.setAspectLocked(True)
## Add image item
item = ImageItem(image)
view.addItem(item)
# Add Line
line = pg.QtGui.QGraphicsLineItem(200, -100, 400, -100, view)
line.setPen(pg.mkPen(color=(255, 0, 0), width=10))
view.addItem(line)
def mouseClicked(evt):
pos = evt[0]
print(pos)
proxyClicked = pg.SignalProxy(w.scene().sigMouseClicked, rateLimit=60, slot=mouseClicked)
## Start Qt event loop unless running in interactive mode.
if __name__ == '__main__':
import sys
if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'):
QtGui.QApplication.instance().exec_()
I ended up borrowing RectItem from drawing a rectangle in pyqtgraph and using its code for LineItem.
With three LineItems I draw the distance meter between the 2 points.
I still have to add some signals and slots to handle resizing, but I'm working on it.
However the core of the solution is here and I'll follow up with my improvements
# import the necessary packages
from PySide2.QtCore import QLineF, Qt, Signal, Slot, QObject, QPointF, QRectF, QSizeF
from PySide2.QtGui import QRegion
from PySide2.QtWidgets import QGraphicsItem, QLabel, QWidget
from pyqtgraph.graphicsItems.ImageItem import ImageItem
from pyqtgraph.graphicsItems.LinearRegionItem import LinearRegionItem
import numpy as np
import cv2
import pyqtgraph as pg
from pyqtgraph.Qt import QtCore, QtGui, QtWidgets
from pyqtgraph.graphicsItems.ViewBox.ViewBox import ViewBox
image = cv2.imread('image.jpg')
image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)
app = QtGui.QApplication([])
## Create window with GraphicsView widget
w = pg.GraphicsView()
w.show()
w.setWindowTitle('Test')
view = pg.ViewBox()
view.setLimits(xMin=0, xMax=image.shape[0],
minXRange=100, maxXRange=2000,
yMin=0, yMax=image.shape[1],
minYRange=100, maxYRange=2000)
w.setCentralItem(view)
## lock the aspect ratio
view.setAspectLocked(True)
## Add image item
item = ImageItem(image)
view.addItem(item)
class LineItem(pg.UIGraphicsItem):
moved = Signal(QPointF)
def __init__(self, line, extend=0, horizontal=False, parent=None):
super().__init__(parent)
self.initialPos = QLineF(line)
self._line = line
self.extend = extend
self.horizontal = horizontal
self._extendLine()
self.picture = QtGui.QPicture()
self._generate_picture()
self.setFlag(QGraphicsItem.ItemIsSelectable)
self.setFlag(QGraphicsItem.ItemIsMovable)
self.setFlag(QGraphicsItem.ItemSendsGeometryChanges)
#property
def line(self):
return self._line
def _extendLine(self):
if (self.extend != 0 and not self.horizontal):
self._line.setP1( QPointF(self._line.x1(), self._line.y1() - abs(self.extend)) )
# if (self.horizontal):
# self.extend = 0
# self._line.setP1( QPointF(self._line.x1(), self._line.y1() - abs(self.extend)) )
def _generate_picture(self):
painter = QtGui.QPainter(self.picture)
painter.setPen(pg.mkPen(color="y", width=2))
painter.drawLine(self.line)
painter.end()
def paint(self, painter, option, widget=None):
painter.drawPicture(0, 0, self.picture)
def boundingRect(self):
lineShape = self.picture.boundingRect()
lineShape.adjust(-10, -10, 10, 10)
return QtCore.QRectF(lineShape)
def itemChange(self, change, value):
if change == QtWidgets.QGraphicsItem.ItemPositionChange:
# value is the new position.
if self.horizontal:
if value.x() != 0:
value = QPointF(0, value.y())
else:
if value.y() != 0:
value = QPointF(value.x(), 0)
self.moved.emit(value)
return pg.UIGraphicsItem.itemChange(self, change, value)
class Distance(QObject):
def __init__(self, A: QPointF, B: QPointF, view: ViewBox, parent: QWidget=None):
super().__init__(parent)
self.A = A
self.B = B
if A.x() > B.x():
self.A, self.B = B, A
self.distance = abs(B.x() - A.x())
print(self.A)
print(self.B)
extend = 50
top = max(self.A.y(), self.B.y()) + 200
self.left = LineItem(QtCore.QLineF(self.A.x(), self.A.y(), self.A.x(), top), extend)
self.right = LineItem(QtCore.QLineF(self.B.x(), self.B.y(), self.B.x(), top), extend)
self.top = LineItem(QtCore.QLineF(self.A.x(), top, self.B.x(), top), horizontal=True)
self.top.setPos(0, 0)
self.left.moved.connect(self.onLeftSegmentMoved)
self.right.moved.connect(self.onRightSegmentMoved)
self.top.moved.connect(self.onTopSegmentMoved)
self.label = pg.TextItem(str(round(self.distance, 2)), color=(0xFF, 0xFF, 0x00), anchor=(1, 1))
# self.label.setParentItem(self.top)
self.label.setPos(self.A.x()+self.distance/2, top + 5)
view.addItem(self.label)
view.addItem(self.left)
view.addItem(self.top)
view.addItem(self.right)
#Slot(QPointF)
def onLeftSegmentMoved(self, delta: QPointF):
topLeft = self.top.initialPos.p1()
newX = topLeft.x() + delta.x()
newTopLeft = QPointF(newX, topLeft.y())
self.top.line.setP1(newTopLeft)
self.top._generate_picture()
pos = self.label.pos()
self.distance = abs(self.top.line.x2() - self.top.line.x1())
self.label.setPos(newX + (self.top.line.x2() - self.top.line.x1())/2, pos.y())
self.label.setText(str(round(self.distance, 2)))
#Slot(QPointF)
def onTopSegmentMoved(self, delta: QPointF):
leftTop = self.top.initialPos.p1()
newY = leftTop.y() + delta.y()
newLeftTop = QPointF(leftTop.x(), newY)
self.left.line.setP2(newLeftTop)
self.left._generate_picture()
rightTop = self.top.initialPos.p2()
newY = rightTop.y() + delta.y()
newRightTop = QPointF(rightTop.x(), newY)
self.right.line.setP2(newRightTop)
self.right._generate_picture()
pos = self.label.pos()
self.label.setPos(pos.x(), newY)
#Slot(QPointF)
def onRightSegmentMoved(self, delta: QPointF):
topRight = self.top.initialPos.p2()
newX = topRight.x() + delta.x()
newTopRight = QPointF(newX, topRight.y())
self.top.line.setP2(newTopRight)
self.top._generate_picture()
pos = self.label.pos()
self.distance = abs(self.top.line.x2() - self.top.line.x1())
self.label.setPos(newX - (self.top.line.x2() - self.top.line.x1())/2, pos.y())
self.label.setText(str(round(self.distance, 2)))
distance = Distance(QPointF(925, 425), QPointF(138, 500), view)
I'd like to use the wheelEvent to resize an image and place a QGraphicPixmap into a QGraphicsScene.
Before adding the original image, it is resized to around 1/3rd its original size. In the wheelEvent, I'm calling a function that will resize the original image and create a QImage to set the QGraphicsPixmap.
After adding the resized pixmap to the scene, the pixels that were originally under the cursor before the scale have shifted. I'm not sure which positions I need to be mapping to/from the scene to achieve this.
I've tried scaling the graphicsPixmap, scaling and translating the graphicsPixmap, scaling the view and translating the graphicsPixmap/setting an offset.
I clearly don't something about what's happening but I'm not sure what that is..
The WheelEvent below works perfectly until maybe_resize is called.
Depending on the size of the current image in the viewer the maybe_resize method will either resize the current ndarray image, create a new qimage and set a new pixmap in the graphicPixmap, or it exits the method without resizing.
If you run the code as is, the pixmap is in the same place under the cursor, but if you uncomment maybe_resize this is no longer the case.
from PyQt5.QtCore import QRectF, QSize, Qt, pyqtSignal
import cv2
import numpy as np
from PyQt5.QtCore import QRectF, QSize, Qt, pyqtSignal
from PyQt5.QtGui import QImage, QPixmap
from PyQt5.QtWidgets import (QApplication,
QFrame,
QGraphicsPixmapItem,
QGraphicsScene,
QGraphicsView,
QMainWindow,
QSizePolicy)
class GraphicsView(QGraphicsView):
def __init__(self, parent):
super(GraphicsView, self).__init__(parent)
self.pixmap = QPixmap()
self._zoom_level = 0
self._scene = Scene(self)
self.setScene(self._scene)
self.gpm = QGraphicsPixmapItem()
self._scene.addItem(self.gpm)
self.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
self.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
self.setFrameShape(QFrame.NoFrame)
self.has_image = False
def maybe_resize(self, factor):
self.resize_requested(factor)
def read_image(self, path):
self.base_image = cv2.imread(path, -1)
self._original_res = self.base_image.shape
h, w = self.base_image.shape[0], self.base_image.shape[1]
self.resized_image = cv2.resize(self.base_image, (w // 4, h // 4))
self.has_image = True
self.set_image(self.resized_image)
return self.resized_image
def resize_requested(self, factor):
factor = max(1. * (self._zoom_level * factor), 1)
h = int(self.resized_image.shape[0] * factor)
w = int(self.resized_image.shape[1] * factor)
src = cv2.resize(self.base_image, (w, h))
dst = np.ndarray(src.shape, src.dtype)
dst[:, :, :] = src
self.set_image(dst)
def wheelEvent(self, event):
factor = 1.1
if event.angleDelta().y() < 0:
factor = 0.9
self._zoom_level-=1
else:
self._zoom_level+=1
view_pos = event.pos()
scene_pos = self.mapToScene(view_pos)
self.centerOn(scene_pos)
self.scale(factor, factor)
delta = self.mapToScene(view_pos) - self.mapToScene(self.viewport().rect().center())
self.centerOn(scene_pos - delta)
# self.maybe_resize(factor)
def set_image(self, img):
if not self.has_image:
return
shape = img.shape
w = shape[1]
h = shape[0]
self._image = img
q_img_format = QImage.Format_RGB888
try:
bands = shape[2]
except IndexError:
bands = 1
q_img = QImage(img, w, h, w * bands, q_img_format)
self.pixmap = self.pixmap.fromImage(q_img)
self.setSceneRect(QRectF(self.pixmap.rect()))
self.gpm.setPixmap(self.pixmap)
class Scene(QGraphicsScene):
zoom_changed = pyqtSignal(float)
def __init__(self, parent=None):
super(Scene, self).__init__(parent)
class Window(QMainWindow):
def __init__(self):
super(Window, self).__init__()
self.gv = GraphicsView(self)
self.setCentralWidget(self.gv)
def load_image(self, path):
self.gv.read_image(path)
def sizeHint(self):
return QSize(800, 800)
if __name__ == "__main__":
app = QApplication([])
w = Window()
w.load_image('test.jpg')
w.show()
app.exit(app.exec_())
In my chart, I need to store many info in every QPointF, but when I subclass QPointF, it seems that it has no effect, can somebody give me good solution or idea?
The code is:
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
from PyQt5.QtPrintSupport import *
from PyQt5.QtChart import *
import random
class MyPoint(QPointF):
def __init__(self, *args):
super().__init__(*args)
def set_name(self, name):
self.name = name
def set_info(self, info):
self.info = info
class DemoChar(QChartView):
def __init__(self):
super().__init__()
self.setRenderHint(QPainter.Antialiasing)
self.chart = QChart()
self.chart.setTitle('Demo')
self.chart.setAnimationOptions(QChart.SeriesAnimations)
self.setChart(self.chart)
self.lineItem = QGraphicsLineItem(self.chart)
series = QLineSeries(name="random serie")
series.setPointsVisible(True)
series.clicked.connect(self.on_click)
self.series = series
for i in range(20):
#series << QPointF(0.1 * i, random.uniform(-10, 10))
pt = MyPoint( 0.1 * i, random.uniform(-10, 10) )
pt.set_name(str(pt))
pt.set_info(str(random.randint(1, 100)))
series << pt
self.chart.addSeries(series)
self.chart.createDefaultAxes()
def on_click(self, pt):
one_pt = self.series.pointsVector()[0]
print(one_pt)
# print(one_pt.name) #the point have no name, info attribute
app = QApplication([])
demo = DemoChar()
demo.show()
app.exec()
When you pass a QPointF to a QLineSeries, the copy constructor is used that only copies the x and y values, that is, it does not copy the object but only some attributes.
So instead of implementing a custom QPointF it is better to implement a model that is mapped to a QLineSeries using QVXYModelMapper:
import random
from PyQt5.QtCore import pyqtSlot, QPointF, Qt
from PyQt5.QtGui import QPainter, QStandardItem, QStandardItemModel
from PyQt5.QtWidgets import QApplication
from PyQt5.QtChart import QChart, QChartView, QLineSeries, QVXYModelMapper
class CustomModel(QStandardItemModel):
def add_point(self, pt, name="", info=""):
items = []
for value in (pt.x(), pt.y(), name, info):
it = QStandardItem()
it.setData(value, Qt.DisplayRole)
items.append(it)
self.appendRow(items)
def get_data(self, row):
if 0 <= row < self.rowCount():
pt = QPointF(
self.item(row, 0).data(Qt.DisplayRole),
self.item(row, 1).data(Qt.DisplayRole),
)
name = self.item(row, 2).data(Qt.DisplayRole)
info = self.item(row, 3).data(Qt.DisplayRole)
return pt, name, info
class DemoChar(QChartView):
def __init__(self, parent=None):
super().__init__(parent)
self.setRenderHint(QPainter.Antialiasing)
self.chart = QChart()
self.chart.setTitle("Demo")
self.chart.setAnimationOptions(QChart.SeriesAnimations)
self.setChart(self.chart)
self.model = CustomModel()
self.series = QLineSeries(name="random serie")
self.series.setPointsVisible(True)
self.series.clicked.connect(self.on_click)
self.mapper = QVXYModelMapper(xColumn=0, yColumn=1)
self.mapper.setModel(self.model)
self.mapper.setSeries(self.series)
for i in range(20):
pt = QPointF(0.1 * i, random.uniform(-10, 10))
name = "name-{}".format(i)
info = str(random.randint(1, 100))
self.model.add_point(pt, name, info)
self.chart.addSeries(self.series)
self.chart.createDefaultAxes()
#pyqtSlot(QPointF)
def on_click(self, pt):
# first point
index = 0
value = self.model.get_data(index)
if value is not None:
pt, name, info = value
print(pt, name, info)
def main(args):
app = QApplication(args)
demo = DemoChar()
demo.show()
ret = app.exec()
if __name__ == "__main__":
import sys
sys.exit(main(sys.argv))
I am creating a program to play videos and then process them. I am able to play the videos with QMediaPlayer. How do I access specific frames as images or something similar. My end goal would be to format the video into a 4-d tensor of size [Num of frames, width_of_video, height_of_video, channels].
Here is the code that loads my video.:
self.clear_layout(self.vlayout)
videoItem = QVideoWidget()
self.mediaPlayer = QMediaPlayer(None, QMediaPlayer.VideoSurface)
self.mediaPlayer.durationChanged.connect(self.update_duration)
self.mediaPlayer.positionChanged.connect(self.update_slider_position)
self.vlayout.addWidget(videoItem)
self.mediaPlayer.setVideoOutput(videoItem)
local = QUrl.fromLocalFile(self.video_paths)
media = QMediaContent(local)
self.mediaPlayer.setMedia(media)
self.play_video()
Here is a working example that I converted to Python from the C++ version available in this question: How to save a frame using QMediaPlayer?
import sys
import uuid
import PyQt5
from PyQt5 import QtCore, QtWidgets
from PyQt5.QtCore import Qt, QObject, QUrl, QRect, pyqtSignal, QPoint
from PyQt5.QtGui import QPainter, QImage
from PyQt5.QtWidgets import QWidget, QApplication, QMainWindow, QGridLayout, QToolBar, QAction
from PyQt5.QtMultimedia import QMediaPlayer, QMediaContent, QAbstractVideoBuffer, \
QVideoFrame, QVideoSurfaceFormat, QAbstractVideoSurface
from PyQt5.QtMultimediaWidgets import QVideoWidget
class VideoFrameGrabber(QAbstractVideoSurface):
frameAvailable = pyqtSignal(QImage)
def __init__(self, widget: QWidget, parent: QObject):
super().__init__(parent)
self.widget = widget
def supportedPixelFormats(self, handleType):
return [QVideoFrame.Format_ARGB32, QVideoFrame.Format_ARGB32_Premultiplied,
QVideoFrame.Format_RGB32, QVideoFrame.Format_RGB24, QVideoFrame.Format_RGB565,
QVideoFrame.Format_RGB555, QVideoFrame.Format_ARGB8565_Premultiplied,
QVideoFrame.Format_BGRA32, QVideoFrame.Format_BGRA32_Premultiplied, QVideoFrame.Format_BGR32,
QVideoFrame.Format_BGR24, QVideoFrame.Format_BGR565, QVideoFrame.Format_BGR555,
QVideoFrame.Format_BGRA5658_Premultiplied, QVideoFrame.Format_AYUV444,
QVideoFrame.Format_AYUV444_Premultiplied, QVideoFrame.Format_YUV444,
QVideoFrame.Format_YUV420P, QVideoFrame.Format_YV12, QVideoFrame.Format_UYVY,
QVideoFrame.Format_YUYV, QVideoFrame.Format_NV12, QVideoFrame.Format_NV21,
QVideoFrame.Format_IMC1, QVideoFrame.Format_IMC2, QVideoFrame.Format_IMC3,
QVideoFrame.Format_IMC4, QVideoFrame.Format_Y8, QVideoFrame.Format_Y16,
QVideoFrame.Format_Jpeg, QVideoFrame.Format_CameraRaw, QVideoFrame.Format_AdobeDng]
def isFormatSupported(self, format):
imageFormat = QVideoFrame.imageFormatFromPixelFormat(format.pixelFormat())
size = format.frameSize()
return imageFormat != QImage.Format_Invalid and not size.isEmpty() and \
format.handleType() == QAbstractVideoBuffer.NoHandle
def start(self, format: QVideoSurfaceFormat):
imageFormat = QVideoFrame.imageFormatFromPixelFormat(format.pixelFormat())
size = format.frameSize()
if imageFormat != QImage.Format_Invalid and not size.isEmpty():
self.imageFormat = imageFormat
self.imageSize = size
self.sourceRect = format.viewport()
super().start(format)
self.widget.updateGeometry()
self.updateVideoRect()
return True
else:
return False
def stop(self):
self.currentFrame = QVideoFrame()
self.targetRect = QRect()
super().stop()
self.widget.update()
def present(self, frame):
if frame.isValid():
cloneFrame = QVideoFrame(frame)
cloneFrame.map(QAbstractVideoBuffer.ReadOnly)
image = QImage(cloneFrame.bits(), cloneFrame.width(), cloneFrame.height(),
QVideoFrame.imageFormatFromPixelFormat(cloneFrame.pixelFormat()))
self.frameAvailable.emit(image) # this is very important
cloneFrame.unmap()
if self.surfaceFormat().pixelFormat() != frame.pixelFormat() or \
self.surfaceFormat().frameSize() != frame.size():
self.setError(QAbstractVideoSurface.IncorrectFormatError)
self.stop()
return False
else:
self.currentFrame = frame
self.widget.repaint(self.targetRect)
return True
def updateVideoRect(self):
size = self.surfaceFormat().sizeHint()
size.scale(self.widget.size().boundedTo(size), Qt.KeepAspectRatio)
self.targetRect = QRect(QPoint(0, 0), size)
self.targetRect.moveCenter(self.widget.rect().center())
def paint(self, painter):
if self.currentFrame.map(QAbstractVideoBuffer.ReadOnly):
oldTransform = self.painter.transform()
if self.surfaceFormat().scanLineDirection() == QVideoSurfaceFormat.BottomToTop:
self.painter.scale(1, -1)
self.painter.translate(0, -self.widget.height())
image = QImage(self.currentFrame.bits(), self.currentFrame.width(), self.currentFrame.height(),
self.currentFrame.bytesPerLine(), self.imageFormat)
self.painter.drawImage(self.targetRect, image, self.sourceRect)
self.painter.setTransform(oldTransform)
self.currentFrame.unmap()
class App(QApplication):
def __init__(self, sys_argv):
super().__init__(sys_argv)
# Show main window
self.view = QMainWindow()
self.centralWidget = QWidget(self.view)
self.gridLayout = QGridLayout(self.centralWidget)
self.gridLayout.setContentsMargins(0, 0, 0, 0)
self.gridLayout.setSpacing(0)
self.video_item = QVideoWidget()
self.gridLayout.addWidget(self.video_item)
self.view.setCentralWidget(self.centralWidget)
self.mediaPlayer = QMediaPlayer(None, QMediaPlayer.VideoSurface)
self.grabber = VideoFrameGrabber(self.video_item, self)
self.mediaPlayer.setVideoOutput(self.grabber)
self.grabber.frameAvailable.connect(self.process_frame)
self.mediaPlayer.durationChanged.connect(self.update_duration)
self.mediaPlayer.positionChanged.connect(self.update_slider_position)
local = QUrl.fromLocalFile('c:/temp/lorem.mp4')
media = QMediaContent(local)
self.mediaPlayer.setMedia(media)
self.mediaPlayer.play()
self.view.show()
def process_frame(self, image):
# Save image here
image.save('c:/temp/{}.jpg'.format(str(uuid.uuid4())))
def update_duration(self):
pass
def update_slider_position(self):
pass
if __name__ == '__main__':
def except_hook(cls, exception, traceback):
sys.__excepthook__(cls, exception, traceback)
if hasattr(QtCore.Qt, 'AA_EnableHighDpiScaling'):
PyQt5.QtWidgets.QApplication.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling, True)
if hasattr(QtCore.Qt, 'AA_UseHighDpiPixmaps'):
PyQt5.QtWidgets.QApplication.setAttribute(QtCore.Qt.AA_UseHighDpiPixmaps, True)
app = App(sys.argv)
app.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling)
sys.excepthook = except_hook
sys.exit(app.exec_())