We Keep Coding

Python API to C++ - return pointer to object instead of full object

I created a python module to call some c++ functions of my library.
It use opencv to manipulate some matrix.
My python code
image = "zebre_1080p.png"
intput_image = cv2.resize(image, (int(cols - cols % ratio), int(rows - rows % ratio)))
output_image = image
cv2.imshow('input',intput_image)
cv2.waitKey(0)
test = py_module.testProcess(input_image, output_image) //increase the resolution of the input image
cv2.imshow('output',output_image) // same as image
cv2.waitKey(0)
cv2.imshow('output',test) // input_image with new resolution
cv2.waitKey(0)
My C++ code
static PyObject* testProcess(PyObject* self, PyObject* args) {
PyArrayObject *input, *output;
if ( ! PyArg_ParseTuple(args, "OO", &input, &output) ) return NULL;
cv::Mat in = cv::Mat(cv::Size(in_dims[0], in_dims[1]), CV_8UC3, PyArray_DATA(input));
cv::Mat out = cv::Mat(cv::Size(out_dims[0], out_dims[1]), CV_8UC3);
in ret = Process(in, out);
if(ret < 0){
printf("Erreur Process : %x\n", ret);
return NULL;
}
npy_intp dimens[] = {out.rows, out.cols};
PyObject* obj = PyArray_SimpleNewFromData(out.dims, dimens, NPY_UINT8, out.data);
return obj;
So this solution works but I want to return the same value as my C++ function and like the c++ function return the out matrix in output_image object.
Like that :
ret = py_module.testModule(input_image, output_image) //increase the resolution of the input image
cv2.imshow('output',output_image) // intput_image with new resolution
cv2.waitKey(0)
I don't know if it is possible, maybe with :
Py_BuildValue("O", obj);
or something else ?
Thanks.

I succeeded to do it like that :
static PyObject* testProcess(PyObject* self, PyObject* args) {
_import_array();
PyArrayObject *input, *output;
if ( ! PyArg_ParseTuple(args, "OO", &input, &output) ) return NULL;
npy_intp* in_shape = PyArray_SHAPE(input);
npy_intp* out_shape = PyArray_SHAPE(output);
cv::Mat in = cv::Mat(in_shape[0], in_shape[1], CV_8UC3, PyArray_DATA(input));
cv::Mat out = cv::Mat(out_shape[0], out_shape[1], CV_8UC3, PyArray_DATA(output));
int ret = Process(in, out);
if(ret < 0){
printf("Erreur Process : %x\n", ret);
return NULL;
}
PyArrayObject *obj = (PyArrayObject *)PyArray_SimpleNewFromData(out.channels(), out_shape, PyArray_TYPE(output), out.data);
PyArray_MoveInto(output, obj);
return PyLong_FromLong( 0 );
}
My python script :
def main():
ratio = 2
image = "zebre.png"
#Read image
orgHRImage = cv2.imread(image)
cols, rows, dim = orgHRImage.shape
#crop to %2
orgHRImage = cv2.resize(orgHRImage, (int(cols - cols % ratio), int(rows - rows % ratio)))
#downscale image
downSize = (int(cols/ratio), int(rows/ratio))
downScaledImage = cv2.resize(orgHRImage, downSize)
#create empty output image
output = np.zeros(orgHRImage.shape, orgHRImage.dtype)
#cv2.imshow('downScaledImage',downScaledImage)
#cv2.waitKey(0)
#cv2.imshow('output',output)
#cv2.waitKey(0)
py_module.testProcess(downScaledImage, output)
#cv2.imshow('output',output)
#cv2.waitKey(0)
So I create one input matrix (cv::Mat in) with the informations of &input (which is a image) and one output matrix (cv::Mat out) with the informations of &output (which was initialized at zeros in my python script.
I call the Process function to fill my output matrix.
I create a new PyArrayObject with the same properties of &output and with the fresh data of the matrix out.
I move the pointer to the new object to my old output pointer to have the new object.
I was able to see the picture resized its size divided by 2, next a black image and at the end the resized image from output.

Ctypes Cuda - pointer multiplication does not result in product

I implemented a Cuda matrix multiplication solely in C which successfully runs. Now I am trying to shift the Matrix initialization to numpy and use Python's ctypes library to execute the c code. It seems like the array with the pointer does not contain the multiplied values. I am not quite sure where the problem lies, but already in the CUDA code - even after calling the Kernel method and loading back the values from device to host, values are still zeroes.
The CUDA code:
#include <stdio.h>
#include <stdlib.h>
#define BLOCK_SIZE 16
#define RANDOM_MN_RANGE 64
struct Matrix {
int width;
int height;
// contiguously stored Matrix, in row first order
float *elements;
};
__global__ void MatMulKernel(Matrix A, Matrix B, Matrix C){
// runs for each col - row pair
float tmpVal = 0;
int col = blockIdx.x * blockDim.x + threadIdx.x;
int row = blockIdx.y * blockDim.y + threadIdx.y;
for (int i = 0; i < A.width; ++i)
tmpVal += A.elements[row * A.width + i] *
B.elements[i * B.width + col];
C.elements[ row * C.width + col ] = tmpVal;
}
extern "C" {
void mMul( Matrix *A, Matrix *B, Matrix *C ){
Matrix d_A, d_B, d_C;
// Matrix d_A
d_A.width = A->width;
d_A.height = A->height;
size_t sizeA = A->width * A->height * sizeof(float);
// dynamically allocate cudaMemory for elemenst array
cudaMalloc(&d_A.elements, sizeA);
cudaMemcpy(d_A.elements, A->elements, sizeA, cudaMemcpyHostToDevice);
// Matrix d_B
d_B.width = B->width;
d_B.height = B->height;
size_t sizeB = B->width * B->height * sizeof(float);
// dynamically allocate cudaMemory for elemenst array
cudaMalloc(&d_B.elements, sizeB);
cudaMemcpy(d_B.elements, B->elements, sizeB, cudaMemcpyHostToDevice);
// Matrix d_C
d_C.width = C->width;
d_C.height = C->height;
size_t sizeC = C->width * C->height * sizeof(float);
// dynamically allocate cudaMemory for elemenst array
cudaMalloc(&d_C.elements, sizeC);
// 16 * 16 = 256 threads per block
dim3 dimBlock(BLOCK_SIZE, BLOCK_SIZE);
// Blocks per grid
dim3 dimGrid(B->width / dimBlock.x, A->height / dimBlock.y);
// calling the Kernel
MatMulKernel<<<dimGrid, dimBlock>>>(d_A, d_B, d_C);
// copy results from result matrix C to the host again
cudaMemcpy(C->elements, d_C.elements, sizeC, cudaMemcpyDeviceToHost);
// C->elements[0] contains still 0, the values do not seem to be loaded back to host memory.
printf("A is %f\n", A->elements[0]);
printf("B is %f\n", B->elements[0]);
printf("C is %f\n", C->elements[0]);
// free the cuda memory
cudaFree(d_A.elements);
cudaFree(d_B.elements);
cudaFree(d_C.elements);
}
}
To compile the code I use the following command
nvcc --shared --compiler-options '-fPIC' -o Sequential_Cuda_Python.so Sequential_Cuda_Python.cu
ctypes Python code
import numpy as np
from numpy.ctypeslib import ndpointer
from ctypes import *
class Matrix(Structure):
_fields_ = [("width", c_int),
("height", c_int),
("elements", POINTER(c_float))]
libc = CDLL("./Sequential_Cuda_Python.so")
libc.mMul.argtypes = [ POINTER(Matrix), POINTER(Matrix), POINTER(Matrix) ]
libc.mMul.restype = None
def npArrtoMatrixPtr(data: np.ndarray) -> (POINTER(Matrix), tuple):
"""
numpy arr to Matrix pointer
#return (pointer to arr, shape)
"""
c_float_p = POINTER(c_float)
data = data.astype(np.float32)
w, h = np.shape(data)
# print((w, h))
mXp = Matrix(height=h, width=w, elements=data.ctypes.data_as(c_float_p))
return (pointer(mXp), np.shape(data))
def matMulSeqCuda( _mA, _mB, _mC ):
"""
multiplies mA with mB sequentually using mC
"""
pmA, mASz = ( _mA[0], _mA[-1] )
pmB, mBSz = ( _mB[0], _mB[-1] )
pmC, mCSz = ( _mC[0], _mC[-1] )
assert np.shape( mASz )[0] == 2 and \
np.shape( mBSz )[0] == 2 and \
np.shape( mCSz )[0] == 2, "Only 2D arrays accepted"
#assert np.shape(mA)[0] == np.shape(mB)[1], "Rows of mA need to be the same as Cols of mB"
libc.mMul( pmA, pmB, pmC )
c = pmC.contents
mtxC = np.ctypeslib.as_array(c.elements, shape=(c.height, c.height))
# The array still contains only 0. values
print(mtxC)
return 0
if __name__ == '__main__':
a = np.ones( (16, 8) )
b = np.ones( (16, 8) )
c = np.zeros( (16, 16) )
mA = npArrtoMatrixPtr(a)
mB = npArrtoMatrixPtr(b)
mC = npArrtoMatrixPtr(c)
matMulSeqCuda(mA, mB, mC)
Solution:
As #Mark Tolonen pointed out the error lied in the Python script. By calling npArrtoMatrixPptr(creates and returns a Pointer to a Matrix struct) within the same scope as the CUDA function libc.mMul I was able to retrieve the correct resulting Matrix mtxC.
import numpy as np
from numpy.ctypeslib import ndpointer
from ctypes import *
class Matrix(Structure):
_fields_ = [("width", c_int),
("height", c_int),
("elements", POINTER(c_float))]
libc = CDLL("./Sequential_Cuda_Python.so")
libc.mMul.argtypes = [ POINTER(Matrix), POINTER(Matrix), POINTER(Matrix) ]
libc.mMul.restype = None
def npArrtoMatrixPtr(data: np.ndarray) -> (POINTER(Matrix), tuple):
"""
numpy arr to Matrix pointer
#return (pointer to arr, shape)
"""
#c_float_p = POINTER(c_float)
data = data.astype(np.float32)
h, w = data.shape
mXp = Matrix(w, h, data.ctypes.data_as(POINTER(c_float)))
return (pointer(mXp), np.shape(data))
def matMulSeqCuda( npMa, npMb, npMc ):
"""
multiplies mA with mB sequentually using mC
"""
assert len(np.shape( npMa )) == 2 and \
len(np.shape( npMb )) == 2 and \
len(np.shape( npMc )) == 2, "Only 2D arrays accepted"
pmA, szA = npArrtoMatrixPtr(npMa)
pmB, szB = npArrtoMatrixPtr(npMb.T)
pmC, szC = npArrtoMatrixPtr(npMc) # the resulting array
libc.mMul( pmA, pmB, pmC )
c = pmC.contents
mtxC = np.ctypeslib.as_array(c.elements, shape=(c.height, c.width))
# the result is correct
print(mtxC)
return 0
if __name__ == '__main__':
a = np.ones( (16, 8) )
b = np.ones( (16, 8) )
c = np.zeros( (16, 16) )
matMulSeqCuda(a, b, c)

I can't compile your code as is, but the problem is that np.shape returns (rows,columns) or the equivalent (height,width), not (width,height):
w, h = np.shape(data) # should be h,w
And also on visual inspection this line is wrong (c.height twice).
mtxC = np.ctypeslib.as_array(c.elements, shape=(c.height, c.height))
The CUDA code is extraneous to the question, which is passing a Matrix* correctly and receiving back the modifications. I made a minimal reproducible example below that concentrates on passing a Matrix correctly:
test.cpp - receives a Matrix structure and doubles the values in it.
#ifdef _WIN32
# define API __declspec(dllexport)
#else
# define API
#endif
struct Matrix {
int width;
int height;
float *elements;
};
extern "C" API
void doubleit(Matrix *A) {
for(int r = 0; r < A->height; ++r)
for(int c = 0; c < A->width; ++c) {
A->elements[r * A->width + c] *= 2;
}
}
test.py
import numpy as np
from ctypes import *
class Matrix(Structure):
_fields_ = [("width", c_int),
("height", c_int),
("elements", POINTER(c_float))]
libc = CDLL('./test')
libc.doubleit.argtypes = POINTER(Matrix),
libc.doubleit.restype = None
def doubleit(a):
h,w = a.shape # note h,w not w,h
m = Matrix(w,h,a.ctypes.data_as(POINTER(c_float)))
libc.doubleit(m)
a = np.arange(0.0,0.6,0.1,dtype=np.float32).reshape((2,3))
print(a)
doubleit(a)
print(a)
Output:
[[0. 0.1 0.2]
[0.3 0.4 0.5]]
[[0. 0.2 0.4]
[0.6 0.8 1. ]]

reading an opencv image in python through a socket

I am trying to read an opencv image in a python socket that is sent from c++.
I am able to read the image into another c++ program or VB program and build an image but with python I don't understand what's happening.
The sending code where I send the mat.data:
char *start_s = "<S><size>43434234<cols>64<rows>64<SE>";//plus I send the image size, cols, rows, which varies, not like the static char string shown
char *end_e = "<E>";
cv::Mat image_send = some_mat;
iResult = send( ConnectSocket, start_s, (int)strlen(start_s), 0 );
iResult = send( ConnectSocket, (const char *) image_send.data, i_buffer_size, 0 );
iResult = send( ConnectSocket, end_e, (int)strlen(end_e), 0 );
This is what I have tried with the python, but haven't had any success yet. The image_cols and Image_rows are filtered from the socket, not shown here, and only the image_mat.data from the c++ mat is in the socket that I am trying to put into the image:
data = conn.recv(4757560)
if(i_Read_Image == 2) & (image_cols != 0) & (image_rows != 0):
print ("Entering")
#print(data)
data2 = np.fromstring(data, dtype='uint8')
img_np = cv2.imdecode(data2,cv2.IMREAD_COLOR )
cv2.imshow('image',img_np)
cv2.waitKey(0)
#Also tried this
#img = Image.new('RGB', (image_cols, image_rows))
#img.putdata(data)
#img5 = np.reshape(data2,(image_rows,image_cols))
i_Read_Image = 0

With the help of the comments I was able to get a working answer. The original image is in a single array RGB, this needs to be reshaped and placed into a 'RGB' image, it can be done in one line:
img = Image.fromarray(data2.reshape(image_rows,image_cols,3), 'RGB')
and when reading an opencv data array from a socket: this works:
data = conn.recv(567667)
if(i_Read_Image == 2) & (image_cols != 0) & (image_rows != 0):
data2 = np.fromstring(data, dtype='uint8')
img = Image.fromarray(data2.reshape(image_rows,image_cols,3), 'RGB')
img.show()

Convert PyQt5 QPixmap to numpy ndarray

I have pixmap:
pixmap = self._screen.grabWindow(0,
self._x, self._y,
self._width, self._height)
I want to convert it to OpenCV format.
I tried to convert it to numpy.ndarray as described here but I got error sip.voidptr object has an unknown size
Is there any way to get numpy array (same format as cv2.VideoCapture read method returns)?

I got numpy array using this code:
channels_count = 4
pixmap = self._screen.grabWindow(0, self._x, self._y, self._width, self._height)
image = pixmap.toImage()
s = image.bits().asstring(self._width * self._height * channels_count)
arr = np.fromstring(s, dtype=np.uint8).reshape((self._height, self._width, channels_count))

The copy can be avoided by doing:
channels_count = 4
pixmap = self._screen.grabWindow(0, self._x, self._y, self._width, self._height)
image = pixmap.toImage()
b = image.bits()
# sip.voidptr must know size to support python buffer interface
b.setsize(self._height * self._width * channels_count)
arr = np.frombuffer(b, np.uint8).reshape((self._height, self._width, channels_count))

Heres a function:
def QPixmapToArray(pixmap):
## Get the size of the current pixmap
size = pixmap.size()
h = size.width()
w = size.height()
## Get the QImage Item and convert it to a byte string
qimg = pixmap.toImage()
byte_str = qimg.bits().tobytes()
## Using the np.frombuffer function to convert the byte string into an np array
img = np.frombuffer(byte_str, dtype=np.uint8).reshape((w,h,4))
return img

data type errors for input images of cv2.calcOpticalFlowPyrLK

I'm running opencv 2.4.1 using python bindings and am having difficulty calculating the optical flow.
Specifically this section of code:
#calculate the opticalflow
if prev_saturation_thresh_img==None:
prev_saturation_thresh_img=saturation_img
if i >=0:
prev_img=prev_saturation_thresh_img
next_img=saturation_thresh_img
p1, st, err = cv2.calcOpticalFlowPyrLK(prev_img,next_img,tracks_np,**lk_params)
Returns the error:
<unknown> is not a numpy array
So then I try to convert the images to numpy arrays:
prev_img=prev_saturation_thresh_img
next_img=saturation_thresh_img
Now I have a new error:
<unknown> data type = 17 is not supported
In a last-ditch effort I convert the images to cvmat (from iplimage) before converting it to a numpy array, just to see what happens
error: ..\..\..\OpenCV-2.4.1\modules\video\src\lkpyramid.cpp:607: error: (-215) nextPtsMat.checkVector(2, CV_32F, true) == npoints
So now I'm stuck. Below is the code in it's entirety for reference
import cv
import cv2
import numpy as np
class Target:
def __init__(self):
self.capture = cv.CaptureFromFile("raw_gait_cropped.avi")
def run(self):
#initiate font
font = cv.InitFont(cv.CV_FONT_HERSHEY_SIMPLEX, 1, 1, 0, 3, 8)
#instantiate images
img_size=cv.GetSize(cv.QueryFrame(self.capture))
hsv_img=cv.CreateImage(img_size,8,3)
saturation_img=cv.CreateImage(img_size,8,1)
saturation_thresh_img=cv.CreateImage(img_size,8,1)
prev_saturation_thresh_img=None
#create params for GoodFeaturesToTrack and calcOpticalFlowPyrLK
gftt_params = dict( cornerCount=11,
qualityLevel=0.2,
minDistance=5,
mask=None,
useHarris=True
)
lk_params = dict( winSize = (15, 15),
maxLevel = 2,
criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03),
flags = cv2.OPTFLOW_USE_INITIAL_FLOW,
minEigThreshold=1
)
tracks=[]
writer=cv.CreateVideoWriter("angle_tracking.avi",cv.CV_FOURCC('M','J','P','G'),30,cv.GetSize(hsv_img),1)
i=0
while True:
#grab a frame from the video capture
img=cv.QueryFrame(self.capture)
#break the loop when the video is over
if img == None:
break
#convert the image to HSV
cv.CvtColor(img,hsv_img,cv.CV_BGR2HSV)
#Get Saturation channel
cv.MixChannels([hsv_img],[saturation_img],[(1,0)])
#Apply threshold to saturation channel
cv.InRangeS(saturation_img,145,255,saturation_thresh_img)
#locate initial features to track
if i==0:
eig_image=temp_image = cv.CreateMat(img.height, img.width, cv.CV_32FC1)
for (x,y) in cv.GoodFeaturesToTrack(saturation_thresh_img, eig_image, temp_image, **gftt_params):
tracks.append([(x,y)])
cv.Circle(saturation_thresh_img,(int(x),int(y)),5,(255,255,255),-1,cv.CV_AA,0)
tracks_np=np.float32(tracks).reshape(-1,2)
print tracks
#calculate the opticalflow
if prev_saturation_thresh_img==None:
prev_saturation_thresh_img=saturation_img
if i >=0:
prev_img=prev_saturation_thresh_img
next_img=saturation_thresh_img
p1, st, err = cv2.calcOpticalFlowPyrLK(prev_img,next_img,tracks_np,**lk_params)
prev_saturation_thresh_img=saturation_img
i=i+1
print i
#display frames to users
cv.ShowImage("Raw Video",img)
cv.ShowImage("Saturation Channel",saturation_img)
cv.ShowImage("Saturation Thresholded",saturation_thresh_img)
# Listen for ESC or ENTER key
c = cv.WaitKey(7) % 0x100
if c == 27 or c == 10:
break
#close all windows once video is done
cv.DestroyAllWindows()
if __name__=="__main__":
t = Target()
t.run()

OpenCV can be very picky about the data formats it accepts. The following code extract works for me:
prev = cv.LoadImage('images/'+file_list[0])
prev = np.asarray(prev[:,:])
prev_gs = cv2.cvtColor(prev, cv2.COLOR_BGR2GRAY)
current = cv.LoadImage('images/'+file)
current = np.asarray(current[:,:])
current_gs = cv2.cvtColor(current, cv2.COLOR_BGR2GRAY)
features, status, track_error = cv2.calcOpticalFlowPyrLK(prev_gs, current_gs, good_features, None,
**lk_params)
Note the [:,:] when converting from images to numpy arrays, I have found that they are required.
I hope that this may solve your problem.