How can I detect / check the Python-Type of a PyObject?
PyObject_HasAttrString can be used to check, but how do I get the AttrString of a given PyObject?
int modify(cv::cuda::GpuMat* cVal, py::object &pobj)
{
py::handle ptype = pobj.get_type(); //?? ...
PyObject *pobject = pobj.ptr();
auto typeCheck = PyObject_HasAttrString(pobject, "cv2.cuda_GpuMat"); //??
if (!typeCheck)
return(0);
PyObject *pobject1 = PyObject_GetAttrString( pobject, "cv2.cuda_GpuMat");
cv::cuda::GpuMat *pyVal = (cv::cuda::GpuMat *)PyLong_AsVoidPtr(pobject1);
// *pyVal = *cVal; // kidding
return(1);
}
Presumably you know what type it should be so you can check if you can convert to that type, for example: PyTuple_Check, PyUnicode_Check, etc.
Assuming you are using numpy arrays, I would look at PyArray_FROM_OTF which will return an ndarray from any Python object that can be converted into an array:
PyArrayObject* pPyArrayA = reinterpret_cast<PyArrayObject*>(PyArray_FROM_OTF(pPyA, NPY_DOUBLE, NPY_ARRAY_IN_ARRAY));
if(pPyArrayA == nullptr)
{
PyErr_SetString(PyExc_ValueError, "Failed to convert a to nd array.");
return nullptr;
}
size_t iSizeA = static_cast<size_t>(PyArray_SIZE(pPyArrayA));
const double* pcdA = static_cast<const double*>(PyArray_DATA(pPyArrayA));
I'm working on the python with ctypes to call the c so file, but the c file define the structure with function pointer
// mem ==================================================================
typedef struct StdMemFunc
{
void* (*const malloc) (unsigned long size);
void (*const free) (void* ptr);
void* (*const realloc) (void* ptr, unsigned long size);
void* (*const calloc) (unsigned long count, unsigned long size);
void* (*const set) (void* ptr, int value, unsigned long num);
void* (*const copy) (void* dest, const void* src, unsigned long num);
}*StdMemFunc;
typedef struct StdLib
{
const uint32_t version;
bool (*const is_version_compatible) (uint32_t version, uint32_t func_mask);
void (*const delay) (int32_t milli_sec);
const StdMemFunc mem;
}*StdLib;
and mock the function in another file as below
void *std_malloc(unsigned long size)
{
return malloc(size);
}
void std_free(void *ptr)
{
free(ptr);
}
void *std_realloc(void *ptr, unsigned long size)
{
return realloc(ptr, size);
}
void *std_calloc(unsigned long count, unsigned long size)
{
return calloc(count, size);
}
void *std_memset(void *ptr, int value, unsigned long num)
{
return memset(ptr, value, num);
}
void *std_memcopy(void *dest, const void *src, unsigned long num)
{
return memcpy(dest, src, num);
}
struct StdMemFunc mem_func =
{
.malloc = std_malloc,
.free = std_free,
.realloc = std_realloc,
.calloc = std_calloc,
.set = std_memset,
.copy = std_memcopy
};
then the python need to call another method with std_lib as paramater, the std_lib with call mem->malloc() method in C part, so how to define the class in the python with ctypes?
I have tried the below one, but it was not work
class StdMemFunc(Structure):
_fields_ = [
("malloc", ctypes.CFUNCTYPE(c_void_p, c_ulong)),
("free", ctypes.CFUNCTYPE(None, c_void_p)),
("realloc", ctypes.CFUNCTYPE(c_void_p, c_void_p, c_ulong)),
("calloc", ctypes.CFUNCTYPE(c_void_p, c_ulong, c_ulong)),
("set", ctypes.CFUNCTYPE(c_void_p, c_void_p, c_int, c_ulong)),
("copy", ctypes.CFUNCTYPE(c_void_p, c_void_p, c_ulong))
]
class StdLib(Structure):
_fields_ = [
("version", c_uint32),
("is_version_compatible", c_bool),
("delay", c_void_p),
("mem", POINTER(StdMemFunc)),
]
libc_std_lib = CDLL('/home/linus/code/galileo/mock_std_lib.so')
std_lib = StdLib()
std_lib.mem.malloc = libc_std_lib.std_malloc
libc_modbus.modbus_create_server_station.argtypes = [POINTER(ModbusNodeDef), c_int, StdLib, PlcDrvAccessor]
libc_modbus.modbus_create_server_station.restype = POINTER(ModbusStation)
libc_modbus.modbus_create_server_station(node_def, node_num, std_lib, plc_drv_accessor)
It looks like there are two problems here:
The is_version_compatible and delay fields in the StdLib struct are functions, but you are defining them as constants.
You are not instantiating all the fields in the struct, meaning that the program might be trying to dereference a null pointer, as null pointers are the default value for pointer types.
The StdLib struct definition should look something like this:
class StdLib(Structure):
_fields_ = [
("version", c_uint32),
("is_version_compatible", CFUNCTYPE(POINTER(c_bool), c_uint32, _uint32)),
("delay", CFUNCTYPE(c_void_p, c_int32)),
("mem", POINTER(StdMemFunc)),
]
For the instantiation, I would do something like this:
libc_std_lib = CDLL('/home/linus/code/galileo/mock_std_lib.so')
std_mem_func = StdMemFunc(
libc_std_lib.std_malloc,
libc_std_lib.std_free,
libc_std_lib.std_realloc,
libc_std_lib.std_calloc,
libc_std_lib.std_set,
libc_std_lib.std_copy
)
std_lib = StdLib(
1,
reference_to_is_version_compatible_func,
reference_to_delay_func,
std_mem_func
)
Of course, you need to pass the correct params/function references to StdLib. Maybe you will need to mock the is_version_compatible and delay functions as well.
Disclaimer: this is entirely untested, so I don't guarantee it will work.
The OP's code isn't quite reproducible, but I was able to get the same error message on the following line:
std_lib.mem.malloc = libc_std_lib.std_malloc
If I am following correctly, the OP wants to initialize the C structure with functions that are provided in C, but libc.std_lib.std_malloc isn't wrapped properly to do that. It is a function that wraps a C function that is callable from Python, not C.
ctypes function prototypes can be instantiated a number of ways, and the one that works is:
prototype(func_spec[, paramflags])
Returns a foreign function exported by a shared library. func_spec must be a 2-tuple
(name_or_ordinal, library). The first item is the name of the exported
function as string, or the ordinal of the exported function as small
integer. The second item is the shared library instance.
For example:
std_lib.mem.malloc = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_ulong)(('std_malloc',libc_std_lib))
Here's a working set of files:
test.cpp
#include <stdlib.h>
#include <stdint.h>
#include <memory.h>
#include <stdio.h>
#ifdef _WIN32
# define API __declspec(dllexport)
#else
# define API
#endif
extern "C" {
typedef struct StdMemFunc {
void* (*const malloc)(unsigned long size);
void (*const free)(void* ptr);
void* (*const realloc)(void* ptr, unsigned long size);
void* (*const calloc)(unsigned long count, unsigned long size);
void* (*const set)(void* ptr, int value, unsigned long num);
void* (*const copy)(void* dest, const void* src, unsigned long num);
} *StdMemFuncPtr;
typedef struct StdLib {
const uint32_t version;
bool (*const is_version_compatible)(uint32_t version, uint32_t func_mask);
void (*const delay)(int32_t milli_sec);
const StdMemFunc mem;
} *StdLibPtr;
API void* std_malloc(unsigned long size) {
return malloc(size);
}
API void std_free(void* ptr) {
free(ptr);
}
API void* std_realloc(void* ptr, unsigned long size) {
return realloc(ptr, size);
}
API void* std_calloc(unsigned long count, unsigned long size) {
return calloc(count, size);
}
API void* std_memset(void* ptr, int value, unsigned long num) {
return memset(ptr, value, num);
}
API void* std_memcopy(void* dest, const void* src, unsigned long num) {
return memcpy(dest, src, num);
}
// A couple of test functions that accepts the initialized structure
// and calls sum of the function pointers.
API char* testit(StdLib* test) {
// This is how I debugged this, by comparing the *actual*
// function pointer value to the one received from Python.
// Once they matched the code worked.
printf("%p %p\n", std_malloc, test->mem.malloc);
char* p = static_cast<char*>(test->mem.malloc(10));
test->mem.set(p, 'A', 9);
p[9] = 0;
return p;
}
API void freeit(StdLib* test, char* p) {
test->mem.free(p);
}
}
test.py
import ctypes as ct
# prototypes
MALLOC = ct.CFUNCTYPE(ct.c_void_p,ct.c_ulong)
FREE = ct.CFUNCTYPE(None,ct.c_void_p)
REALLOC = ct.CFUNCTYPE(ct.c_void_p, ct.c_void_p, ct.c_ulong)
CALLOC = ct.CFUNCTYPE(ct.c_void_p, ct.c_ulong, ct.c_ulong)
SET = ct.CFUNCTYPE(ct.c_void_p,ct.c_void_p,ct.c_int,ct.c_ulong)
COPY = ct.CFUNCTYPE(ct.c_void_p, ct.c_void_p, ct.c_ulong)
class StdMemFunc(ct.Structure):
_fields_ = [("malloc", MALLOC),
("free", FREE),
("realloc", REALLOC),
("calloc", CALLOC),
("set", SET),
("copy", COPY)]
class StdLib(ct.Structure):
_fields_ = [("version", ct.c_uint32),
# Note these two fields were function pointers as well.
# Declared correctly now.
("is_version_compatible", ct.CFUNCTYPE(ct.c_bool, ct.c_uint32, ct.c_uint32)),
("delay", ct.CFUNCTYPE(None, ct.c_int32)),
("mem", StdMemFunc)]
dll = ct.CDLL('./test')
dll.testit.argtypes = ct.POINTER(StdLib),
dll.testit.restype = ct.POINTER(ct.c_char)
dll.freeit.argtypes = ct.POINTER(StdLib), ct.c_char_p
dll.freeit.restype = None
lib = StdLib()
lib.mem.malloc = MALLOC(('std_malloc', dll))
lib.mem.realloc = REALLOC(('std_realloc', dll))
lib.mem.calloc = CALLOC(('std_calloc', dll))
lib.mem.free = FREE(('std_free', dll))
lib.mem.set = SET(('std_memset', dll))
lib.mem.copy = COPY(('std_memcopy', dll))
p = dll.testit(lib)
# One way to access the data in the returned pointer is to slice it to the known length
print(p[:10])
# If known to be null-terminated, can also cast to c_char_p, which expects
# null-terminated data, and extract the value.
print(ct.cast(p,ct.c_char_p).value)
dll.freeit(lib,p)
Output:
b'AAAAAAAAA\x00'
b'AAAAAAAAA'
I'm figuring out the Python/C API for a more complex task. Initially, I wrote a simple example of adding two ndarrays of shape = (2,3) and type = float32.
I am able to pass two numpy arrays into c functions, read their dimensions and data and perform custom addion on data. But when I try to wrap the resulting data using PyArray_SimpleNewFromData, code hangs (returns NULL?)
To replicate the issue, create three files: mymath.c, setup.py, test.py in a folder as follows and run test.py (it runs setup.py to compile and install the module and then runs a simple test).
I'm using python in windows, inside an anaconda environment. I'm new to the Python/C API. So, any help would be much appreciated.
// mymath.c
#include <Python.h>
#include <stdio.h>
#include "numpy/arrayobject.h"
#include "numpy/npy_math.h"
#include <math.h>
#include <omp.h>
/*
C functions
*/
float* arr_add(float* d1, float* d2, int M, int N){
float * result = (float *) malloc(sizeof(float)*M*N);
for (int m=0; m<M; m++)
for (int n=0; n<N; n++)
result [m*N+ n] = d1[m*N+ n] + d2[m*N+ n];
return result;
}
/*
Unwrap apply and wrap pyObjects
*/
void capsule_cleanup(PyObject *capsule) {
void *memory = PyCapsule_GetPointer(capsule, NULL);
free(memory);
}
// add two 2d arrays (float32)
static PyObject *arr_add_fn(PyObject *self, PyObject *args)
{
PyArrayObject *arr1, *arr2;
if (!PyArg_ParseTuple(args, "OO", &arr1, &arr2))
return NULL;
// get data as flat list
float *d1, *d2;
d1 = (float *) arr1->data;
d2 = (float *) arr2->data;
int M, N;
M = (int)arr1->dimensions[0];
N = (int)arr1->dimensions[1];
printf("Dimensions, %d, %d \n\n", M,N);
PyObject *result, *capsule;
npy_intp dim[2];
dim[0] = M;
dim[1] = N;
float * d3 = arr_add(d1, d2, M, N);
result = PyArray_SimpleNewFromData(2, dim, NPY_FLOAT, (void *)d3);
if (result == NULL)
return NULL;
// -----------This is not executed. code hangs--------------------
for (int m=0; m<M; m++)
for (int n=0; n<N; n++)
printf("%f \n", d3[m*N+n]);
capsule = PyCapsule_New(d3, NULL, capsule_cleanup);
PyArray_SetBaseObject((PyArrayObject *) result, capsule);
return result;
}
/*
Bundle functions into module
*/
static PyMethodDef MyMethods [] ={
{"arr_add", arr_add_fn, METH_VARARGS, "Array Add two numbers"},
{NULL,NULL,0,NULL}
};
/*
Create module
*/
static struct PyModuleDef mymathmodule = {
PyModuleDef_HEAD_INIT,
"mymath", "My doc of mymath", -1, MyMethods
};
PyMODINIT_FUNC PyInit_mymath(void){
return PyModule_Create(&mymathmodule);
}
# setup.py
from distutils.core import setup, Extension
import numpy
module1 = Extension('mymath',
sources = ['mymath.c'],
# define_macros = [('NPY_NO_DEPRECATED_API', 'NPY_1_7_API_VERSION')],
include_dirs=[numpy.get_include()],
extra_compile_args = ['-fopenmp'],
extra_link_args = ['-lgomp'])
setup (name = 'mymath',
version = '1.0',
description = 'My math',
ext_modules = [module1])
# test.py
import os
os.system("python .\setup.py install")
import numpy as np
import mymath
a = np.arange(6,dtype=np.float32).reshape(2,3)
b = np.arange(6,dtype=np.float32).reshape(2,3)
c = mymath.arr_add(a,b)
print(c)
I have a C library adapted for python, it is part of the project rdpy(https://github.com/citronneur/rdpy).
However, there is a memory leak, that I cant kill in python code: deleting references to objects (input=None, output=None), gc.collect(), etc. Leaks analyzers, such as pympler, also dont show objects that occupy memory. I also found that the same code is used in the rdesktop project (https://github.com/rdesktop/rdesktop/blob/master/bitmap.c), but it works correctly there, maybe it's a matter of integration with python. What in this function can make leak a memory?
/* Specific rename for RDPY integration */
#define uint8 unsigned char
#define uint16 unsigned short
#define unimpl(str, code)
#define RD_BOOL int
#define False 0
#define True 1
/* end specific rename */
......................................
static PyObject*
bitmap_decompress_wrapper(PyObject* self, PyObject* args)
{
Py_buffer output, input;
int width = 0, height = 0, bpp = 0;
if (!PyArg_ParseTuple(args, "s*iis*i", &output, &width, &height, &input, &bpp))
return NULL;
if(bitmap_decompress((uint8*)output.buf, width, height, (uint8*)input.buf, input.len, bpp) == False)
return NULL;
Py_RETURN_NONE;
}
static PyMethodDef rle_methods[] =
{
{"bitmap_decompress", bitmap_decompress_wrapper, METH_VARARGS, "decompress bitmap from microsoft rle algorithm."},
{NULL, NULL, 0, NULL}
};
PyMODINIT_FUNC
initrle(void)
{
(void) Py_InitModule("rle", rle_methods);
}
I would like to integrate C modules in the Python, so my choice fell on the interface Python.h. Everything compiled without errors and warnings, so I can not understand what the problem is.
C side:
#include <python3.5m/Python.h>
...
#define PyInt_AsLong(x) (PyLong_AsLong((x)))
typedef PyObject* Py;
static Py getSumma(Py self, Py args){
Py nums;
if (!PyArg_ParseTuple(args, "O", &nums)){
return NULL;
}
size_t numsAmount = PyList_Size(args);
int32_t summa = 0;
for (size_t i = 0; i < numsAmount; i++){
Py temp = PyList_GetItem(nums, i);
int32_t num = PyInt_AsLong(temp);
summa += num;
}
return Py_BuildValue("l", summa);
}
static PyMethodDef moduleMethods[] = {
{"getSumma", (PyCFunction)getSumma, METH_VARARGS, NULL},
{NULL, NULL, 0, NULL}
};
static PyModuleDef SummaLogic = {
PyModuleDef_HEAD_INIT,
"SummaLogic",
"",
-1,
moduleMethods
};
PyMODINIT_FUNC PyInit_SummaLogic(void){
return PyModule_Create(&SummaLogic);
}
setup.py:
from distutils.core import setup, Extension
SummaLogic = Extension("SummaLogic", sources=['SummaLogic.c'])
setup(ext_modules=[SummaLogic])
Python side:
from SummaLogic import getSumma
if __name__ == "__main__":
a = [1, 2, 3]
b = getSumma(a)
print(b)
It seems right, but when I start it in terminal - nothing happens, just hanging without any activity. What could I miss?
It boils down to PyList_Size and that you don't check for errors there.
You probably wanted to use it on nums, not args as argument. However you used on args and a very interesting thing happened:
args is a tuple,
therefore PyList_Size failed and returned -1
that -1 which was cast to an unsigned size_t which probably resulted in a very huge number, probably 2**64-1
therefore your iteration runs a "very long time" because it takes quite a while to iterate over 2**64-1 items (apart from all the out-of-bound memory accesses).
The quick fix would be to use:
Py_ssize_t listlength = PyList_Size(nums); /* nums instead of args */
if (listlength == -1) { /* check for errors */
return NULL;
}
size_t numsAmount = (size_t)listlength /* cast to size_t only after you checked for errors */
However you should check what the error conditions are and test for them after every python C API function call otherwise you'll get a lot of undefined behaviours. Also I probably would stick to the defined return types instead of int32_t (PyInt_AsLong returns long so you might get weird casting errors there as well!), size_t, ... and the typedef PyObject* Py; makes things really tricky for someone who regularly writes C extensions.