We Keep Coding

How set a C char** pointer in Python

I want to set a C char** pointer, called results, in Python. The variable is in a dll I have loaded. I want to set results so that it points to a string in Python. I want to get the string I created in Python (or at least a copy of it since ctypes does a lot of copying) to be pointed to by the C variable results. So I have in Python product_class = (ctypes.c_char_p)(b"321"). I want to set results to the value "321".
Here is the code I have written. It does not work. It does not even change the C-variable results.
# py_parse_pdl_func function is a callback which is called from a c dll which has been loaded into the python prorgram.
# Here is the declaration of the callback in c
# typedef int (*tsl_pdl_cb_t)(void *pz_prv, const char **results, const char* query);
# so am trying to set results to point to a string "321"
def py_parse_pdl_func(pz_prv, py_results, query):
global product_class_void
product_class = (ctypes.c_char_p)(b"321")
product_class_void = ctypes.cast(product_class, ctypes.c_void_p)
py_results.contents = ctypes.c_long(product_class_void.value)
return 1

Here's a reproducible example. You may need to keep a reference to the string returned since Python could deallocate it at any time.
test.c
#include <stdio.h>
typedef int (*tsl_pdl_cb_t)(void *pz_prv, const char **results, const char* query);
__declspec(dllexport)
int function_using_callback(tsl_pdl_cb_t callback) {
char* results = NULL;
int retval = 0;
if(callback)
retval = callback(NULL, &results, "the query");
printf("results = '%s'\n", results);
return retval;
}
test.py
import ctypes as ct
CALLBACK = ct.CFUNCTYPE(ct.c_int, ct.c_void_p, ct.POINTER(ct.c_char_p), ct.c_char_p)
dll = ct.CDLL('./test')
dll.function_using_callback.argtypes = CALLBACK,
dll.function_using_callback.restype = ct.c_int
#CALLBACK
def py_parse_pdl_func(pz_prv, py_results, query):
py_results[0] = b'321'
return 1
retval = dll.function_using_callback(py_parse_pdl_func)
print('retval =', retval)
Output:
results = '321'
retval = 1

Loading c DLL with python, problems with pointer

Currently I have C++ loading DLL. I need replace C++ code with python. My problems are:
in callback function device_ID1_callback, all values seems empty, i'm guessing i did not use pointer correctly.
after call device_get_info, all values are 0, I suppose get some values none zero back.
I have tried anything I can think of for weeks but with very little luck.
To simplified the problem, here's partial of my code. Thanks for your time and help!!
in my lib.h file, i have
typedef unsigned int DeviceHandler;
typedef struct {
unsigned int fpga_version;
}DeviceInfo_t;
typedef struct {
unsigned int check_id;
float distance[256];
}MeasureResult_t;
DLLEPXORT int EXCALL device_add(DeviceHandler* outHandler, char* device_ip, MeasureModeCallback callback);
DLLEPXORT void EXCALL device_get_info(DeviceHandler handler, DeviceInfo_t* p_device_info);
in sample C++ file:
"""
void device_ID1_callback(const void *out,unsigned int out_num){
MeasureResult_t *ptr = (MeasureResult_t *)out;
printf("[ChechID:0x%x] %d pack's data\n",ptr[0].check_id,out_num);
}
void demo_callback_mode(){
int ret;
DeviceHandler device_handler;
DeviceInfo_t device_info;
ret = device_add(&device_handler,"192.168.1.2",&device_ID1_callback);
device_get_info(device_handler,&device_info);
printf("[FPGA] version : %d\n", device_info.fpga_version);
}
"""
*end of c++ *
Here's my python code:
"""
import ctypes as c
class MeasureResult_t(c.Structure):
_fields_ = [
('check_id', c.c_int),
('distance[256]', c.c_float)]
class DeviceInfo_t(c.Structure):
_fields_ = [
('fpga_version', c.c_int)
]
def device_ID1_callback(out, out_num):
print("---enter device call back function---")
print(dir(out))
print("out: ",out.contents)
#print(dir(out))
print("out_num:",out_num)
print("---exit device call back function---\n\n")
return 0
_dev = c.CDLL("./OPSensor/osp_lidar")
T_device_handler = c.c_int
T_device_handler_ptr = c.POINTER(T_device_handler)
_dev.device_add.argtypes = [T_device_handler_ptr, c.c_char_p]
_dev.device_add.restype = c.c_int
device_handler = c.c_int()
ip_val = c.c_char_p("192.168.1.2".encode('utf-8'))
out = MeasureResult_t()
out_num = c.c_int()
CMPFUNC_t = c.CFUNCTYPE(None, c.POINTER(MeasureResult_t), c.c_int)
MeasureModeCallback = CMPFUNC_t(device_ID1_callback)
ret = _dev.device_add(c.byref(device_handler), (ip_val), MeasureModeCallback(c.byref(out), out_num))
_dev.device_get_info.argtypes = [T_device_handler_ptr, c.POINTER(DeviceInfo_t)]
_dev.device_get_info.restype = c.c_void_p # assume it returns C int
p_device_info = DeviceInfo_t()
#_dev.device_get_info(c.byref(device_handler), c.byref(p_device_info)) # does not work
_dev.device_get_info((device_handler), c.byref(p_device_info)) #does not work either
print(device_handler) # I have correct device_handler value
print(p_device_info.fpga_version) # the value i got is 0, does seem right
"""

Here's my attempt at a minimal reproducible example. I implemented dummy functions that demonstrate the callback you described:
// lib.c
#define DLLEXPORT __declspec(dllexport)
#define EXCALL
typedef unsigned int DeviceHandler;
typedef struct {
unsigned int fpga_version;
} DeviceInfo_t;
typedef struct {
unsigned int check_id;
float distance[256];
} MeasureResult_t;
typedef void (*MeasureModeCallback)(MeasureResult_t*, unsigned int);
DLLEXPORT int EXCALL device_add(DeviceHandler* outHandler, char* device_ip, MeasureModeCallback callback) {
*outHandler = 123; // dummy device ID
MeasureResult_t m; // some fake measurement results
m.check_id = 456;
for(int i = 0; i < 256; ++i)
m.distance[i] = (float)(i * .25);
callback(&m, 789); // call the callback
return 1;
}
DLLEXPORT void EXCALL device_get_info(DeviceHandler handler, DeviceInfo_t* p_device_info) {
p_device_info->fpga_version = handler * 2; // fake fpga version
}
To create a callback in Python, assign the CFUNCTYPE prototype to the callback type, decorate the callback function with that type, and use that type in the callback argument definition, and the actual function name when passing it as an argument.
Also note the that float distance[256] is declared in Python as c.c_float * 256 to create an array and the differences in the .argtypes/.restype attributes for the functions. device_get_info takes a DeviceHandler, not a c.POINTER(DeviceHandler) for example.
# test.py
import ctypes as c
class MeasureResult_t(c.Structure):
_fields_ = (('check_id', c.c_uint),
('distance', c.c_float * 256))
def __repr__(self): # defines how to display this class
return f'MeasureResult_t(check_id={self.check_id}, distance=[{self.distance[0]}, ..., {self.distance[255]}])'
class DeviceInfo_t(c.Structure):
_fields_ = ('fpga_version', c.c_uint),
def __repr__(self): # defines how to display this class
return f'DeviceInfo_t(fpga_version={self.fpga_version})'
# Declare the callback type
MeasureModeCallback = c.CFUNCTYPE(None, c.POINTER(MeasureResult_t), c.c_uint)
DeviceHandler = c.c_uint
# apply the decorator so this function can be called from C
#MeasureModeCallback
def device_ID1_callback(out, out_num):
print('---enter device call back function---')
print('out: ',out.contents)
print('out_num:',out_num)
print('---exit device call back function---')
_dev = c.CDLL('./lib')
# Use the argument type
_dev.device_add.argtypes = c.POINTER(DeviceHandler), c.c_char_p, MeasureModeCallback
_dev.device_add.restype = c.c_int
_dev.device_get_info.argtypes = DeviceHandler, c.POINTER(DeviceInfo_t)
_dev.device_get_info.restype = None
device_handler = DeviceHandler()
ip_val = b'192.168.1.2'
# Use the callback function name when calling the function
ret = _dev.device_add(c.byref(device_handler), ip_val, device_ID1_callback)
device_info = DeviceInfo_t()
_dev.device_get_info(device_handler, c.byref(device_info))
print(f'{device_handler.value=}')
print(f'{device_info=}')
Output. Note the classes know how to display themselves and the fake data agrees with my implementation:
---enter device call back function---
out: MeasureResult_t(check_id=456, distance=[0.0, ..., 63.75])
out_num: 789
---exit device call back function---
device_handler.value=123
device_info=DeviceInfo_t(fpga_version=246)

Python ctypes pointer to struct as identifier without member access

I have two different C functions and I would like to use them with ctypes in Python.
One function is establishing a connection and returns a pointer to an truct. The pointer shall be used as an argument in the second function to reuse the established connection.
C Code:
customStruct * connect()
{
customStruct *obj = connection_helper();
return obj;
}
void foo(customStruct * obj)
{
foo_helper(obj);
}
Python code:
from ctypes import *
lib = CDLL("./test.dll")
obj = lib.connect()
lib.foo(obj)
Unfortunately, I retrieve access violation errors when I call lib.foo(). I could recreate the customStruct struct in Python using a class with the _fields_ attribute, but since the struct consists of many other structs and since I don't want to access the struct members in Python itself, I'm thinking about an alternative how to create an identifier that can be reused.
I can change the definition of connect() as well as foo() as I'd like. I could also create another "identifier" struct if that would allow me to not have to recreate the struct in python.
Update:
It looks like I have to use the function byref() to achieve what I want.
https://docs.python.org/3/library/ctypes.html#ctypes.byref
The documentation states "The returned object can only be used as a foreign function call parameter" but I am not sure what to return in connect() then.

If you have an opaque structure (you do not know its members, or do not want to know its members), you should still create a class to represent that struct in python. You can then use this class to properly type your functions. This will help prevent bugs where you accidentally pass the wrong object as a "CustomStruct" pointer.
For example:
from ctypes import cdll, c_int, c_void_p
mylib = cdll.LoadLibrary('mylib')
class CustomStructP(c_void_p):
# subclassing c_void_p creates an opaque pointer type that is distinct
# from c_void_p, and can only be instantiated as a pointer
pass
create = mylib.create
create.argtypes = [c_int]
create.restype = CustomStructP
display = mylib.display
display.argtypes = [CustomStructP]
display.restype = None
delete = mylib.delete
delete.argtypes = [CustomStructP]
delete.restype = None
obj = create(10)
display(obj)
delete(obj)
display(CustomStructP()) # passing a null pointer
Now, if you tried something like: display(c_void_p()), you would get:
Traceback (most recent call last):
File "C:\Users\User\Documents\python\src\main.py", line 31, in <module>
display(c_void_p())
ctypes.ArgumentError: argument 1: <class 'TypeError'>: wrong type
The C code I used was:
#include <stdio.h>
#include <stdlib.h>
struct customStruct {
int val;
};
struct customStruct *
create(int val) {
struct customStruct *obj = malloc(sizeof(struct customStruct));
obj->val = val;
return obj;
}
void
display(struct customStruct *obj) {
if (obj) {
printf("customStruct(%d) # %p\n", obj->val, obj);
}
else {
puts("customStruct is NULL");
}
}
void
delete(struct customStruct *obj) {
free(obj);
}

Like mentioned in comments already you need to set restype for the connect function and argtypes for the foo function on Python side.
In code it would look like this:
from ctypes import *
lib = cdll.LoadLibrary("some.dll")
lib.connect.restype = c_void_p
lib.foo.argtypes = c_void_p,
obj = lib.connect()
lib.foo(obj)
Test
A short test should verify that this gives the same pointer in your connection and foo function on the C side.
A slightly modified version of your code might look like this:
#include <stdlib.h>
#include <stdio.h>
typedef struct {
int x;
} customStruct;
static customStruct *connection_helper() {
return malloc(sizeof(customStruct));
}
customStruct *connect()
{
customStruct *obj = connection_helper();
printf("connect: %p\n", obj);
return obj;
}
void foo(customStruct * obj)
{
printf("foo: %p\n", obj);
//do something
}
If you run this you get something like:
connect: 0x7fa219e094a0
foo: 0x7fa219e094a0

ctypes: extract members of structure returned by C library

I am trying to use ctypes to extract a structure initialized by a C library (see for example: https://tentacles666.wordpress.com/2012/01/21/python-ctypes-dereferencing-a-pointer-to-a-c).
The 'prototype' is:
mytype * createMyType();
The structure in C is:
typedef struct
{
int a;
void *b;
}mytype;
from which in python(3)! I have created a ctypes structure thus:
class mytype(ctypes.Structure):
_fields_ = [("a", ctypes.c_int),
("b", ctypes.POINTER(None))]
The C call is:
mytype*myinstance = createMyType()
The Python call is as follows:
import ctypes
f=mylib.createMyType
f.argtypes=()
f.restype=(ctypes.POINTER(mytype),)
x=f()
The problem is that x seems to be an integer; how do I interpret this as a pointer, or - as required - extract the members of x themselves?
How do I access and then modify x.a and x.b?
[See also Accessing data from a structure returned by C function in Python using ctypes, which led nowhere]

Mainly you need c_void_p for the void* and must dereference the return with .contents.
Here's a working example (Windows)...
Edit: I added an example of casting the void pointer member.
test.h
#ifdef EXPORT
#define API __declspec(dllexport)
#else
#define API __declspec(dllimport)
#endif
typedef struct
{
int a;
void* b;
} mytype;
API mytype* createMyType();
API void destroyMyType(mytype* p);
test.c
#define EXPORT
#include <stdlib.h>
#include <stdio.h>
#include "test.h"
API mytype* createMyType()
{
int* tmp;
mytype* p = malloc(sizeof(mytype));
p->a = 5;
tmp = malloc(sizeof(int));
*tmp = 123;
p->b = tmp;
printf("%d %p\n",p->a,p->b);
return p;
}
API void destroyMyType(mytype* p)
{
free(p->b);
free(p);
}
test.py
from ctypes import *
class mytype(Structure):
_fields_ = [('a',c_int),
('b',c_void_p)]
test = CDLL('test')
createMyType = test.createMyType
createMyType.argtypes = None
createMyType.restype = POINTER(mytype)
destroyMyType = test.destroyMyType
destroyMyType.argtypes = POINTER(mytype),
destroyMyType.restype = None
t = createMyType()
print('t is',t)
print('t.a is',t.contents.a)
print('t.b is',hex(t.contents.b))
b = cast(t.contents.b,POINTER(c_int))
print('*b is',b.contents)
destroyMyType(t)
Output: Note that the void* b address output in the C code matches the integer returned by t.contents.b. The cast turns that integer into a POINTER(c_int) where the contents can be extracted.
5 00000216C0E2A5D0
t is <__main__.LP_mytype object at 0x00000216C30C4A48>
t.a is 5
t.b is 0x216c0e2a5d0
*b is c_long(123)

boost python returning tuple containing custom types

I have a C++ myObject class that I expose via boost python using a wrapper structure:
struct myObjectWrapper{
static tuple compute(myObject& o,const Container& x0, const double& t0, Container& x){
double t;
int stat = o.evaluate(x0,t0,x,t);
return make_tuple(stat,t);
}
}
BOOST_PYTHON_MODULE(myModule)
{
// not shown here is code to expose Container class
class_<myObject>("MyObject")
.def("compute",&myObjectWrapper::compute)
;
}
Container is currently defined as:
typedef std::valarray<double> Container
and is exposed to python.
Now in python I can do.
x = Container()
(status,t) = obj.compute(Container([0.,0.,0.]),0.0,x)
print status, t, x[0]
This is not very pythonic. I would prefer to do:
(status,t,x) = obj.compute(Container([0.,0.,0.]),0.0)
print status, t, x[0]
I could write an additional wrapper in python, but I would prefer to avoid adding more wrappers.
The following code does't compile:
struct myObjectWrapper{
static tuple compute(myObject& o,const Container& x0, const double& t0){
double t;
Container x;
int stat = o.evaluate(x0,t0,x,t);
return make_tuple(stat,t,x);
}
}
Also I would prefer to steal the content of the local variable x and have python manage it rather than copy it:
return make_tuple(stat,t,std::move(x));
How do I achieve this?

In short, allocate the wrapper on the free store and use the manage_new_object result convert to transfer ownership to a Python object. This will cause Boost.Python to copy the pointer when constructing the Python object, rather than copying the pointee. For more details, see this answer.
Here is an auxiliary function that will transfer ownership to a Python object:
/// #brief Transfer ownership to a Python object. If the transfer fails,
/// then object will be destroyed and an exception is thrown.
template <typename T>
boost::python::object transfer_to_python(T* t)
{
// Transfer ownership to a smart pointer, allowing for proper cleanup
// incase Boost.Python throws.
std::unique_ptr<T> ptr(t);
// Use the manage_new_object generator to transfer ownership to Python.
namespace python = boost::python;
typename python::manage_new_object::apply<T*>::type converter;
// Transfer ownership to the Python handler and release ownership
// from C++.
python::handle<> handle(converter(*ptr));
ptr.release();
return python::object(handle);
}
And one could use it as follows:
boost::python::tuple myObjectWrapper::compute(
myObject& o, const Container& x0, const double& t0)
{
auto x1 = std::make_unique<container>();
double t1 = 0;
int stat = self.evaluate(x0, t0, *x1, t1);
return boost::python::make_tuple(stat, t1, transfer_to_python(x1.release()));
}
Here is a complete example based on the original question that demonstrates using the transfer_to_python auxiliary function.
#include <boost/python.hpp>
#include <cassert>
#include <memory> // std::unique_ptr
// Mock legacy API.
struct container
{
container() {}
container(boost::python::object) {}
container(const container&)
{
// For this example, guarantee copy is not made.
assert(false);
}
};
struct my_object
{
int evaluate(container&, double&, container&, double&) { return 42; }
};
/// #brief Transfer ownership to a Python object. If the transfer fails,
/// then object will be destroyed and an exception is thrown.
template <typename T>
boost::python::object transfer_to_python(T* t)
{
// Transfer ownership to a smart pointer, allowing for proper cleanup
// incase Boost.Python throws.
std::unique_ptr<T> ptr(t);
// Use the manage_new_object generator to transfer ownership to Python.
namespace python = boost::python;
typename python::manage_new_object::apply<T*>::type converter;
// Transfer ownership to the Python handler and release ownership
// from C++.
python::handle<> handle(converter(*ptr));
ptr.release();
return python::object(handle);
}
// API wrapper.
boost::python::tuple my_object_compute(
my_object& self, container& x0, double t0)
{
auto x1 = std::make_unique<container>();
double t1 = 21;
int stat = self.evaluate(x0, t0, *x1, t1);
return boost::python::make_tuple(stat, t1, transfer_to_python(x1.release()));
}
BOOST_PYTHON_MODULE(example)
{
namespace python = boost::python;
python::class_<container>("Container")
.def(python::init<python::object>())
;
python::class_<my_object>("MyObject")
.def("compute", &my_object_compute)
;
}
Interactive usage:
>>> import example
>>> my_object = example.MyObject()
>>> status, t, x = my_object.compute(example.Container([1, 2, 3]), 4)
>>> assert(status == 42)
>>> assert(t == 21)
>>> assert(isinstance(x, example.Container))