What would be the C equivalent to this python code?
Thanks.
data = gather_me_some_data()
# where data = [ (metic, datapoints), ... ]
# and datapoints = [ (timestamp, value), ... ]
serialized_data = cPickle.dumps(data, protocol=-1)
length_prefix = struct.pack("!L", len(serialized_data))
message = length_prefix + serialized_data
C doesn't supports direct serialization mechanism because in C you can't get type information at run-time. You must yourself inject some type info at run-time and then construct required object by that type info. So define all your possible structs:
typedef struct {
int myInt;
float myFloat;
unsigned char myData[MY_DATA_SIZE];
} MyStruct_1;
typedef struct {
unsigned char myUnsignedChar;
double myDouble;
} MyStruct_2;
Then define enum which collects info about what structs in total you have:
typedef enum {
ST_MYSTRUCT_1,
ST_MYSTRUCT_2
} MyStructType;
Define helper function which lets to determine any struct size:
int GetStructSize(MyStructType structType) {
switch (structType) {
case ST_MYSTRUCT_1:
return sizeof(MyStruct_1);
case ST_MYSTRUCT_2:
return sizeof(MyStruct_2);
default:
// OOPS no such struct in our pocket
return 0;
}
}
Then define serialize function:
void BinarySerialize(
MyStructType structType,
void * structPointer,
unsigned char * serializedData) {
int structSize = GetStructSize(structType);
if (structSize != 0) {
// copy struct metadata to serialized bytes
memcpy(serializedData, &structType, sizeof(structType));
// copy struct itself
memcpy(serializedData+sizeof(structType), structPointer, structSize);
}
}
And de-serialization function:
void BinaryDeserialize(
MyStructType structTypeDestination,
void ** structPointer,
unsigned char * serializedData)
{
// get source struct type
MyStructType structTypeSource;
memcpy(&structTypeSource, serializedData, sizeof(structTypeSource));
// get source struct size
int structSize = GetStructSize(structTypeSource);
if (structTypeSource == structTypeDestination && structSize != 0) {
*structPointer = malloc(structSize);
memcpy(*structPointer, serializedData+sizeof(structTypeSource), structSize);
}
}
Serialization usage example:
MyStruct_2 structInput = {0x69, 0.1};
MyStruct_1 * structOutput_1 = NULL;
MyStruct_2 * structOutput_2 = NULL;
unsigned char testSerializedData[SERIALIZED_DATA_MAX_SIZE] = {0};
// serialize structInput
BinarySerialize(ST_MYSTRUCT_2, &structInput, testSerializedData);
// try to de-serialize to something
BinaryDeserialize(ST_MYSTRUCT_1, &structOutput_1, testSerializedData);
BinaryDeserialize(ST_MYSTRUCT_2, &structOutput_2, testSerializedData);
// determine which object was de-serialized
// (plus you will get code-completion support about object members from IDE)
if (structOutput_1 != NULL) {
// do something with structOutput_1
free(structOutput_1);
}
else if (structOutput_2 != NULL) {
// do something with structOutput_2
free(structOutput_2);
}
I think this is most simple serialization approach in C. But it has some problems:
struct must not have pointers, because you will never know how much memory one needs to allocate when serializing pointers and from where/how to serialize data into pointers.
this example has issues with system endianess - you need to be careful about how data is stored in memory - in big-endian or little-endian fashion and reverse bytes if needed [when casting char * to integal type such as enum] (...or refactor code to be more portable).
If you can use C++, there is the PicklingTools library
Related
How can I detect / check the Python-Type of a PyObject?
PyObject_HasAttrString can be used to check, but how do I get the AttrString of a given PyObject?
int modify(cv::cuda::GpuMat* cVal, py::object &pobj)
{
py::handle ptype = pobj.get_type(); //?? ...
PyObject *pobject = pobj.ptr();
auto typeCheck = PyObject_HasAttrString(pobject, "cv2.cuda_GpuMat"); //??
if (!typeCheck)
return(0);
PyObject *pobject1 = PyObject_GetAttrString( pobject, "cv2.cuda_GpuMat");
cv::cuda::GpuMat *pyVal = (cv::cuda::GpuMat *)PyLong_AsVoidPtr(pobject1);
// *pyVal = *cVal; // kidding
return(1);
}
Presumably you know what type it should be so you can check if you can convert to that type, for example: PyTuple_Check, PyUnicode_Check, etc.
Assuming you are using numpy arrays, I would look at PyArray_FROM_OTF which will return an ndarray from any Python object that can be converted into an array:
PyArrayObject* pPyArrayA = reinterpret_cast<PyArrayObject*>(PyArray_FROM_OTF(pPyA, NPY_DOUBLE, NPY_ARRAY_IN_ARRAY));
if(pPyArrayA == nullptr)
{
PyErr_SetString(PyExc_ValueError, "Failed to convert a to nd array.");
return nullptr;
}
size_t iSizeA = static_cast<size_t>(PyArray_SIZE(pPyArrayA));
const double* pcdA = static_cast<const double*>(PyArray_DATA(pPyArrayA));
Currently I have C++ loading DLL. I need replace C++ code with python. My problems are:
in callback function device_ID1_callback, all values seems empty, i'm guessing i did not use pointer correctly.
after call device_get_info, all values are 0, I suppose get some values none zero back.
I have tried anything I can think of for weeks but with very little luck.
To simplified the problem, here's partial of my code. Thanks for your time and help!!
in my lib.h file, i have
typedef unsigned int DeviceHandler;
typedef struct {
unsigned int fpga_version;
}DeviceInfo_t;
typedef struct {
unsigned int check_id;
float distance[256];
}MeasureResult_t;
DLLEPXORT int EXCALL device_add(DeviceHandler* outHandler, char* device_ip, MeasureModeCallback callback);
DLLEPXORT void EXCALL device_get_info(DeviceHandler handler, DeviceInfo_t* p_device_info);
in sample C++ file:
"""
void device_ID1_callback(const void *out,unsigned int out_num){
MeasureResult_t *ptr = (MeasureResult_t *)out;
printf("[ChechID:0x%x] %d pack's data\n",ptr[0].check_id,out_num);
}
void demo_callback_mode(){
int ret;
DeviceHandler device_handler;
DeviceInfo_t device_info;
ret = device_add(&device_handler,"192.168.1.2",&device_ID1_callback);
device_get_info(device_handler,&device_info);
printf("[FPGA] version : %d\n", device_info.fpga_version);
}
"""
*end of c++ *
Here's my python code:
"""
import ctypes as c
class MeasureResult_t(c.Structure):
_fields_ = [
('check_id', c.c_int),
('distance[256]', c.c_float)]
class DeviceInfo_t(c.Structure):
_fields_ = [
('fpga_version', c.c_int)
]
def device_ID1_callback(out, out_num):
print("---enter device call back function---")
print(dir(out))
print("out: ",out.contents)
#print(dir(out))
print("out_num:",out_num)
print("---exit device call back function---\n\n")
return 0
_dev = c.CDLL("./OPSensor/osp_lidar")
T_device_handler = c.c_int
T_device_handler_ptr = c.POINTER(T_device_handler)
_dev.device_add.argtypes = [T_device_handler_ptr, c.c_char_p]
_dev.device_add.restype = c.c_int
device_handler = c.c_int()
ip_val = c.c_char_p("192.168.1.2".encode('utf-8'))
out = MeasureResult_t()
out_num = c.c_int()
CMPFUNC_t = c.CFUNCTYPE(None, c.POINTER(MeasureResult_t), c.c_int)
MeasureModeCallback = CMPFUNC_t(device_ID1_callback)
ret = _dev.device_add(c.byref(device_handler), (ip_val), MeasureModeCallback(c.byref(out), out_num))
_dev.device_get_info.argtypes = [T_device_handler_ptr, c.POINTER(DeviceInfo_t)]
_dev.device_get_info.restype = c.c_void_p # assume it returns C int
p_device_info = DeviceInfo_t()
#_dev.device_get_info(c.byref(device_handler), c.byref(p_device_info)) # does not work
_dev.device_get_info((device_handler), c.byref(p_device_info)) #does not work either
print(device_handler) # I have correct device_handler value
print(p_device_info.fpga_version) # the value i got is 0, does seem right
"""
Here's my attempt at a minimal reproducible example. I implemented dummy functions that demonstrate the callback you described:
// lib.c
#define DLLEXPORT __declspec(dllexport)
#define EXCALL
typedef unsigned int DeviceHandler;
typedef struct {
unsigned int fpga_version;
} DeviceInfo_t;
typedef struct {
unsigned int check_id;
float distance[256];
} MeasureResult_t;
typedef void (*MeasureModeCallback)(MeasureResult_t*, unsigned int);
DLLEXPORT int EXCALL device_add(DeviceHandler* outHandler, char* device_ip, MeasureModeCallback callback) {
*outHandler = 123; // dummy device ID
MeasureResult_t m; // some fake measurement results
m.check_id = 456;
for(int i = 0; i < 256; ++i)
m.distance[i] = (float)(i * .25);
callback(&m, 789); // call the callback
return 1;
}
DLLEXPORT void EXCALL device_get_info(DeviceHandler handler, DeviceInfo_t* p_device_info) {
p_device_info->fpga_version = handler * 2; // fake fpga version
}
To create a callback in Python, assign the CFUNCTYPE prototype to the callback type, decorate the callback function with that type, and use that type in the callback argument definition, and the actual function name when passing it as an argument.
Also note the that float distance[256] is declared in Python as c.c_float * 256 to create an array and the differences in the .argtypes/.restype attributes for the functions. device_get_info takes a DeviceHandler, not a c.POINTER(DeviceHandler) for example.
# test.py
import ctypes as c
class MeasureResult_t(c.Structure):
_fields_ = (('check_id', c.c_uint),
('distance', c.c_float * 256))
def __repr__(self): # defines how to display this class
return f'MeasureResult_t(check_id={self.check_id}, distance=[{self.distance[0]}, ..., {self.distance[255]}])'
class DeviceInfo_t(c.Structure):
_fields_ = ('fpga_version', c.c_uint),
def __repr__(self): # defines how to display this class
return f'DeviceInfo_t(fpga_version={self.fpga_version})'
# Declare the callback type
MeasureModeCallback = c.CFUNCTYPE(None, c.POINTER(MeasureResult_t), c.c_uint)
DeviceHandler = c.c_uint
# apply the decorator so this function can be called from C
#MeasureModeCallback
def device_ID1_callback(out, out_num):
print('---enter device call back function---')
print('out: ',out.contents)
print('out_num:',out_num)
print('---exit device call back function---')
_dev = c.CDLL('./lib')
# Use the argument type
_dev.device_add.argtypes = c.POINTER(DeviceHandler), c.c_char_p, MeasureModeCallback
_dev.device_add.restype = c.c_int
_dev.device_get_info.argtypes = DeviceHandler, c.POINTER(DeviceInfo_t)
_dev.device_get_info.restype = None
device_handler = DeviceHandler()
ip_val = b'192.168.1.2'
# Use the callback function name when calling the function
ret = _dev.device_add(c.byref(device_handler), ip_val, device_ID1_callback)
device_info = DeviceInfo_t()
_dev.device_get_info(device_handler, c.byref(device_info))
print(f'{device_handler.value=}')
print(f'{device_info=}')
Output. Note the classes know how to display themselves and the fake data agrees with my implementation:
---enter device call back function---
out: MeasureResult_t(check_id=456, distance=[0.0, ..., 63.75])
out_num: 789
---exit device call back function---
device_handler.value=123
device_info=DeviceInfo_t(fpga_version=246)
I have a list of strings in python which I'm trying to pass down to a C extension for character analysis. I've gotten so far as to have the list broken up into their individual string PyObjects. Next, I'm hoping to split these strings into their individual characters so that every string PyObject is now a corresponding C-type character array. I can't seem to figure out how to do this though.
Here's what I have so far: Currently after building the .pyd file it will return a list of 1's as a filler to Python (so everything else works), I just don't know how to split a string PyObject into the C-type character array.
--- cExt.c ---
#include <Python.h>
#include <stdio.h>
static int *CitemCheck(PyObject *commandString, int commandStringLength) {
// HAALP
//char* commandChars = (char*) malloc(commandStringLength*sizeof(char*));
// char c[] = PyString_AsString("c", commandString);
// printf("%c" , c);
// printf("%s", PyString_AsString(commandString));
// for (int i=0; i<sizeof(commandChars)/sizeof(*commandChars); i++) {
// printf("%s", PyString_AsString(commandString));
// printf("%c", commandChars[i]);
// }
return 1; // TODO: RETURN PROPER RESULTANT
}
static PyObject *ClistCheck(PyObject *commandList, int commandListLength) {
PyObject *results = PyList_New(commandListLength);
for (int index = 0; index < commandListLength; index++) {
PyObject *commandString;
commandString = PyList_GetItem(commandList, index);
int commandStringLength = PyObject_Length(commandString);
// CitemCheck should take string PyObject and its length as int
int x = CitemCheck(commandString, commandStringLength);
PyObject* pyItem = Py_BuildValue("i", x);
PyList_SetItem(results, index, pyItem);
}
return results;
}
static PyObject *parseListCheck(PyObject *self, PyObject *args) {
PyObject *commandList;
int commandListLength;
if (!PyArg_ParseTuple(args, "O", &commandList)){
return NULL;
}
commandListLength = PyObject_Length(commandList);
return Py_BuildValue("O", ClistCheck(commandList, commandListLength));
}
static char listCheckDocs[] =
""; // TODO: ADD DOCSTRING
static PyMethodDef listCheck[] = {
{"listCheck", (PyCFunction) parseListCheck, METH_VARARGS, listCheckDocs},
{NULL,NULL,0,NULL}
};
static struct PyModuleDef DCE = {
PyModuleDef_HEAD_INIT,
"listCheck",
NULL,
-1,
listCheck
};
PyMODINIT_FUNC PyInit_cExt(void){
return PyModule_Create(&DCE);
}
for reference, my temporary extension build file:
--- _c_setup.py ---
(located in same folder as cExt.c)
"""
to build C files, pass:
python _c_setup.py build_ext --inplace clean --all
in command prompt which is cd'd to the file's dierctory
"""
import glob
from setuptools import setup, Extension, find_packages
from os import path
here = path.abspath(path.dirname(__file__))
files = [path.split(x)[1] for x in glob.glob(path.join(here, '**.c'))]
extensions = [Extension(
path.splitext(x)[0], [x]
) for x in files]
setup(
ext_modules = extensions,
)
You can use PyUnicode_AsEncodedString, which
Encode a Unicode object and return the result as Python bytes object. encoding and errors have the same meaning as the parameters of the same name in the Unicode encode() method. The codec to be used is looked up using the Python codec registry. Return NULL if an exception was raised by the codec.
see https://docs.python.org/3/c-api/unicode.html#c.PyUnicode_AsEncodedString
Then with PyBytes_AsString you get a pointer to internal buffer with a terminating NUL byte. This buffer must neither be deallocated nor modified. If you need a copy you could use e.g. strdup.
see https://docs.python.org/3/c-api/bytes.html#c.PyBytes_AsString
Slightly modifying your code it could look like this:
PyObject *encodedString = PyUnicode_AsEncodedString(commandString, "UTF-8", "strict");
if (encodedString) { //returns NULL if an exception was raised
char *commandChars = PyBytes_AsString(encodedString); //pointer refers to the internal buffer of encodedString
if(commandChars) {
printf("the string '%s' consists of the following chars:\n", commandChars);
for (int i = 0; commandChars[i] != '\0'; i++) {
printf("%c ", commandChars[i]);
}
printf("\n");
}
Py_DECREF(encodedString);
}
If one would test with:
import cExt
fruits = ["apple", "pears", "cherry", "pear", "blueberry", "strawberry"]
res = cExt.listCheck(fruits)
print(res)
The output would be:
the string 'apple' consists of the following chars:
a p p l e
the string 'pears' consists of the following chars:
p e a r s
the string 'cherry' consists of the following chars:
c h e r r y
the string 'pear' consists of the following chars:
p e a r
the string 'blueberry' consists of the following chars:
b l u e b e r r y
the string 'strawberry' consists of the following chars:
s t r a w b e r r y
[1, 1, 1, 1, 1, 1]
Side note not directly related to the question:
Your CitemCheck function returns a pointer to int, but if looking at how it is called, it seems that you want to return an int value. The function signature should look more like this:
static int CitemCheck(PyObject *commandString, int commandStringLength)
(note the removed * after int).
I have a struct:
struct some_struct_s {
int arg1;
int arg2;
};
I have a C function:
int func(some_struct_s *output);
Both are %included into my SWIG file.
I want some_struct_s *output to be treated like an output parameter. Python example:
int_val, some_struct_output = func()
"Output parameters" is covered in the manual for POD-types (sec 10.1.3), but not for non-POD types.
How do I tell SWIG I want some_struct_s *output to be an output parameter?
From the documentation:
11.5.7 "argout" typemap
The "argout" typemap is used to return values from arguments. This is most commonly used to write wrappers for C/C++ functions that need to return multiple values. The "argout" typemap is almost always combined with an "in" typemap---possibly to ignore the input value....
Here's a complete example for your code (no error checking for brevity):
%module test
// Declare an input typemap that suppresses requiring any input and
// declare a temporary stack variable to hold the return data.
%typemap(in,numinputs=0) some_struct_s* (some_struct_s tmp) %{
$1 = &tmp;
%}
// Declare an output argument typemap. In this case, we'll use
// a tuple to hold the structure data (no error checking).
%typemap(argout) some_struct_s* (PyObject* o) %{
o = PyTuple_New(2);
PyTuple_SET_ITEM(o,0,PyLong_FromLong($1->arg1));
PyTuple_SET_ITEM(o,1,PyLong_FromLong($1->arg2));
$result = SWIG_Python_AppendOutput($result,o);
%}
// Instead of a header file, we'll just declare this code inline.
// This includes the code in the wrapper, as well as telling SWIG
// to create wrappers in the target language.
%inline %{
struct some_struct_s {
int arg1;
int arg2;
};
int func(some_struct_s *output) {
output->arg1 = 1;
output->arg2 = 2;
return 0;
}
%}
Demo below. Note that the int return value of zero as well as the output parameter as a tuple are returned as a list.
>>> import test
>>> test.func()
[0, (1, 2)]
If you don't want typemaps, you can also inject code to create the object and return it to hide it from the user:
%module test
%rename(_func) func; // Give the wrapper a different name
%inline %{
struct some_struct_s {
int arg1;
int arg2;
};
int func(struct some_struct_s *output)
{
output->arg1 = 1;
output->arg2 = 2;
return 0;
}
%}
// Declare your interface
%pythoncode %{
def func():
s = some_struct_s()
r = _func(s)
return r, s
%}
Demo:
>>> import test
>>> r, s = test.func()
>>> r
0
>>> s
<test.some_struct_s; proxy of <Swig Object of type 'some_struct_s *' at 0x000001511D70A880> >
>>> s.arg1
1
>>> s.arg2
2
You can make the typemap language agnostic if you carefully select SWIG macros:
%module test
%typemap(in,numinputs=0) struct some_struct_s *output %{
$1 = malloc(sizeof(struct some_struct_s));
%}
%typemap(argout) struct some_struct_s* output {
// The last parameter passes ownership of the pointer
// to Python so it will be freed when the object's
// reference count goes to zero.
%append_output(SWIG_NewPointerObj($1, $1_descriptor, SWIG_POINTER_OWN));
}
%inline %{
struct some_struct_s {
int arg1;
int arg2;
};
int func(struct some_struct_s *output)
{
output->arg1 = 1;
output->arg2 = 2;
return 0;
}
%}
Demo:
>>> import test
>>> r, s = test.func()
>>> r
0
>>> s
<test.some_struct_s; proxy of <Swig Object of type 'some_struct_s *' at 0x000001DD0425A700> >
>>> s.arg1
1
>>> s.arg2
2
I am creating a Python wrapper for a C DLL using Python ctypes.
In the Python code below I am creating a array connectionString of c_ubyte that I need to fill int the individual. For example 1,2,3,4,5,6... This connection string is passed to the DLL's DoCallBack function and printed. A buffer is created for the callback function to fill in and everything is passed to the python call back function.
I am looking for a way to update the connectionString bytes before passing them to the DLL's DoCallBack.
Then how to extract the bytes from the connectionString in the python callbackFnk function.
I am looking for a way to update the bytes in outBuffer from the callbackFnk python function
A continuation of this question
In python how do I set the value of a LP_c_ubyte
C DLL Code
typedef void(*FPCallback)(unsigned char * outBuffer, unsigned short MaxOutBufferLength, unsigned char * connectionString);
FPCallback g_Callback;
extern "C" __declspec( dllexport ) void RegisterCallback(void(*p_Callback)( unsigned char * outBuffer, unsigned short MaxOutBufferLength, unsigned char * connectionString)) {
g_Callback = p_Callback ;
}
extern "C" __declspec( dllexport ) void DoCallBack( unsigned char connectionString) {
printf( "connectionString=[%02x %02x %02x %02x %02x %02x...]\n", connectionString[0], connectionString[1], connectionString[2], connectionString[3], connectionString[4], connectionString[5] );
const unsigned short MAX_BUFFER_SIZE = 6 ;
unsigned char outBuffer[MAX_BUFFER_SIZE];
g_Callback( outBuffer, MAX_BUFFER_SIZE, connectionString, 6 );
// Print the results.
printf( "buffer=[%02x %02x %02x %02x %02x %02x...]\n", buffer[0], buffer[1], buffer[2], buffer[3], buffer[4], buffer[5] );
}
Python code
def callbackFnk( outBuffer, outBufferMaxSize, connectionString )
# (Q2) How do I extract individual bytes of the connectionString?
# (Q3) How do I update individual bytes of the out buffer?
customDLL = cdll.LoadLibrary ("customeDLL.dll")
# RegisterCallback
CustomDLLCallbackFUNC = CFUNCTYPE(None, POINTER( c_ubyte), c_ushort, POINTER( c_ubyte) )
CustomDLLCallback_func = CustomDLLCallbackFUNC( callbackFnk )
RegisterCallback = customDLL.RegisterCallback
RegisterCallback.argtypes = [ CustomDLLCallbackFUNC ]
RegisterCallback( CustomDLLCallback_func )
# DoCallBack
DoCallBack = customDLL.DoCallBack
DoCallBack.argtypes = [ POINTER( c_ubyte) ]
connectionString = c_ubyte(6)
# (Q1) How do I update this array of bytes?
# Call the callback
DoCallBack(connectionString)
The OP's example has a number of errors and doesn't compile, so I put this together. I assume connectionString is just a nul-terminated input string, and demonstrate updating the output string in the callback.
Note with an input string, c_char_p can be the type and a Python byte string can be passed. c_wchar_p is used for Python Unicode strings. The string must not be modified in the C code. The callback will receive it as a Python string as well, making it easy to read.
The output buffer can just be indexed, being careful to not index past the length of the buffer. Output buffers allocated by the caller should always be passed as a pointer-and-length.
C++ DLL
#include <stdio.h>
typedef void (*CALLBACK)(const char* string, unsigned char* buffer, size_t size);
CALLBACK g_pCallback;
extern "C" __declspec(dllexport) void RegisterCallback(CALLBACK pCallback) {
g_pCallback = pCallback;
}
extern "C" __declspec(dllexport) void DoCallBack(char* string) {
unsigned char buf[6];
printf("string = %s\n", string);
g_pCallback(string, buf, sizeof(buf));
printf("buf = [%02x %02x %02x %02x %02x %02x]\n", buf[0], buf[1], buf[2], buf[3], buf[4], buf[5]);
}
Python
from ctypes import *
CALLBACK = CFUNCTYPE(None,c_char_p,POINTER(c_ubyte),c_size_t)
#CALLBACK
def callback(string,buf,length):
print(string)
for i in range(length):
buf[i] = i * 2
dll = CDLL('test')
# RegisterCallback
RegisterCallback = dll.RegisterCallback
RegisterCallback.argtypes = [CALLBACK]
RegisterCallback.restype = None
RegisterCallback(callback)
# DoCallBack
DoCallBack = dll.DoCallBack
DoCallBack.argtypes = [c_char_p]
DoCallBack.restype = None
DoCallBack(b'test string')
Output
string = test string
b'test string'
buf = [00 02 04 06 08 0a]