We Keep Coding

Numpy Python/C API - PyArray_SimpleNewFromData hangs

I'm figuring out the Python/C API for a more complex task. Initially, I wrote a simple example of adding two ndarrays of shape = (2,3) and type = float32.
I am able to pass two numpy arrays into c functions, read their dimensions and data and perform custom addion on data. But when I try to wrap the resulting data using PyArray_SimpleNewFromData, code hangs (returns NULL?)
To replicate the issue, create three files: mymath.c, setup.py, test.py in a folder as follows and run test.py (it runs setup.py to compile and install the module and then runs a simple test).
I'm using python in windows, inside an anaconda environment. I'm new to the Python/C API. So, any help would be much appreciated.
​
// mymath.c
#include <Python.h>
#include <stdio.h>
#include "numpy/arrayobject.h"
#include "numpy/npy_math.h"
#include <math.h>
#include <omp.h>
/*
C functions
*/
float* arr_add(float* d1, float* d2, int M, int N){
float * result = (float *) malloc(sizeof(float)*M*N);
for (int m=0; m<M; m++)
for (int n=0; n<N; n++)
result [m*N+ n] = d1[m*N+ n] + d2[m*N+ n];
return result;
}
/*
Unwrap apply and wrap pyObjects
*/
void capsule_cleanup(PyObject *capsule) {
void *memory = PyCapsule_GetPointer(capsule, NULL);
free(memory);
}
// add two 2d arrays (float32)
static PyObject *arr_add_fn(PyObject *self, PyObject *args)
{
PyArrayObject *arr1, *arr2;
if (!PyArg_ParseTuple(args, "OO", &arr1, &arr2))
return NULL;
// get data as flat list
float *d1, *d2;
d1 = (float *) arr1->data;
d2 = (float *) arr2->data;
int M, N;
M = (int)arr1->dimensions[0];
N = (int)arr1->dimensions[1];
printf("Dimensions, %d, %d \n\n", M,N);
PyObject *result, *capsule;
npy_intp dim[2];
dim[0] = M;
dim[1] = N;
float * d3 = arr_add(d1, d2, M, N);
result = PyArray_SimpleNewFromData(2, dim, NPY_FLOAT, (void *)d3);
if (result == NULL)
return NULL;
// -----------This is not executed. code hangs--------------------
for (int m=0; m<M; m++)
for (int n=0; n<N; n++)
printf("%f \n", d3[m*N+n]);
capsule = PyCapsule_New(d3, NULL, capsule_cleanup);
PyArray_SetBaseObject((PyArrayObject *) result, capsule);
return result;
}
/*
Bundle functions into module
*/
static PyMethodDef MyMethods [] ={
{"arr_add", arr_add_fn, METH_VARARGS, "Array Add two numbers"},
{NULL,NULL,0,NULL}
};
/*
Create module
*/
static struct PyModuleDef mymathmodule = {
PyModuleDef_HEAD_INIT,
"mymath", "My doc of mymath", -1, MyMethods
};
PyMODINIT_FUNC PyInit_mymath(void){
return PyModule_Create(&mymathmodule);
}
​
# setup.py
from distutils.core import setup, Extension
import numpy
module1 = Extension('mymath',
sources = ['mymath.c'],
# define_macros = [('NPY_NO_DEPRECATED_API', 'NPY_1_7_API_VERSION')],
include_dirs=[numpy.get_include()],
extra_compile_args = ['-fopenmp'],
extra_link_args = ['-lgomp'])
setup (name = 'mymath',
version = '1.0',
description = 'My math',
ext_modules = [module1])
​
# test.py
import os
os.system("python .\setup.py install")
import numpy as np
import mymath
a = np.arange(6,dtype=np.float32).reshape(2,3)
b = np.arange(6,dtype=np.float32).reshape(2,3)
c = mymath.arr_add(a,b)
print(c)

Error in method argv[] SWIG C++ / Python, what happens?

I'm using swig to connect a C ++ function with my Python libraries. I managed to compile everything and create the .so file. But after having done the import of my C ++ function in python I have some errors on the argv ... maybe I'm wrong to pass the values to it from the outside or maybe when I compiled and created the .i swig file I got something wrong. The C ++ function creates a TCP socket and sends a Hex code to a device that replies with another Hex code, from a terminal with the C ++ code i write:
connect_PE_func 192.168.1.170 600000060
and it works perfectly. and it works perfectly. I would expect a similar syntax once I set my c ++ function via swig in python, type:
answer= connect_PE_func.connect_PE_func("192.168.1.170", 600000060,2)
but I get this error:
test=connect_PE_func.connect_pe_func(["192.168.2.170"],["2600000026"])
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-12-f58f79b27fe5> in <module>
----> 1 test=connect_PE_func.connect_pe_func(["192.168.2.170"],["2600000026"])
~/mqtt_atenapy/C_connect_PE_dev/test_cpython/connect_PE_func.py in connect_pe_func(argv, argc)
64
65 def connect_pe_func(argv, argc):
---> 66 return _connect_PE_func.connect_pe_func(argv, argc)
67
68
TypeError: in method 'connect_pe_func', argument 1 of type 'char *[]'
I attach the code to the files .c .h e .i which I used with swig to get the .so.
thanks.
connect_PE.func.c :
#include <stdio.h>
#include <errno.h>
#include <string>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <cstring>
#include <unistd.h>
#include <iostream>
#include <unistd.h>
#include <sstream>
#include "connect_PE_func.h"
using namespace std;
// to compile gcc connect_PE_func.cpp -lstdc++ -c
char* connect_pe_func(char *argv[],int argc)
{
int sockfd, n;
int connected = 0;
struct sockaddr_in servaddr;
std::string serveraddr = argv[1];
sockfd = socket(AF_INET, SOCK_STREAM, 0);
bzero(&servaddr, sizeof(servaddr));
servaddr.sin_family = AF_INET;
servaddr.sin_addr.s_addr = inet_addr(serveraddr.c_str());
servaddr.sin_port = htons(9761);
connect(sockfd, (struct sockaddr *)&servaddr, sizeof(servaddr));
std::string pref_hex;
std::string hex("0x");
std::string test = argv[2];
size_t numbytes = test.size() / 2;
uint8_t command[numbytes];
for (size_t w = 0, x = 0; w < numbytes; ++w, x += 2)
{
pref_hex = hex + test.substr(x, 2);
cout << pref_hex;
command[w] = stoi(pref_hex, nullptr, 16);
//cout << test.substr(x, 2);
//cout << "\n";
//cout << command[w];
//cout << "\n";
}
//uint8_t command[] = {0x26, 0x00, 0x00, 0x00, 0x26};
int bytes_to_send = sizeof(command);
send(sockfd, command, bytes_to_send, 0);
uint8_t output_command[numbytes];
recv(sockfd, output_command, bytes_to_send, 0);
char test_out[10];
for (size_t w = 0, x = 0; w < numbytes; ++w, x += 2)
{
test_out[x] = (char)output_command[w];
//cout << unsigned(test_out[x]);
}
return test_out;
}
connect_PE_func.h:
// file: connect_PE_func.h
char* connect_pe_func(char *argv[], int argc);
connect_PE_func.i:
/* file: connect_PE_func.i */
%module connect_PE_func
%{
/* Everything in this block will be copied in the wrapper file. We include the C header file necessary to compile the interface
*/
#include "connect_PE_func.h"
// extern char *connect_pe_func(int argc, char *argv[]);
%}
/* list functions to be interfaced: */
char* connect_pe_func(char *argv[], int argc);

You've misplaced a comma func(["192.168.2.170"] , ["2600000026"])
My guess is that you need to pass func(["192.168.2.170", "2600000026"], 2)

ctypes: extract members of structure returned by C library

I am trying to use ctypes to extract a structure initialized by a C library (see for example: https://tentacles666.wordpress.com/2012/01/21/python-ctypes-dereferencing-a-pointer-to-a-c).
The 'prototype' is:
mytype * createMyType();
The structure in C is:
typedef struct
{
int a;
void *b;
}mytype;
from which in python(3)! I have created a ctypes structure thus:
class mytype(ctypes.Structure):
_fields_ = [("a", ctypes.c_int),
("b", ctypes.POINTER(None))]
The C call is:
mytype*myinstance = createMyType()
The Python call is as follows:
import ctypes
f=mylib.createMyType
f.argtypes=()
f.restype=(ctypes.POINTER(mytype),)
x=f()
The problem is that x seems to be an integer; how do I interpret this as a pointer, or - as required - extract the members of x themselves?
How do I access and then modify x.a and x.b?
[See also Accessing data from a structure returned by C function in Python using ctypes, which led nowhere]

Mainly you need c_void_p for the void* and must dereference the return with .contents.
Here's a working example (Windows)...
Edit: I added an example of casting the void pointer member.
test.h
#ifdef EXPORT
#define API __declspec(dllexport)
#else
#define API __declspec(dllimport)
#endif
typedef struct
{
int a;
void* b;
} mytype;
API mytype* createMyType();
API void destroyMyType(mytype* p);
test.c
#define EXPORT
#include <stdlib.h>
#include <stdio.h>
#include "test.h"
API mytype* createMyType()
{
int* tmp;
mytype* p = malloc(sizeof(mytype));
p->a = 5;
tmp = malloc(sizeof(int));
*tmp = 123;
p->b = tmp;
printf("%d %p\n",p->a,p->b);
return p;
}
API void destroyMyType(mytype* p)
{
free(p->b);
free(p);
}
test.py
from ctypes import *
class mytype(Structure):
_fields_ = [('a',c_int),
('b',c_void_p)]
test = CDLL('test')
createMyType = test.createMyType
createMyType.argtypes = None
createMyType.restype = POINTER(mytype)
destroyMyType = test.destroyMyType
destroyMyType.argtypes = POINTER(mytype),
destroyMyType.restype = None
t = createMyType()
print('t is',t)
print('t.a is',t.contents.a)
print('t.b is',hex(t.contents.b))
b = cast(t.contents.b,POINTER(c_int))
print('*b is',b.contents)
destroyMyType(t)
Output: Note that the void* b address output in the C code matches the integer returned by t.contents.b. The cast turns that integer into a POINTER(c_int) where the contents can be extracted.
5 00000216C0E2A5D0
t is <__main__.LP_mytype object at 0x00000216C30C4A48>
t.a is 5
t.b is 0x216c0e2a5d0
*b is c_long(123)

Pass Variable from Python to C

I have Python Script embedded in C which I run in a thread. I need to pass the variable 'a' from the Python-Class 'Detect Motion' to my C program continuously. (Not as a return value)
I know I could do this with a fifo or something like that, but is there a way to pass it directly to C, maybe by calling a C function?
C:
#include <Python.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <pthread.h>
pthread_t mythread;
void *ThreadProc();
PyObject *pName, *pModule, *pDict, *pFunc, *pFunc2;
int main(int argc, char *argv[])
{
py_callback = PyCFunction_New(&callback_descr, NULL);
char *script = "motion";
char *functionUse = "get_values";
Py_Initialize();
pName = PyString_FromString(script);
pModule = PyImport_Import(pName);
// pDict and pFunc are borrowed references
pDict = PyModule_GetDict(pModule);
pFunc = PyDict_GetItemString(pDict, functionUse);
// POSIX code
pthread_create( &mythread, NULL, ThreadProc, NULL);
// Random testing code
for(int i = 0; i < 10; i++)
{
printf("Printed from the main thread.\n");
sleep(1);
}
printf("Main Thread waiting for My Thread to complete...\n");
// Join and wait for the created thread to complete...
// POSIX code
pthread_join(mythread, NULL);
printf("Main thread finished gracefully.\n");
return 0;
}
void *ThreadProc()
{
if (PyCallable_Check(pFunc))
{
PyObject_CallObject(pFunc, NULL);
}
else {
PyErr_Print();
}
// Clean up
Py_DECREF(pModule);
Py_DECREF(pName);
Py_Finalize();
printf("My thread is finishing...\n");
}
Python:
import numpy as np
import picamera
import picamera.array
class DetectMotion(picamera.array.PiMotionAnalysis):
def analyse(self, a):
a = np.sqrt(
np.square(a['x'].astype(np.float)) +
np.square(a['y'].astype(np.float))
).clip(0, 255).astype(np.uint8)
# If there're more than 10 vectors with a magnitude greater
# than 60, then say we've detected motion
print a
if (a > 60).sum() > 10:
print 'Motion detected!'
def get_values():
with picamera.PiCamera() as camera:
with DetectMotion(camera) as output:
camera.resolution = (640, 480)
camera.start_preview()
camera.start_recording(
'/dev/null', format='h264', motion_output=output)
camera.wait_recording(10)
camera.stop_recording()
camera.stop_preview()

What methods can I use to return a struct to a Python Ctypes call to the function in a shared object?

I have the following C file that I am compiling to a shared object. I then load the .so shared object via ctypes in python. I can call the function from ctypes, and the function prints the correct temp and humidity, however I can't seem to get the struct back from the main code. How can I get the struct back from the C function and how can I retrieve the fields from it within python.
#!/bin/python
from ctypes import *
class HMTEMP(Structure):
_fields_ = [ ("temp", c_double) , ("humidity", c_double) ]
dhtlib = 'libdht4py.so'
hlibc = CDLL(dhtlib)
HMTEMP = hlibc.readDHT()
print HMTEMP.temp
#define BCM2708_PERI_BASE 0x20000000
#define GPIO_BASE (BCM2708_PERI_BASE + 0x200000) /* GPIO controller */
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <dirent.h>
#include <fcntl.h>
#include <assert.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <bcm2835.h>
#include <unistd.h>
#define MAXTIMINGS 100
struct DHStruct {
double temp;
double humidity;
} ;
struct DHStruct readDHT();
int bits[250], data[100];
int bitidx = 0;
struct DHStruct readDHT() {
bcm2835_init() ;
int type = 11 ;
int pin = 4 ;
struct DHStruct dhts;
int counter = 0;
int laststate = HIGH;
int j=0;
// Set GPIO pin to output
bcm2835_gpio_fsel(pin, BCM2835_GPIO_FSEL_OUTP);
bcm2835_gpio_write(pin, HIGH);
usleep(500000); // 500 ms
bcm2835_gpio_write(pin, LOW);
usleep(20000);
bcm2835_gpio_fsel(pin, BCM2835_GPIO_FSEL_INPT);
data[0] = data[1] = data[2] = data[3] = data[4] = 0;
// wait for pin to drop?
while (bcm2835_gpio_lev(pin) == 1) {
usleep(1);
} //while
// read data!
for (int i=0; i< MAXTIMINGS; i++) {
counter = 0;
while ( bcm2835_gpio_lev(pin) == laststate) {
counter++;
//nanosleep(1); // overclocking might change this?
if (counter == 1000)
break;
}//while
laststate = bcm2835_gpio_lev(pin);
if (counter == 1000) break;
bits[bitidx++] = counter;
if ((i>3) && (i%2 == 0)) {
// shove each bit into the storage bytes
data[j/8] <<= 1;
if (counter > 200)
data[j/8] |= 1;
j++;
}//if
} //for
dhts.temp = data[2] ;
dhts.humidity = data[0] ;
printf("Temp = %5.2f *C, Hum = %5.2f \%\n", dhts.temp , dhts.humidity );
return dhts;
}//function

Ok I got it - and using ctypes was very fast. The python code:
#!/bin/python
from ctypes import *
# define the struct and it's fields
class DHStruct(Structure):
_fields_ = [("temp",c_double),("humidity",c_double)]
#reference the library
dhtlib = CDLL("libdht4py.so")
# set the return type as the object above
dhtlib.readDHT.restype = POINTER(DHStruct)
# dereference the pointer using ctype's -contents and access the struct fields.
print ( dhtlib.readDHT().contents.temp , dhtlib.readDHT().contents.humidity )
The C code : the key was to convert the function to return a pointer.
#define BCM2708_PERI_BASE 0x20000000
#define GPIO_BASE (BCM2708_PERI_BASE + 0x200000) /* GPIO controller */
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <dirent.h>
#include <fcntl.h>
#include <assert.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <bcm2835.h>
#include <unistd.h>
#define MAXTIMINGS 100
//define the struct
struct DHStruct {
double temp;
double humidity;
} ;
struct DHStruct *readDHT(); // define the function prototype to return the pointer
int bits[250], data[100];
int bitidx = 0;
//make sure to return a POINTER!!
struct DHStruct *readDHT() {
bcm2835_init() ;
int type = 11 ;
int pin = 4 ;
struct DHStruct *dhts; // here is the key - define the pointer to the struct
int counter = 0;
int laststate = HIGH;
int j=0;
// Set GPIO pin to output
bcm2835_gpio_fsel(pin, BCM2835_GPIO_FSEL_OUTP);
bcm2835_gpio_write(pin, HIGH);
usleep(500000); // 500 ms
bcm2835_gpio_write(pin, LOW);
usleep(20000);
bcm2835_gpio_fsel(pin, BCM2835_GPIO_FSEL_INPT);
data[0] = data[1] = data[2] = data[3] = data[4] = 0;
// wait for pin to drop?
while (bcm2835_gpio_lev(pin) == 1) {
usleep(1);
} //while
// read data!
for (int i=0; i< MAXTIMINGS; i++) {
counter = 0;
while ( bcm2835_gpio_lev(pin) == laststate) {
counter++;
//nanosleep(1); // overclocking might change this?
if (counter == 1000)
break;
}//while
laststate = bcm2835_gpio_lev(pin);
if (counter == 1000) break;
bits[bitidx++] = counter;
if ((i>3) && (i%2 == 0)) {
// shove each bit into the storage bytes
data[j/8] <<= 1;
if (counter > 200)
data[j/8] |= 1;
j++;
}//if
} //for
dhts->temp = data[2] ;
dhts->humidity = data[0] ;
//for debug printf("Temp = %5.2f *C, Hum = %5.2f \%\n", dhts->temp , dhts->humidity );
return dhts;
}//function

To combine C/C++ and Python I would recommend to use Cython.
With Cython you are able to pass objects (eg. numpy arrays) to C/C++, fill it with your data and get it back to your python-code.
Here is a minmal example:
The C-skript: (c_example.c)
#include <stdlib.h>
#include <math.h>
void c_claculate(double *x, int N) {
int i;
for (i = 0; i<N;i++) {
x[i]+=i*i;
}
}
The python-skript: (example.py)
from numpy import *
from example import *
data=zeros(10)
calculate(data)
print data
The .pyx file: (example.pyx)
import cython
import numpy
cimport numpy
# declare the interface to the C code
cdef extern void c_claculate(double *x, int N)
# Cython interface to C function
def calculate(numpy.ndarray[double, ndim=1, mode='c'] x not None):
cdef int N = x.shape[0]
c_claculate(&x[0],N)
return x
and the setup file: (setup.py)
from distutils.core import setup
from distutils.extension import Extension
from Cython.Distutils import build_ext
import numpy
setup(
cmdclass = {'build_ext': build_ext},
ext_modules = [
Extension("example",
sources=["example.pyx", "c_example.c"],
include_dirs=[numpy.get_include()]
)
],
)
Now you can compile the skript by running
python setup.py build_ext -fi
and then execute the python skript.
Cython should be available via pip on your PI.