We Keep Coding

Python Method Call in a C++ Code through Python C-API

I am working on my Project which implies the use of Empirical Mode Decomposition in C++ for EEG Signals. The input Data is Eigen::MatrixXd, where the rows are the Channels and the columns are the samples.
I did not found a good C++ library for EMD so I want to use a Python one (dsatools). I have downloaded the package through Pip installer from the setup.py file on Xubuntu... so it's a system package now.
the problem is that the program can't read the module.
this is the code:
std::vector <Eigen::MatrixXd> DataAquisition::EMD (Eigen::MatrixXd array, int order, int iterations, int locality) {
std::vector <Eigen::MatrixXd> IMFs;
for (int i = 0; i < array.rows(); i++) {
Eigen::MatrixXd Kanals = array.row(i);
Eigen::MatrixXd IMFs_Cpp;
Py_Initialize();
//PyRun_SimpleString("from dsatools._base._imf_decomposition import * ");
PyObject* sys_path = PySys_GetObject("path");
PyObject* ProgrammName = PyUnicode_FromString("/home/user/Schreibtisch/mne-cpp-main/applications/mne_bci/MNE-BCI-QT/dsatools-master/dsatools/_base/_imf_decomposition/_emd.py");
PyList_Append(sys_path, ProgrammName);
PyObject* pModuleString = PyUnicode_FromString ((char*)"_emd.py");
PyObject* pModule = PyImport_Import(pModuleString);
PyObject* pFunction = PyObject_GetAttrString(pModule,(char*)"emd");
//PyObject* pDict = PyModule_GetDict(pModule);
//PyObject* pFunc = PyDict_GetItemString(pDict, (char*)"emd");
if (PyCallable_Check(pFunction))
{
PyObject* Signal = Py_BuildValue("(d)",(double*)Kanals.data());
PyObject* Order = Py_BuildValue("(i)",order);
PyObject* method = Py_BuildValue("(z)",(char*)"cubic");
PyObject* max_itter = Py_BuildValue("(i)",iterations);
PyObject* args = PyTuple_Pack(4,Signal,Order,method,max_itter);
PyErr_Print();
PyObject* IMFs_Py = PyObject_CallObject(pFunction,args);
PyErr_Print();
if (PyArray_Check(IMFs_Py))
std::cout << "EMD Output is NOT Array \n";
PyArrayObject *np_ret = reinterpret_cast <PyArrayObject*> (IMFs_Py);
int Rows = PyArray_SHAPE(np_ret)[0];
int Cols = PyArray_SHAPE(np_ret)[1];
double* c_out = reinterpret_cast<double*>(PyArray_DATA(np_ret));
Eigen::MatrixXd IMFs_Cpp = Eigen::Map <Eigen::MatrixXd> (c_out,Rows,Cols);
IMFs.push_back(IMFs_Cpp);
}
else
std::cout << "Python did not call the function \n";
Py_Finalize();
}
return IMFs;}
this is how the code in Python should look like and I just want to call the emd function:

Numpy Python/C API - PyArray_SimpleNewFromData hangs

I'm figuring out the Python/C API for a more complex task. Initially, I wrote a simple example of adding two ndarrays of shape = (2,3) and type = float32.
I am able to pass two numpy arrays into c functions, read their dimensions and data and perform custom addion on data. But when I try to wrap the resulting data using PyArray_SimpleNewFromData, code hangs (returns NULL?)
To replicate the issue, create three files: mymath.c, setup.py, test.py in a folder as follows and run test.py (it runs setup.py to compile and install the module and then runs a simple test).
I'm using python in windows, inside an anaconda environment. I'm new to the Python/C API. So, any help would be much appreciated.
​
// mymath.c
#include <Python.h>
#include <stdio.h>
#include "numpy/arrayobject.h"
#include "numpy/npy_math.h"
#include <math.h>
#include <omp.h>
/*
C functions
*/
float* arr_add(float* d1, float* d2, int M, int N){
float * result = (float *) malloc(sizeof(float)*M*N);
for (int m=0; m<M; m++)
for (int n=0; n<N; n++)
result [m*N+ n] = d1[m*N+ n] + d2[m*N+ n];
return result;
}
/*
Unwrap apply and wrap pyObjects
*/
void capsule_cleanup(PyObject *capsule) {
void *memory = PyCapsule_GetPointer(capsule, NULL);
free(memory);
}
// add two 2d arrays (float32)
static PyObject *arr_add_fn(PyObject *self, PyObject *args)
{
PyArrayObject *arr1, *arr2;
if (!PyArg_ParseTuple(args, "OO", &arr1, &arr2))
return NULL;
// get data as flat list
float *d1, *d2;
d1 = (float *) arr1->data;
d2 = (float *) arr2->data;
int M, N;
M = (int)arr1->dimensions[0];
N = (int)arr1->dimensions[1];
printf("Dimensions, %d, %d \n\n", M,N);
PyObject *result, *capsule;
npy_intp dim[2];
dim[0] = M;
dim[1] = N;
float * d3 = arr_add(d1, d2, M, N);
result = PyArray_SimpleNewFromData(2, dim, NPY_FLOAT, (void *)d3);
if (result == NULL)
return NULL;
// -----------This is not executed. code hangs--------------------
for (int m=0; m<M; m++)
for (int n=0; n<N; n++)
printf("%f \n", d3[m*N+n]);
capsule = PyCapsule_New(d3, NULL, capsule_cleanup);
PyArray_SetBaseObject((PyArrayObject *) result, capsule);
return result;
}
/*
Bundle functions into module
*/
static PyMethodDef MyMethods [] ={
{"arr_add", arr_add_fn, METH_VARARGS, "Array Add two numbers"},
{NULL,NULL,0,NULL}
};
/*
Create module
*/
static struct PyModuleDef mymathmodule = {
PyModuleDef_HEAD_INIT,
"mymath", "My doc of mymath", -1, MyMethods
};
PyMODINIT_FUNC PyInit_mymath(void){
return PyModule_Create(&mymathmodule);
}
​
# setup.py
from distutils.core import setup, Extension
import numpy
module1 = Extension('mymath',
sources = ['mymath.c'],
# define_macros = [('NPY_NO_DEPRECATED_API', 'NPY_1_7_API_VERSION')],
include_dirs=[numpy.get_include()],
extra_compile_args = ['-fopenmp'],
extra_link_args = ['-lgomp'])
setup (name = 'mymath',
version = '1.0',
description = 'My math',
ext_modules = [module1])
​
# test.py
import os
os.system("python .\setup.py install")
import numpy as np
import mymath
a = np.arange(6,dtype=np.float32).reshape(2,3)
b = np.arange(6,dtype=np.float32).reshape(2,3)
c = mymath.arr_add(a,b)
print(c)

Python/C API - The result is not displayed

I would like to integrate C modules in the Python, so my choice fell on the interface Python.h. Everything compiled without errors and warnings, so I can not understand what the problem is.
C side:
#include <python3.5m/Python.h>
...
#define PyInt_AsLong(x) (PyLong_AsLong((x)))
typedef PyObject* Py;
static Py getSumma(Py self, Py args){
Py nums;
if (!PyArg_ParseTuple(args, "O", &nums)){
return NULL;
}
size_t numsAmount = PyList_Size(args);
int32_t summa = 0;
for (size_t i = 0; i < numsAmount; i++){
Py temp = PyList_GetItem(nums, i);
int32_t num = PyInt_AsLong(temp);
summa += num;
}
return Py_BuildValue("l", summa);
}
static PyMethodDef moduleMethods[] = {
{"getSumma", (PyCFunction)getSumma, METH_VARARGS, NULL},
{NULL, NULL, 0, NULL}
};
static PyModuleDef SummaLogic = {
PyModuleDef_HEAD_INIT,
"SummaLogic",
"",
-1,
moduleMethods
};
PyMODINIT_FUNC PyInit_SummaLogic(void){
return PyModule_Create(&SummaLogic);
}
setup.py:
from distutils.core import setup, Extension
SummaLogic = Extension("SummaLogic", sources=['SummaLogic.c'])
setup(ext_modules=[SummaLogic])
Python side:
from SummaLogic import getSumma
if __name__ == "__main__":
a = [1, 2, 3]
b = getSumma(a)
print(b)
It seems right, but when I start it in terminal - nothing happens, just hanging without any activity. What could I miss?

It boils down to PyList_Size and that you don't check for errors there.
You probably wanted to use it on nums, not args as argument. However you used on args and a very interesting thing happened:
args is a tuple,
therefore PyList_Size failed and returned -1
that -1 which was cast to an unsigned size_t which probably resulted in a very huge number, probably 2**64-1
therefore your iteration runs a "very long time" because it takes quite a while to iterate over 2**64-1 items (apart from all the out-of-bound memory accesses).
The quick fix would be to use:
Py_ssize_t listlength = PyList_Size(nums); /* nums instead of args */
if (listlength == -1) { /* check for errors */
return NULL;
}
size_t numsAmount = (size_t)listlength /* cast to size_t only after you checked for errors */
However you should check what the error conditions are and test for them after every python C API function call otherwise you'll get a lot of undefined behaviours. Also I probably would stick to the defined return types instead of int32_t (PyInt_AsLong returns long so you might get weird casting errors there as well!), size_t, ... and the typedef PyObject* Py; makes things really tricky for someone who regularly writes C extensions.

How to use multi-threading to two c++ functions with embedding python?

I am learning embedding python in c++ code. I have trouble to use multi-threading to parallelize two c++ functions with embedding python.
My sample codes are shown below:
thread_test.py
import time
def test1():
time.sleep(5) # delays for 5 seconds
print 1935
return 'happy'
def test2():
time.sleep(10) # delays for 10 seconds
print 3000
py_thread.h
string test_func1(string file_dir){
string result_dir;
string str = "import sys; sys.path.insert(0," "\'"+file_dir+"\'"+")";
const char * c = str.c_str();
PyRun_SimpleString (c);
PyObject * pModule,* pFunc, *pName, *presult, *pArgs;
pName = PyString_FromString("thread_test");
pModule = PyImport_Import(pName);
Py_DECREF(pName);
pFunc = PyObject_GetAttrString(pModule, "test1");
if(pFunc != NULL) {
presult=PyObject_CallObject(pFunc,NULL);
result_dir = PyString_AsString(presult);
}
else {
printf("pFunc returned NULL\n");
}
Py_DECREF(pModule);
Py_DECREF(pFunc);
return result_dir;
}
void test_func2(string file_dir){
// Almost the same test_func1 except replacing "test1" with "test2" and no return value of result_dir
}
In main class, if I don't use multi-threading and just run the two functions and other normal c++ functions in serious, it works. But if I use some c++ threading techniques, such as OPENMP, it will give me SEGMENTATION FAULT. (code is shown below)
main.cpp
int main(){
Py_Initialize();
#pragma omp parallel num_threads(2)
{
int i = omp_get_thread_num();
if(i == 0)
{
test_func1("../");
}
if(i == 1 || omp_get_num_threads() != 2)
{
ANOTHER_C++_ONLY_SIMPLE_FUNCTION();
test_func2("../");
}
}
Py_Finalize();
return 0;
}
I also have tried thread in c++11 and pthread. They all give me segmentation fault. So how can I parallelize the two function???
Thank you!

Pass Variable from Python to C

I have Python Script embedded in C which I run in a thread. I need to pass the variable 'a' from the Python-Class 'Detect Motion' to my C program continuously. (Not as a return value)
I know I could do this with a fifo or something like that, but is there a way to pass it directly to C, maybe by calling a C function?
C:
#include <Python.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <pthread.h>
pthread_t mythread;
void *ThreadProc();
PyObject *pName, *pModule, *pDict, *pFunc, *pFunc2;
int main(int argc, char *argv[])
{
py_callback = PyCFunction_New(&callback_descr, NULL);
char *script = "motion";
char *functionUse = "get_values";
Py_Initialize();
pName = PyString_FromString(script);
pModule = PyImport_Import(pName);
// pDict and pFunc are borrowed references
pDict = PyModule_GetDict(pModule);
pFunc = PyDict_GetItemString(pDict, functionUse);
// POSIX code
pthread_create( &mythread, NULL, ThreadProc, NULL);
// Random testing code
for(int i = 0; i < 10; i++)
{
printf("Printed from the main thread.\n");
sleep(1);
}
printf("Main Thread waiting for My Thread to complete...\n");
// Join and wait for the created thread to complete...
// POSIX code
pthread_join(mythread, NULL);
printf("Main thread finished gracefully.\n");
return 0;
}
void *ThreadProc()
{
if (PyCallable_Check(pFunc))
{
PyObject_CallObject(pFunc, NULL);
}
else {
PyErr_Print();
}
// Clean up
Py_DECREF(pModule);
Py_DECREF(pName);
Py_Finalize();
printf("My thread is finishing...\n");
}
Python:
import numpy as np
import picamera
import picamera.array
class DetectMotion(picamera.array.PiMotionAnalysis):
def analyse(self, a):
a = np.sqrt(
np.square(a['x'].astype(np.float)) +
np.square(a['y'].astype(np.float))
).clip(0, 255).astype(np.uint8)
# If there're more than 10 vectors with a magnitude greater
# than 60, then say we've detected motion
print a
if (a > 60).sum() > 10:
print 'Motion detected!'
def get_values():
with picamera.PiCamera() as camera:
with DetectMotion(camera) as output:
camera.resolution = (640, 480)
camera.start_preview()
camera.start_recording(
'/dev/null', format='h264', motion_output=output)
camera.wait_recording(10)
camera.stop_recording()
camera.stop_preview()