I am working on my Project which implies the use of Empirical Mode Decomposition in C++ for EEG Signals. The input Data is Eigen::MatrixXd, where the rows are the Channels and the columns are the samples.
I did not found a good C++ library for EMD so I want to use a Python one (dsatools). I have downloaded the package through Pip installer from the setup.py file on Xubuntu... so it's a system package now.
the problem is that the program can't read the module.
this is the code:
std::vector <Eigen::MatrixXd> DataAquisition::EMD (Eigen::MatrixXd array, int order, int iterations, int locality) {
std::vector <Eigen::MatrixXd> IMFs;
for (int i = 0; i < array.rows(); i++) {
Eigen::MatrixXd Kanals = array.row(i);
Eigen::MatrixXd IMFs_Cpp;
Py_Initialize();
//PyRun_SimpleString("from dsatools._base._imf_decomposition import * ");
PyObject* sys_path = PySys_GetObject("path");
PyObject* ProgrammName = PyUnicode_FromString("/home/user/Schreibtisch/mne-cpp-main/applications/mne_bci/MNE-BCI-QT/dsatools-master/dsatools/_base/_imf_decomposition/_emd.py");
PyList_Append(sys_path, ProgrammName);
PyObject* pModuleString = PyUnicode_FromString ((char*)"_emd.py");
PyObject* pModule = PyImport_Import(pModuleString);
PyObject* pFunction = PyObject_GetAttrString(pModule,(char*)"emd");
//PyObject* pDict = PyModule_GetDict(pModule);
//PyObject* pFunc = PyDict_GetItemString(pDict, (char*)"emd");
if (PyCallable_Check(pFunction))
{
PyObject* Signal = Py_BuildValue("(d)",(double*)Kanals.data());
PyObject* Order = Py_BuildValue("(i)",order);
PyObject* method = Py_BuildValue("(z)",(char*)"cubic");
PyObject* max_itter = Py_BuildValue("(i)",iterations);
PyObject* args = PyTuple_Pack(4,Signal,Order,method,max_itter);
PyErr_Print();
PyObject* IMFs_Py = PyObject_CallObject(pFunction,args);
PyErr_Print();
if (PyArray_Check(IMFs_Py))
std::cout << "EMD Output is NOT Array \n";
PyArrayObject *np_ret = reinterpret_cast <PyArrayObject*> (IMFs_Py);
int Rows = PyArray_SHAPE(np_ret)[0];
int Cols = PyArray_SHAPE(np_ret)[1];
double* c_out = reinterpret_cast<double*>(PyArray_DATA(np_ret));
Eigen::MatrixXd IMFs_Cpp = Eigen::Map <Eigen::MatrixXd> (c_out,Rows,Cols);
IMFs.push_back(IMFs_Cpp);
}
else
std::cout << "Python did not call the function \n";
Py_Finalize();
}
return IMFs;}
this is how the code in Python should look like and I just want to call the emd function:
Related
I'm figuring out the Python/C API for a more complex task. Initially, I wrote a simple example of adding two ndarrays of shape = (2,3) and type = float32.
I am able to pass two numpy arrays into c functions, read their dimensions and data and perform custom addion on data. But when I try to wrap the resulting data using PyArray_SimpleNewFromData, code hangs (returns NULL?)
To replicate the issue, create three files: mymath.c, setup.py, test.py in a folder as follows and run test.py (it runs setup.py to compile and install the module and then runs a simple test).
I'm using python in windows, inside an anaconda environment. I'm new to the Python/C API. So, any help would be much appreciated.
// mymath.c
#include <Python.h>
#include <stdio.h>
#include "numpy/arrayobject.h"
#include "numpy/npy_math.h"
#include <math.h>
#include <omp.h>
/*
C functions
*/
float* arr_add(float* d1, float* d2, int M, int N){
float * result = (float *) malloc(sizeof(float)*M*N);
for (int m=0; m<M; m++)
for (int n=0; n<N; n++)
result [m*N+ n] = d1[m*N+ n] + d2[m*N+ n];
return result;
}
/*
Unwrap apply and wrap pyObjects
*/
void capsule_cleanup(PyObject *capsule) {
void *memory = PyCapsule_GetPointer(capsule, NULL);
free(memory);
}
// add two 2d arrays (float32)
static PyObject *arr_add_fn(PyObject *self, PyObject *args)
{
PyArrayObject *arr1, *arr2;
if (!PyArg_ParseTuple(args, "OO", &arr1, &arr2))
return NULL;
// get data as flat list
float *d1, *d2;
d1 = (float *) arr1->data;
d2 = (float *) arr2->data;
int M, N;
M = (int)arr1->dimensions[0];
N = (int)arr1->dimensions[1];
printf("Dimensions, %d, %d \n\n", M,N);
PyObject *result, *capsule;
npy_intp dim[2];
dim[0] = M;
dim[1] = N;
float * d3 = arr_add(d1, d2, M, N);
result = PyArray_SimpleNewFromData(2, dim, NPY_FLOAT, (void *)d3);
if (result == NULL)
return NULL;
// -----------This is not executed. code hangs--------------------
for (int m=0; m<M; m++)
for (int n=0; n<N; n++)
printf("%f \n", d3[m*N+n]);
capsule = PyCapsule_New(d3, NULL, capsule_cleanup);
PyArray_SetBaseObject((PyArrayObject *) result, capsule);
return result;
}
/*
Bundle functions into module
*/
static PyMethodDef MyMethods [] ={
{"arr_add", arr_add_fn, METH_VARARGS, "Array Add two numbers"},
{NULL,NULL,0,NULL}
};
/*
Create module
*/
static struct PyModuleDef mymathmodule = {
PyModuleDef_HEAD_INIT,
"mymath", "My doc of mymath", -1, MyMethods
};
PyMODINIT_FUNC PyInit_mymath(void){
return PyModule_Create(&mymathmodule);
}
# setup.py
from distutils.core import setup, Extension
import numpy
module1 = Extension('mymath',
sources = ['mymath.c'],
# define_macros = [('NPY_NO_DEPRECATED_API', 'NPY_1_7_API_VERSION')],
include_dirs=[numpy.get_include()],
extra_compile_args = ['-fopenmp'],
extra_link_args = ['-lgomp'])
setup (name = 'mymath',
version = '1.0',
description = 'My math',
ext_modules = [module1])
# test.py
import os
os.system("python .\setup.py install")
import numpy as np
import mymath
a = np.arange(6,dtype=np.float32).reshape(2,3)
b = np.arange(6,dtype=np.float32).reshape(2,3)
c = mymath.arr_add(a,b)
print(c)
I'm writing C code for python (Python C API), and I noticed that python is not releasing the memory of the file, I'm wondering if the issue is in my code.
I want to simplify as much as passable, but I hope that no details will be missing.
The file is a binary file with buffers, first 4 bytes is the buffer size, then the buffer.
The binary file (big_file.comp):
du ~/Desktop/TEST_FILES/big_file.comp
4175416 ~/Desktop/TEST_FILES/big_file.comp
The python code (test.py):
#!/usr/bin/env python3
from struct import unpack_from
from psutil import Process
from os import getpid
import decomplib
def file_handler(file_name):
with open(file_name, 'rb') as reader:
while True:
next_4_bytes = reader.read(4)
if next_4_bytes == b'':
break
next_size, *_ = unpack_from("I", next_4_bytes)
buffer = reader.read(next_size)
yield buffer, next_size
def main():
args = _parse_args()
decompress = decomplib.Decompress()
for buf, buf_size in file_handler(args.file):
for msg in decompress.decompress_buffer(buf, buf_size):
print(msg)
if __name__ == "__main__":
pid = getpid()
ps = Process(pid)
main()
print(ps.memory_info())
Some of the C code simplified:
#include <Python.h>
#include "structmember.h"
typedef struct {
PyObject_HEAD
uint32_t arr_size;
} DecompressObject;
static int Decompress_init(DecompressObject *self, PyObject *args, PyObject *kwds){
return 0;
}
static PyObject* Decompress_handle_buffer(DecompressObject* self, PyObject* args){
uint32_t buf_size = 0;
uint8_t *buf = NULL;
// get buffer and buffer length from python function
if(!PyArg_ParseTuple(args, "y*i", &buf, &buf_size)){
PyErr_SetString(PyExc_Exception, "Failed to parse function arguments");
return NULL;
}
self->arr_size = 10;
Py_XINCREF(self);
return (PyObject *) self;
}
static PyObject* Decompress_next(DecompressObject *self, PyObject *Py_UNUSED(ignored)){
static uint32_t seq_index = 0;
if (seq_index < self->arr_size) {
seq_index++;
Py_RETURN_NONE;
}
seq_index = 0;
return NULL;
}
static void Decompress_dealloc(DecompressObject *self){
Py_TYPE(self)->tp_free((PyObject *) self);
}
static PyMethodDef Decompress_methods[] = {
{"decompress_buffer", (PyCFunction) Decompress_handle_buffer, METH_VARARGS, "Decompress a buffer to asc data."},
{NULL} /* Sentinel */
};
static PyTypeObject DecompressType = {
PyVarObject_HEAD_INIT(NULL, 0)
.tp_name = "decomplib.Decompress",
.tp_doc = "Decompress object",
.tp_basicsize = sizeof(DecompressObject),
.tp_itemsize = 0,
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
.tp_alloc = PyType_GenericAlloc,
.tp_new = PyType_GenericNew,
.tp_iter = PyObject_SelfIter,
.tp_init = (initproc) Decompress_init,
.tp_dealloc = (destructor) Decompress_dealloc,
.tp_iternext = (iternextfunc) Decompress_next,
.tp_methods = Decompress_methods,
};
static PyModuleDef Decompressmodule = {
PyModuleDef_HEAD_INIT,
.m_name = "decomplib",
.m_doc = "Decompress an compressed file.",
.m_size = -1,
};
PyMODINIT_FUNC PyInit_decomplib(void){
PyObject *d;
if (PyType_Ready(&DecompressType) < 0)
return NULL;
d = PyModule_Create(&Decompressmodule);
if (d == NULL)
return NULL;
Py_INCREF(&DecompressType);
if (PyModule_AddObject(d, "Decompress", (PyObject *) &DecompressType) < 0) {
Py_DECREF(&DecompressType);
Py_DECREF(d);
return NULL;
}
return d;
}
As a result, I got the following output:
./test.py -f ~/Desktop/TEST_CAN_OPT/big_fie.comp
None
None
None
...
None
None
None
pmem(rss=4349915136, vms=4412583936, shared=6270976, text=2867200, lib=0, data=4344135680, dirty=0)
While playing around I noticed that if I change in the C function Decompress_handle_buffer the call to the function PyArg_ParseTuple the second argument from "y*i" to "Si", Python do cleanup the memory...
./test.py -f ~/Desktop/TEST_CAN_OPT/big_fie.comp
None
None
None
...
None
None
None
pmem(rss=22577152, vms=84869120, shared=6361088, text=2867200, lib=0, data=16420864, dirty=0)
However, The buffer is NOT correctly read.
Any ideas?!
Extra Info:
I'm using a virtual machine (VMware Workstation 15)
OS Ubuntu 18.4
Python 3.6.9
y* does not correspond to uint8_t like you're using it. As stated in the documentation, it fills a Py_buffer struct that you're supposed to provide.
You need to actually provide a Py_buffer, and when you're done with it, you need to release the buffer with PyBuffer_Release.
I need to send an OpenCv image from C++ to Python to do some processing on it.
The Mat will be received through the code but for simplicity I am using imread here for the question.
What I did in the C++ part of the code was:
#include <Python.h>
#include <arrayobject.h>
#include <iostream>
#include <opencv2/opencv.hpp>
#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
using namespace cv;
using namespace std;
int main(int argc, char *argv[])
{
QCoreApplication a(argc, argv);
Mat image = imread("test.jpg");
Py_Initialize();
PyObject *pName, *pModule, *pDict, *pFunc, *pArgs, *pValue;
pName = PyUnicode_FromString("prog");
if (pName == NULL)
{
PyErr_Print();
return 0;
}
pModule = PyImport_Import(pName);
if (pModule == NULL)
{
PyErr_Print();
return 0;
}
pDict = PyModule_GetDict(pModule);
pFunc = PyDict_GetItemString(pDict, "add");
if (pFunc == NULL)
{
PyErr_Print();
return 0;
}
pArgs = PyTuple_New(1);
import_array ();
npy_intp dimensions[3] = {image.rows, image.cols, image.channels()};
pValue = PyArray_SimpleNewFromData(image.dims + 1, (npy_intp*)&dimensions, NPY_UINT8, image.data);
PyTuple_SetItem(pArgs, 0, pValue);
PyObject* pResult = PyObject_CallObject(pFunc, pArgs);
if(pResult == NULL)
cout<<"Calling the add method failed"<<endl;
long result = PyLong_AsLong(pResult);
cout<<"Result = "<<result<<endl;
Py_Finalize();
return 0;
}
This code compiles and runs.
For the Python part:
import cv2
import numpy as np
def add (a):
print ("Contents of a :")
print (a)
# mat_array = cv2.fromarray(a, numpy.float32)
vis0 = cv.fromarray(a)
return 0
The Python code receives the numpy array from C++ (I think) and when I print the contents of a, I have an output (so I think I am receiving the image from C++).
Now I need to convert the data in a to a cv2 Mat in Python so that I can work on it.
Once I reach the mat_array = cv2.fromarray(a, numpy.float32) line or vis0 = cv.fromarray(a) the code crashes with the following output:
Exception ignored in: <module 'threading' from '/usr/lib/python3.5/threading.py'>
Traceback (most recent call last):
File "/usr/lib/python3.5/threading.py", line 1283, in _shutdown
assert tlock.locked()
SystemError: <built-in method locked of _thread.lock object at 0x7ff0f34d20d0> returned a result with an error set
How do I correctly send / receive the Mat object?
Please find my answer here. You can also find other answer here. for converting numpy -> cv::Mat and cv::Mat -> numpy.
I'm trying to embed Python in a C++ multithreaded program.
What I do is calling two statistical functions from the Python C API to perform the Two Sample Kolmogorov-Smirnov Test and the Two Sample Anderson-Darling Test on some data that I collect. So I'm just embedding Python in my code, I'm not extending it or using my own Python functions.
I recently found out that in order to run a multithreaded program that uses the Python C API you need to handle properly the Global Interpreter Lock (GIL) and when ever you use a Python C API function you need to acquire the GIL and then release it when you're done using the API functions.
The thing that I still don't understand is how to properly release the GIL from the main thread in order to let the others execute the Python code.
I tried this (option 1):
int main(int argc, const char * argv[]) {
int n = 4;
std::thread threads[n];
Py_Initialize();
PyEval_InitThreads();
PyEval_SaveThread();
for (int i = 0; i < n; i++) {
threads[i] = std::thread(exec, i);
}
for (int i = 0; i < n; i++) {
threads[i].join();
}
Py_Finalize();
return 0;
}
But it gives me a segmentation fault when calling Py_Finalize().
So I tried this (option 2):
int main(int argc, const char * argv[]) {
int n = 4;
std::thread threads[n];
Py_Initialize();
PyEval_InitThreads();
PyThreadState * Py_UNBLOCK_THREADS
for (int i = 0; i < n; i++) {
threads[i] = std::thread(exec, i);
}
for (int i = 0; i < n; i++) {
threads[i].join();
}
Py_BLOCK_THREADS
Py_Finalize();
return 0;
}
and this (option 3):
int main(int argc, const char * argv[]) {
int n = 4;
std::thread threads[n];
Py_Initialize();
PyEval_InitThreads();
Py_BEGIN_ALLOW_THREADS
for (int i = 0; i < n; i++) {
threads[i] = std::thread(exec, i);
}
for (int i = 0; i < n; i++) {
threads[i].join();
}
Py_END_ALLOW_THREADS
Py_Finalize();
return 0;
}
With both these last two options the code runs but ends with this error:
Exception ignored in: <module 'threading' from '/usr/local/opt/python3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/threading.py'>
Traceback (most recent call last):
File "/usr/local/opt/python3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/threading.py", line 1289, in _shutdown
assert tlock.locked()
AssertionError:
EDIT:
The code that is executed by the spawned threads is this:
double limited_rand(double lower_bound, double upper_bound) {
return lower_bound + (rand() / (RAND_MAX / (upper_bound-lower_bound) ) );
}
double exec_1(std::vector<int> &left_sample, std::vector<int> &right_sample) {
PyGILState_STATE gstate = PyGILState_Ensure(); // Acquiring GIL for thread-safe usage Python C API
PyObject* scipy_stats_module = PyImport_ImportModule("scipy.stats"); // importing "scipy.stats" module
import_array();
npy_intp left_nparray_shape[] = {(npy_intp)left_sample.size()}; // Size of left nparray's first dimension
PyObject* left_sample_nparray = PyArray_SimpleNewFromData(1, left_nparray_shape, NPY_INT, &left_sample[0]); // Creating numpy array with 1 dimension, taking "dim" as a dummy, elements are integers, and the data is taken from "sample1" as a int* pointer
npy_intp right_nparray_shape[] = {(npy_intp)right_sample.size()}; // Size of right nparray's first dimension
PyObject* right_sample_nparray = PyArray_SimpleNewFromData(1, right_nparray_shape, NPY_INT, &right_sample[0]);
PyObject* ks_2samp = PyObject_GetAttrString(scipy_stats_module, "ks_2samp");
Py_DecRef(scipy_stats_module);
PyObject* ks_2samp_return_val = PyObject_CallFunctionObjArgs(ks_2samp, left_sample_nparray, right_sample_nparray, NULL);
Py_DecRef(ks_2samp);
Py_DecRef(right_sample_nparray);
Py_DecRef(left_sample_nparray);
double p_value = PyFloat_AsDouble(PyTuple_GetItem(ks_2samp_return_val, 1));
Py_DecRef(ks_2samp_return_val);
PyGILState_Release(gstate); // Releasing GIL
return p_value;
}
void initialize_c_2d_int_array(int*& c_array, unsigned long row_length_c_array, std::vector<int> &row1, std::vector<int> &row2) {
for (unsigned int i = 0; i < row_length_c_array; i++) {
c_array[i] = row1[i];
c_array[row_length_c_array + i] = row2[i];
}
}
double exec_2(std::vector<int> &left_sample, std::vector<int> &right_sample){
PyGILState_STATE gstate = PyGILState_Ensure(); // Acquiring GIL for thread-safe usage Python C API
PyObject* scipy_stats_module = PyImport_ImportModule("scipy.stats"); // importing "scipy.stats" module
// import_array();
unsigned long n_cols = std::min(left_sample.size(), right_sample.size());
int* both_samples = (int*) (malloc(2 * n_cols * sizeof(int)));
initialize_c_2d_int_array(both_samples, n_cols, left_sample, right_sample);
npy_intp dim3[] = {2, (npy_intp) n_cols};
PyObject* both_samples_nparray = PyArray_SimpleNewFromData(2, dim3, NPY_INT, both_samples);
PyObject* anderson_ksamp = PyObject_GetAttrString(scipy_stats_module, "anderson_ksamp");
Py_DecRef(scipy_stats_module);
PyObject* anderson_2samp_return_val = PyObject_CallFunctionObjArgs(anderson_ksamp, both_samples_nparray, NULL);
Py_DecRef(anderson_ksamp);
Py_DecRef(both_samples_nparray);
free(both_samples);
double p_value = PyFloat_AsDouble(PyTuple_GetItem(anderson_2samp_return_val, 2));
Py_DecRef(anderson_2samp_return_val);
PyGILState_Release(gstate); // Releasing GIL
return p_value;
}
void exec(int thread_id) {
std::vector<int> left_sample;
std::vector<int> right_sample;
int n = 50;
for (int j = 0; j < n; j++) {
int size = 100;
for (int i = 0; i < size; i++) {
left_sample.push_back(limited_rand(0, 100));
right_sample.push_back(limited_rand(0, 100));
}
exec_1(left_sample, right_sample);
exec_2(left_sample, right_sample);
}
}
The functions where I use the Python C API are only exec_1 and exec_2, while exec has just the job to call the repeatedly on new random data. This is the simplest code I could think of that mimics the behavior of my real code. I've also left out every type of error checking when using the Python APIs for a better readability.
Without any other choice I'll run my code like option 2 or option 3 and forget about the error, but I would really like to understand what's going on. Can you help me?
P.S. I'm running Python 3.6.1 under a macOS 10.12.5 system using Xcode 8.3.3. If you need more details let me know.
option1:
I think is giving you a segmentation fault because you called PyEval_SaveThread() (which releases the gil, returns a saved thread state, and sets the current thread state to NULL).
Py_Finalize will try to free all memory associated with the interpreter, and I guess this included the main thread state. So you can either capture this state with:
PyEval_InitThreads(); //initialize and aquire the GIL
//release the GIL, store thread state, set the current thread state to NULL
PyThreadState *mainThreadState = PyEval_SaveThread();
*main code segment*
//re-aquire the GIL (re-initialize the current thread state)
PyEval_RestoreThread(mainThreadState);
Py_Finalize();
return 0;
Or you can immediately call PyEval_ReleaseLock() after calling PyEval_InitThreads() since it looks like the main code segment does not use any embedded python. I had a similar problem and that seemed to fix it.
NOTE: Other threads will still need to aquire/release the GIL wherever necessary
I am hosting python interpreter(2.7.13) in a C application and running a python script that has some calls to opencv-python functions.
The problem is that when the script is run in the main interpreter, it works fine. But, when the script is run in a sub-interpreter, it hangs.
Is there any known incompatibility between opencv-python package and sub-interpreter ?
The C application code is as follows:-
const char *pythonSource = "abc_module";
const char *funcName = "abc_function";
Py_Initialize();
PyEval_InitThreads();
if(run_in_subinterpreter){
PyThreadState* threadState = Py_NewInterpreter();
PyThreadState *mainState = PyThreadState_Swap(threadState);
}
pName = PyString_FromString(pythonSource);
pModule = PyImport_Import(pName);
pDict = PyModule_GetDict(pModule);
pFunc = PyDict_GetItemString(pDict, funcName);
PyObject *pyList = PyList_New(gArray.size()); //std::vector<int> gArray;
for (size_t i = 0; i < gArray.size(); ++i)
{
PyObject *elem = PyInt_FromLong(gArray[i]);
PyList_SetItem(pyList, i, elem);
}
PyObject* pArgs = PyTuple_New(1);
PyTuple_SetItem(pArgs, 0, pyList);
pValue = PyObject_CallObject(pFunc, pArgs);
Py_Finalize();
return;
The python script code is as follows:-
import sys
import cv2
import numpy as np
def abc_function(list):
array = np.asarray(list)
array = np.reshape(array, (100, 200, 4)).astype(np.uint8)
array = abc_function_core(array)
return array
def abc_function_core(array):
# array is manipulated by calling the below functions
cv2.resize(....)
cv2.cvtColor(...)
cv2.equalizeHist(...)
cv2.CascadeClassifier.detectMultiScale(...)
cv2.rectangle(...)
return array