We Keep Coding

How to wrap c++ code calling python as .dll or .so?

I want to wrap my c++ code as .so and .dll file.I know how to wrap c++ code as dynamic library, But my c++ code is calling python, normally we call embedding python.
I write a basic simple code.
python code:
def init_test(env, mode):
print(env)
print(mode)
return 1
c++ code calling python:
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include <iostream>
#include <exception>
/**
* #description: run risk performance use Python
* #param {string} env
* #param {string } mode
* #return {*}
*/
extern "C" int init_python_test(char* env, char* mode) {
std::cout << "start" <<std::endl;
if(Py_IsInitialized == 0){
std::cout << "not init" << std::endl;
}
else{
std::cout << "init already" <<std::endl;
//std::cout << Py_FinalizeEx() <<std::endl;
Py_Finalize();
}
std::cout << "init:"<<Py_IsInitialized() << std::endl;
Py_Initialize();
PyErr_Print();
std::cout <<"second" <<std::endl;
PyRun_SimpleString("import sys");
PyRun_SimpleString("sys.path.append('./')");
std::cout <<"ok" <<std::endl;
//int res;
PyObject *pModule,*pFunc = NULL;
PyObject *pArgs, *pValue = NULL;
pModule = PyImport_ImportModule("mini");//0x7ffff64b9cc0
if(!pModule)
std::cout << "can't open python file" << std::endl;
PyErr_Print();
pFunc = PyObject_GetAttrString(pModule, "init_test");
PyErr_Print();
if(pFunc && PyCallable_Check(pFunc)){
PyErr_Print();
pValue = PyObject_CallObject(pFunc, Py_BuildValue("(ss)", env, mode));
PyErr_Print();
}
Py_FinalizeEx();
return 1;
}
int main(){
char *env = (char*)"prod";
char * mode = (char*)"prod";
init_python_test(env, mode);
std::cout << "ok" <<std::endl;
}
I am able to run my c++ code properly with g++ command linked with python dynamic library. And I can use g++ to wrap my c++ code as .so file. When I use another c++ code and python code to test the init_python_test function. Segmentation fault occurs when the code runs into Py_Initialize().
So, how to resolve this question? and did I wrap c++ code properly with g++? here is my shell.
g++ -fPIC -shared -Wall -o libtest.so ./mini_test.cpp -DLINUX -D_GLIBCXX_USE_CXX11_ABI=0 -I /usr/include/python3.8 -L/usr/lib/python3 -L/usr/lib/python3.8 -lpython3.8
Somebody helps me! plz!!! thank u!!!!

Embedding Python in C++ from anaconda, getting error

Trying to compile this a.cpp file, using
g++ a.cpp -I ~/anaconda3/include/python3.7m/ -l ~/anaconda3/lib/python3.7/
Error I am getting is
/usr/bin/ld: cannot find -l/home/rverma/anaconda3/lib/python3.7/
collect2: error: ld returned 1 exit status
Tried g++ a.cpp -I ~/anaconda3/include/python3.7m/ -l ~/anaconda3/lib/ this as well for giving
/usr/bin/ld: cannot find -l/home/rverma/anaconda3/lib/ collect2: error: ld returned 1 exit status
Where my a.cpp file looks like this:
#include <Python.h>
#include <stdlib.h>
int main()
{
// Set PYTHONPATH TO working directory
setenv("PYTHONPATH",".",1);
PyObject *pName, *pModule, *pDict, *pFunc, *pValue, *presult;
// Initialize the Python Interpreter
Py_Initialize();
// Build the name object
pName = PyUnicode_FromString((char*)"arbName");
// Load the module object
pModule = PyImport_Import(pName);
// pDict is a borrowed reference
pDict = PyModule_GetDict(pModule);
// pFunc is also a borrowed reference
pFunc = PyDict_GetItemString(pDict, (char*)"someFunction");
if (PyCallable_Check(pFunc))
{
pValue=Py_BuildValue("(z)",(char*)"something");
PyErr_Print();
printf("Let's give this a shot!\n");
presult=PyObject_CallObject(pFunc,pValue);
PyErr_Print();
} else
{
PyErr_Print();
}
printf("Result is %ld\n",PyLong_AsLong(presult));
Py_DECREF(pValue);
// Clean up
Py_DECREF(pModule);
Py_DECREF(pName);
// Finish the Python Interpreter
Py_Finalize();
return 0;
}
And my argName.py looks like this:
def someFunction(text):
print 'You passed this Python program '+text+' from C! Congratulations!'
return 12345
Please help

How to solve: lto1: fatal error: bytecode stream in file '...' generated with LTO version 6.0 instead of the expected 7.1

I am trying to embed a simple (test) Python script into C++. See this question for reference: Undefined reference to `main` error when embedding Python in C++ I'm trying to embed Python in C++. This is my Python file (with the name EmbedTest.py):
from __future__ import division
class model:
def init(self,a,b):
self.a = a
self.b = b
def test_method(a,b):
m = model(a,b)
m.add(1)
print("a: ",a,"b: ",b)
return (a+b,a-b,a*b)
This is my C++ file (with the name EmbedTest.cpp and located in the same folder as EmbedTest.py)
#include <Python.h>
int main(int argc, char *argv[]) {
PyObject *pName, *pModule, *pFunc;
PyObject *pArgs, *pValue, *pValue_1, *pValue_2;
double sum,diff,prod;
double a = atof(argv[1]);
double b = atof(argv[2]);
Py_Initialize();
pName = PyUnicode_DecodeFSDefault("EmbedTest.py");
pModule = PyImport_Import(pName);
Py_DECREF(pName);
if(pModule != NULL) {
pFunc = PyObject_GetAttrString(pModule,"test_method");
if(pFunc && PyCallable_Check(pFunc)) {
pArgs = PyTuple_New(2);
pValue_1 = PyFloat_FromDouble(a);
pValue_2 = PyFloat_FromDouble(b);
if (!pValue_1) {
Py_DECREF(pArgs);
Py_DECREF(pModule);
fprintf(stderr, "Cannot convert argument\n");
return 1;
}
if (!pValue_2) {
Py_DECREF(pArgs);
Py_DECREF(pModule);
fprintf(stderr, "Cannot convert argument\n");
return 1;
}
PyTuple_SetItem(pArgs, 0, pValue_1);
PyTuple_SetItem(pArgs, 1, pValue_2);
pValue = PyObject_CallObject(pFunc, pArgs);
Py_DECREF(pArgs);
if (pValue != NULL) {
sum = PyFloat_AsDouble(PyTuple_GetItem(pValue,0));
diff = PyFloat_AsDouble(PyTuple_GetItem(pValue,1));
prod = PyFloat_AsDouble(PyTuple_GetItem(pValue,2));
printf("a: %f b: %f sum: %f diff: %f prod: %f",a,b,sum,diff,prod);
Py_DECREF(pValue);
}
else {
Py_DECREF(pFunc);
Py_DECREF(pModule);
PyErr_Print();
fprintf(stderr,"Call failed\n");
return 1;
}
} else {
if (PyErr_Occurred())
PyErr_Print();
fprintf(stderr, "Cannot find function \"%s\"\n", argv[2]);
}
Py_XDECREF(pFunc);
Py_DECREF(pModule);
}
else {
PyErr_Print();
fprintf(stderr, "Failed to load \"%s\"\n", argv[1]);
return 1;
}
if (Py_FinalizeEx() < 0) {
return 120;
}
return 0; }
I try to compile and link by running
gcc -c $(python3.8-config --cflags --embed) EmbedTest.cpp
gcc EmbedTest.o $(python3.8-config --embed --ldflags)
where python3.8-config --cflags expands to
-I/home/MyUserName/anaconda3/include/python3.8 -I/home/MyUserName/anaconda3/include/python3.8 -Wno-unused-result -Wsign-compare -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O3 -ffunction-sections -pipe -isystem /home/mbm/anaconda3/include -fdebug-prefix-map=/tmp/build/80754af9/python_1593706424329/work=/usr/local/src/conda/python-3.8.3 -fdebug-prefix-map=/home/MyUserName/anaconda3=/usr/local/src/conda-prefix -fuse-linker-plugin -ffat-lto-objects -flto-partition=none -flto -DNDEBUG -fwrapv -O3 -Wall
and where python3.8-config --ldflags expands to
python3.8-config --ldflags
-L/home/MyUserName/anaconda3/lib/python3.8/config-3.8-x86_64-linux-gnu -L/home/MyUserName/anaconda3/lib -lcrypt -lpthread -ldl -lutil -lrt -lm -lm
However, I get the following error message:
lto1: fatal error: bytecode stream in file ‘/home/MyUserName/anaconda3/lib/python3.8/config-3.8-x86_64-linux-gnu/libpython3.8.a’ generated with LTO version 6.0 instead of the expected 7.1
compilation terminated.
lto-wrapper: fatal error: gcc returned 1 exit status
compilation terminated.
/usr/bin/ld: error: lto-wrapper failed
collect2: error: ld returned 1 exit status
I understand this has to with the link-time-optimization and the fact that the python code appears to be compiled with an old vesion off gcc. My version of gcc is 8.3.0. To address this I tried to recompile but removing any extra flags (that have to do with LTO). So I tried
gcc -c -I/home/MyUserName/anaconda3/include/python3.8 -I/home/MyUserName/anaconda3/include/python3.8 EmbedTest.cpp
gcc EmbedTest.o $(python3.8-config --embed --ldflags)
but I get the same error message as before. Any idea how to solve this problem?

Turning off LTO with -fno-lto solved it.

I had the same problem when I misdirected PYTHON_INCLUDE_DIR and PYTHON_LIBRARIES in my CMakeList.txt. The problem was solved when I removed the manual location designation.

Undeclared identifier: PyUnicode_DecodeFSDefault

I try to compile the example code from Python website https://docs.python.org/3/extending/embedding.html. Everything works fine except the following line:
pName = PyUnicode_DecodeFSDefault(argv[1]);
I have Python 3.6 installed on my MacOS El Captain. My make file looks like the following:
call_function: call_function.o
gcc -o call_function call_function.o -export-dynamic -L/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/config-3.6m-darwin -lpython3.6m -lpthread -lm -ldl -lutil
call_function.o:call_function.cpp
gcc -c call_function.cpp -I/Library/Frameworks/Python.framework/Versions/3.6/include/python3.6m
When I compile the C++ code, I get the following error:
gcc -c call_function.cpp -I/Library/Frameworks/Python.framework/Versions/3.6/include/python3.6m
call_function.cpp:16:13: error: use of undeclared identifier 'PyUnicode_DecodeFSDefault'
pName = PyUnicode_DecodeFSDefault(argv[1]);
^
1 error generated.
Does anyone know how to fix the above error? I would greatly appreciated for your help.
Here is the complete example code:
#include <Python/Python.h>
#include <Python/unicodeobject.h>
int main(int argc, char *argv[])
{
PyObject *pName, *pModule, *pDict, *pFunc;
PyObject *pArgs, *pValue;
int i;
if (argc < 3) {
fprintf(stderr,"Usage: call pythonfile funcname [args]\n");
return 1;
}
Py_Initialize();
pName = PyUnicode_DecodeFSDefault(argv[1]);
// pName = PyUnicode_FromString(argv[1]); <-- also gives me an error
/* Error checking of pName left out */
pModule = PyImport_Import(pName);
Py_DECREF(pName);
if (pModule != NULL) {
pFunc = PyObject_GetAttrString(pModule, argv[2]);
/* pFunc is a new reference */
if (pFunc && PyCallable_Check(pFunc)) {
pArgs = PyTuple_New(argc - 3);
for (i = 0; i < argc - 3; ++i) {
pValue = PyLong_FromLong(atoi(argv[i + 3]));
if (!pValue) {
Py_DECREF(pArgs);
Py_DECREF(pModule);
fprintf(stderr, "Cannot convert argument\n");
return 1;
}
/* pValue reference stolen here: */
PyTuple_SetItem(pArgs, i, pValue);
}
pValue = PyObject_CallObject(pFunc, pArgs);
Py_DECREF(pArgs);
if (pValue != NULL) {
printf("Result of call: %ld\n", PyLong_AsLong(pValue));
Py_DECREF(pValue);
}
else {
Py_DECREF(pFunc);
Py_DECREF(pModule);
PyErr_Print();
fprintf(stderr,"Call failed\n");
return 1;
}
}
else {
if (PyErr_Occurred())
PyErr_Print();
fprintf(stderr, "Cannot find function \"%s\"\n", argv[2]);
}
Py_XDECREF(pFunc);
Py_DECREF(pModule);
}
else {
PyErr_Print();
fprintf(stderr, "Failed to load \"%s\"\n", argv[1]);
return 1;
}
Py_Finalize();
return 0;
}

I figure out what the problem is. The include <Python/Python.h> statement is actually linked to /System/Library/Frameworks/Python.framework/Versions/Current which currently is Python2.7. I have installed Python 3.6 on my Mac and even I include /Library/Frameworks/Python.framework/Versions/3.6/include/python3.6m in my make file, it's still referred to the Python.h file in MacOS's Python 2.7 folder. I tried to change the symbolic link of the Current to the new 3.6 folder and it's not letting me make the changes. Does anybody know how to change the symbolic link of the Python2.7 current folder so my gcc would look into the Python 3.6 folder for Python.h instead?

It is been a long time since this question got posted, but the solution that I came up is as follows,
char * full_cls_name = argv[1];
Py_Initialize();
pName = PyString_FromString(full_cls_name);
You should also check whether argv[1] exists or not?

This is what worked for me.
gcc -o call $(python3-config --cflags --embed) multiply_embed.c $(python3-config --ldflags --embed)
I got the answer from this post - answer by Ciro Santilli Путлер.
I also got module not found error when running the code, which was overcome by setting the PYTHONPATH variable in the zshrc file as described here.

How to solve the 'Segmentation fault' when hybrid programming of C & Python?

Under my Ubuntu:
$ cat test.py
#Filename test.py
def Hello():
print "Hello, world!"
$ cat tom.cpp
#include <Python.h>
int main()
{
Py_Initialize();
PyObject * pModule = NULL;
PyObject * pFunc = NULL;
pModule = PyImport_ImportModule("test");
pFunc = PyObject_GetAttrString(pModule, "Hello");
PyEval_CallObject(pFunc, NULL);
Py_Finalize();
return 0;
}
And then compile it:
g++ tom.cpp -I/usr/include/python2.7 -L/usr/lib/python2.7 -lpython2.7
Run:
$ ./a.out
Segmentation fault
Why?
Could anyone help?
Thanks!
BR,
Tom

The previous poster is probably right, so my comment is more "generic"...but in C/C++, you should NEVER accept a pointer back from a function without confirming it's not NULL before attempting to de-refence it. The above code should more rightly be:
pModule = PyImport_ImportModule("test");
if (pModule == NULL) {
printf("ERROR importing module");
exit(-1);
}
pFunc = PyObject_GetAttrString(pModule, "Hello");
if (pFunc == NULL) {
printf("ERROR getting Hello attribute");
exit(-1);
}
PyEval_CallObject(pFunc, NULL);

The issue is caused by PyObject_GetAttrString returning NULL. I have also added the directory path using PyRun_SimpleString as my dev dir was not under python path
#include <Python.h>
int main() {
Py_Initialize();
PyRun_SimpleString("import sys; sys.path.insert(0, 'add the directory path here')");
PyObject * pModule = NULL;
PyObject * pFunc = NULL;
pModule = PyImport_ImportModule("test");
pFunc = PyObject_GetAttrString(pModule, "Hello");
if (pFunc != NULL) {
PyEval_CallObject(pFunc, NULL);
Py_Finalize();
}
else {
printf("pFunc returned NULL\n");
}
return 0;
}