I'm trying to use future from C++ STL via cppyy (a C++-python binding packet).
For example, I could run this following code in C++ (which is adapted from this answer)
#include <future>
#include <thread>
#include <chrono>
#include <iostream>
using namespace std;
using namespace chrono_literals;
int main () {
promise<int> p;
future<int> f = p.get_future();
thread t([&p]() {
this_thread::sleep_for(10s);
p.set_value(2);
});
auto status = f.wait_for(10ms);
if (status == future_status::ready) {
cout << "task is read" << endl;
} else {
cout << "task is running" << endl;
}
t.join();
return 0;
}
A similar implementation of the above in Python is
import cppyy
cppyy.cppdef(r'''
#include <future>
#include <thread>
#include <chrono>
#include <iostream>
using namespace std;
int test () {
promise<int> p;
future<int> f = p.get_future();
thread t([&p]() {
this_thread::sleep_for(10s);
p.set_value(2);
});
auto status = f.wait_for(10ms);
if (status == future_status::ready) {
cout << "task is read" << endl;
} else {
cout << "task is running" << endl;
}
t.join();
return 0;
}
''')
cppyy.gbl.test()
And the above code yields
IncrementalExecutor::executeFunction: symbol '__emutls_v._ZSt15__once_callable' unresolved while linking symbol '__cf_4'!
IncrementalExecutor::executeFunction: symbol '__emutls_v._ZSt11__once_call' unresolved while linking symbol '__cf_4'!
It looks like it's caused by using future in cppyy.
Any solutions to this?
Clang9's JIT does not support thread local storage the way the modern g++ implements it, will check again when the (on-going) upgrade to Clang13 is finished, which may resolve this issue.
Otherwise, cppyy mixes fine with threaded code (e.g. the above example runs fine on MacOS, with Clang the system compiler). Just that any TLS use needs to sit in compiled library code while the JIT has this limitation.
Related
I've been struggling to get an example of pybind11 with Qt working. I can import other libraries like VTK fine, but when I include a Qt library, say QString, and create a simple QString object inside one of my functions, the built library has an import error when it's being imported in Python. I am not sure how to debug these issues, as there is no useful error anywhere that I can see. I tried to look at docs, but they don't show a way to debug these errors. There are no warnings or other issues when building the library.
>>> import pyLib
ImportError: DLL load failed while importing pyLib: The specified module could not be found.
I tried to create a minimal example below. The executable target pyLib2 builds and runs just fine, but the pyLib python library target doesn't work when imported due to this line QString x;. Without it, it works fine:
CMake
cmake_minimum_required (VERSION 3.24)
project (pybindTest)
include_directories(${PROJECT_BINARY_DIR} src)
# set C++ settings
set (CXX_VERSION 20) # sets which standard of C we are using, e.g. C++20
set (CMAKE_CXX_FLAGS "/EHsc /O2 /favor:INTEL64 /W4 /MP -std:c++${CXX_VERSION}")
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
add_subdirectory (external/pybind11)
add_executable(pyLib2 main.cpp)
pybind11_add_module(pyLib pyLib.cpp)
target_include_directories(pyLib PUBLIC
"C:/.../external/pybind11/include"
)
find_package (Qt5 5.15.2 EXACT COMPONENTS CONFIG REQUIRED Core Widgets SerialPort Network)
target_link_libraries(pyLib PUBLIC
Qt5::Core
Qt5::Widgets
Qt5::SerialPort
)
target_link_libraries(pyLib2 PUBLIC
Qt5::Core
Qt5::Widgets
Qt5::SerialPort
)
pyLib.cpp
#include <pybind11/pybind11.h>
#include <QString>
#include <array>
#include <iostream>
namespace py = pybind11;
float test(float a)
{
QString x; // Commenting this line out works fine
return a * 2.0;
}
void test2()
{
std::cout << "test2!" << std::endl;
}
void init_pyLib(py::module& handle)
{
std::cout << "here!" << std::endl;
}
PYBIND11_MODULE(pyLib, handle)
{
handle.doc() = "test doc";
handle.def("testpy", &test, py::arg("i"));
handle.def("testpy2", &test2);
init_pyLib(handle);
}
main.cpp
#include <QString>
#include <array>
#include <iostream>
float test(float a)
{
QString x;
return a * 2.0;
}
void test2()
{
std::cout << "test2!" << std::endl;
}
void init_pyLib()
{
std::cout << "here!" << std::endl;
}
int main()
{
std::cout << "hello!\n";
QString x;
test(5.0f);
std::cout << "goodbye!\n";
}
I want to wrap my c++ code as .so and .dll file.I know how to wrap c++ code as dynamic library, But my c++ code is calling python, normally we call embedding python.
I write a basic simple code.
python code:
def init_test(env, mode):
print(env)
print(mode)
return 1
c++ code calling python:
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include <iostream>
#include <exception>
/**
* #description: run risk performance use Python
* #param {string} env
* #param {string } mode
* #return {*}
*/
extern "C" int init_python_test(char* env, char* mode) {
std::cout << "start" <<std::endl;
if(Py_IsInitialized == 0){
std::cout << "not init" << std::endl;
}
else{
std::cout << "init already" <<std::endl;
//std::cout << Py_FinalizeEx() <<std::endl;
Py_Finalize();
}
std::cout << "init:"<<Py_IsInitialized() << std::endl;
Py_Initialize();
PyErr_Print();
std::cout <<"second" <<std::endl;
PyRun_SimpleString("import sys");
PyRun_SimpleString("sys.path.append('./')");
std::cout <<"ok" <<std::endl;
//int res;
PyObject *pModule,*pFunc = NULL;
PyObject *pArgs, *pValue = NULL;
pModule = PyImport_ImportModule("mini");//0x7ffff64b9cc0
if(!pModule)
std::cout << "can't open python file" << std::endl;
PyErr_Print();
pFunc = PyObject_GetAttrString(pModule, "init_test");
PyErr_Print();
if(pFunc && PyCallable_Check(pFunc)){
PyErr_Print();
pValue = PyObject_CallObject(pFunc, Py_BuildValue("(ss)", env, mode));
PyErr_Print();
}
Py_FinalizeEx();
return 1;
}
int main(){
char *env = (char*)"prod";
char * mode = (char*)"prod";
init_python_test(env, mode);
std::cout << "ok" <<std::endl;
}
I am able to run my c++ code properly with g++ command linked with python dynamic library. And I can use g++ to wrap my c++ code as .so file. When I use another c++ code and python code to test the init_python_test function. Segmentation fault occurs when the code runs into Py_Initialize().
So, how to resolve this question? and did I wrap c++ code properly with g++? here is my shell.
g++ -fPIC -shared -Wall -o libtest.so ./mini_test.cpp -DLINUX -D_GLIBCXX_USE_CXX11_ABI=0 -I /usr/include/python3.8 -L/usr/lib/python3 -L/usr/lib/python3.8 -lpython3.8
Somebody helps me! plz!!! thank u!!!!
I'm trying to use pybind11 in order to make a 3rd party C++ library call a Python method. The library is multithreaded, and each thread creates a Python object, and then does numerous calls to the object's methods.
My problem is that the call to py::gil_scoped_acquire acquire; deadlocks. A minimal code which reproduces the problem is given below. What am I doing wrong?
// main.cpp
class Wrapper
{
public:
Wrapper()
{
py::gil_scoped_acquire acquire;
auto obj = py::module::import("main").attr("PythonClass")();
_get_x = obj.attr("get_x");
_set_x = obj.attr("set_x");
}
int get_x()
{
py::gil_scoped_acquire acquire;
return _get_x().cast<int>();
}
void set_x(int x)
{
py::gil_scoped_acquire acquire;
_set_x(x);
}
private:
py::object _get_x;
py::object _set_x;
};
void thread_func()
{
Wrapper w;
for (int i = 0; i < 10; i++)
{
w.set_x(i);
std::cout << "thread: " << std::this_thread::get_id() << " w.get_x(): " << w.get_x() << std::endl;
std::this_thread::sleep_for(100ms);
}
}
int main() {
py::scoped_interpreter python;
std::vector<std::thread> threads;
for (int i = 0; i < 5; ++i)
threads.push_back(std::thread(thread_func));
for (auto& t : threads)
t.join();
return 0;
}
and the Python code:
// main.py
class PythonClass:
def __init__(self):
self._x = 0
def get_x(self):
return self._x
def set_x(self, x):
self._x = x
Related questions can be found here and here, but did not help me solve the problem.
I managed to resolve the issue by releasing the GIL in the main thread, before starting the worker threads (added py::gil_scoped_release release;). For anybody who is interested, the following now works (also added cleaning up Python objects):
#include <pybind11/embed.h>
#include <iostream>
#include <thread>
#include <chrono>
#include <sstream>
namespace py = pybind11;
using namespace std::chrono_literals;
class Wrapper
{
public:
Wrapper()
{
py::gil_scoped_acquire acquire;
_obj = py::module::import("main").attr("PythonClass")();
_get_x = _obj.attr("get_x");
_set_x = _obj.attr("set_x");
}
~Wrapper()
{
_get_x.release();
_set_x.release();
}
int get_x()
{
py::gil_scoped_acquire acquire;
return _get_x().cast<int>();
}
void set_x(int x)
{
py::gil_scoped_acquire acquire;
_set_x(x);
}
private:
py::object _obj;
py::object _get_x;
py::object _set_x;
};
void thread_func(int iteration)
{
Wrapper w;
for (int i = 0; i < 10; i++)
{
w.set_x(i);
std::stringstream msg;
msg << "iteration: " << iteration << " thread: " << std::this_thread::get_id() << " w.get_x(): " << w.get_x() << std::endl;
std::cout << msg.str();
std::this_thread::sleep_for(100ms);
}
}
int main() {
py::scoped_interpreter python;
py::gil_scoped_release release; // add this to release the GIL
std::vector<std::thread> threads;
for (int i = 0; i < 5; ++i)
threads.push_back(std::thread(thread_func, 1));
for (auto& t : threads)
t.join();
return 0;
}
Related to #bavaza's answer above, there is a way to self-contain both initialization and GIL release into a single class. You have to be careful, as that class is a singleton now (no different than scoped_interpreter), but it's possible. Here's the idea:
#include <pybind11/embed.h>
#include <memory>
using py = pybind11;
class PythonWrapper {
public:
PythonWrapper() : m_interpreter() {
// Do whatever one-time module/object initialization you want here
py::object obj = py::module::import("main").attr("PythonClass")(); // Speeds up importing later
// Last line of constructor releases the GIL
mp_gil_release = std::make_unique<py::gil_scoped_release>();
}
private:
py::scoped_interpreter m_interpreter;
// Important that this is the LAST member, so it gets destructed first, re-acquiring the GIL
std::unique_ptr<py::gil_scoped_release> mp_gil_release;
};
This would replace the two objects on the stack in main, leaving the Wrapper class unchanged! And if you wanted to have a true singleton for all your Python calls, this would help there too.
Again, thanks to #bavaza for the original solution. It helped me get my head around the right way to use the scoped locks for my own cross-thread usage.
Python is known to have a Global Interpreter Lock.
So you basically need to write your own Python interpreter from scratch, or download the source code of Python and improve it a lot.
If you are on Linux, you could consider running many Python interpreters (using appropriate syscalls(2), with pipe(7) or unix(7) for interprocess communication) - perhaps one Python process communicating with each of your C++ threads.
What am I doing wrong?
Coding in Python something which should be coded otherwise. Did you consider trying SBCL?
Some libraries (e.g. Tensorflow) can be called from both Python and C++. Maybe you could take inspiration from them...
In practice, if you have just a dozen C++ threads on a powerful Linux machine, you could afford having one Python process per C++ thread. So each C++ thread would have its own companion Python process.
Otherwise, budget several years of work to improve the source code of Python to remove its GIL. You might code your GCC plugin to help you on that task -analyzing and understanding the C code of Python.
I have a question regarding the below code.
It's an example how to pass a custom class via shared_ptr to embedded python code and it works when boost is dynamically linked.
Unfortunately the same code with statically linked boost doesn't work with the following error message:
"No to_python (by-value) converter found for C++ type: class boost::shared_ptr".
I don't understand why a different linking can affect type recognition of a registered converter. What am I missing?
Can anybody help me out?
Thanks,
Dominik
Example from here.
#include <boost/shared_ptr.hpp>
#include <boost/make_shared.hpp>
#include <boost/python.hpp>
#include <string>
#include <iostream>
namespace bp = boost::python;
struct Foo{
Foo(){}
Foo(std::string const& s) : m_string(s){}
void doSomething() {
std::cout << "Foo:" << m_string << std::endl;
}
std::string m_string;
};
typedef boost::shared_ptr<Foo> foo_ptr;
BOOST_PYTHON_MODULE(hello)
{
bp::class_<Foo, foo_ptr>("Foo")
.def("doSomething", &Foo::doSomething)
;
};
int main(int argc, char **argv)
{
Py_Initialize();
try {
PyRun_SimpleString(
"a_foo = None\n"
"\n"
"def setup(a_foo_from_cxx):\n"
" print 'setup called with', a_foo_from_cxx\n"
" global a_foo\n"
" a_foo = a_foo_from_cxx\n"
"\n"
"def run():\n"
" a_foo.doSomething()\n"
"\n"
"print 'main module loaded'\n"
);
foo_ptr a_cxx_foo = boost::make_shared<Foo>("c++");
inithello();
bp::object main = bp::object(bp::handle<>(bp::borrowed(
PyImport_AddModule("__main__")
)));
// pass the reference to a_cxx_foo into python:
bp::object setup_func = main.attr("setup");
setup_func(a_cxx_foo);
// now run the python 'main' function
bp::object run_func = main.attr("run");
run_func();
}
catch (bp::error_already_set) {
PyErr_Print();
}
Py_Finalize();
return 0;
}
I far as I understand the documentation about Boost Python linkage, it seems that the conversion registry used for automatic conversion of Python object into C++ object is not available when statically linked. I'm facing the same issue and that's a pity it is actually the case. I would have imagined at least the required converter to be bundle but I'm afraid it is not the case for some reason.
I'm having problems getting the NumPy C API to properly initialize. I think I've isolated the problem to calling import_array from a different translation unit, but I don't know why this should matter.
Minimal working example:
header1.hpp
#ifndef HEADER1_HPP
#define HEADER1_HPP
#include <Python.h>
#include <numpy/npy_3kcompat.h>
#include <numpy/arrayobject.h>
void initialize();
#endif
file1.cpp
#include "header1.hpp"
void* wrap_import_array()
{
import_array();
return (void*) 1;
}
void initialize()
{
wrap_import_array();
}
file2.cpp
#include "header1.hpp"
#include <iostream>
void* loc_wrap_import_array()
{
import_array();
return (void*) 1;
}
void loc_initialize()
{
loc_wrap_import_array();
}
int main()
{
Py_Initialize();
#ifdef USE_LOC_INIT
loc_initialize();
#else
initialize();
#endif
npy_intp dims[] = {5};
std::cout << "creating descr" << std::endl;
PyArray_Descr* dtype = PyArray_DescrFromType(NPY_FLOAT64);
std::cout << "zeros" << std::endl;
PyArray_Zeros(1, dims, dtype, 0);
std::cout << "cleanup" << std::endl;
return 0;
}
Compiler commands:
g++ file1.cpp file2.cpp -o segissue -lpython3.4m -I/usr/include/python3.4m -DUSE_LOC_INIT
./segissue
# runs fine
g++ file1.cpp file2.cpp -o segissue -lpython3.4m -I/usr/include/python3.4m
./segissue
# segfaults
I've tested this with Clang 3.6.0, GCC 4.9.2, Python 2.7, and Python 3.4 (with a suitably modified wrap_import_array because this is different between Python 2.x and 3.x). The various combinations all give the same result: if I don't call loc_initialize, the program will segfault in the PyArray_DescrFromType call. I have NumPy version 1.8.2. For reference, I'm running this in Ubuntu 15.04.
What baffles me most of all is this C++ NumPy wrapper appears to get away with calling import_array in a different translation unit.
What am I missing? Why must I call import_array from the same translation unit in order for it to actually take effect? More importantly, how do I get it to work when I call import_array from a different translation unit like the Boost.NumPy wrapper does?
After digging through the NumPy headers, I think I've found a solution:
in numpy/__multiarray_api.h, there's a section dealing with where an internal API buffer should be. For conciseness, here's the relevant snippet:
#if defined(PY_ARRAY_UNIQUE_SYMBOL)
#define PyArray_API PY_ARRAY_UNIQUE_SYMBOL
#endif
#if defined(NO_IMPORT) || defined(NO_IMPORT_ARRAY)
extern void **PyArray_API;
#else
#if defined(PY_ARRAY_UNIQUE_SYMBOL)
void **PyArray_API;
#else
static void **PyArray_API=NULL;
#endif
#endif
It looks like this is intended to allow multiple modules define their own internal API buffer, in which each module must call their own import_array define.
A consistent way to get several translation units to use the same internal API buffer is in every module, define PY_ARRAY_UNIQUE_SYMBOL to some library unique name, then every translation unit other than the one where the import_array wrapper is defined defines NO_IMPORT or NO_IMPORT_ARRAY. Incidentally, there are similar macros for the ufunc features: PY_UFUNC_UNIQUE_SYMBOL, and NO_IMPORT/NO_IMPORT_UFUNC.
The modified working example:
header1.hpp
#ifndef HEADER1_HPP
#define HEADER1_HPP
#ifndef MYLIBRARY_USE_IMPORT
#define NO_IMPORT
#endif
#define PY_ARRAY_UNIQUE_SYMBOL MYLIBRARY_ARRAY_API
#define PY_UFUNC_UNIQUE_SYMBOL MYLIBRARY_UFUNC_API
#include <Python.h>
#include <numpy/npy_3kcompat.h>
#include <numpy/arrayobject.h>
void initialize();
#endif
file1.cpp
#define MYLIBRARY_USE_IMPORT
#include "header1.hpp"
void* wrap_import_array()
{
import_array();
return (void*) 1;
}
void initialize()
{
wrap_import_array();
}
file2.cpp
#include "header1.hpp"
#include <iostream>
int main()
{
Py_Initialize();
initialize();
npy_intp dims[] = {5};
std::cout << "creating descr" << std::endl;
PyArray_Descr* dtype = PyArray_DescrFromType(NPY_FLOAT64);
std::cout << "zeros" << std::endl;
PyArray_Zeros(1, dims, dtype, 0);
std::cout << "cleanup" << std::endl;
return 0;
}
I don't know what pitfalls there are with this hack or if there are any better alternatives, but this appears to at least compile and run without any segfaults.