I have this simple wrapper working and I am trying to make changes in order
to get the array all at once to use it in python.
test.cpp
#include <iostream>
using namespace std;
class Test{
public:
int ret(void);
};
int vec[10] = {0,1,2,3,4,5,6,7,8,9};
int Test::ret(){
return *vec; // This obviously won't work
}
extern "C" {
Test* Test_new(){ return new Test();}
int Test_ret(Test* test){ return test->ret();}
}
test.py
from ctypes import cdll
lib = cdll.LoadLibrary('./libtest.so')
class Test(object):
def __init__(self):
self.obj = lib.Test_new()
def ret(self):
return lib.Test_ret(self.obj)
dev = Test()
ret = dev.ret()
print ("Return", ret)
Compiling script
g++ -c -fPIC test.cpp -o test.o
g++ -shared -Wl,-soname,libtest.so -o libtest.so test.o
How can I do it?
Related
I am trying to get a minimal example running calling c++ routines from python. I am referring to a related blog here. I am not able to get any output from the c++ code. Also I am not getting any error or warning which can guide me.
max.cpp
#include <iostream>
using namespace std;
// function declaration
int max(int num1, int num2);
int main () {
// local variable declaration:
int a = 100;
int b = 200;
int ret;
// calling a function to get max value.
ret = max(a, b);
cout << "In CPP : Max value is : " << ret << endl;
return 0;
}
// function returning the max between two numbers
int max(int num1, int num2) {
// local variable declaration
int result;
if (num1 > num2)
result = num1;
else
result = num2;
return result;
}
tasks.py
import invoke
#invoke.task()
def build_max(c):
"Build the shared library for cpp cmult code"
print_banner("Building C++ Library")
invoke.run(
"g++ -O3 -Wall -Werror -shared -std=c++11 -fPIC max.cpp "
"-o libmax.so"
)
print("* Complete")
def compile_python_module(cpp_name, extension_name):
invoke.run(
"g++ -O3 -Wall -Werror -shared -std=c++11 -fPIC "
"`python3 -m pybind11 --includes` "
"-I /usr/include/python3.8 -I . "
"{0} "
"-o {1}`python3.8-config --extension-suffix` "
"-L. -lmax -Wl,-rpath,.".format(cpp_name, extension_name)
)
#invoke.task(build_max)
def build_pybind11_max(c):
""" Build the pybind11 wrapper library """
print_banner("Building PyBind11 Module")
compile_python_module("pybind11_max.cpp", "pybind11_max")
print("* Complete")
#invoke.task()
def test_pybind11_max(c):
""" Run the script to test PyBind11 """
print_banner("Testing PyBind11 Module")
invoke.run("python3.8 pybind11_max_test.py", pty=True)
max.hpp header file
int main();
pybind11 code(pybind11_max.cpp) that wraps C++ code
#include <pybind11/pybind11.h>
#include <max.hpp>
PYBIND11_MODULE(pybind11_max, m) {
m.doc() = "pybind11 max plugin";
m.def("cpp_function", &main, "find max");
}
I am able to compile and build it invoke build-pybind11-max
Response
==================================================
= Building C++ Library
* Complete
==================================================
= Building PyBind11 Module
* Complete
Code to test
#!/usr/bin/env python
import pybind11_max
if __name__ == "__main__":
answer = pybind11_max.cpp_function()
print(f" In Python {answer}")
To run the test, I run invoke test-pybind11-max I get
==================================================
= Testing PyBind11 Module
Adding updates here-
when I run the test file python pybind11_max_test.py from terminal, I get Segmentation fault (core dumped) error.
When running python3.8 pybind11_max_test.py, I get the following output
Fatal Python error: _PyArgv_AsWstrList: memory allocation failed
Python runtime state: initialized
Current thread 0x00007fc24d885080 (most recent call first):
File "pybind11_max_test.py", line 5 in <module>
I'm trying to make a simple example to wrap an abstract C++ class with pybind. The code is:
#include <pybind11/pybind11.h>
#include <ostream>
#include <iostream>
namespace py = pybind11;
using namespace std;
class Base
{
public:
virtual void test() = 0;
};
class Derived: public Base
{
public:
void test() {cout << "Test";}
};
PYBIND11_MODULE(example,m) {
py::class_<Base, Derived>(m, "Base")
.def(py::init<>())
.def("test", &Derived::test);
}
And while I run the following command
c++ -O3 -Wall -shared -std=c++11 -fPIC `python3 -m pybind11 --includes` abstract_test.cpp -o example`python3-config --extension-suffix`\n
I get the error:
In file included from abstrakt_test.cpp:1:
/home/anaconda3/envs/pybind/lib/python3.8/site-packages/pybind11/include/pybind11/pybind11.h: In instantiation of ‘Return (Derived::* pybind11::method_adaptor(Return (Class::*)(Args ...)))(Args ...) [with Derived = Base; Return = void; Class = Derived; Args = {}]’:
/home/anaconda3/envs/pybind/lib/python3.8/site-packages/pybind11/include/pybind11/pybind11.h:1118:45: required from ‘pybind11::class_<type_, options>& pybind11::class_<type_, options>::def(const char*, Func&&, const Extra& ...) [with Func = void (Derived::*)(); Extra = {}; type_ = Base; options = {Derived}]’
abstrakt_test.cpp:23:36: required from here
/home/anaconda3/envs/pybind/lib/python3.8/site-packages/pybind11/include/pybind11/pybind11.h:1032:19: error: static assertion failed: Cannot bind an inaccessible base class method; use a lambda definition instead
static_assert(detail::is_accessible_base_of<Class, Derived>::value,
^~~~~~
/home/anaconda3/envs/pybind/lib/python3.8/site-packages/pybind11/include/pybind11/pybind11.h:1034:12: error: cannot convert ‘void (Derived::*)()’ to ‘void (Base::*)()’ in return
return pmf;
^~~
You need to "wrap" Base as well. Otherwise you are going to get following exception at import time:
ImportError: generic_type: type "Derived" referenced unknown base type "Base"
Also, wrapping order of Derived is wrong:
py::class_<Derived, Base>(m, "Derived")
Full example:
PYBIND11_MODULE(example,m) {
py::class_<Base>(m, "Base");
py::class_<Derived, Base>(m, "Derived")
.def(py::init<>())
.def("test", &Derived::test);
}
I have a little problem with the compilation of multiple C++ on Windows.
I implemented four classes in C++ for cryptography with gmp. I want to call them from Python with ctypes.
I wrote a cpp file with the extern keyword:
#include "integer.h"
#include "modular_number.h"
#include "padic_number.h"
#include "rational_number.h"
extern "C" {
__declspec(dllexport) ModNum* newModNum(const char * n, const char * p) { return new ModNum(Integer(n), Integer(p)); }
__declspec(dllexport) const char* getModValue(const ModNum& mod){ return mod.getValue().getValue(); }
__declspec(dllexport) RationalNum* newRationalNum(const char* mpq) { return new RationalNum(mpq); }
__declspec(dllexport) const char* getRationalValue(const RationalNum& rat){ return rat.getValue(); }
__declspec(dllexport) PadicNum* newPadicNum(const char* n, const char* base) { return new PadicNum(Integer(n), Integer(base)); }
__declspec(dllexport) const char* getPadicValue(const PadicNum& padic){ return padic.getValue().getValue(); }
}
I compiled my files with:
mingw32-g++ -fexceptions -g -fexpensive-optimizations -flto -O3 -Weffc++ -Wextra -Wall -std=c++14 -fPIC -Og -IC:\MinGW\include -flto -s -lgmp -lmpfr -lpthread -c -fPIC *.cpp -I"C:\Program Files\Python38-32\include" -I"C:\Program Files\Python38-32\libs"
mingw32-g++.exe -shared -Wl,-dll -o numeric.dll *.o -lgmp -lmpfr -lgmpxx -static
But when I use these commands in Python:
import ctypes;
x = ctypes.DLL("./numeric.dll");
The variable x does not have the functions: newModNum, getModValue, etc...
Could anyone tell me what I'm doing wrong? I get no error and I do not understand.
My other files are common C ++ files with header and implementation.
Thanks in advance and have a nice day!
ctypes functions are imported on first use. Using libc as an example:
>>> import ctypes
>>> libc = ctypes.CDLL("libc.so.06")
>>> "printf" in dir(libc)
False
>>> libc.printf
<_FuncPtr object at 0x7f6512c23430>
>>> "printf" in dir(libc)
True
ctypes assumes all parameters and the return value are int. You should give type hints which also conveniently import the functions.
import ctypes
x = ctypes.DLL("./numeric.dll")
x.newModNum.argtypes = [ctypes.c_char_p, ctypes.c_char_p] # <-- also imports
x.newModNum.rettype = ctypes.c_void_p
And remove the semicolons from the end of lines. It causes dangerous blood pressure spikes in python programmers.
I'm getting different results for the same shared library when a function (from the shared library) is executed by a standalone executable and when using Python's ctypes for calling the exact same function.
The difference was narrowed down to be the version of libm used by the library.
When using the library from a standalone executable we link against openlibm, but when the library is linked by ctypes, it's linked against Ubuntu's libm.
How can I make ctypes link my shared library against openlibm?
The shared library header (fft.h):
extern "C" {
void fft(const char* srcPath);
};
The shared library implementation (fft.cpp):
#include <iostream>
#include <fstream>
#include <nlohmann/json.hpp>
#include <Eigen/Eigen>
#include <unsupported/Eigen/FFT>
void fft(const char* srcPath) {
std::cout.precision(17);
std::ifstream fs(srcPath, std::ofstream::in);
// Get length of file
fs.seekg(0, std::ifstream::end);
auto length = fs.tellg();
fs.seekg(0, std::ifstream::beg);
// Read the file
auto srcRaw = new char[length];
fs.read(srcRaw, length);
// Convert data to vector of doubles
nlohmann::json j = nlohmann::json::parse(srcRaw);
std::vector<double> srcDoubles = j;
int n = 65536;
Eigen::VectorXd srcEigen(n);
Eigen::VectorXcd dstEigen(n);
std::copy(srcDoubles.data(), srcDoubles.data() + srcDoubles.size(), srcEigen.data());
Eigen::FFT<double> fft;
fft.fwd(dstEigen, srcEigen);
std::vector<std::complex<double>> dstTmp(dstEigen.data(), dstEigen.data() + dstEigen.size());
for (size_t i = 0; i < dstTmp.size(); i++) {
std::cout << "Result[" << i << "] = " << dstTmp[i].real() << "+i*" << dstTmp[i].imag() << std::endl;
}
delete[] srcRaw;
}
Using the library from a standalone executable:
#include <fft.h>
int main(int argc, char** argv) {
fft("input_data");
}
Using Python's ctypes:
from ctypes import c_char_p, CDLL
class LibCFft(CDLL):
def __init__(self):
super(LibCFft, self).__init__("libexample.so")
self._addfunc('fft', [c_char_p], None)
def _addfunc(self, name, argtypes, restype):
attr = getattr(self, name)
attr.argtypes = argtypes
attr.restype = restype
return lambda x: x
class CFft(object):
def fft(self):
LibCFft().fft("input_data")
if __name__ == '__main__':
CFft().fft()
Lib build:
clang++ /opt/eigen-eigen-43d9075b23ef -isystem /opt/spdlog/include -isystem /opt/nlohmann/include -O3 -DNDEBUG -fPIC -std=gnu++14 -o CMakeFiles/example.dir/fft.cpp.o -c /home/user/fft.cpp
clang++ -fPIC -O3 -DNDEBUG -shared -Wl,-soname,libexample.so -o libbugatone.so CMakeFiles/example.dir/generated/fft.cpp.o -lopenlibm
Executable build:
clang++ -O3 -DNDEBUG -latomic -nodefaultlibs -lopenlibm -lc -lc++ -lgcc -lgcc_s -lstdc++ -latomic -lm CMakeFiles/eigen_tester.dir/eigen_tester.cpp.o -o eigen_tester libexample.so -lopenlibm
Thanks in advance for the help.
In the executable link step you use BOTH -lm and -lopenlibm which is probably not what you want.
-nodefaultlibs disables automatic inclusion of all standard libs, so if you simply not include -lm but keep -lopenlibm this should probably do what you want.
You colud update LD_LIBRARY_PATH in your environment, to point to your local library folder first, and keep your libopenlibm.so there and also create a soft link to it: 'ln -s libopenlibm.so libm.so'.
In most cases that would cause any shared objects to use this as a substitution to a regular libm.so
I would like to use a shared library (with python ctypes) which expects that some symbols are defined in the program that uses this shared library.
In the following example, var is defined in test.c, while nm libadd.so shows that var is undefined;
/* libadd.c */
#include <stdio.h>
extern int var;
int add(int a, int b) {
return (a + b + var);
}
gcc -c -Wall -Werror -fpic libadd.c
gcc -shared -o libadd.so libadd.o
/* test.c */
#include <stdio.h>
#include "libadd.h"
int var = 1;
int main(void) {
printf("%d", add(1,2));
return 0;
}
gcc -L/path/to/libadd -Wall -o test test.c -ladd
Similarly, how to do this with python ctypes;
# test.py
from ctypes import *
#what to do to define var?
sharedlib = CDLL("sharedlib/libadd.so", mode = RTLD_GLOBAL)
print( sharedlib.add(1,2) )
OSError: sharedlib/libadd.so: undefined symbol: var
One possibility is to create a seperate shared library to define var and load this before libadd.so.
/* resolve.c */
int var = 1;
gcc -c -Wall -Werror -fpic resolve.c
gcc -shared -o resolve.so resolve.o
from ctypes import *
sharedlib = CDLL("sharedlib/resolve.so", mode = RTLD_GLOBAL)
sharedlib = CDLL("sharedlib/libadd.so", mode = RTLD_GLOBAL)