I want to know how to wrap a C++ object with Python Extension API (and distutils) without external tools (like Cython, Boost, SWIG, ...). Just in pure Python way without creating a dll.
Note that my C++ object has memory allocations so destructor has to be called to avoid memory leaks.
#include "Voice.h"
namespace transformation
{
Voice::Voice(int fftSize) { mem=new double[fftSize]; }
Voice::~Voice() { delete [] mem; }
int Voice::method1() { /*do stuff*/ return (1); }
}
I just want to do somethings like that in Python :
import voice
v=voice.Voice(512)
result=v.method1()
Seems that the answer was in fact here : https://docs.python.org/3.6/extending/newtypes.html
With examples, but not really easy.
EDIT 1 :
In fact, it is not really for wrapping a C++ object in a Python object, but rather to create a Python object with C code. (edit2 : and so you can wrap C++ object!)
EDIT 2 :
Here is a solution using the Python newtypes
.
Original C++ file : Voice.cpp
#include <cstdio>
#include "Voice.h"
namespace transformation
{
Voice::Voice(int fftSize) {
printf("c++ constructor of voice\n");
this->fftSize=fftSize;
mem=new double[fftSize];
}
Voice::~Voice() { delete [] mem; }
int Voice::filter(int freq) {
printf("c++ voice filter method\n");
return (doubleIt(3));
}
int Voice::doubleIt(int i) { return 2*i; }
}
.
Original h file : Voice.h
namespace transformation {
class Voice {
public:
double *mem;
int fftSize;
Voice(int fftSize);
~Voice();
int filter(int freq);
int doubleIt(int i);
};
}
.
C++ Python wrapper file : voiceWrapper.cpp
#include <Python.h>
#include <cstdio>
//~ #include "structmember.h"
#include "Voice.h"
using transformation::Voice;
typedef struct {
PyObject_HEAD
Voice * ptrObj;
} PyVoice;
static PyModuleDef voicemodule = {
PyModuleDef_HEAD_INIT,
"voice",
"Example module that wrapped a C++ object",
-1,
NULL, NULL, NULL, NULL, NULL
};
static int PyVoice_init(PyVoice *self, PyObject *args, PyObject *kwds)
// initialize PyVoice Object
{
int fftSize;
if (! PyArg_ParseTuple(args, "i", &fftSize))
return -1;
self->ptrObj=new Voice(fftSize);
return 0;
}
static void PyVoice_dealloc(PyVoice * self)
// destruct the object
{
delete self->ptrObj;
Py_TYPE(self)->tp_free(self);
}
static PyObject * PyVoice_filter(PyVoice* self, PyObject* args)
{
int freq;
int retval;
if (! PyArg_ParseTuple(args, "i", &freq))
return Py_False;
retval = (self->ptrObj)->filter(freq);
return Py_BuildValue("i",retval);
}
static PyMethodDef PyVoice_methods[] = {
{ "filter", (PyCFunction)PyVoice_filter, METH_VARARGS, "filter the mem voice" },
{NULL} /* Sentinel */
};
static PyTypeObject PyVoiceType = { PyVarObject_HEAD_INIT(NULL, 0)
"voice.Voice" /* tp_name */
};
PyMODINIT_FUNC PyInit_voice(void)
// create the module
{
PyObject* m;
PyVoiceType.tp_new = PyType_GenericNew;
PyVoiceType.tp_basicsize=sizeof(PyVoice);
PyVoiceType.tp_dealloc=(destructor) PyVoice_dealloc;
PyVoiceType.tp_flags=Py_TPFLAGS_DEFAULT;
PyVoiceType.tp_doc="Voice objects";
PyVoiceType.tp_methods=PyVoice_methods;
//~ PyVoiceType.tp_members=Noddy_members;
PyVoiceType.tp_init=(initproc)PyVoice_init;
if (PyType_Ready(&PyVoiceType) < 0)
return NULL;
m = PyModule_Create(&voicemodule);
if (m == NULL)
return NULL;
Py_INCREF(&PyVoiceType);
PyModule_AddObject(m, "Voice", (PyObject *)&PyVoiceType); // Add Voice object to the module
return m;
}
.
distutils file : setup.py
from distutils.core import setup, Extension
setup(name='voicePkg', version='1.0', \
ext_modules=[Extension('voice', ['voiceWrapper.cpp','Voice.cpp'])])
.
python test file : test.py
import voice
v=voice.Voice(512)
result=v.filter(5)
print('result='+str(result))
.
and magic :
sudo python3 setup.py install
python3 test.py
Output is :
c++ constructor of voice
c++ voice filter method
result=6
Enjoy !
Doom
Related
I wrote a c++ module witch should be imported into Python. Below are both Codes, the C++ part and the Python part. The C++ function method_sum should return the double of a value to python.
module.cpp:
#define PY_SSIZE_T_CLEAN
#include <Python.h>
static PyObject *method_sum(PyObject *self, PyObject *args) {
const int *prop;
if (!PyArg_ParseTuple(args, "i", &prop)) return NULL;
int result = *prop + *prop;
return Py_BuildValue("i", result);
}
static PyMethodDef ModuleMethods[] = {
{"sum", method_sum, METH_VARARGS, "description of the function"},
{NULL, NULL, 0, NULL}
};
static struct PyModuleDef module = {
PyModuleDef_HEAD_INIT,
"module",
"description of the module",
-1,
ModuleMethods
};
PyMODINIT_FUNC PyInit_module(void) {
return PyModule_Create(&module);
}
main.py:
import module
print(module.sum(18))
setup.py:
from distutils.core import setup, Extension
setup(name='module', version='1.0', ext_modules=[Extension('module', ['module.cpp'])])
I changed method_sum to the following and main.py prints 36 instead of segfaulting.
static PyObject *method_sum(PyObject *self, PyObject *args) {
int prop;
if (!PyArg_ParseTuple(args, "i", &prop)) return NULL;
int result = prop + prop;
return Py_BuildValue("i", result);
}
The following also works and prop is still a pointer like in the code in the question.
static PyObject *method_sum(PyObject *self, PyObject *args) {
const int *prop = new int;
if (!PyArg_ParseTuple(args, "i", prop)) {
delete prop;
return NULL;
}
int result = *prop + *prop;
delete prop;
return Py_BuildValue("i", result);
}
Everywhere, I can easily find an example about writing a method with Python C Extensions and use it in Python. Like this one: Python 3 extension example
$ python3
>>> import hello
>>> hello.hello_world()
Hello, world!
>>> hello.hello('world')
Hello, world!
How to do write a hello word full featured Python class (not just a module method)?
I think this How to wrap a C++ object using pure Python Extension API (python3)? question has an example, but it does not seem minimal as he is using (or wrapping?) C++ classes on it.
For example:
class ClassName(object):
"""docstring for ClassName"""
def __init__(self, hello):
super().__init__()
self.hello = hello
def talk(self, world):
print( '%s %s' % ( self.hello, world ) )
What is the equivalent of this Python class example with C Extensions?
I would use it like this:
from .mycextensionsmodule import ClassName
classname = ClassName("Hello")
classname.talk( 'world!' )
# prints "Hello world!"
My goal is to write a class fully written in C for performance (all other classes in my project will be in Python, except this one). I am not looking for portability as using ctypes, neither black boxes as using Boost.Python or SWIG. Just a high-performance class purely written with Python C Extensions.
After I got this Hello word working, I can figure my self out within Python Extensive documentation:
https://docs.python.org/3/c-api/
https://docs.python.org/3/extending/extending.html
See also: Python instance method in C
Create the file called MANIFEST.in
include README.md
include LICENSE.txt
recursive-include source *.h
Create the file called setup.py
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from setuptools import setup, Extension
__version__ = '0.1.0'
setup(
name = 'custom',
version = __version__,
package_data = {
'': [ '**.txt', '**.md', '**.py', '**.h', '**.hpp', '**.c', '**.cpp' ],
},
ext_modules = [
Extension(
name = 'custom',
sources = [
'source/custom.cpp',
],
include_dirs = ['source'],
)
],
)
Create the file called source/custom.cpp
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include "structmember.h"
typedef struct {
PyObject_HEAD
PyObject *first; /* first name */
PyObject *last; /* last name */
int number;
} CustomObject;
static int
Custom_traverse(CustomObject *self, visitproc visit, void *arg)
{
Py_VISIT(self->first);
Py_VISIT(self->last);
return 0;
}
static int
Custom_clear(CustomObject *self)
{
Py_CLEAR(self->first);
Py_CLEAR(self->last);
return 0;
}
static void
Custom_dealloc(CustomObject *self)
{
PyObject_GC_UnTrack(self);
Custom_clear(self);
Py_TYPE(self)->tp_free((PyObject *) self);
}
static PyObject *
Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
CustomObject *self;
self = (CustomObject *) type->tp_alloc(type, 0);
if (self != NULL) {
self->first = PyUnicode_FromString("");
if (self->first == NULL) {
Py_DECREF(self);
return NULL;
}
self->last = PyUnicode_FromString("");
if (self->last == NULL) {
Py_DECREF(self);
return NULL;
}
self->number = 0;
}
return (PyObject *) self;
}
static int
Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"first", "last", "number", NULL};
PyObject *first = NULL, *last = NULL, *tmp;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist,
&first, &last,
&self->number))
return -1;
if (first) {
tmp = self->first;
Py_INCREF(first);
self->first = first;
Py_DECREF(tmp);
}
if (last) {
tmp = self->last;
Py_INCREF(last);
self->last = last;
Py_DECREF(tmp);
}
return 0;
}
static PyMemberDef Custom_members[] = {
{"number", T_INT, offsetof(CustomObject, number), 0,
"custom number"},
{NULL} /* Sentinel */
};
static PyObject *
Custom_getfirst(CustomObject *self, void *closure)
{
Py_INCREF(self->first);
return self->first;
}
static int
Custom_setfirst(CustomObject *self, PyObject *value, void *closure)
{
if (value == NULL) {
PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
return -1;
}
if (!PyUnicode_Check(value)) {
PyErr_SetString(PyExc_TypeError,
"The first attribute value must be a string");
return -1;
}
Py_INCREF(value);
Py_CLEAR(self->first);
self->first = value;
return 0;
}
static PyObject *
Custom_getlast(CustomObject *self, void *closure)
{
Py_INCREF(self->last);
return self->last;
}
static int
Custom_setlast(CustomObject *self, PyObject *value, void *closure)
{
if (value == NULL) {
PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute");
return -1;
}
if (!PyUnicode_Check(value)) {
PyErr_SetString(PyExc_TypeError,
"The last attribute value must be a string");
return -1;
}
Py_INCREF(value);
Py_CLEAR(self->last);
self->last = value;
return 0;
}
static PyGetSetDef Custom_getsetters[] = {
{"first", (getter) Custom_getfirst, (setter) Custom_setfirst,
"first name", NULL},
{"last", (getter) Custom_getlast, (setter) Custom_setlast,
"last name", NULL},
{NULL} /* Sentinel */
};
static PyObject *
Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored))
{
return PyUnicode_FromFormat("%S %S", self->first, self->last);
}
static PyMethodDef Custom_methods[] = {
{"name", (PyCFunction) Custom_name, METH_NOARGS,
"Return the name, combining the first and last name"
},
{NULL} /* Sentinel */
};
static PyTypeObject CustomType = {
PyVarObject_HEAD_INIT(NULL, 0)
.tp_name = "custom.Custom",
.tp_doc = "Custom objects",
.tp_basicsize = sizeof(CustomObject),
.tp_itemsize = 0,
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
.tp_new = Custom_new,
.tp_init = (initproc) Custom_init,
.tp_dealloc = (destructor) Custom_dealloc,
.tp_traverse = (traverseproc) Custom_traverse,
.tp_clear = (inquiry) Custom_clear,
.tp_members = Custom_members,
.tp_methods = Custom_methods,
.tp_getset = Custom_getsetters,
};
static PyModuleDef custommodule = {
PyModuleDef_HEAD_INIT,
.m_name = "custom",
.m_doc = "Example module that creates an extension type.",
.m_size = -1,
};
PyMODINIT_FUNC
PyInit_custom(void)
{
PyObject *m;
if (PyType_Ready(&CustomType) < 0)
return NULL;
m = PyModule_Create(&custommodule);
if (m == NULL)
return NULL;
Py_INCREF(&CustomType);
PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
return m;
}
Then, to compile it and install you can run either:
pip3 install . -v
python3 setup.py install
As side note from this question How to use setuptools packages and ext_modules with the same name? do not mix on the same project *.py files and Python C Extensions, i.e., use only purely C/C++, building Python C Extensions without adding packages = [ 'package_name' ] entries because they cause the Python C Extensions code run 30%, i.e., if the program would take 7 seconds to run, now with *.py files, it will take 11 seconds.
References:
https://docs.python.org/3/extending/newtypes_tutorial.html#supporting-cyclic-garbage-collection
I currently have a function that uses a struct as a buffer to return some information, like so:
int example_reader(int code, void* return_struct);
My goal is to make it so that when I wrap this function using SWIG so that it can be used in Python, I will return the struct along with the function's regular return value. Thus far, I have been doing so using the %apply command like so:
%apply struct ret_struct *OUTPUT {void* return_struct};
However, when I add the above line to my .i file and try to run SWIG, I get the following warning:
"warning 453: Can't apply (struct ret_struct *OUTPUT. No typemaps are defined"
I believe I'm including the .h file that defines the struct I'm trying to return, so I've had trouble pinpointing the issue. Please correct me if the issue seems to involve the improper inclusion of the struct. I've tried reading through the SWIG documentation as well as other Stack Overflow posts to get some inkling of what the problem might be, but I haven't been able to figure it out thus far. The problem is made slightly trickier because I am trying to return a void pointer to a struct, and the code I'm trying to wrap could have multiple kinds of structs for me to return. What would be a wise way of handling the return of this struct? Thank you!
I have given here a full C example, where an interface is used for returning a struct to the target language together with a return value. In this way you can make a proper interface, where no implementation is given in the header. That is no default implementation of a virtual destructor. If you don't want to use an interface, you can let SWIG and Python know how data are represented.
Interface header: foo.h
typedef struct _Foo Foo;
int foo_new(Foo **obj);
int foo_free(Foo *obj);
int foo_get_value_a(Foo *obj, int *result);
int foo_set_value_a(Foo *obj, int value);
int foo_get_value_b(Foo *obj, char **result);
int foo_set_value_b(Foo *obj, char *value);
SWIG interface: foo.i
%module foo
%{
#include "foo.h"
%}
%include "typemaps.i"
%typemap(in, numinputs=0) Foo ** (Foo *temp) {
$1 = &temp;
}
%typemap(argout) Foo ** {
PyObject* temp = NULL;
if (!PyList_Check($result)) {
temp = $result;
$result = PyList_New(1);
PyList_SetItem($result, 0, temp);
}
temp = SWIG_NewPointerObj(*$1, SWIGTYPE_p__Foo, SWIG_POINTER_NEW);
PyList_Append($result, temp);
Py_DECREF(temp);
}
%delobject foo_free; // Protect for double deletion
struct _Foo {};
%extend _Foo {
~_Foo() {
foo_free($self);
}
};
%ignore _Foo;
Some implementation of the interface: foo.c
%include "foo.h"
#include "foo.h"
#include "stdlib.h"
#include "string.h"
struct FooImpl {
char* c;
int i;
};
int foo_new(Foo **obj)
{
struct FooImpl* f = (struct FooImpl*) malloc(sizeof(struct FooImpl));
f->c = NULL;
*obj = (Foo*) f;
return 0;
}
int foo_free(Foo *obj)
{
struct FooImpl* impl = (struct FooImpl*) obj;
if (impl) {
if (impl->c) {
free(impl->c);
impl->c = NULL;
}
}
return 0;
}
int foo_get_value_a(Foo *obj, int *result)
{
struct FooImpl* impl = (struct FooImpl*) obj;
*result = impl->i;
return 0;
}
int foo_set_value_a(Foo *obj, int value)
{
struct FooImpl* impl = (struct FooImpl*) obj;
impl->i = value;
return 0;
}
int foo_get_value_b(Foo *obj, char **result)
{
struct FooImpl* impl = (struct FooImpl*) obj;
*result = impl->c;
return 0;
}
int foo_set_value_b(Foo *obj, char *value)
{
struct FooImpl* impl = (struct FooImpl*) obj;
int len = strlen(value);
if (impl->c) {
free(impl->c);
}
impl->c = (char*)malloc(len+1);
strcpy(impl->c,value);
return 0;
}
Script for building
#!/usr/bin/env python
from distutils.core import setup, Extension
import os
os.environ['CC'] = 'gcc';
setup(name='foo',
version='1.0',
ext_modules =[Extension('_foo',
['foo.i','foo.c'])])
Usage:
import foo
OK, f = foo.foo_new()
OK = foo.foo_set_value_b(f, 'Hello world!')
OK = foo.foo_free(f)
OK, f = foo.foo_new()
# Test safe to double delete
del f
I embedded python in my C++ program.
I use PyImport_ImportModule to load my module written in a .py file.
But how can I load it from memory? Let's say my .py file is encrypted, so I need to first decrypt it and feed the code to python to execute.
Moreover, it'd be nice if I could bypass/intercept or modify the import mechanism, so that doesn't load modules from the filesystem but my own memory blocks, how/can I do that?
The following example shows how to define a module from a C string:
#include <stdio.h>
#include <Python.h>
int main(int argc, char *argv[])
{
Py_Initialize();
PyRun_SimpleString("print('hello from python')");
// fake module
char *source = "__version__ = '2.0'";
char *filename = "test_module.py";
// perform module load
PyObject *builtins = PyEval_GetBuiltins();
PyObject *compile = PyDict_GetItemString(builtins, "compile");
PyObject *code = PyObject_CallFunction(compile, "sss", source, filename, "exec");
PyObject *module = PyImport_ExecCodeModule("test_module", code);
PyRun_SimpleString("import test_module; print(test_module.__version__)");
Py_Finalize();
return 0;
}
output:
hello from python
version: 2.0
You can read about import hooks in the docs. You will need to define a class with find_module and load_module methods. Something like the following should work:
PyObject* find_module(PyObject* self, PyObject* args) {
// ... lookup args in available special modules ...
return Py_BuildValue("B", found);
}
PyObject* load_module(PyObject* self, PyObject* args) {
// ... convert args into filname, source ...
PyObject *builtins = PyEval_GetBuiltins();
PyObject *compile = PyDict_GetItemString(builtins, "compile");
PyObject *code = PyObject_CallFunction(compile, "sss", source, filename, "exec");
PyObject *module = PyImport_ExecCodeModule("test_module", code);
return Py_BuildValue("O", module);
}
static struct PyMethodDef methods[] = {
{ "find_module", find_module, METH_VARARGS, "Returns module_loader if this is an encrypted module"},
{ "load_module", load_module, METH_VARARGS, "Load an encrypted module" },
{ NULL, NULL, 0, NULL }
};
static struct PyModuleDef modDef = {
PyModuleDef_HEAD_INIT, "embedded", NULL, -1, methods,
NULL, NULL, NULL, NULL
};
static PyObject* PyInit_embedded(void)
{
return PyModule_Create(&modDef);
}
int main() {
...
PyImport_AppendInittab("embedded", &PyInit_embedded);
PyRun_SimpleString("\
import embedded, sys\n\
class Importer:\n\
def find_module(self, fullpath):\n\
return self if embedded.find_module(fullpath) else None\n\
def load_module(self, fullpath):\n\
return embedded.load_module(fullpath)\n\
sys.path_hooks.insert(0, Importer())\n\
");
...
}
My problem is as follows:
I would like to call a C function from my Python file and return a value back to that Python file.
I have tried the following method of using embedded C in Python (the following code is the C code called "mod1.c). I am using Python3.4 so the format follows that given in the documentation guidelines. The problem comes when I call my setup file (second code below).
#include
#include "sum.h"
static PyObject*
mod_sum(PyObject *self, PyObject *args)
{
int a;
int b;
int s;
if (!PyArg_ParseTuple(args,"ii",&a,&b))
return NULL;
s = sum(a,b);
return Py_BuildValue("i",s);
}
/* DECLARATION OF METHODS */
static PyMethodDef ModMethods[] = {
{"sum", mod_sum, METH_VARARGS, "Descirption"}, // {"methName", modName_methName, METH_VARARGS, "Description.."}, modName is name of module and methName is name of method
{NULL,NULL,0,NULL}
};
// Module Definition Structure
static struct PyModuleDef summodule = {
PyModuleDef_HEAD_INIT,
"sum",
NULL,
-1,
ModMethods
};
/* INITIALIZATION FUNCTION */
PyMODINIT_FUNC initmod(void)
{
PyObject *m;
m = PyModule_Create(&summodule);
if (m == NULL)
return m;
}
Setup.py
from distutils.core import setup, Extension
setup(name='buildsum', version='1.0', \
ext_modules=[Extension('buildsum', ['mod1.c'])])
The result that I get when I compile my code using gcc is the following error: Cannot export PyInit_buildsum: symbol not defined
I would greatly appreciate any insight or help on this problem, or any suggestion in how to call C from Python. Thank you!
---------------------------------------EDIT ---------------------------------
Thank you for the comments:
I have tried the following now:
static PyObject*
PyInit_sum(PyObject *self, PyObject *args)
{
int a;
int b;
int s;
if (!PyArg_ParseTuple(args,"ii",&a,&b))
return NULL;
s = sum(a,b);
return Py_BuildValue("i",s);
}
For the first function; however, I still get the same error of PyInit_sum: symbol not defined
The working code from above in case anyone runs into the same error: the answer from #dclarke is correct. The initialization function in python 3 must have PyInit_(name) as its name.
#include <Python.h>
#include "sum.h"
static PyObject* mod_sum(PyObject *self, PyObject *args)
{
int a;
int b;
int s;
if (!PyArg_ParseTuple(args,"ii",&a,&b))
return NULL;
s = sum(a,b);
return Py_BuildValue("i",s);
}
/* DECLARATION OF METHODS*/
static PyMethodDef ModMethods[] = {
{"modsum", mod_sum, METH_VARARGS, "Descirption"},
{NULL,NULL,0,NULL}
};
// Module Definition Structure
static struct PyModuleDef summodule = {
PyModuleDef_HEAD_INIT,"modsum", NULL, -1, ModMethods
};
/* INITIALIZATION FUNCTION*/
PyMODINIT_FUNC PyInit_sum(void)
{
PyObject *m;
m = PyModule_Create(&summodule);
return m;
}