I'm creating my own python module in C++. The module source code :
#include <Python.h>
#include <numpy/arrayobject.h>
#include "FMM.h"
struct module_state {
PyObject *error;
};
#define GETSTATE(m) ((struct module_state*)PyModule_GetState(m))
static PyObject* FMM(PyObject* self, PyObject* args)
{
PyObject *model_obj;
PyObject *time_obj;
PyObject *accepted_obj;
PyObject *lat_obj;
PyObject *lon_obj;
PyObject *h_obj;
int N;
if (!PyArg_ParseTuple(args, "OOOOOOi", &model_obj, &time_obj, &accepted_obj, &lat_obj, &lon_obj, &h_obj, &N))
{
Py_INCREF(Py_None);
return Py_None;
}
PyObject *model = PyArray_FROM_OTF(model_obj, NPY_FLOAT, NPY_INOUT_ARRAY);
PyObject *time = PyArray_FROM_OTF(time_obj, NPY_FLOAT, NPY_INOUT_ARRAY);
PyObject *accepted = PyArray_FROM_OTF(accepted_obj, NPY_BOOL, NPY_INOUT_ARRAY);
PyObject *lat = PyArray_FROM_OTF(lon_obj, NPY_DOUBLE, NPY_INOUT_ARRAY);
PyObject *lon = PyArray_FROM_OTF(lon_obj, NPY_DOUBLE, NPY_INOUT_ARRAY);
PyObject *h = PyArray_FROM_OTF(h_obj, NPY_DOUBLE, NPY_INOUT_ARRAY);
float *MODEL = static_cast<float *>(PyArray_DATA(model_obj));
float *TIME = static_cast<float *>(PyArray_DATA(time_obj));
bool *ACCEPTED = static_cast<bool *>(PyArray_DATA(accepted_obj));
double *LAT = static_cast<double *>(PyArray_DATA(lat_obj));
double *LON = static_cast<double *>(PyArray_DATA(lon_obj));
double *H = static_cast<double *>(PyArray_DATA(h_obj));
_FMM(MODEL, TIME, ACCEPTED, LAT, LON, H, N);
Py_INCREF(Py_None);
return Py_None;
}
static PyObject* SetModelSize(PyObject* self, PyObject* args)
{
int x, y, z;
if (!PyArg_ParseTuple(args, "iii", &x, &y, &z))
{
Py_INCREF(Py_None);
return Py_None;
}
_SaveModelSize(x, y, z);
// Rerutn values
Py_INCREF(Py_None);
return Py_None;
}
static PyMethodDef FMMMethods[] = {
{"FMM", FMM, METH_VARARGS, "Some Text"},
{"SetModelSize", SetModelSize, METH_VARARGS, "Some Text"},
{NULL, NULL, 0, NULL}
};
static int FMM_traverse(PyObject *m, visitproc visit, void *arg) {
Py_VISIT(GETSTATE(m)->error);
return 0;
}
static int FMM_clear(PyObject *m) {
Py_CLEAR(GETSTATE(m)->error);
return 0;
}
static struct PyModuleDef moduledef = {
PyModuleDef_HEAD_INIT,
"FMM",
NULL,
sizeof(struct module_state),
FMMMethods,
NULL,
FMM_traverse,
FMM_clear,
NULL
};
extern "C" PyObject * PyInit_FMM(void)
{
PyObject *module = PyModule_Create(&moduledef);
if (module == NULL)
return NULL;
struct module_state *st = GETSTATE(module);
st->error = PyErr_NewException("FMM.Error", NULL, NULL);
if (st->error == NULL)
{
Py_DECREF(module);
return NULL;
}
import_array();
Py_INCREF(module);
return module;
}
My function _FMM does numerical computation on large data sets (two floats and bool has size of 2e9 elements.
The module is compiled using setup.py:
from distutils.core import setup, Extension
import numpy.distutils.misc_util
import os
os.environ["CC"] = "g++"
os.environ["CXX"] = "g++"
module1 = Extension('FMM', sources = ['FMMmodule.c', 'FMM.c'])
setup (name = 'PackageName',
version = '1.0',
description = 'This is a demo package',
ext_modules = [module1],
include_dirs=numpy.distutils.misc_util.get_numpy_include_dirs())
When I run it from python I get Segmentation fault after about 60 seconds of calculation.
Every time I run in different step of calculation. I have read that it might be related to misuse of Py_INCREF() and Py_DECREF() - tried to play with that with no luck.
Anybody has experience how to deal with such behavior?
I'm testing using python 3.4.
I have finally sorted that out. I had to modify the code to make it compatible with newer numpy c-api: see more here: Migrating to numpy api 1.7
Related
From reading another post, I am trying to embbed some some Python code into C:
main.c
#include <Python.h>
int callModuleFunc(int array[], size_t size) {
PyObject *mymodule = PyImport_ImportModule("py_function");
PyObject *myfunc = PyObject_GetAttrString(mymodule, "printlist");
PyObject *mylist = PyList_New(size);
for (size_t i = 0; i != size; ++i) {
PyList_SET_ITEM(mylist, i, PyInt_FromLong(array[i]));
}
PyObject *arglist = Py_BuildValue("(o)", mylist);
PyObject *result = PyObject_CallObject(myfunc, arglist);
int retval = (int)PyInt_AsLong(result);
Py_DECREF(result);
Py_DECREF(arglist);
Py_DECREF(mylist);
Py_DECREF(myfunc);
Py_DECREF(mymodule);
return retval;
}
int main(int argc, char *argv[])
{
int a[] = {1,2,3,4};
callModuleFunc(a, 4);
return 0;
}
py_function.py
'''py_function.py - Python source designed to '''
'''demonstrate the use of python embedding'''
def printlist(mylist):
print mylist
Then I compiled with:
gcc main.c -I/usr/include/python2.7 -lpython2.7
But then I ran the app, it gives me a segmentation fault error:
/a.out
[1] 18890 segmentation fault ./a.out
Is there something that I am missing?
There were several problems with your code:
Py_Initialize() was not called.
PyImport_ImportModule() failed to find your python file, since in embedded Python you start without an initial module, relative to which the search can work. The fix is to explicitly include the current directory in sys.path.
"(O)" in Py_BuildValue() should use a capital 'O'.
The printlist function should return a value (since that is what the C-code expects).
This should work:
main.c
#include <Python.h>
void initPython()
{
Py_Initialize();
PyObject *sysmodule = PyImport_ImportModule("sys");
PyObject *syspath = PyObject_GetAttrString(sysmodule, "path");
PyList_Append(syspath, PyString_FromString("."));
Py_DECREF(syspath);
Py_DECREF(sysmodule);
}
int callModuleFunc(int array[], size_t size) {
PyObject *mymodule = PyImport_ImportModule("py_function");
assert(mymodule != NULL);
PyObject *myfunc = PyObject_GetAttrString(mymodule, "printlist");
assert(myfunc != NULL);
PyObject *mylist = PyList_New(size);
for (size_t i = 0; i != size; ++i) {
PyList_SET_ITEM(mylist, i, PyInt_FromLong(array[i]));
}
PyObject *arglist = Py_BuildValue("(O)", mylist);
assert(arglist != NULL);
PyObject *result = PyObject_CallObject(myfunc, arglist);
assert(result != NULL);
int retval = (int)PyInt_AsLong(result);
Py_DECREF(result);
Py_DECREF(arglist);
Py_DECREF(mylist);
Py_DECREF(myfunc);
Py_DECREF(mymodule);
return retval;
}
int main(int argc, char *argv[])
{
initPython();
int a[] = {1,2,3,4,5,6,7};
callModuleFunc(a, 4);
callModuleFunc(a+2, 5);
Py_Finalize();
return 0;
}
py_function.py
'''py_function.py - Python source designed to '''
'''demonstrate the use of python embedding'''
def printlist(mylist):
print mylist
return 0
I wrote a c++ module witch should be imported into Python. Below are both Codes, the C++ part and the Python part. The C++ function method_sum should return the double of a value to python.
module.cpp:
#define PY_SSIZE_T_CLEAN
#include <Python.h>
static PyObject *method_sum(PyObject *self, PyObject *args) {
const int *prop;
if (!PyArg_ParseTuple(args, "i", &prop)) return NULL;
int result = *prop + *prop;
return Py_BuildValue("i", result);
}
static PyMethodDef ModuleMethods[] = {
{"sum", method_sum, METH_VARARGS, "description of the function"},
{NULL, NULL, 0, NULL}
};
static struct PyModuleDef module = {
PyModuleDef_HEAD_INIT,
"module",
"description of the module",
-1,
ModuleMethods
};
PyMODINIT_FUNC PyInit_module(void) {
return PyModule_Create(&module);
}
main.py:
import module
print(module.sum(18))
setup.py:
from distutils.core import setup, Extension
setup(name='module', version='1.0', ext_modules=[Extension('module', ['module.cpp'])])
I changed method_sum to the following and main.py prints 36 instead of segfaulting.
static PyObject *method_sum(PyObject *self, PyObject *args) {
int prop;
if (!PyArg_ParseTuple(args, "i", &prop)) return NULL;
int result = prop + prop;
return Py_BuildValue("i", result);
}
The following also works and prop is still a pointer like in the code in the question.
static PyObject *method_sum(PyObject *self, PyObject *args) {
const int *prop = new int;
if (!PyArg_ParseTuple(args, "i", prop)) {
delete prop;
return NULL;
}
int result = *prop + *prop;
delete prop;
return Py_BuildValue("i", result);
}
Everywhere, I can easily find an example about writing a method with Python C Extensions and use it in Python. Like this one: Python 3 extension example
$ python3
>>> import hello
>>> hello.hello_world()
Hello, world!
>>> hello.hello('world')
Hello, world!
How to do write a hello word full featured Python class (not just a module method)?
I think this How to wrap a C++ object using pure Python Extension API (python3)? question has an example, but it does not seem minimal as he is using (or wrapping?) C++ classes on it.
For example:
class ClassName(object):
"""docstring for ClassName"""
def __init__(self, hello):
super().__init__()
self.hello = hello
def talk(self, world):
print( '%s %s' % ( self.hello, world ) )
What is the equivalent of this Python class example with C Extensions?
I would use it like this:
from .mycextensionsmodule import ClassName
classname = ClassName("Hello")
classname.talk( 'world!' )
# prints "Hello world!"
My goal is to write a class fully written in C for performance (all other classes in my project will be in Python, except this one). I am not looking for portability as using ctypes, neither black boxes as using Boost.Python or SWIG. Just a high-performance class purely written with Python C Extensions.
After I got this Hello word working, I can figure my self out within Python Extensive documentation:
https://docs.python.org/3/c-api/
https://docs.python.org/3/extending/extending.html
See also: Python instance method in C
Create the file called MANIFEST.in
include README.md
include LICENSE.txt
recursive-include source *.h
Create the file called setup.py
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from setuptools import setup, Extension
__version__ = '0.1.0'
setup(
name = 'custom',
version = __version__,
package_data = {
'': [ '**.txt', '**.md', '**.py', '**.h', '**.hpp', '**.c', '**.cpp' ],
},
ext_modules = [
Extension(
name = 'custom',
sources = [
'source/custom.cpp',
],
include_dirs = ['source'],
)
],
)
Create the file called source/custom.cpp
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include "structmember.h"
typedef struct {
PyObject_HEAD
PyObject *first; /* first name */
PyObject *last; /* last name */
int number;
} CustomObject;
static int
Custom_traverse(CustomObject *self, visitproc visit, void *arg)
{
Py_VISIT(self->first);
Py_VISIT(self->last);
return 0;
}
static int
Custom_clear(CustomObject *self)
{
Py_CLEAR(self->first);
Py_CLEAR(self->last);
return 0;
}
static void
Custom_dealloc(CustomObject *self)
{
PyObject_GC_UnTrack(self);
Custom_clear(self);
Py_TYPE(self)->tp_free((PyObject *) self);
}
static PyObject *
Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
CustomObject *self;
self = (CustomObject *) type->tp_alloc(type, 0);
if (self != NULL) {
self->first = PyUnicode_FromString("");
if (self->first == NULL) {
Py_DECREF(self);
return NULL;
}
self->last = PyUnicode_FromString("");
if (self->last == NULL) {
Py_DECREF(self);
return NULL;
}
self->number = 0;
}
return (PyObject *) self;
}
static int
Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"first", "last", "number", NULL};
PyObject *first = NULL, *last = NULL, *tmp;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist,
&first, &last,
&self->number))
return -1;
if (first) {
tmp = self->first;
Py_INCREF(first);
self->first = first;
Py_DECREF(tmp);
}
if (last) {
tmp = self->last;
Py_INCREF(last);
self->last = last;
Py_DECREF(tmp);
}
return 0;
}
static PyMemberDef Custom_members[] = {
{"number", T_INT, offsetof(CustomObject, number), 0,
"custom number"},
{NULL} /* Sentinel */
};
static PyObject *
Custom_getfirst(CustomObject *self, void *closure)
{
Py_INCREF(self->first);
return self->first;
}
static int
Custom_setfirst(CustomObject *self, PyObject *value, void *closure)
{
if (value == NULL) {
PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
return -1;
}
if (!PyUnicode_Check(value)) {
PyErr_SetString(PyExc_TypeError,
"The first attribute value must be a string");
return -1;
}
Py_INCREF(value);
Py_CLEAR(self->first);
self->first = value;
return 0;
}
static PyObject *
Custom_getlast(CustomObject *self, void *closure)
{
Py_INCREF(self->last);
return self->last;
}
static int
Custom_setlast(CustomObject *self, PyObject *value, void *closure)
{
if (value == NULL) {
PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute");
return -1;
}
if (!PyUnicode_Check(value)) {
PyErr_SetString(PyExc_TypeError,
"The last attribute value must be a string");
return -1;
}
Py_INCREF(value);
Py_CLEAR(self->last);
self->last = value;
return 0;
}
static PyGetSetDef Custom_getsetters[] = {
{"first", (getter) Custom_getfirst, (setter) Custom_setfirst,
"first name", NULL},
{"last", (getter) Custom_getlast, (setter) Custom_setlast,
"last name", NULL},
{NULL} /* Sentinel */
};
static PyObject *
Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored))
{
return PyUnicode_FromFormat("%S %S", self->first, self->last);
}
static PyMethodDef Custom_methods[] = {
{"name", (PyCFunction) Custom_name, METH_NOARGS,
"Return the name, combining the first and last name"
},
{NULL} /* Sentinel */
};
static PyTypeObject CustomType = {
PyVarObject_HEAD_INIT(NULL, 0)
.tp_name = "custom.Custom",
.tp_doc = "Custom objects",
.tp_basicsize = sizeof(CustomObject),
.tp_itemsize = 0,
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
.tp_new = Custom_new,
.tp_init = (initproc) Custom_init,
.tp_dealloc = (destructor) Custom_dealloc,
.tp_traverse = (traverseproc) Custom_traverse,
.tp_clear = (inquiry) Custom_clear,
.tp_members = Custom_members,
.tp_methods = Custom_methods,
.tp_getset = Custom_getsetters,
};
static PyModuleDef custommodule = {
PyModuleDef_HEAD_INIT,
.m_name = "custom",
.m_doc = "Example module that creates an extension type.",
.m_size = -1,
};
PyMODINIT_FUNC
PyInit_custom(void)
{
PyObject *m;
if (PyType_Ready(&CustomType) < 0)
return NULL;
m = PyModule_Create(&custommodule);
if (m == NULL)
return NULL;
Py_INCREF(&CustomType);
PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
return m;
}
Then, to compile it and install you can run either:
pip3 install . -v
python3 setup.py install
As side note from this question How to use setuptools packages and ext_modules with the same name? do not mix on the same project *.py files and Python C Extensions, i.e., use only purely C/C++, building Python C Extensions without adding packages = [ 'package_name' ] entries because they cause the Python C Extensions code run 30%, i.e., if the program would take 7 seconds to run, now with *.py files, it will take 11 seconds.
References:
https://docs.python.org/3/extending/newtypes_tutorial.html#supporting-cyclic-garbage-collection
I embedded python in my C++ program.
I use PyImport_ImportModule to load my module written in a .py file.
But how can I load it from memory? Let's say my .py file is encrypted, so I need to first decrypt it and feed the code to python to execute.
Moreover, it'd be nice if I could bypass/intercept or modify the import mechanism, so that doesn't load modules from the filesystem but my own memory blocks, how/can I do that?
The following example shows how to define a module from a C string:
#include <stdio.h>
#include <Python.h>
int main(int argc, char *argv[])
{
Py_Initialize();
PyRun_SimpleString("print('hello from python')");
// fake module
char *source = "__version__ = '2.0'";
char *filename = "test_module.py";
// perform module load
PyObject *builtins = PyEval_GetBuiltins();
PyObject *compile = PyDict_GetItemString(builtins, "compile");
PyObject *code = PyObject_CallFunction(compile, "sss", source, filename, "exec");
PyObject *module = PyImport_ExecCodeModule("test_module", code);
PyRun_SimpleString("import test_module; print(test_module.__version__)");
Py_Finalize();
return 0;
}
output:
hello from python
version: 2.0
You can read about import hooks in the docs. You will need to define a class with find_module and load_module methods. Something like the following should work:
PyObject* find_module(PyObject* self, PyObject* args) {
// ... lookup args in available special modules ...
return Py_BuildValue("B", found);
}
PyObject* load_module(PyObject* self, PyObject* args) {
// ... convert args into filname, source ...
PyObject *builtins = PyEval_GetBuiltins();
PyObject *compile = PyDict_GetItemString(builtins, "compile");
PyObject *code = PyObject_CallFunction(compile, "sss", source, filename, "exec");
PyObject *module = PyImport_ExecCodeModule("test_module", code);
return Py_BuildValue("O", module);
}
static struct PyMethodDef methods[] = {
{ "find_module", find_module, METH_VARARGS, "Returns module_loader if this is an encrypted module"},
{ "load_module", load_module, METH_VARARGS, "Load an encrypted module" },
{ NULL, NULL, 0, NULL }
};
static struct PyModuleDef modDef = {
PyModuleDef_HEAD_INIT, "embedded", NULL, -1, methods,
NULL, NULL, NULL, NULL
};
static PyObject* PyInit_embedded(void)
{
return PyModule_Create(&modDef);
}
int main() {
...
PyImport_AppendInittab("embedded", &PyInit_embedded);
PyRun_SimpleString("\
import embedded, sys\n\
class Importer:\n\
def find_module(self, fullpath):\n\
return self if embedded.find_module(fullpath) else None\n\
def load_module(self, fullpath):\n\
return embedded.load_module(fullpath)\n\
sys.path_hooks.insert(0, Importer())\n\
");
...
}
I'm trying to write a C-Extension for python. What I'd like to write is a ModPolynomial class which represents a polynomial on (Z/nZ)[x]/x^r-1[even though you may answer to my question without knowing anything of such polynomials].
I've written some code, which seems to work. Basically I just store three PyObject* in my ModPoly structure. Now I'd like to add the storage for the coefficients of the polynomial.
Since I want the coefficients to be read-only, I'd like to add a getter/setter pair of functions through PyGetSetDef. But when I access the getter from python(e.g print pol.coefficients) I receive a Segmentation Fault.
The original code, without the "coefficients" can be found here.
The code with the coefficients is here.
I hope someone of you can tell me where I'm doing wrong here.
By the way, also comments on the code are welcome. This is my first extension and I know that I'm probably doing things quite badly.
As ecatmur says in the comments PyVarObject store a certain number of "slots" at the end of the struct. So I've decided to avoid them.
The relevant code is:
typedef struct {
PyObject_HEAD
/* Type specific fields */
Py_ssize_t ob_size;
PyObject **ob_item;
Py_ssize_t allocated;
PyObject *r_modulus;
PyObject *n_modulus;
PyObject *degree;
} ModPoly;
static PyObject *
ModPoly_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
ModPoly *self;
self = (ModPoly *)type->tp_alloc(type, 0);
if (self != NULL) {
[...]
self->ob_size = 0;
self->ob_item = NULL;
self->allocated = 0;
}
return (PyObject *)self;
}
static int
ModPoly_init(ModPoly *self, PyObject *args, PyObject *kwds)
{
PyObject *r_modulus=NULL, *n_modulus=NULL, *coefs=NULL, *tmp;
PyObject **tmp_ar;
static char *kwlist[] = {"r_modulus", "n_modulus", "coefficients", NULL};
if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O", kwlist,
&r_modulus, &n_modulus, &coefs))
return -1;
[...]
// The polynomial defaults to "x", so the coefficients should be [0, 1].
tmp_ar = (PyObject **)malloc(2 * sizeof(PyObject*));
if (tmp_ar == NULL) {
Py_DECREF(self->r_modulus);
Py_DECREF(self->n_modulus);
Py_DECREF(self->degree);
return -1;
}
tmp_ar[0] = PyInt_FromLong(0);
if (tmp_ar[0] != NULL) {
tmp_ar[1] = PyInt_FromLong(1);
}
if (tmp_ar[0] == NULL || tmp_ar[0] == NULL) {
Py_DECREF(self->r_modulus);
Py_DECREF(self->n_modulus);
Py_DECREF(self->degree);
Py_XDECREF(tmp_ar[0]);
Py_XDECREF(tmp_ar[1]);
free(tmp_ar);
return -1;
}
self->ob_size = 2;
self->allocated = 2;
return 0;
}
[...]
static PyObject *
ModPoly_getcoefs(ModPoly *self, void *closure)
{
printf("here"); // "here" is never printed
PyTupleObject *res=(PyTupleObject*)PyTuple_New(self->ob_size);
Py_ssize_t i;
PyObject *tmp;
if (res == NULL)
return NULL;
for (i=0; i < self->ob_size; i++) {
tmp = self->ob_item[i];
Py_INCREF(tmp);
PyTuple_SET_ITEM(res, i, tmp);
}
return (PyObject *)res;
}
static PyObject *
ModPoly_setcoefs(ModPoly *self, PyObject *value, void* closure)
{
PyErr_SetString(PyExc_AttributeError,
"Cannot set the coefficients of a polynomial.");
return NULL;
}
[...]
static PyGetSetDef ModPoly_getsetters[] = {
{"coefficients",
(getter)ModPoly_getcoefs, (setter)ModPoly_setcoefs,
"The polynomial coefficients.", NULL},
{NULL, 0, 0, NULL, NULL}
};
static PyTypeObject ModPolyType = {
PyObject_HEAD_INIT(NULL)
0, /* ob_size */
[...]
ModPoly_members, /* tp_members */
ModPoly_getsetters, /* tp_getset */
0, /* tp_base */
[...]
};
[...]
edit
I tried to reimplement the getter instruction by instruction, and I understood what I wasn't doing. In the ModPoly_init function I create the tmp_ar where I store the coefficients, but I do not assign it to self->ob_item.
-facepalm-
You only seem to be assigning to ModPoly.ob_item in ModPoly_new() (setting it to NULL).
ModPoly_getcoefs() then dereferences the null pointer, which would give you your segfault. It looks like you intended to assign to ob_item in ModPoly_init(), but don't actually get around to doing so.