Consider the following example code that I got off the Python documentation page. It takes one argument right now. I want to make it accept a named argument like "mode=2". So that i can call myext.do_something('/home/crypto/something.jpg', mode=2). The argument passed with mode should be an integer and if it gets something else, I want it to raise an error in Python. How do I go about doing this? I'm thinking I should edit module_functions[] but I don't know what to put in there.
#include <Python.h>
static PyObject *
do_something(PyObject *self, PyObject *args)
{
char *filename;
if (!PyArg_ParseTuple(args, "s", &filename))
return NULL;
char *result = do_somethingelse(filename);
return Py_BuildValue("s", result);
}
static PyMethodDef
module_functions[] = {
{ "do_something", do_something, METH_VARARGS, "do something" },
{ NULL }
};
PyMODINIT_FUNC
initmyext(void)
{
Py_InitModule("myext", module_functions);
}
This is a minimal example:
#include <Python.h>
static PyObject *
do_something(PyObject *self, PyObject *args, PyObject *kwargs)
{
char *filename;
int mode = 2;
static char *keywords[] = {"filename", "mode", NULL};
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|i", keywords, &filename, &mode))
return NULL;
char *result = do_somethingelse(filename);
return Py_BuildValue("s", result);
}
static PyMethodDef
module_functions[] = {
{ "do_something", (PyCFunction)do_something, METH_VARARGS | METH_KEYWORDS, "do something" },
{ NULL }
};
PyMODINIT_FUNC
initmyext(void)
{
Py_InitModule("myext", module_functions);
}
Related
I wrote a c++ module witch should be imported into Python. Below are both Codes, the C++ part and the Python part. The C++ function method_sum should return the double of a value to python.
module.cpp:
#define PY_SSIZE_T_CLEAN
#include <Python.h>
static PyObject *method_sum(PyObject *self, PyObject *args) {
const int *prop;
if (!PyArg_ParseTuple(args, "i", &prop)) return NULL;
int result = *prop + *prop;
return Py_BuildValue("i", result);
}
static PyMethodDef ModuleMethods[] = {
{"sum", method_sum, METH_VARARGS, "description of the function"},
{NULL, NULL, 0, NULL}
};
static struct PyModuleDef module = {
PyModuleDef_HEAD_INIT,
"module",
"description of the module",
-1,
ModuleMethods
};
PyMODINIT_FUNC PyInit_module(void) {
return PyModule_Create(&module);
}
main.py:
import module
print(module.sum(18))
setup.py:
from distutils.core import setup, Extension
setup(name='module', version='1.0', ext_modules=[Extension('module', ['module.cpp'])])
I changed method_sum to the following and main.py prints 36 instead of segfaulting.
static PyObject *method_sum(PyObject *self, PyObject *args) {
int prop;
if (!PyArg_ParseTuple(args, "i", &prop)) return NULL;
int result = prop + prop;
return Py_BuildValue("i", result);
}
The following also works and prop is still a pointer like in the code in the question.
static PyObject *method_sum(PyObject *self, PyObject *args) {
const int *prop = new int;
if (!PyArg_ParseTuple(args, "i", prop)) {
delete prop;
return NULL;
}
int result = *prop + *prop;
delete prop;
return Py_BuildValue("i", result);
}
Everywhere, I can easily find an example about writing a method with Python C Extensions and use it in Python. Like this one: Python 3 extension example
$ python3
>>> import hello
>>> hello.hello_world()
Hello, world!
>>> hello.hello('world')
Hello, world!
How to do write a hello word full featured Python class (not just a module method)?
I think this How to wrap a C++ object using pure Python Extension API (python3)? question has an example, but it does not seem minimal as he is using (or wrapping?) C++ classes on it.
For example:
class ClassName(object):
"""docstring for ClassName"""
def __init__(self, hello):
super().__init__()
self.hello = hello
def talk(self, world):
print( '%s %s' % ( self.hello, world ) )
What is the equivalent of this Python class example with C Extensions?
I would use it like this:
from .mycextensionsmodule import ClassName
classname = ClassName("Hello")
classname.talk( 'world!' )
# prints "Hello world!"
My goal is to write a class fully written in C for performance (all other classes in my project will be in Python, except this one). I am not looking for portability as using ctypes, neither black boxes as using Boost.Python or SWIG. Just a high-performance class purely written with Python C Extensions.
After I got this Hello word working, I can figure my self out within Python Extensive documentation:
https://docs.python.org/3/c-api/
https://docs.python.org/3/extending/extending.html
See also: Python instance method in C
Create the file called MANIFEST.in
include README.md
include LICENSE.txt
recursive-include source *.h
Create the file called setup.py
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from setuptools import setup, Extension
__version__ = '0.1.0'
setup(
name = 'custom',
version = __version__,
package_data = {
'': [ '**.txt', '**.md', '**.py', '**.h', '**.hpp', '**.c', '**.cpp' ],
},
ext_modules = [
Extension(
name = 'custom',
sources = [
'source/custom.cpp',
],
include_dirs = ['source'],
)
],
)
Create the file called source/custom.cpp
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include "structmember.h"
typedef struct {
PyObject_HEAD
PyObject *first; /* first name */
PyObject *last; /* last name */
int number;
} CustomObject;
static int
Custom_traverse(CustomObject *self, visitproc visit, void *arg)
{
Py_VISIT(self->first);
Py_VISIT(self->last);
return 0;
}
static int
Custom_clear(CustomObject *self)
{
Py_CLEAR(self->first);
Py_CLEAR(self->last);
return 0;
}
static void
Custom_dealloc(CustomObject *self)
{
PyObject_GC_UnTrack(self);
Custom_clear(self);
Py_TYPE(self)->tp_free((PyObject *) self);
}
static PyObject *
Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
CustomObject *self;
self = (CustomObject *) type->tp_alloc(type, 0);
if (self != NULL) {
self->first = PyUnicode_FromString("");
if (self->first == NULL) {
Py_DECREF(self);
return NULL;
}
self->last = PyUnicode_FromString("");
if (self->last == NULL) {
Py_DECREF(self);
return NULL;
}
self->number = 0;
}
return (PyObject *) self;
}
static int
Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"first", "last", "number", NULL};
PyObject *first = NULL, *last = NULL, *tmp;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist,
&first, &last,
&self->number))
return -1;
if (first) {
tmp = self->first;
Py_INCREF(first);
self->first = first;
Py_DECREF(tmp);
}
if (last) {
tmp = self->last;
Py_INCREF(last);
self->last = last;
Py_DECREF(tmp);
}
return 0;
}
static PyMemberDef Custom_members[] = {
{"number", T_INT, offsetof(CustomObject, number), 0,
"custom number"},
{NULL} /* Sentinel */
};
static PyObject *
Custom_getfirst(CustomObject *self, void *closure)
{
Py_INCREF(self->first);
return self->first;
}
static int
Custom_setfirst(CustomObject *self, PyObject *value, void *closure)
{
if (value == NULL) {
PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
return -1;
}
if (!PyUnicode_Check(value)) {
PyErr_SetString(PyExc_TypeError,
"The first attribute value must be a string");
return -1;
}
Py_INCREF(value);
Py_CLEAR(self->first);
self->first = value;
return 0;
}
static PyObject *
Custom_getlast(CustomObject *self, void *closure)
{
Py_INCREF(self->last);
return self->last;
}
static int
Custom_setlast(CustomObject *self, PyObject *value, void *closure)
{
if (value == NULL) {
PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute");
return -1;
}
if (!PyUnicode_Check(value)) {
PyErr_SetString(PyExc_TypeError,
"The last attribute value must be a string");
return -1;
}
Py_INCREF(value);
Py_CLEAR(self->last);
self->last = value;
return 0;
}
static PyGetSetDef Custom_getsetters[] = {
{"first", (getter) Custom_getfirst, (setter) Custom_setfirst,
"first name", NULL},
{"last", (getter) Custom_getlast, (setter) Custom_setlast,
"last name", NULL},
{NULL} /* Sentinel */
};
static PyObject *
Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored))
{
return PyUnicode_FromFormat("%S %S", self->first, self->last);
}
static PyMethodDef Custom_methods[] = {
{"name", (PyCFunction) Custom_name, METH_NOARGS,
"Return the name, combining the first and last name"
},
{NULL} /* Sentinel */
};
static PyTypeObject CustomType = {
PyVarObject_HEAD_INIT(NULL, 0)
.tp_name = "custom.Custom",
.tp_doc = "Custom objects",
.tp_basicsize = sizeof(CustomObject),
.tp_itemsize = 0,
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
.tp_new = Custom_new,
.tp_init = (initproc) Custom_init,
.tp_dealloc = (destructor) Custom_dealloc,
.tp_traverse = (traverseproc) Custom_traverse,
.tp_clear = (inquiry) Custom_clear,
.tp_members = Custom_members,
.tp_methods = Custom_methods,
.tp_getset = Custom_getsetters,
};
static PyModuleDef custommodule = {
PyModuleDef_HEAD_INIT,
.m_name = "custom",
.m_doc = "Example module that creates an extension type.",
.m_size = -1,
};
PyMODINIT_FUNC
PyInit_custom(void)
{
PyObject *m;
if (PyType_Ready(&CustomType) < 0)
return NULL;
m = PyModule_Create(&custommodule);
if (m == NULL)
return NULL;
Py_INCREF(&CustomType);
PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
return m;
}
Then, to compile it and install you can run either:
pip3 install . -v
python3 setup.py install
As side note from this question How to use setuptools packages and ext_modules with the same name? do not mix on the same project *.py files and Python C Extensions, i.e., use only purely C/C++, building Python C Extensions without adding packages = [ 'package_name' ] entries because they cause the Python C Extensions code run 30%, i.e., if the program would take 7 seconds to run, now with *.py files, it will take 11 seconds.
References:
https://docs.python.org/3/extending/newtypes_tutorial.html#supporting-cyclic-garbage-collection
I want to know how to wrap a C++ object with Python Extension API (and distutils) without external tools (like Cython, Boost, SWIG, ...). Just in pure Python way without creating a dll.
Note that my C++ object has memory allocations so destructor has to be called to avoid memory leaks.
#include "Voice.h"
namespace transformation
{
Voice::Voice(int fftSize) { mem=new double[fftSize]; }
Voice::~Voice() { delete [] mem; }
int Voice::method1() { /*do stuff*/ return (1); }
}
I just want to do somethings like that in Python :
import voice
v=voice.Voice(512)
result=v.method1()
Seems that the answer was in fact here : https://docs.python.org/3.6/extending/newtypes.html
With examples, but not really easy.
EDIT 1 :
In fact, it is not really for wrapping a C++ object in a Python object, but rather to create a Python object with C code. (edit2 : and so you can wrap C++ object!)
EDIT 2 :
Here is a solution using the Python newtypes
.
Original C++ file : Voice.cpp
#include <cstdio>
#include "Voice.h"
namespace transformation
{
Voice::Voice(int fftSize) {
printf("c++ constructor of voice\n");
this->fftSize=fftSize;
mem=new double[fftSize];
}
Voice::~Voice() { delete [] mem; }
int Voice::filter(int freq) {
printf("c++ voice filter method\n");
return (doubleIt(3));
}
int Voice::doubleIt(int i) { return 2*i; }
}
.
Original h file : Voice.h
namespace transformation {
class Voice {
public:
double *mem;
int fftSize;
Voice(int fftSize);
~Voice();
int filter(int freq);
int doubleIt(int i);
};
}
.
C++ Python wrapper file : voiceWrapper.cpp
#include <Python.h>
#include <cstdio>
//~ #include "structmember.h"
#include "Voice.h"
using transformation::Voice;
typedef struct {
PyObject_HEAD
Voice * ptrObj;
} PyVoice;
static PyModuleDef voicemodule = {
PyModuleDef_HEAD_INIT,
"voice",
"Example module that wrapped a C++ object",
-1,
NULL, NULL, NULL, NULL, NULL
};
static int PyVoice_init(PyVoice *self, PyObject *args, PyObject *kwds)
// initialize PyVoice Object
{
int fftSize;
if (! PyArg_ParseTuple(args, "i", &fftSize))
return -1;
self->ptrObj=new Voice(fftSize);
return 0;
}
static void PyVoice_dealloc(PyVoice * self)
// destruct the object
{
delete self->ptrObj;
Py_TYPE(self)->tp_free(self);
}
static PyObject * PyVoice_filter(PyVoice* self, PyObject* args)
{
int freq;
int retval;
if (! PyArg_ParseTuple(args, "i", &freq))
return Py_False;
retval = (self->ptrObj)->filter(freq);
return Py_BuildValue("i",retval);
}
static PyMethodDef PyVoice_methods[] = {
{ "filter", (PyCFunction)PyVoice_filter, METH_VARARGS, "filter the mem voice" },
{NULL} /* Sentinel */
};
static PyTypeObject PyVoiceType = { PyVarObject_HEAD_INIT(NULL, 0)
"voice.Voice" /* tp_name */
};
PyMODINIT_FUNC PyInit_voice(void)
// create the module
{
PyObject* m;
PyVoiceType.tp_new = PyType_GenericNew;
PyVoiceType.tp_basicsize=sizeof(PyVoice);
PyVoiceType.tp_dealloc=(destructor) PyVoice_dealloc;
PyVoiceType.tp_flags=Py_TPFLAGS_DEFAULT;
PyVoiceType.tp_doc="Voice objects";
PyVoiceType.tp_methods=PyVoice_methods;
//~ PyVoiceType.tp_members=Noddy_members;
PyVoiceType.tp_init=(initproc)PyVoice_init;
if (PyType_Ready(&PyVoiceType) < 0)
return NULL;
m = PyModule_Create(&voicemodule);
if (m == NULL)
return NULL;
Py_INCREF(&PyVoiceType);
PyModule_AddObject(m, "Voice", (PyObject *)&PyVoiceType); // Add Voice object to the module
return m;
}
.
distutils file : setup.py
from distutils.core import setup, Extension
setup(name='voicePkg', version='1.0', \
ext_modules=[Extension('voice', ['voiceWrapper.cpp','Voice.cpp'])])
.
python test file : test.py
import voice
v=voice.Voice(512)
result=v.filter(5)
print('result='+str(result))
.
and magic :
sudo python3 setup.py install
python3 test.py
Output is :
c++ constructor of voice
c++ voice filter method
result=6
Enjoy !
Doom
I embedded python in my C++ program.
I use PyImport_ImportModule to load my module written in a .py file.
But how can I load it from memory? Let's say my .py file is encrypted, so I need to first decrypt it and feed the code to python to execute.
Moreover, it'd be nice if I could bypass/intercept or modify the import mechanism, so that doesn't load modules from the filesystem but my own memory blocks, how/can I do that?
The following example shows how to define a module from a C string:
#include <stdio.h>
#include <Python.h>
int main(int argc, char *argv[])
{
Py_Initialize();
PyRun_SimpleString("print('hello from python')");
// fake module
char *source = "__version__ = '2.0'";
char *filename = "test_module.py";
// perform module load
PyObject *builtins = PyEval_GetBuiltins();
PyObject *compile = PyDict_GetItemString(builtins, "compile");
PyObject *code = PyObject_CallFunction(compile, "sss", source, filename, "exec");
PyObject *module = PyImport_ExecCodeModule("test_module", code);
PyRun_SimpleString("import test_module; print(test_module.__version__)");
Py_Finalize();
return 0;
}
output:
hello from python
version: 2.0
You can read about import hooks in the docs. You will need to define a class with find_module and load_module methods. Something like the following should work:
PyObject* find_module(PyObject* self, PyObject* args) {
// ... lookup args in available special modules ...
return Py_BuildValue("B", found);
}
PyObject* load_module(PyObject* self, PyObject* args) {
// ... convert args into filname, source ...
PyObject *builtins = PyEval_GetBuiltins();
PyObject *compile = PyDict_GetItemString(builtins, "compile");
PyObject *code = PyObject_CallFunction(compile, "sss", source, filename, "exec");
PyObject *module = PyImport_ExecCodeModule("test_module", code);
return Py_BuildValue("O", module);
}
static struct PyMethodDef methods[] = {
{ "find_module", find_module, METH_VARARGS, "Returns module_loader if this is an encrypted module"},
{ "load_module", load_module, METH_VARARGS, "Load an encrypted module" },
{ NULL, NULL, 0, NULL }
};
static struct PyModuleDef modDef = {
PyModuleDef_HEAD_INIT, "embedded", NULL, -1, methods,
NULL, NULL, NULL, NULL
};
static PyObject* PyInit_embedded(void)
{
return PyModule_Create(&modDef);
}
int main() {
...
PyImport_AppendInittab("embedded", &PyInit_embedded);
PyRun_SimpleString("\
import embedded, sys\n\
class Importer:\n\
def find_module(self, fullpath):\n\
return self if embedded.find_module(fullpath) else None\n\
def load_module(self, fullpath):\n\
return embedded.load_module(fullpath)\n\
sys.path_hooks.insert(0, Importer())\n\
");
...
}
I'm creating my own python module in C++. The module source code :
#include <Python.h>
#include <numpy/arrayobject.h>
#include "FMM.h"
struct module_state {
PyObject *error;
};
#define GETSTATE(m) ((struct module_state*)PyModule_GetState(m))
static PyObject* FMM(PyObject* self, PyObject* args)
{
PyObject *model_obj;
PyObject *time_obj;
PyObject *accepted_obj;
PyObject *lat_obj;
PyObject *lon_obj;
PyObject *h_obj;
int N;
if (!PyArg_ParseTuple(args, "OOOOOOi", &model_obj, &time_obj, &accepted_obj, &lat_obj, &lon_obj, &h_obj, &N))
{
Py_INCREF(Py_None);
return Py_None;
}
PyObject *model = PyArray_FROM_OTF(model_obj, NPY_FLOAT, NPY_INOUT_ARRAY);
PyObject *time = PyArray_FROM_OTF(time_obj, NPY_FLOAT, NPY_INOUT_ARRAY);
PyObject *accepted = PyArray_FROM_OTF(accepted_obj, NPY_BOOL, NPY_INOUT_ARRAY);
PyObject *lat = PyArray_FROM_OTF(lon_obj, NPY_DOUBLE, NPY_INOUT_ARRAY);
PyObject *lon = PyArray_FROM_OTF(lon_obj, NPY_DOUBLE, NPY_INOUT_ARRAY);
PyObject *h = PyArray_FROM_OTF(h_obj, NPY_DOUBLE, NPY_INOUT_ARRAY);
float *MODEL = static_cast<float *>(PyArray_DATA(model_obj));
float *TIME = static_cast<float *>(PyArray_DATA(time_obj));
bool *ACCEPTED = static_cast<bool *>(PyArray_DATA(accepted_obj));
double *LAT = static_cast<double *>(PyArray_DATA(lat_obj));
double *LON = static_cast<double *>(PyArray_DATA(lon_obj));
double *H = static_cast<double *>(PyArray_DATA(h_obj));
_FMM(MODEL, TIME, ACCEPTED, LAT, LON, H, N);
Py_INCREF(Py_None);
return Py_None;
}
static PyObject* SetModelSize(PyObject* self, PyObject* args)
{
int x, y, z;
if (!PyArg_ParseTuple(args, "iii", &x, &y, &z))
{
Py_INCREF(Py_None);
return Py_None;
}
_SaveModelSize(x, y, z);
// Rerutn values
Py_INCREF(Py_None);
return Py_None;
}
static PyMethodDef FMMMethods[] = {
{"FMM", FMM, METH_VARARGS, "Some Text"},
{"SetModelSize", SetModelSize, METH_VARARGS, "Some Text"},
{NULL, NULL, 0, NULL}
};
static int FMM_traverse(PyObject *m, visitproc visit, void *arg) {
Py_VISIT(GETSTATE(m)->error);
return 0;
}
static int FMM_clear(PyObject *m) {
Py_CLEAR(GETSTATE(m)->error);
return 0;
}
static struct PyModuleDef moduledef = {
PyModuleDef_HEAD_INIT,
"FMM",
NULL,
sizeof(struct module_state),
FMMMethods,
NULL,
FMM_traverse,
FMM_clear,
NULL
};
extern "C" PyObject * PyInit_FMM(void)
{
PyObject *module = PyModule_Create(&moduledef);
if (module == NULL)
return NULL;
struct module_state *st = GETSTATE(module);
st->error = PyErr_NewException("FMM.Error", NULL, NULL);
if (st->error == NULL)
{
Py_DECREF(module);
return NULL;
}
import_array();
Py_INCREF(module);
return module;
}
My function _FMM does numerical computation on large data sets (two floats and bool has size of 2e9 elements.
The module is compiled using setup.py:
from distutils.core import setup, Extension
import numpy.distutils.misc_util
import os
os.environ["CC"] = "g++"
os.environ["CXX"] = "g++"
module1 = Extension('FMM', sources = ['FMMmodule.c', 'FMM.c'])
setup (name = 'PackageName',
version = '1.0',
description = 'This is a demo package',
ext_modules = [module1],
include_dirs=numpy.distutils.misc_util.get_numpy_include_dirs())
When I run it from python I get Segmentation fault after about 60 seconds of calculation.
Every time I run in different step of calculation. I have read that it might be related to misuse of Py_INCREF() and Py_DECREF() - tried to play with that with no luck.
Anybody has experience how to deal with such behavior?
I'm testing using python 3.4.
I have finally sorted that out. I had to modify the code to make it compatible with newer numpy c-api: see more here: Migrating to numpy api 1.7