I embedded python in my C++ program.
I use PyImport_ImportModule to load my module written in a .py file.
But how can I load it from memory? Let's say my .py file is encrypted, so I need to first decrypt it and feed the code to python to execute.
Moreover, it'd be nice if I could bypass/intercept or modify the import mechanism, so that doesn't load modules from the filesystem but my own memory blocks, how/can I do that?
The following example shows how to define a module from a C string:
#include <stdio.h>
#include <Python.h>
int main(int argc, char *argv[])
{
Py_Initialize();
PyRun_SimpleString("print('hello from python')");
// fake module
char *source = "__version__ = '2.0'";
char *filename = "test_module.py";
// perform module load
PyObject *builtins = PyEval_GetBuiltins();
PyObject *compile = PyDict_GetItemString(builtins, "compile");
PyObject *code = PyObject_CallFunction(compile, "sss", source, filename, "exec");
PyObject *module = PyImport_ExecCodeModule("test_module", code);
PyRun_SimpleString("import test_module; print(test_module.__version__)");
Py_Finalize();
return 0;
}
output:
hello from python
version: 2.0
You can read about import hooks in the docs. You will need to define a class with find_module and load_module methods. Something like the following should work:
PyObject* find_module(PyObject* self, PyObject* args) {
// ... lookup args in available special modules ...
return Py_BuildValue("B", found);
}
PyObject* load_module(PyObject* self, PyObject* args) {
// ... convert args into filname, source ...
PyObject *builtins = PyEval_GetBuiltins();
PyObject *compile = PyDict_GetItemString(builtins, "compile");
PyObject *code = PyObject_CallFunction(compile, "sss", source, filename, "exec");
PyObject *module = PyImport_ExecCodeModule("test_module", code);
return Py_BuildValue("O", module);
}
static struct PyMethodDef methods[] = {
{ "find_module", find_module, METH_VARARGS, "Returns module_loader if this is an encrypted module"},
{ "load_module", load_module, METH_VARARGS, "Load an encrypted module" },
{ NULL, NULL, 0, NULL }
};
static struct PyModuleDef modDef = {
PyModuleDef_HEAD_INIT, "embedded", NULL, -1, methods,
NULL, NULL, NULL, NULL
};
static PyObject* PyInit_embedded(void)
{
return PyModule_Create(&modDef);
}
int main() {
...
PyImport_AppendInittab("embedded", &PyInit_embedded);
PyRun_SimpleString("\
import embedded, sys\n\
class Importer:\n\
def find_module(self, fullpath):\n\
return self if embedded.find_module(fullpath) else None\n\
def load_module(self, fullpath):\n\
return embedded.load_module(fullpath)\n\
sys.path_hooks.insert(0, Importer())\n\
");
...
}
Related
I wrote a c++ module witch should be imported into Python. Below are both Codes, the C++ part and the Python part. The C++ function method_sum should return the double of a value to python.
module.cpp:
#define PY_SSIZE_T_CLEAN
#include <Python.h>
static PyObject *method_sum(PyObject *self, PyObject *args) {
const int *prop;
if (!PyArg_ParseTuple(args, "i", &prop)) return NULL;
int result = *prop + *prop;
return Py_BuildValue("i", result);
}
static PyMethodDef ModuleMethods[] = {
{"sum", method_sum, METH_VARARGS, "description of the function"},
{NULL, NULL, 0, NULL}
};
static struct PyModuleDef module = {
PyModuleDef_HEAD_INIT,
"module",
"description of the module",
-1,
ModuleMethods
};
PyMODINIT_FUNC PyInit_module(void) {
return PyModule_Create(&module);
}
main.py:
import module
print(module.sum(18))
setup.py:
from distutils.core import setup, Extension
setup(name='module', version='1.0', ext_modules=[Extension('module', ['module.cpp'])])
I changed method_sum to the following and main.py prints 36 instead of segfaulting.
static PyObject *method_sum(PyObject *self, PyObject *args) {
int prop;
if (!PyArg_ParseTuple(args, "i", &prop)) return NULL;
int result = prop + prop;
return Py_BuildValue("i", result);
}
The following also works and prop is still a pointer like in the code in the question.
static PyObject *method_sum(PyObject *self, PyObject *args) {
const int *prop = new int;
if (!PyArg_ParseTuple(args, "i", prop)) {
delete prop;
return NULL;
}
int result = *prop + *prop;
delete prop;
return Py_BuildValue("i", result);
}
Everywhere, I can easily find an example about writing a method with Python C Extensions and use it in Python. Like this one: Python 3 extension example
$ python3
>>> import hello
>>> hello.hello_world()
Hello, world!
>>> hello.hello('world')
Hello, world!
How to do write a hello word full featured Python class (not just a module method)?
I think this How to wrap a C++ object using pure Python Extension API (python3)? question has an example, but it does not seem minimal as he is using (or wrapping?) C++ classes on it.
For example:
class ClassName(object):
"""docstring for ClassName"""
def __init__(self, hello):
super().__init__()
self.hello = hello
def talk(self, world):
print( '%s %s' % ( self.hello, world ) )
What is the equivalent of this Python class example with C Extensions?
I would use it like this:
from .mycextensionsmodule import ClassName
classname = ClassName("Hello")
classname.talk( 'world!' )
# prints "Hello world!"
My goal is to write a class fully written in C for performance (all other classes in my project will be in Python, except this one). I am not looking for portability as using ctypes, neither black boxes as using Boost.Python or SWIG. Just a high-performance class purely written with Python C Extensions.
After I got this Hello word working, I can figure my self out within Python Extensive documentation:
https://docs.python.org/3/c-api/
https://docs.python.org/3/extending/extending.html
See also: Python instance method in C
Create the file called MANIFEST.in
include README.md
include LICENSE.txt
recursive-include source *.h
Create the file called setup.py
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from setuptools import setup, Extension
__version__ = '0.1.0'
setup(
name = 'custom',
version = __version__,
package_data = {
'': [ '**.txt', '**.md', '**.py', '**.h', '**.hpp', '**.c', '**.cpp' ],
},
ext_modules = [
Extension(
name = 'custom',
sources = [
'source/custom.cpp',
],
include_dirs = ['source'],
)
],
)
Create the file called source/custom.cpp
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include "structmember.h"
typedef struct {
PyObject_HEAD
PyObject *first; /* first name */
PyObject *last; /* last name */
int number;
} CustomObject;
static int
Custom_traverse(CustomObject *self, visitproc visit, void *arg)
{
Py_VISIT(self->first);
Py_VISIT(self->last);
return 0;
}
static int
Custom_clear(CustomObject *self)
{
Py_CLEAR(self->first);
Py_CLEAR(self->last);
return 0;
}
static void
Custom_dealloc(CustomObject *self)
{
PyObject_GC_UnTrack(self);
Custom_clear(self);
Py_TYPE(self)->tp_free((PyObject *) self);
}
static PyObject *
Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
CustomObject *self;
self = (CustomObject *) type->tp_alloc(type, 0);
if (self != NULL) {
self->first = PyUnicode_FromString("");
if (self->first == NULL) {
Py_DECREF(self);
return NULL;
}
self->last = PyUnicode_FromString("");
if (self->last == NULL) {
Py_DECREF(self);
return NULL;
}
self->number = 0;
}
return (PyObject *) self;
}
static int
Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"first", "last", "number", NULL};
PyObject *first = NULL, *last = NULL, *tmp;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist,
&first, &last,
&self->number))
return -1;
if (first) {
tmp = self->first;
Py_INCREF(first);
self->first = first;
Py_DECREF(tmp);
}
if (last) {
tmp = self->last;
Py_INCREF(last);
self->last = last;
Py_DECREF(tmp);
}
return 0;
}
static PyMemberDef Custom_members[] = {
{"number", T_INT, offsetof(CustomObject, number), 0,
"custom number"},
{NULL} /* Sentinel */
};
static PyObject *
Custom_getfirst(CustomObject *self, void *closure)
{
Py_INCREF(self->first);
return self->first;
}
static int
Custom_setfirst(CustomObject *self, PyObject *value, void *closure)
{
if (value == NULL) {
PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
return -1;
}
if (!PyUnicode_Check(value)) {
PyErr_SetString(PyExc_TypeError,
"The first attribute value must be a string");
return -1;
}
Py_INCREF(value);
Py_CLEAR(self->first);
self->first = value;
return 0;
}
static PyObject *
Custom_getlast(CustomObject *self, void *closure)
{
Py_INCREF(self->last);
return self->last;
}
static int
Custom_setlast(CustomObject *self, PyObject *value, void *closure)
{
if (value == NULL) {
PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute");
return -1;
}
if (!PyUnicode_Check(value)) {
PyErr_SetString(PyExc_TypeError,
"The last attribute value must be a string");
return -1;
}
Py_INCREF(value);
Py_CLEAR(self->last);
self->last = value;
return 0;
}
static PyGetSetDef Custom_getsetters[] = {
{"first", (getter) Custom_getfirst, (setter) Custom_setfirst,
"first name", NULL},
{"last", (getter) Custom_getlast, (setter) Custom_setlast,
"last name", NULL},
{NULL} /* Sentinel */
};
static PyObject *
Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored))
{
return PyUnicode_FromFormat("%S %S", self->first, self->last);
}
static PyMethodDef Custom_methods[] = {
{"name", (PyCFunction) Custom_name, METH_NOARGS,
"Return the name, combining the first and last name"
},
{NULL} /* Sentinel */
};
static PyTypeObject CustomType = {
PyVarObject_HEAD_INIT(NULL, 0)
.tp_name = "custom.Custom",
.tp_doc = "Custom objects",
.tp_basicsize = sizeof(CustomObject),
.tp_itemsize = 0,
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
.tp_new = Custom_new,
.tp_init = (initproc) Custom_init,
.tp_dealloc = (destructor) Custom_dealloc,
.tp_traverse = (traverseproc) Custom_traverse,
.tp_clear = (inquiry) Custom_clear,
.tp_members = Custom_members,
.tp_methods = Custom_methods,
.tp_getset = Custom_getsetters,
};
static PyModuleDef custommodule = {
PyModuleDef_HEAD_INIT,
.m_name = "custom",
.m_doc = "Example module that creates an extension type.",
.m_size = -1,
};
PyMODINIT_FUNC
PyInit_custom(void)
{
PyObject *m;
if (PyType_Ready(&CustomType) < 0)
return NULL;
m = PyModule_Create(&custommodule);
if (m == NULL)
return NULL;
Py_INCREF(&CustomType);
PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
return m;
}
Then, to compile it and install you can run either:
pip3 install . -v
python3 setup.py install
As side note from this question How to use setuptools packages and ext_modules with the same name? do not mix on the same project *.py files and Python C Extensions, i.e., use only purely C/C++, building Python C Extensions without adding packages = [ 'package_name' ] entries because they cause the Python C Extensions code run 30%, i.e., if the program would take 7 seconds to run, now with *.py files, it will take 11 seconds.
References:
https://docs.python.org/3/extending/newtypes_tutorial.html#supporting-cyclic-garbage-collection
I want to know how to wrap a C++ object with Python Extension API (and distutils) without external tools (like Cython, Boost, SWIG, ...). Just in pure Python way without creating a dll.
Note that my C++ object has memory allocations so destructor has to be called to avoid memory leaks.
#include "Voice.h"
namespace transformation
{
Voice::Voice(int fftSize) { mem=new double[fftSize]; }
Voice::~Voice() { delete [] mem; }
int Voice::method1() { /*do stuff*/ return (1); }
}
I just want to do somethings like that in Python :
import voice
v=voice.Voice(512)
result=v.method1()
Seems that the answer was in fact here : https://docs.python.org/3.6/extending/newtypes.html
With examples, but not really easy.
EDIT 1 :
In fact, it is not really for wrapping a C++ object in a Python object, but rather to create a Python object with C code. (edit2 : and so you can wrap C++ object!)
EDIT 2 :
Here is a solution using the Python newtypes
.
Original C++ file : Voice.cpp
#include <cstdio>
#include "Voice.h"
namespace transformation
{
Voice::Voice(int fftSize) {
printf("c++ constructor of voice\n");
this->fftSize=fftSize;
mem=new double[fftSize];
}
Voice::~Voice() { delete [] mem; }
int Voice::filter(int freq) {
printf("c++ voice filter method\n");
return (doubleIt(3));
}
int Voice::doubleIt(int i) { return 2*i; }
}
.
Original h file : Voice.h
namespace transformation {
class Voice {
public:
double *mem;
int fftSize;
Voice(int fftSize);
~Voice();
int filter(int freq);
int doubleIt(int i);
};
}
.
C++ Python wrapper file : voiceWrapper.cpp
#include <Python.h>
#include <cstdio>
//~ #include "structmember.h"
#include "Voice.h"
using transformation::Voice;
typedef struct {
PyObject_HEAD
Voice * ptrObj;
} PyVoice;
static PyModuleDef voicemodule = {
PyModuleDef_HEAD_INIT,
"voice",
"Example module that wrapped a C++ object",
-1,
NULL, NULL, NULL, NULL, NULL
};
static int PyVoice_init(PyVoice *self, PyObject *args, PyObject *kwds)
// initialize PyVoice Object
{
int fftSize;
if (! PyArg_ParseTuple(args, "i", &fftSize))
return -1;
self->ptrObj=new Voice(fftSize);
return 0;
}
static void PyVoice_dealloc(PyVoice * self)
// destruct the object
{
delete self->ptrObj;
Py_TYPE(self)->tp_free(self);
}
static PyObject * PyVoice_filter(PyVoice* self, PyObject* args)
{
int freq;
int retval;
if (! PyArg_ParseTuple(args, "i", &freq))
return Py_False;
retval = (self->ptrObj)->filter(freq);
return Py_BuildValue("i",retval);
}
static PyMethodDef PyVoice_methods[] = {
{ "filter", (PyCFunction)PyVoice_filter, METH_VARARGS, "filter the mem voice" },
{NULL} /* Sentinel */
};
static PyTypeObject PyVoiceType = { PyVarObject_HEAD_INIT(NULL, 0)
"voice.Voice" /* tp_name */
};
PyMODINIT_FUNC PyInit_voice(void)
// create the module
{
PyObject* m;
PyVoiceType.tp_new = PyType_GenericNew;
PyVoiceType.tp_basicsize=sizeof(PyVoice);
PyVoiceType.tp_dealloc=(destructor) PyVoice_dealloc;
PyVoiceType.tp_flags=Py_TPFLAGS_DEFAULT;
PyVoiceType.tp_doc="Voice objects";
PyVoiceType.tp_methods=PyVoice_methods;
//~ PyVoiceType.tp_members=Noddy_members;
PyVoiceType.tp_init=(initproc)PyVoice_init;
if (PyType_Ready(&PyVoiceType) < 0)
return NULL;
m = PyModule_Create(&voicemodule);
if (m == NULL)
return NULL;
Py_INCREF(&PyVoiceType);
PyModule_AddObject(m, "Voice", (PyObject *)&PyVoiceType); // Add Voice object to the module
return m;
}
.
distutils file : setup.py
from distutils.core import setup, Extension
setup(name='voicePkg', version='1.0', \
ext_modules=[Extension('voice', ['voiceWrapper.cpp','Voice.cpp'])])
.
python test file : test.py
import voice
v=voice.Voice(512)
result=v.filter(5)
print('result='+str(result))
.
and magic :
sudo python3 setup.py install
python3 test.py
Output is :
c++ constructor of voice
c++ voice filter method
result=6
Enjoy !
Doom
My problem is as follows:
I would like to call a C function from my Python file and return a value back to that Python file.
I have tried the following method of using embedded C in Python (the following code is the C code called "mod1.c). I am using Python3.4 so the format follows that given in the documentation guidelines. The problem comes when I call my setup file (second code below).
#include
#include "sum.h"
static PyObject*
mod_sum(PyObject *self, PyObject *args)
{
int a;
int b;
int s;
if (!PyArg_ParseTuple(args,"ii",&a,&b))
return NULL;
s = sum(a,b);
return Py_BuildValue("i",s);
}
/* DECLARATION OF METHODS */
static PyMethodDef ModMethods[] = {
{"sum", mod_sum, METH_VARARGS, "Descirption"}, // {"methName", modName_methName, METH_VARARGS, "Description.."}, modName is name of module and methName is name of method
{NULL,NULL,0,NULL}
};
// Module Definition Structure
static struct PyModuleDef summodule = {
PyModuleDef_HEAD_INIT,
"sum",
NULL,
-1,
ModMethods
};
/* INITIALIZATION FUNCTION */
PyMODINIT_FUNC initmod(void)
{
PyObject *m;
m = PyModule_Create(&summodule);
if (m == NULL)
return m;
}
Setup.py
from distutils.core import setup, Extension
setup(name='buildsum', version='1.0', \
ext_modules=[Extension('buildsum', ['mod1.c'])])
The result that I get when I compile my code using gcc is the following error: Cannot export PyInit_buildsum: symbol not defined
I would greatly appreciate any insight or help on this problem, or any suggestion in how to call C from Python. Thank you!
---------------------------------------EDIT ---------------------------------
Thank you for the comments:
I have tried the following now:
static PyObject*
PyInit_sum(PyObject *self, PyObject *args)
{
int a;
int b;
int s;
if (!PyArg_ParseTuple(args,"ii",&a,&b))
return NULL;
s = sum(a,b);
return Py_BuildValue("i",s);
}
For the first function; however, I still get the same error of PyInit_sum: symbol not defined
The working code from above in case anyone runs into the same error: the answer from #dclarke is correct. The initialization function in python 3 must have PyInit_(name) as its name.
#include <Python.h>
#include "sum.h"
static PyObject* mod_sum(PyObject *self, PyObject *args)
{
int a;
int b;
int s;
if (!PyArg_ParseTuple(args,"ii",&a,&b))
return NULL;
s = sum(a,b);
return Py_BuildValue("i",s);
}
/* DECLARATION OF METHODS*/
static PyMethodDef ModMethods[] = {
{"modsum", mod_sum, METH_VARARGS, "Descirption"},
{NULL,NULL,0,NULL}
};
// Module Definition Structure
static struct PyModuleDef summodule = {
PyModuleDef_HEAD_INIT,"modsum", NULL, -1, ModMethods
};
/* INITIALIZATION FUNCTION*/
PyMODINIT_FUNC PyInit_sum(void)
{
PyObject *m;
m = PyModule_Create(&summodule);
return m;
}
I'm writing a Python C Extension that needs to return a CTypes pointer to a char array in memory (I need to interface with another Python library that expects a CTypes pointer).
I cannot find any documentation on any kind of CTypes interface for C. Are there any workarounds, like calling the Python pointer constructor?
static PyObject *get_pointer(myObject *self)
{
char *my_pointer = self->internal_pointer;
return PyCTypes_Pointer(my_pointer); /* how do I turn 'my_pointer' into
a Python object representing a CTypes pointer? */
}
Thanks in advance.
Even though you're programming in C, try using Python itself. Sadly I don't know of any kind of CTypes interface for C and couldn't find any hint in the sources. Maybe you'll be able to write your own PyCTypes_Pointer method with the following.
I didn't do any cleanup, error checking or handling here, but it's working with Python 2.7 and 3.4 for me.
#include <Python.h>
static const char *test = "Testing Python ctypes char pointer ...";
static PyObject *c_char_p = NULL;
static PyObject *c_void_p = NULL;
static PyObject *cast = NULL;
static PyObject * char_p ( void ) {
PyObject *p = PyLong_FromVoidPtr((void *) test);
p = PyObject_CallFunction(c_void_p, "O", p);
return PyObject_CallFunction(cast, "OO", p, c_char_p);
}
static PyObject *len ( void ) {
return PyLong_FromSize_t(strlen(test));
}
static PyMethodDef methods[] = {
{"char_p", (PyCFunction) char_p, METH_NOARGS, ""},
{"len", (PyCFunction) len, METH_NOARGS, ""},
{ NULL }
};
#if PY_MAJOR_VERSION >= 3
static PyModuleDef module = {PyModuleDef_HEAD_INIT, "pyt", "", -1, methods,
NULL, NULL, NULL, NULL};
#define INIT_FUNC PyInit_pyp
#else
#define INIT_FUNC initpyp
#endif
PyMODINIT_FUNC INIT_FUNC ( void ) {
PyObject* m;
PyObject *ctypes = PyImport_ImportModule("ctypes");
PyObject *c_char = PyObject_GetAttrString(ctypes, "c_char");
c_char_p = PyObject_CallMethod(ctypes, "POINTER", "O", c_char);
c_void_p = PyObject_GetAttrString(ctypes, "c_void_p");
cast = PyObject_GetAttrString(ctypes, "cast");
#if PY_MAJOR_VERSION >= 3
m = PyModule_Create(&module);
#else
m = Py_InitModule("pyp", methods);
#endif
return (PyMODINIT_FUNC) m;
}
Just build with
from __future__ import print_function
from distutils.core import Extension, setup
import sys
sys.argv.extend(["build_ext", "-i"])
setup(ext_modules = [Extension('pyp', ['pyp.c'])])
import pyp
c = pyp.char_p()
print(c[:pyp.len()])
and get an output like
<class 'ctypes.LP_c_char'> Testing Python ctypes char pointer ...
If you're handling zero-terminated strings only, you might be able to use ctypes.c_char_p directly, maybe ...