We Keep Coding

Creating a module subclass in a Python extension

I am trying to create a Python extension module with multi-phase initialization, following the advice I got from a previous question. PEP 489 suggests that it is preferable for the Py_mod_create function to return a module subclass, which presumably means a subclass of PyModule, but I cannot figure out how to do this. In all my attempts, the module segfaults when it is imported. It works fine if Py_mod_create returns some other object, (one which is not a subclass of PyModule), but I am not sure if this will cause problems in future, since isinstance(mymodule, types.ModuleType) returns false in this case.
Following the docs on subclassing built-in types, I set tp_base to PyModule_Type, and my tp_init function calls PyModule_Type.tp_init. The docs also suggest that my structure should contain the superclass structure at the beginning, which in this case is PyModuleObject. This structure is not in the public Python header files, (it is defined in moduleobject.c in the Python sources), so for now I copied and paste the definitions of the PyModuleObject fields at the start of my structure. The complete code looks like this:
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include <stdio.h>
struct testmod_s {
// Fields copied from PyModuleObject in moduleobject.c
PyObject_HEAD
PyObject *md_dict;
struct PyModuleDef *md_def;
void *md_state;
PyObject *md_weaklist;
PyObject *md_name;
};
static int testmod_init(PyObject *self, PyObject *args, PyObject *kwds);
static PyObject *testmod_create(PyObject *spec, PyModuleDef *def);
static PyModuleDef_Slot testmod_slots[] = {
{Py_mod_create, testmod_create},
{0, 0} /* Sentinel */
};
static struct PyModuleDef testmod_def = {
PyModuleDef_HEAD_INIT, /* m_base */
"testmod", /* m_name */
NULL, /* m_doc */
sizeof(struct testmod_s), /* m_size */
NULL, /* m_methods */
testmod_slots, /* m_slots */
NULL, /* m_traverse */
NULL, /* m_clear */
NULL /* m_free */
};
static PyTypeObject testmodtype = {
PyVarObject_HEAD_INIT (NULL, 0)
"testmodtype", /* tp_name */
sizeof (struct testmod_s), /* tp_basicsize */
/* fields omitted for brevity, all set to zero */
Py_TPFLAGS_DEFAULT |
Py_TPFLAGS_BASETYPE, /* tp_flags */
/* fields omitted for brevity, all set to zero */
testmod_init, /* tp_init */
0, /* tp_alloc */
0, /* tp_new */
};
PyMODINIT_FUNC
PyInit_testmod(void)
{
testmodtype.tp_base = &PyModule_Type;
if (PyType_Ready(&testmodtype)) {
return NULL;
}
PyObject *moduledef = PyModuleDef_Init(&testmod_def);
if (moduledef == NULL) {
return NULL;
}
return moduledef;
}
static int testmod_init(PyObject *self, PyObject *args, PyObject *kwds)
{
if (PyModule_Type.tp_init((PyObject *)self, args, kwds) < 0) {
return -1;
}
return 0;
}
static PyObject *testmod_create(PyObject *spec, PyModuleDef *def)
{
struct testmod_s *module = PyObject_New(struct testmod_s, &testmodtype);
if (module == NULL) {
return NULL;
}
return (PyObject *) module;
}
Importing this module causes a segfault. What am I doing wrong?
I am running Python 3.8.5 on macOS 12.0.1 with a build from Anaconda:
>>> sys.version
'3.8.5 (default, Sep 4 2020, 02:22:02) \n[Clang 10.0.0 ]'

After some tests I could build a custom module type by copying parts of code from moduleobject.c
Your problem is that your code does create an instance of a subclass of module, but never initializes it and gets random values in key members. Additionaly, modules are expected to be gc collectables, so you have to create your custom module with PyObject_GC_New.
The following code replaces your initial testmod_create function with a full initialization of the module:
...
// copied from moduleobject.c
static int
module_init_dict(struct testmod_s* mod, PyObject* md_dict,
PyObject* name, PyObject* doc)
{
_Py_IDENTIFIER(__name__);
_Py_IDENTIFIER(__doc__);
_Py_IDENTIFIER(__package__);
_Py_IDENTIFIER(__loader__);
_Py_IDENTIFIER(__spec__);
if (md_dict == NULL)
return -1;
if (doc == NULL)
doc = Py_None;
if (_PyDict_SetItemId(md_dict, &PyId___name__, name) != 0)
return -1;
if (_PyDict_SetItemId(md_dict, &PyId___doc__, doc) != 0)
return -1;
if (_PyDict_SetItemId(md_dict, &PyId___package__, Py_None) != 0)
return -1;
if (_PyDict_SetItemId(md_dict, &PyId___loader__, Py_None) != 0)
return -1;
if (_PyDict_SetItemId(md_dict, &PyId___spec__, Py_None) != 0)
return -1;
if (PyUnicode_CheckExact(name)) {
Py_INCREF(name);
Py_XSETREF(mod->md_name, name);
}
return 0;
}
static PyObject* testmod_create(PyObject* spec, PyModuleDef* def)
{
struct testmod_s* module = PyObject_GC_New(struct testmod_s, &testmodtype);
if (module == NULL) {
return NULL;
}
PyObject* name = PyUnicode_FromString("testmod");
if (name == NULL) {
Py_DECREF(module);
return 0;
}
module->md_def = NULL;
module->md_state = NULL;
module->md_weaklist = NULL;
module->md_name = NULL;
module->md_dict = PyDict_New();
int cr = module_init_dict(module, module->md_dict, name, NULL);
Py_DECREF(name);
if (cr != 0) {
Py_DECREF(module);
return NULL;
}
return (PyObject*)module;
}

Use closure in PyGetSetDef for reusing attribute getter-settter

As the title suggests, I'm trying to create a bunch of attributes but the code is getting repetitive and messy. I want to use the closure argument to make the code more compact.
According to the C API reference, the closure is a function pointer that provides additional information for getters/setters. I have not been able to find an example of it in use.
This is how I am currently using it:
static void closure_1() {};
static void closure_2() {};
...
static PyObject *
FOO_getter(FOO* self, void *closure) {
if (closure == &closure_1) {
return self->bar_1;
} else if (closure == &closure_2) {
return self->bar_2;
}
}
static int
FOO_setter(FOO* self, PyObject *value, void *closure) {
if (closure == &closure_1) {
if (somehow value is invalid) {
PyErr_SetString(PyExc_ValueError, "invalid value for bar_1.");
return -1;
}
} else if (closure == closure_2) {
if (somehow value is invalid) {
PyErr_SetString(PyExc_ValueError, "invalid value for bar_2.");
return -1;
}
}
return 0;
}
static PyGetSetDef FOO_getsetters[] = {
{"bar_1", (getter) FOO_getter, (setter) FOO_setter, "bar_1 attribute", closure_1},
{"bar_2", (getter) FOO_getter, (setter) FOO_setter, "bar_2 attribute", closure_2},
{NULL} /* Sentinel */
};
...
It works the way I want it to, but it looks more like a hack than something "pythonic". Is there a better way to handle this? e.g. call the closure in some way.

I guess this "closure" is used to pass an extra context to Foo_getter. It should be something that simplifies accessing members of Foo. Documentation it likely wrong. It should be "optional pointer", not "optional function pointer".
Consider passing offsets of the members. Offsets to struct members can be easily obtained with standard offsetof macro defined in stddef.h. It is a small unsigned integer that will fit to void* type.
static PyGetSetDef FOO_getsetters[] = {
{"bar_1", (getter) FOO_getter, (setter) FOO_setter, "bar_1 attribute", (void*)offsetof(FOO, bar_1)},
{"bar_2", (getter) FOO_getter, (setter) FOO_setter, "bar_2 attribute", (void*)offsetof(FOO, bar_2)},
{NULL} /* Sentinel */
};
Now the getter could be:
static PyObject *
FOO_getter(FOO* self, void *closure) {
// pointer to location where the FOO's member is stored
char *memb_ptr = (char*)self + (size_t)closure;
// cast to `PyObject**` because `mem_ptr` points to location where a pointer to `PyObject` is stored
return *(PyObject**)mem_ptr;
}
Use similar schema for the setter.

Despite the documentation, I'm assuming the closure can be any pointer you want. So how about passing an "object", seeing as C doesn't support closures (short of literally generating functions at run-time).
In an object, we can store the offset of the member in FOO, and a pointer to an attribute-specific validator.
typedef int (*Validator)(FOO *, const struct Attribute *, void *);
typedef struct Attribute {
const char *name;
size_t offset;
Validator validator;
} Attribute;
static PyObject **resolve_offset(FOO *self, const Attribute *attr) {
return (PyObject **)( ( (char *)self ) + attr->offset );
}
static PyObject *FOO_getter(FOO *self, void *_attr) {
const Attribute *attr = (const Attribute *)_attr;
return *resolve_offset(self, attr);
}
static int FOO_setter(FOO *self, PyObject *val, void *_attr) {
const Attribute *attr = (const Attribute *)_attr;
if (attr->validator(self, attr, val)) {
*resolve_offset(self, attr) = val;
return 0;
} else {
// Building the string to include attr->name is left to you.
PyErr_SetString(PyExc_ValueError, "invalid value.");
return -1;
}
}
static int FOO_bar_1_validator(FOO *self, const Attribute *attr, void *val) { ... }
static int FOO_bar_2_validator(FOO *self, const Attribute *attr, void *val) { ... }
#define ATTRIBUTE(name) \
static Attribute FOO_ ## name ## attr = { \
#name, \
offsetof(FOO, name), \
FOO_ ## name ## _validator \
};
ATTRIBUTE(bar_1);
ATTRIBUTE(bar_2);
#define PY_ATTR_DEF(name) { \
#name, \
(getter)FOO_getter, \
(setter)FOO_setter, \
#name " attribute", \
&(FOO_ ## name ## attr) \
}
static PyGetSetDef FOO_getsetters[] = {
PY_ATTR_DEF(bar_1),
PY_ATTR_DEF(bar_2),
{ NULL }
};
I originally wrote:
resolve_offset surely relies on undefined behaviour, but it should work fine. The alternative would be to have three functions in our attribute object (get, validate, set) instead of one, but that defies the point of the question.
But #tstanisl points out that it looks like it isn't UB. Awesome!

Python C API: How to check if an object is an instance of a type

I want to check if an object is an instance of a certain class. In Python I can do this with instanceof. In C/C++, I found a function named PyObject_IsInstance. But it seems not to work like isinstance.
In detail (also described as sample codes below):
In C++ I defined my custom type My. The type definition is MyType and the object definition is MyObject.
Add MyType to the exported module with name My.
In Python, create a new instance my = My(), and isinstance(my, My) returns True.
While in C++ we use PyObject_IsInstance(my, (PyObject*)&MyType) to check my, and this returns 0, which means my is not an instance of the class defined by MyType.
Full C++ code:
#define PY_SSIZE_T_CLEAN
#include <python3.6/Python.h>
#include <python3.6/structmember.h>
#include <stddef.h>
typedef struct {
PyObject_HEAD
int num;
} MyObject;
static PyTypeObject MyType = []{
PyTypeObject ret = {
PyVarObject_HEAD_INIT(NULL, 0)
};
ret.tp_name = "cpp.My";
ret.tp_doc = NULL;
ret.tp_basicsize = sizeof(MyObject);
ret.tp_itemsize = 0;
ret.tp_flags = Py_TPFLAGS_DEFAULT;
ret.tp_new = PyType_GenericNew;
return ret;
}();
// check if obj is an instance of MyType
static PyObject *Py_fn_checkMy(PyObject *obj) {
if (PyObject_IsInstance(obj, (PyObject *)&MyType)) Py_RETURN_TRUE;
else Py_RETURN_FALSE;
}
static PyMethodDef modmethodsdef[] = {
{ "checkMy", (PyCFunction)Py_fn_checkMy, METH_VARARGS, NULL },
{ NULL }
};
static PyModuleDef moddef = []{
PyModuleDef ret = {
PyModuleDef_HEAD_INIT
};
ret.m_name = "cpp";
ret.m_doc = NULL;
ret.m_size = -1;
return ret;
}();
PyMODINIT_FUNC
PyInit_cpp(void)
{
PyObject *mod;
if (PyType_Ready(&MyType) < 0)
return NULL;
mod = PyModule_Create(&moddef);
if (mod == NULL)
return NULL;
Py_INCREF(&MyType);
PyModule_AddObject(mod, "My", (PyObject *)&MyType);
PyModule_AddFunctions(mod, modmethodsdef);
return mod;
}
Compile this into cpp.so, and test it in Python:
>>> import cpp
>>> isinstance(cpp.My(), cpp.My)
True
>>> cpp.checkMy(cpp.My())
False

METH_VARARGS
This is the typical calling convention, where the methods have the type PyCFunction. The function expects two PyObject* values. The first one is the self object for methods; for module functions, it is the module object. The second parameter (often called args) is a tuple object representing all arguments. This parameter is typically processed using PyArg_ParseTuple() or PyArg_UnpackTuple().
The function signature of Py_fn_checkMy does not match this. It should take two arguments. The first is the module, and this is what you are checking against MyType. The second argument (which you don't actually accept) is a tuple containing the object you passed. You should extract the argument from the tuple and check the type of this.
You'd probably be better using METH_O to specify a single argument instead of extracting arguments from a tuple:
static PyObject *Py_fn_checkMy(PyObject *self, PyObject *obj) {
if (PyObject_IsInstance(obj, (PyObject *)&MyType)) Py_RETURN_TRUE;
else Py_RETURN_FALSE;
}
static PyMethodDef modmethodsdef[] = {
{ "checkMy", (PyCFunction)Py_fn_checkMy, METH_O, NULL },
{ NULL }
};

A full and minimal example for a class (not method) with Python C Extension?

Everywhere, I can easily find an example about writing a method with Python C Extensions and use it in Python. Like this one: Python 3 extension example
$ python3
>>> import hello
>>> hello.hello_world()
Hello, world!
>>> hello.hello('world')
Hello, world!
How to do write a hello word full featured Python class (not just a module method)?
I think this How to wrap a C++ object using pure Python Extension API (python3)? question has an example, but it does not seem minimal as he is using (or wrapping?) C++ classes on it.
For example:
class ClassName(object):
"""docstring for ClassName"""
def __init__(self, hello):
super().__init__()
self.hello = hello
def talk(self, world):
print( '%s %s' % ( self.hello, world ) )
What is the equivalent of this Python class example with C Extensions?
I would use it like this:
from .mycextensionsmodule import ClassName
classname = ClassName("Hello")
classname.talk( 'world!' )
# prints "Hello world!"
My goal is to write a class fully written in C for performance (all other classes in my project will be in Python, except this one). I am not looking for portability as using ctypes, neither black boxes as using Boost.Python or SWIG. Just a high-performance class purely written with Python C Extensions.
After I got this Hello word working, I can figure my self out within Python Extensive documentation:
https://docs.python.org/3/c-api/
https://docs.python.org/3/extending/extending.html

See also: Python instance method in C
Create the file called MANIFEST.in
include README.md
include LICENSE.txt
recursive-include source *.h
Create the file called setup.py
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
from setuptools import setup, Extension
__version__ = '0.1.0'
setup(
name = 'custom',
version = __version__,
package_data = {
'': [ '**.txt', '**.md', '**.py', '**.h', '**.hpp', '**.c', '**.cpp' ],
},
ext_modules = [
Extension(
name = 'custom',
sources = [
'source/custom.cpp',
],
include_dirs = ['source'],
)
],
)
Create the file called source/custom.cpp
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include "structmember.h"
typedef struct {
PyObject_HEAD
PyObject *first; /* first name */
PyObject *last; /* last name */
int number;
} CustomObject;
static int
Custom_traverse(CustomObject *self, visitproc visit, void *arg)
{
Py_VISIT(self->first);
Py_VISIT(self->last);
return 0;
}
static int
Custom_clear(CustomObject *self)
{
Py_CLEAR(self->first);
Py_CLEAR(self->last);
return 0;
}
static void
Custom_dealloc(CustomObject *self)
{
PyObject_GC_UnTrack(self);
Custom_clear(self);
Py_TYPE(self)->tp_free((PyObject *) self);
}
static PyObject *
Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
CustomObject *self;
self = (CustomObject *) type->tp_alloc(type, 0);
if (self != NULL) {
self->first = PyUnicode_FromString("");
if (self->first == NULL) {
Py_DECREF(self);
return NULL;
}
self->last = PyUnicode_FromString("");
if (self->last == NULL) {
Py_DECREF(self);
return NULL;
}
self->number = 0;
}
return (PyObject *) self;
}
static int
Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"first", "last", "number", NULL};
PyObject *first = NULL, *last = NULL, *tmp;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist,
&first, &last,
&self->number))
return -1;
if (first) {
tmp = self->first;
Py_INCREF(first);
self->first = first;
Py_DECREF(tmp);
}
if (last) {
tmp = self->last;
Py_INCREF(last);
self->last = last;
Py_DECREF(tmp);
}
return 0;
}
static PyMemberDef Custom_members[] = {
{"number", T_INT, offsetof(CustomObject, number), 0,
"custom number"},
{NULL} /* Sentinel */
};
static PyObject *
Custom_getfirst(CustomObject *self, void *closure)
{
Py_INCREF(self->first);
return self->first;
}
static int
Custom_setfirst(CustomObject *self, PyObject *value, void *closure)
{
if (value == NULL) {
PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
return -1;
}
if (!PyUnicode_Check(value)) {
PyErr_SetString(PyExc_TypeError,
"The first attribute value must be a string");
return -1;
}
Py_INCREF(value);
Py_CLEAR(self->first);
self->first = value;
return 0;
}
static PyObject *
Custom_getlast(CustomObject *self, void *closure)
{
Py_INCREF(self->last);
return self->last;
}
static int
Custom_setlast(CustomObject *self, PyObject *value, void *closure)
{
if (value == NULL) {
PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute");
return -1;
}
if (!PyUnicode_Check(value)) {
PyErr_SetString(PyExc_TypeError,
"The last attribute value must be a string");
return -1;
}
Py_INCREF(value);
Py_CLEAR(self->last);
self->last = value;
return 0;
}
static PyGetSetDef Custom_getsetters[] = {
{"first", (getter) Custom_getfirst, (setter) Custom_setfirst,
"first name", NULL},
{"last", (getter) Custom_getlast, (setter) Custom_setlast,
"last name", NULL},
{NULL} /* Sentinel */
};
static PyObject *
Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored))
{
return PyUnicode_FromFormat("%S %S", self->first, self->last);
}
static PyMethodDef Custom_methods[] = {
{"name", (PyCFunction) Custom_name, METH_NOARGS,
"Return the name, combining the first and last name"
},
{NULL} /* Sentinel */
};
static PyTypeObject CustomType = {
PyVarObject_HEAD_INIT(NULL, 0)
.tp_name = "custom.Custom",
.tp_doc = "Custom objects",
.tp_basicsize = sizeof(CustomObject),
.tp_itemsize = 0,
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
.tp_new = Custom_new,
.tp_init = (initproc) Custom_init,
.tp_dealloc = (destructor) Custom_dealloc,
.tp_traverse = (traverseproc) Custom_traverse,
.tp_clear = (inquiry) Custom_clear,
.tp_members = Custom_members,
.tp_methods = Custom_methods,
.tp_getset = Custom_getsetters,
};
static PyModuleDef custommodule = {
PyModuleDef_HEAD_INIT,
.m_name = "custom",
.m_doc = "Example module that creates an extension type.",
.m_size = -1,
};
PyMODINIT_FUNC
PyInit_custom(void)
{
PyObject *m;
if (PyType_Ready(&CustomType) < 0)
return NULL;
m = PyModule_Create(&custommodule);
if (m == NULL)
return NULL;
Py_INCREF(&CustomType);
PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
return m;
}
Then, to compile it and install you can run either:
pip3 install . -v
python3 setup.py install
As side note from this question How to use setuptools packages and ext_modules with the same name? do not mix on the same project *.py files and Python C Extensions, i.e., use only purely C/C++, building Python C Extensions without adding packages = [ 'package_name' ] entries because they cause the Python C Extensions code run 30%, i.e., if the program would take 7 seconds to run, now with *.py files, it will take 11 seconds.
References:
https://docs.python.org/3/extending/newtypes_tutorial.html#supporting-cyclic-garbage-collection

Return CTypes pointer from C

I'm writing a Python C Extension that needs to return a CTypes pointer to a char array in memory (I need to interface with another Python library that expects a CTypes pointer).
I cannot find any documentation on any kind of CTypes interface for C. Are there any workarounds, like calling the Python pointer constructor?
static PyObject *get_pointer(myObject *self)
{
char *my_pointer = self->internal_pointer;
return PyCTypes_Pointer(my_pointer); /* how do I turn 'my_pointer' into
a Python object representing a CTypes pointer? */
}
Thanks in advance.

Even though you're programming in C, try using Python itself. Sadly I don't know of any kind of CTypes interface for C and couldn't find any hint in the sources. Maybe you'll be able to write your own PyCTypes_Pointer method with the following.
I didn't do any cleanup, error checking or handling here, but it's working with Python 2.7 and 3.4 for me.
#include <Python.h>
static const char *test = "Testing Python ctypes char pointer ...";
static PyObject *c_char_p = NULL;
static PyObject *c_void_p = NULL;
static PyObject *cast = NULL;
static PyObject * char_p ( void ) {
PyObject *p = PyLong_FromVoidPtr((void *) test);
p = PyObject_CallFunction(c_void_p, "O", p);
return PyObject_CallFunction(cast, "OO", p, c_char_p);
}
static PyObject *len ( void ) {
return PyLong_FromSize_t(strlen(test));
}
static PyMethodDef methods[] = {
{"char_p", (PyCFunction) char_p, METH_NOARGS, ""},
{"len", (PyCFunction) len, METH_NOARGS, ""},
{ NULL }
};
#if PY_MAJOR_VERSION >= 3
static PyModuleDef module = {PyModuleDef_HEAD_INIT, "pyt", "", -1, methods,
NULL, NULL, NULL, NULL};
#define INIT_FUNC PyInit_pyp
#else
#define INIT_FUNC initpyp
#endif
PyMODINIT_FUNC INIT_FUNC ( void ) {
PyObject* m;
PyObject *ctypes = PyImport_ImportModule("ctypes");
PyObject *c_char = PyObject_GetAttrString(ctypes, "c_char");
c_char_p = PyObject_CallMethod(ctypes, "POINTER", "O", c_char);
c_void_p = PyObject_GetAttrString(ctypes, "c_void_p");
cast = PyObject_GetAttrString(ctypes, "cast");
#if PY_MAJOR_VERSION >= 3
m = PyModule_Create(&module);
#else
m = Py_InitModule("pyp", methods);
#endif
return (PyMODINIT_FUNC) m;
}
Just build with
from __future__ import print_function
from distutils.core import Extension, setup
import sys
sys.argv.extend(["build_ext", "-i"])
setup(ext_modules = [Extension('pyp', ['pyp.c'])])
import pyp
c = pyp.char_p()
print(c[:pyp.len()])
and get an output like
<class 'ctypes.LP_c_char'> Testing Python ctypes char pointer ...
If you're handling zero-terminated strings only, you might be able to use ctypes.c_char_p directly, maybe ...