I'm trying to add a c extension as a submodule to an existing python package. I've tried following the documentation but most either are focused on a purely C extension package or it doesn't seem to work for me.
Here is what I'm trying to do:
main/
|--> main/
|--> __init__.py
|--> sub/
|--> sub.c
|--> __init__.py
|--> setup.py
Setup.py file has
from setuptools import setup
from setuptools.extension import Extension
extensions = [
Extension(
name="main.sub",
sources=["main/sub/sub.c"])
]
setup(name='main',
version='1.0',
packages=['main'],
ext_modules = extensions,
)
sub.c has
#include <Python.h>
// This is the definition of a method
static PyObject *division(PyObject *self, PyObject *args)
{
long dividend, divisor;
if (!PyArg_ParseTuple(args, "ll", ÷nd, &divisor))
{
return NULL;
}
if (0 == divisor)
{
PyErr_Format(PyExc_ZeroDivisionError, "Dividing %d by zero!", dividend);
return NULL;
}
return PyLong_FromLong(dividend / divisor);
}
// Exported methods are collected in a table
PyMethodDef method_table[] = {
{"division", (PyCFunction)division, METH_VARARGS, "Method docstring"},
{NULL, NULL, 0, NULL} // Sentinel value ending the table
};
// A struct contains the definition of a module
PyModuleDef sub_module = {
PyModuleDef_HEAD_INIT,
"main.sub", // Module name
"This is the module docstring",
-1, // Optional size of the module state memory
method_table,
NULL, // Optional slot definitions
NULL, // Optional traversal function
NULL, // Optional clear function
NULL // Optional module deallocation function
};
// The module init function
PyMODINIT_FUNC PyInit_sub(void)
{
return PyModule_Create(&sub_module);
}
I want to be able to do:
import main
main.sub.division(4,2)
but not matter what I do, I can't see division. It's driving me nuts... I've tried a bunch of different variations with no luck.
Related
I'm pretty familiar with writing C code and I'm comfortable in writing python code. I'm trying to learn how to write modules in C that can be called from Python-3.9.X on OSX 10.15.7. I've gotten a couple 'hello world' type of examples to work, but for complex examples I'm struggling to figure out how I would debug the C-extensions that I write.
MWE:
src/add.c
// The C function that actually does the work
static PyObject * add_c_func(PyObject *self, PyObject *args)
{
int a=0;
int b=0;
int c=0;
// This determines the number arguments used by add_c_func
if (!PyArg_ParseTuple(args, "iii", &a, &b, &c))
{
return NULL;
}
printf("%i\n", a+b+c);
Py_RETURN_NONE;
}
// This defines the function used by
static PyMethodDef AddMethods[] = {
{"add_py_func", add_c_func, METH_VARARGS, "Add three numbers."},
{NULL, NULL, 0, NULL}
};
static struct PyModuleDef addpymod =
{
PyModuleDef_HEAD_INIT,
"addpymod", /* name of module */
"", /* module documentation, may be NULL */
-1, /* size of per-interpreter state of the module, or -1 if the module keeps state in global variables. */
AddMethods
};
PyMODINIT_FUNC PyInit_addpymod(void)
{
return PyModule_Create(&addpymod);
}
setup.py :
from setuptools import setup, Extension
setup(
name='addpymod',
version='1.0',
description='Python Package with Hello World C Extension',
ext_modules=[
Extension(
'addpymod',
sources=['src/add.c'],
py_limited_api=True)
],
)
Compiling / installing (by default it uses clang):
python setup.py install
Trying to debug :
(py-ext-test) local: understand-python-c-ext $ gdb
GNU gdb (GDB) 10.1
.
.
.
(gdb) b add.c : 20
No symbol table is loaded. Use the "file" command.
Make breakpoint pending on future shared library load? (y or [n]) y
Breakpoint 1 (add.c : 20) pending.
(gdb) python
>import addpymod
>addpymod.add_py_func(10,10,10) # Why didn't my breakpoint get hit?
>Quit
# This clearly failed, I'm not even sure where my stdout is
There are multiple levels of complexity here and I'm sure that I'm being tripped by more than one.
Question :
How do I debug my add.c using gdb (preferred) or possibly lldb since it was compiled by default with clang's -g option?
File structure under folder /home/cyan/TEMP
test.py
lib
|--libc_test_module.so
c_test_module.cc
CMakeLists.txt
test.py
import sys
sys.path.append("/home/cyan/TEMP/lib")
import c_test_module
c_test_module.cc
#include <Python.h>
int c_test_function(int a) {
return a + 1;
}
static PyObject * _c_test_function(PyObject *self, PyObject *args)
{
int _a;
int res;
if (!PyArg_ParseTuple(args, "i", &_a))
return NULL;
res = c_test_function(_a);
return PyLong_FromLong(res);
}
/* define functions in module */
static PyMethodDef TestMethods[] =
{
{"c_test_function", _c_test_function, METH_VARARGS,
"this is a c_test_function"},
{NULL, NULL, 0, NULL}
};
static struct PyModuleDef c_test_module = {
PyModuleDef_HEAD_INIT,
"c_test_module", "Some documentation",
-1,
TestMethods
};
PyMODINIT_FUNC PyInit_c_test_module(void) {
PyObject *module;
module = PyModule_Create(&c_test_module);
if(module==NULL) return NULL;
/* IMPORTANT: this must be called */
import_array();
if (PyErr_Occurred()) return NULL;
return module;
}
Error
ModuleNotFoundError: No module named 'c_test_module'
Question
I don't want to use ctype module to import .so file. Instead, I wonder if there is a way to import this file as a module directly.
I created a setup.py file:
from distutils.core import setup, Extension
import numpy
def main():
setup(name="c_test_module",
version="1.0.0",
description="Python interface for the c_test_module C library function",
author="cyan",
author_email="xxx#gmail.com",
ext_modules=[Extension("c_test_module", ["c_test_module.cc"],
include_dirs=[numpy.get_include()])],
)
if __name__ == "__main__":
main()
Now it works to import c_test_module and call the functions.
I am trying to send a string from a function implemented in Python to a function implemented in C, perform some operations using GMP library and return a string to the function implemented in python. Details of code given below:
myModule.c file:
#include<Python.h>
#include<gmp.h>
char* Cnumfunc(char* n)
{
printf("This is the string %s\n",n);
mpz_t p1;
mpz_init(p1);
mpz_set_str(p1,n,10);
gmp_printf("Value of the gmp number is : %Zd \n", p1);
mpz_add_ui(p1,p1,1);//so now p1=11
char retstr[100];//create a buffer of large size
mpz_get_str(retstr,10,p1);
return retstr;//Should return "11"
}
static PyObject* numfunc(PyObject* self, PyObject* args)
{
char* n;
if (!PyArg_ParseTuple(args, "s", &n))
return NULL;
return Py_BuildValue("s", Cnumfunc(n));
}
static PyMethodDef myMethods[] = {
{"numfunc", numfunc, METH_VARARGS, "Calculate the numfunc"},
{NULL, NULL, 0, NULL}
};
static struct PyModuleDef myModule = {
PyModuleDef_HEAD_INIT,
"myModule", //#name of module.
"numfunc Module",
-1,
myMethods
};
PyMODINIT_FUNC PyInit_myModule(void)
{
return PyModule_Create(&myModule);
}
Now setup.py file:
from distutils.core import setup, Extension
module = Extension('myModule', sources = ['myModule.c'])
setup (name = 'PackageName',
version = '1.0',
description = 'This is a package for myModule',
ext_modules = [module])
And finally the test.py file:
import myModule
a=myModule.numfunc("10")
print(a)
Now I run
python3 setup build
And then I copy the myModule.cpython-38-x86_64-linux-gnu.so file in the build directory to the directory which contains test.py and then do
python3 test.py
This gives the error:
Traceback (most recent call last):
File "test.py", line 1, in <module>
import myModule
ImportError: /home/aghosh/pybind/myModule.cpython-38-x86_64-linux-gnu.so: undefined symbol: __gmpz_set_str
I am really new to this kind of coding involving multiple languages and it will be very helpful if anyone has a solution to this problem
Learning how to write a C extension in Python. Running into the following error when executing ptest.py
Traceback (most recent call last):
File "ptest.py", line 1, in <module>
import euler_py as eul
ImportError: /usr/local/lib/python3.6/site-packages/euler_py.cpython-
36m-x86_64-linux-gnu.so: undefined symbol: problem_one
I'm assuming this is some type of linking issue.
setup.py
sources = glob.glob('ext/*.c')
euler = Extension(
'euler_py',
include_dirs=['src'],
sources=sources,
extra_compile_args=['-std=c99']
)
setup(
name='euler_py',
version='0.1',
description='Project Euler Solutions',
ext_modules=[euler]
)
ptest.py
import euler_py as eul
print(eul.problem_one(10))
The underlying functions are in /src and I wrote test.c to test purely in C. My extension is in /ext/euler_py.c shown below
ext/euler_py.c
#include <Python.h>
#include "../src/euler.h"
static char module_docstring[] = "Provides interface to Project Euler problems";
/*
* Function implementations
*/
static PyObject* euler_py_problem_one(PyObject *self, PyObject *args)
{
int max, result;
if (!PyArg_ParseTuple(args, "i", &max))
return NULL;
result = problem_one(max);
return Py_BuildValue("i", result);
}
// END function implementations
// Wire in functions to module
static PyMethodDef module_methods[] = {
{"problem_one", euler_py_problem_one, METH_VARARGS, "Solution to problem 1"},
{NULL, NULL, 0, NULL}
};
// Module definition
static struct PyModuleDef euler_py_module = {
PyModuleDef_HEAD_INIT,
"euler_py",
module_docstring,
-1,
module_methods
};
// Module initialization function
PyMODINIT_FUNC PyInit_euler_py(void)
{
return PyModule_Create(&euler_py_module);
}
Repo is here. I've played around with library_dirs & include_dirs on the Extension() initiation and no luck. Python version 3.6. Need a second look.
EDIT
Repo linked to has changed since original ask. Added linking for other 3 functions in the same manor as previous.
You forget to include all source files:
sources = glob.glob('ext/*.c') + glob.glob('src/*.c')
you could see c extension build detail with setup.py build -fv:
$ python setup.py build -fv
...
clang -bundle -undefined dynamic_lookup build/temp.macosx-10.13-x86_64-3.6/ext/euler_py.o build/temp.macosx-10.13-x86_64-3.6/src/euler.o build/temp.macosx-10.13-x86_64-3.6/src/helpers.o -o build/lib.macosx-10.13-x86_64-3.6/euler_py.cpython-36m-darwin.so
now helpers.o and euler.o properly linked.
I embedded the python 3.6 interpreter successfully in a C++ program, but I have a problem.
I'd like to embed two interpreters in the same program:
One which can use my C++ defined module (MyModule)
One which can not use this module.
Regarding the documentation I should call PyImport_AppendInittab before Py_Initialize function, so the module will be available in the whole program, but I'd like to create separate interpreters with separate built-in modules.
Calling Py_Initialize and Py_Finalize doesn't help, the module will be available in the second interpreter, too. By the way calling init and finalize function multiple times causes huge memory leaks, so I think this wouldn't be a good solution even if it would work.
Do you have any idea how to solve this issue?
Full code:
#include <iostream>
#pragma push_macro("_DEBUG")
#undef _DEBUG
#include "Python.h"
#pragma pop_macro("_DEBUG")
static PyObject* Addition (PyObject *self, PyObject *args)
{
double a = 0.0;
double b = 0.0;
if (!PyArg_ParseTuple (args, "dd", &a, &b)) {
return nullptr;
}
double result = a + b;
return PyFloat_FromDouble (result);
}
static PyMethodDef ModuleMethods[] =
{
{"Add", Addition, METH_VARARGS, "Adds numbers."},
{nullptr, nullptr, 0, nullptr}
};
static PyModuleDef ModuleDef = {
PyModuleDef_HEAD_INIT,
"MyModule",
NULL,
-1,
ModuleMethods,
NULL,
NULL,
NULL,
NULL
};
static PyObject* ModuleInitializer (void)
{
return PyModule_Create (&ModuleDef);
}
int main ()
{
Py_SetPythonHome (L".");
Py_SetPath (L"python36.zip\\Lib");
{ // first interpreter
PyImport_AppendInittab ("MyModule", ModuleInitializer);
Py_Initialize ();
PyRun_SimpleString (
"import MyModule\n"
"print (MyModule.Add (1, 2))"
);
Py_Finalize ();
}
{ // second interpreter without AppendInittab (should not find MyModule, but it does)
Py_Initialize ();
PyRun_SimpleString (
"import MyModule\n"
"print (MyModule.Add (1, 2))"
);
Py_Finalize ();
}
system ("pause");
return 0;
}