We Keep Coding

Python ctypes cannot get long string value returned from .so file

I am learning C and trying to import a .so into my python file for higher performance by using a python package ctypes. So everything going well until I had a hard time when trying to get a string returned from .so file.
C code:
char *convert_to_16(char *characters, int n){
char sub_buffer[3];
char code[3];
char *buffer = (char*)malloc(sizeof(characters) * 2);
for(int i=0; i < n; i++){
strncpy(code, characters+i, 1);
sprintf(sub_buffer, "%x", *code);
strncat(buffer, sub_buffer, 2);
}
return buffer;
}
// main() only for test
int main(){
char param[] = "ABCDEFGHTUIKLL";
printf("%s\n", param);
int length = strlen(param);
printf("%s\n", convert_to_16(param, length));
}
It runs well with output:
41424344454647485455494b4c4c
Python code :
c_convert_to_16 = ctypes.CDLL('./convert_to_16.so').convert_to_16
c_convert_to_16.restype = ctypes.c_char_p
a_string = "ABCDEFGHTUIKLL"
new_16base_string = c_convert_to_16(a_string, len(a_string))
print new_16base_string
It runs but only returns two characters:
41
I read the official doc and set restype as ctypes.c_char_p, and try to set it to other values. But it seems it's the only option, just oddly only two characters were returned.
Is it the problem of my ctypes configuration or my C wasn't written correctly?
Many thanks.

I don't know much about ctypes in python but you should create your string like that c_char_p("ABCDEFGHTUIKLL").
And maybe tell what argument take your function c_convert_to_16.argtypes = [c_char_p, c_size_t]
This will fix your undefined behavior in C:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
char *convert_to_16(char const *array, size_t const len);
char *convert_to_16(char const *array, size_t const len) {
size_t const len_buffer = len * 2 + 1;
char *buffer = malloc(len_buffer);
if (buffer == NULL) {
return NULL;
}
size_t used = 0;
for (size_t i = 0; i < len; i++) {
if (len_buffer < used || len_buffer - used < 3) {
free(buffer);
return NULL;
}
int ret = snprintf(buffer + used, 3, "%.2x", (unsigned char)array[i]);
if (ret != 2) {
free(buffer);
return NULL;
}
used += 2;
}
return buffer;
}
int main(void) {
char const param[] = "ABCDEFGHTUIKLL";
printf("%s\n", param);
char *ret = convert_to_16(param, sizeof param - 1);
if (ret != NULL) {
printf("%s\n", ret);
}
free(ret);
}

SWIG: How to return a struct using %apply? 'No typemaps defined' warning

I currently have a function that uses a struct as a buffer to return some information, like so:
int example_reader(int code, void* return_struct);
My goal is to make it so that when I wrap this function using SWIG so that it can be used in Python, I will return the struct along with the function's regular return value. Thus far, I have been doing so using the %apply command like so:
%apply struct ret_struct *OUTPUT {void* return_struct};
However, when I add the above line to my .i file and try to run SWIG, I get the following warning:
"warning 453: Can't apply (struct ret_struct *OUTPUT. No typemaps are defined"
I believe I'm including the .h file that defines the struct I'm trying to return, so I've had trouble pinpointing the issue. Please correct me if the issue seems to involve the improper inclusion of the struct. I've tried reading through the SWIG documentation as well as other Stack Overflow posts to get some inkling of what the problem might be, but I haven't been able to figure it out thus far. The problem is made slightly trickier because I am trying to return a void pointer to a struct, and the code I'm trying to wrap could have multiple kinds of structs for me to return. What would be a wise way of handling the return of this struct? Thank you!

I have given here a full C example, where an interface is used for returning a struct to the target language together with a return value. In this way you can make a proper interface, where no implementation is given in the header. That is no default implementation of a virtual destructor. If you don't want to use an interface, you can let SWIG and Python know how data are represented.
Interface header: foo.h
typedef struct _Foo Foo;
int foo_new(Foo **obj);
int foo_free(Foo *obj);
int foo_get_value_a(Foo *obj, int *result);
int foo_set_value_a(Foo *obj, int value);
int foo_get_value_b(Foo *obj, char **result);
int foo_set_value_b(Foo *obj, char *value);
SWIG interface: foo.i
%module foo
%{
#include "foo.h"
%}
%include "typemaps.i"
%typemap(in, numinputs=0) Foo ** (Foo *temp) {
$1 = &temp;
}
%typemap(argout) Foo ** {
PyObject* temp = NULL;
if (!PyList_Check($result)) {
temp = $result;
$result = PyList_New(1);
PyList_SetItem($result, 0, temp);
}
temp = SWIG_NewPointerObj(*$1, SWIGTYPE_p__Foo, SWIG_POINTER_NEW);
PyList_Append($result, temp);
Py_DECREF(temp);
}
%delobject foo_free; // Protect for double deletion
struct _Foo {};
%extend _Foo {
~_Foo() {
foo_free($self);
}
};
%ignore _Foo;
Some implementation of the interface: foo.c
%include "foo.h"
#include "foo.h"
#include "stdlib.h"
#include "string.h"
struct FooImpl {
char* c;
int i;
};
int foo_new(Foo **obj)
{
struct FooImpl* f = (struct FooImpl*) malloc(sizeof(struct FooImpl));
f->c = NULL;
*obj = (Foo*) f;
return 0;
}
int foo_free(Foo *obj)
{
struct FooImpl* impl = (struct FooImpl*) obj;
if (impl) {
if (impl->c) {
free(impl->c);
impl->c = NULL;
}
}
return 0;
}
int foo_get_value_a(Foo *obj, int *result)
{
struct FooImpl* impl = (struct FooImpl*) obj;
*result = impl->i;
return 0;
}
int foo_set_value_a(Foo *obj, int value)
{
struct FooImpl* impl = (struct FooImpl*) obj;
impl->i = value;
return 0;
}
int foo_get_value_b(Foo *obj, char **result)
{
struct FooImpl* impl = (struct FooImpl*) obj;
*result = impl->c;
return 0;
}
int foo_set_value_b(Foo *obj, char *value)
{
struct FooImpl* impl = (struct FooImpl*) obj;
int len = strlen(value);
if (impl->c) {
free(impl->c);
}
impl->c = (char*)malloc(len+1);
strcpy(impl->c,value);
return 0;
}
Script for building
#!/usr/bin/env python
from distutils.core import setup, Extension
import os
os.environ['CC'] = 'gcc';
setup(name='foo',
version='1.0',
ext_modules =[Extension('_foo',
['foo.i','foo.c'])])
Usage:
import foo
OK, f = foo.foo_new()
OK = foo.foo_set_value_b(f, 'Hello world!')
OK = foo.foo_free(f)
OK, f = foo.foo_new()
# Test safe to double delete
del f

python3 str object cannot pass PyUnicode_Check

I was writing a C extension function, which was supposed to accept a str object as argument. The code is shown below:
static PyObject *py_print_chars(PyObject *self, PyObject *o) {
PyObject *bytes;
char *s;
if (!PyUnicode_Check(o)) {
PyErr_SetString(PyExc_TypeError, "Expected string");
return NULL;
}
bytes = PyUnicode_AsUTF8String(o);
s = PyBytes_AsString(bytes);
print_chars(s);
Py_DECREF(bytes);
Py_RETURN_NONE;
}
But as I test the module in python3 console, I find str objects can't pass the PyUnicode_Check:
>>> from sample2 import *
>>> print_chars('Hello world')
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
TypeError: Expected string
As far as I know, Python 3’s str() type is called PyUnicode in C and the C code above was written in refer to "python cookbook3" Char15.13. I just can't work out the problem. Can anybody tell me what's wrong with my code.
Here is what "python cookbook3" said:
If for some reason, you are working directly with a PyObject * and can’t use PyArg_ParseTuple(), the following code samples show how you can check and extract a suitable char * reference, from both a bytes and string object:
/* Some Python Object (obtained somehow) */
PyObject *obj;
/* Conversion from bytes */
{
char *s;
s = PyBytes_AsString(o);
if (!s) {
return NULL; /* TypeError already raised */
}
print_chars(s);
}
/* Conversion to UTF-8 bytes from a string */
{
PyObject *bytes;
char *s;
if (!PyUnicode_Check(obj)) {
PyErr_SetString(PyExc_TypeError, "Expected string");
return NULL;
}
bytes = PyUnicode_AsUTF8String(obj);
s = PyBytes_AsString(bytes);
print_chars(s);
Py_DECREF(bytes);
}
And the whole code:
#include "Python.h"
#include "sample.h"
static PyObject *py_print_chars(PyObject *self, PyObject *o) {
PyObject *bytes;
char *s;
if (!PyUnicode_Check(o)) {
PyErr_SetString(PyExc_TypeError, "Expected string");
return NULL;
}
bytes = PyUnicode_AsUTF8String(o);
s = PyBytes_AsString(bytes);
print_chars(s);
Py_DECREF(bytes);
Py_RETURN_NONE;
}
/* Module method table */
static PyMethodDef SampleMethods[] = {
{"print_chars", py_print_chars, METH_VARARGS, "print character"},
{ NULL, NULL, 0, NULL}
};
/* Module structure */
static struct PyModuleDef samplemodule = {
PyModuleDef_HEAD_INIT,
"sample",
"A sample module",
-1,
SampleMethods
};
/* Module initialization function */
PyMODINIT_FUNC
PyInit_sample2(void) {
return PyModule_Create(&samplemodule);
}

If the goal is to accept exactly one argument, the function should be declared as METH_O, not METH_VARARGS; the former passes along the single argument without wrapping, the latter wraps in a tuple which would need to be unpacked or parsed to get the PyUnicode* inside.

extending global c variable array with swig and python

I have a global variable array in c that I'd like to pull into python. And I'm having difficulties with varout typemap:
/* example.c */
int foo[] = {0, 1};
And here is the very vanilla interface:
/* example.i */
%module example
%{
extern int foo[2];
%}
%typemap(varout) int foo[] {
int i;
//$1, $1_dim0, $1_dim1
$result = PyList_New($1_dim0);
for (i = 0; i < $1_dim0; i++) {
PyObject *o = PyInt_FromLong((double) $1[i]);
PyList_SetItem($result,i,o);
}
}
%include "example.c"
When I try to build it with the following SConstruct:
import distutils.sysconfig
env = Environment(SWIGFLAGS='-python -shadow -Wall'.split(),
CPPPATH=[distutils.sysconfig.get_python_inc()],
SHLIBPREFIX="")
env.SharedLibrary('_example.so', ['example.c', 'example.i'])
$1_dim0 special variable is not populated, resulting in the following non-compilable code in example_wrap.c:
SWIGINTERN PyObject *Swig_var_foo_get(void) {
PyObject *pyobj = 0;
{
int i;
//foo, , foo_dim1
pyobj = PyList_New();
for (i = 0; i < ; i++) {
PyObject *o = PyInt_FromLong((double) foo[i]);
PyList_SetItem(pyobj,i,o);
}
}
return pyobj;
}
So clearly the typemap match has happened, but dimensionality of array is missing. What am I missing? Hard coding the dimension does works.
In general, is there any way to extend global cvar variables with swig?
$ swig -version
SWIG Version 2.0.4
Compiled with g++ [i686-pc-linux-gnu]
Configured options: +pcre
Please see http://www.swig.org for reporting bugs and further information

You're almost there with your varout typemap. You need to make two minor changes:
You need to add the size ANY to the int foo[] typemap:
%typemap(varout) int foo[ANY] {
int i;
//$1, $1_dim0, $1_dim1
$result = PyList_New($1_dim0);
for (i = 0; i < $1_dim0; i++) {
PyObject *o = PyInt_FromLong((double) $1[i]);
PyList_SetItem($result,i,o);
}
}
This makes sure your typemap is a match for arrays of (any) known size, not just equivalent to int *foo.
You need to modify example.c to make the size of foo clearer. It's legal and correct C as it stands but tricky to deduce the size of the array unless you happen to be a complete C compiler. Changing it to:
int foo[2] = {0, 1};
is sufficient to make sure that it matches the varout typemap.
With those two changes the generated code works as you'd hope:
SWIGINTERN PyObject *Swig_var_foo_get(void) {
PyObject *pyobj = 0;
{
int i;
//foo, 2, foo_dim1
pyobj = PyList_New(2);
for (i = 0; i < 2; i++) {
PyObject *o = PyInt_FromLong((double) foo[i]);
PyList_SetItem(pyobj,i,o);
}
}
return pyobj;
}
is what gets generated on my machine with those changes.

For those like me who ponders what to do with arrays of non-simple types -- here is one way to do it:
The non-simple type:
typedef struct {
int a;
float b;
} Foo;
and a global array:
extern Foo *foov[40];
%typemap(varout) Foo *foov[ANY] {
int i;
$result = PyList_New($1_dim0);
for (i = 0; i < $1_dim0; i++) {
PyObject *o = SWIG_NewPointerObj($1[i], SWIGTYPE_p_Foo, 0);
PyList_SetItem($result, i, o);
}
}
Just shared this since it took me forever to find out, and this article helped. Just needed to find out how to allocate the SWIG version of my non-simple type -- found that buried here:
http://www.swig.org/Doc2.0/Python.html#Python_nn64

Python extension module with variable number of arguments

I am trying to figure out how in C extension modules to have a variable (and maybe) quite large number of arguments to a function.
Reading about PyArg_ParseTuple it seems you have to know how many to accept, some mandatory and some optional but all with their own variable. I was hoping PyArg_UnpackTuple would be able to handle this but it seems to just give me bus errors when I try and use it in what appears to be the wrong way.
As an example take the following python code that one might want to make into an extension module (in C).
def hypot(*vals):
if len(vals) !=1 :
return math.sqrt(sum((v ** 2 for v in vals)))
else:
return math.sqrt(sum((v ** 2 for v in vals[0])))
This can be called with any number of arguments or iterated over, hypot(3,4,5), hypot([3,4,5]), and hypot(*[3,4,5]) all give the same answer.
The start of my C function looks like this
static PyObject *hypot_tb(PyObject *self, PyObject *args) {
// lots of code
// PyArg_ParseTuple or PyArg_UnpackTuple
}
Many thinks to yasar11732. Here for the next guy is a fully working extension module (_toolboxmodule.c) that simply takes in any number or integer arguments and returns a list made up of those arguments (with a poor name). A toy but illustrates what needed to be done.
#include <Python.h>
int ParseArguments(long arr[],Py_ssize_t size, PyObject *args) {
/* Get arbitrary number of positive numbers from Py_Tuple */
Py_ssize_t i;
PyObject *temp_p, *temp_p2;
for (i=0;i<size;i++) {
temp_p = PyTuple_GetItem(args,i);
if(temp_p == NULL) {return NULL;}
/* Check if temp_p is numeric */
if (PyNumber_Check(temp_p) != 1) {
PyErr_SetString(PyExc_TypeError,"Non-numeric argument.");
return NULL;
}
/* Convert number to python long and than C unsigned long */
temp_p2 = PyNumber_Long(temp_p);
arr[i] = PyLong_AsUnsignedLong(temp_p2);
Py_DECREF(temp_p2);
}
return 1;
}
static PyObject *hypot_tb(PyObject *self, PyObject *args)
{
Py_ssize_t TupleSize = PyTuple_Size(args);
long *nums = malloc(TupleSize * sizeof(unsigned long));
PyObject *list_out;
int i;
if(!TupleSize) {
if(!PyErr_Occurred())
PyErr_SetString(PyExc_TypeError,"You must supply at least one argument.");
return NULL;
}
if (!(ParseArguments(nums, TupleSize, args)) {
free(nums);
return NULL;
}
list_out = PyList_New(TupleSize);
for(i=0;i<TupleSize;i++)
PyList_SET_ITEM(list_out, i, PyInt_FromLong(nums[i]));
free(nums);
return (PyObject *)list_out;
}
static PyMethodDef toolbox_methods[] = {
{ "hypot", (PyCFunction)hypot_tb, METH_VARARGS,
"Add docs here\n"},
// NULL terminate Python looking at the object
{ NULL, NULL, 0, NULL }
};
PyMODINIT_FUNC init_toolbox(void) {
Py_InitModule3("_toolbox", toolbox_methods,
"toolbox module");
}
In python then it is:
>>> import _toolbox
>>> _toolbox.hypot(*range(4, 10))
[4, 5, 6, 7, 8, 9]

I had used something like this earlier. It could be a bad code as I am not an experienced C coder, but it worked for me. The idea is, *args is just a Python tuple, and you can do anything that you could do with a Python tuple. You can check http://docs.python.org/c-api/tuple.html .
int
ParseArguments(unsigned long arr[],Py_ssize_t size, PyObject *args) {
/* Get arbitrary number of positive numbers from Py_Tuple */
Py_ssize_t i;
PyObject *temp_p, *temp_p2;
for (i=0;i<size;i++) {
temp_p = PyTuple_GetItem(args,i);
if(temp_p == NULL) {return NULL;}
/* Check if temp_p is numeric */
if (PyNumber_Check(temp_p) != 1) {
PyErr_SetString(PyExc_TypeError,"Non-numeric argument.");
return NULL;
}
/* Convert number to python long and than C unsigned long */
temp_p2 = PyNumber_Long(temp_p);
arr[i] = PyLong_AsUnsignedLong(temp_p2);
Py_DECREF(temp_p2);
if (arr[i] == 0) {
PyErr_SetString(PyExc_ValueError,"Zero doesn't allowed as argument.");
return NULL;
}
if (PyErr_Occurred()) {return NULL; }
}
return 1;
}
I was calling this function like this:
static PyObject *
function_name_was_here(PyObject *self, PyObject *args)
{
Py_ssize_t TupleSize = PyTuple_Size(args);
Py_ssize_t i;
struct bigcouples *temp = malloc(sizeof(struct bigcouples));
unsigned long current;
if(!TupleSize) {
if(!PyErr_Occurred())
PyErr_SetString(PyExc_TypeError,"You must supply at least one argument.");
free(temp);
return NULL;
}
unsigned long *nums = malloc(TupleSize * sizeof(unsigned long));
if(!ParseArguments(nums, TupleSize, args)){
/* Make a cleanup and than return null*/
return null;
}