We Keep Coding

How to return an array in C into python?

I have been working on a selection sort with C extension on python, which aimed to intake a list in python, sort using C code and return a sorted list in python. Sounds simple, but I just could not get the value of the sorted list correct in python, as I would get a value of 1 when I was trying to print the sorted list.
Here is my code in C:
#include <Python.h>
int selectionSort(int array[], int N){
int i, j, min_element;
for (i = 0; i < N-1; i++) {
min_element = i;
for (j = i+1; j < N; j++)
if (array[j] < array[min_element])
min_element = j;
int temp = array[min_element];
array[min_element] = array[i];
array[i] = temp;
}
return *array;
}
static PyObject* selectSort(PyObject *self, PyObject *args)
{
PyObject* list;
int *array, N;
if (!PyArg_ParseTuple(args, "O", &list))
return NULL;
N = PyObject_Length(list);
if (N < 0)
return NULL;
array = (int *) malloc(sizeof(int *) * N);
if (array == NULL)
return NULL;
for (int index = 0; index < N; index++) {
PyObject *item;
item = PyList_GetItem(list, index);
if (!PyFloat_Check(item))
array[index] = 0;
array[index] = PyFloat_AsDouble(item);
}
return Py_BuildValue("i", selectionSort(array, N));
}
static PyMethodDef myMethods[] = {
{ "selectSort", selectSort, METH_VARARGS, "..." },
{ NULL, NULL, 0, NULL }
};
static struct PyModuleDef myModule = {
PyModuleDef_HEAD_INIT,
"myModule",
"Test Module",
-1,
myMethods
};
PyMODINIT_FUNC PyInit_myModule(void)
{
return PyModule_Create(&myModule);
}
Here is the command line I executed:
>>> import myModule
>>> unsortedList = [1, 4, 3, 90, 22, 34, 32]
>>> sortedList = myModule.selectSort(unsortedList)
>>> print(sortedList)
1
Anyone has any ideas on this? I would really appreciate it! Thanks!

Just change return type from int to int* and return with return array, not return *array.

python: calling c++ with numpy array input

I have the following c++ code and I want to import it into python.
#include "test.h"
#include <vector>
#include <cmath>
double vec_sub(const std::vector<double> &u, const std::vector<double> &v, int ui, int vj)
{
double max{0.0};
double d{0.0};
for(int i = 0; i < u.size(); i++)
{
d = std::fabs(u[i] - v[i]);
if(i == ui || i == vj)
{
d = 0.0;
}
if(d > max)
{
max = d;
}
}
return max;
}
std::vector< std::vector<double> > chebyshev_distance(const std::vector< std::vector<double> > &matrix )
{
unsigned long row{matrix[0].size()}, col{matrix[0].size()};
std::vector<std::vector<double> > dist( row, std::vector<double> (col, 0));
for(int i = 0; i < row; i++)
{
for(int j = i + 1; j < row; j++)
{
dist[i][j] = vec_sub(matrix[i], matrix[j], i, j);
dist[j][i] = dist[i][j];
}
}
return dist;
}
For the function "chebyshev_distance", lets assume that the corresponding function in python is the same "chebyshev_distance"
dat = np.array(...) # n by n 2D array
dist = chebyshev_distance(dat)
In the above code, the input is a numpy array. Does the python automatically convert the vector in c into numpy in python? If not, how do I change the c++ code?

Using swig for python list input and output

I am using SWIG to build a Python module for some functions' evaluation based on their C code.
The main function I need is defined as follow:
void eval(double *x, int nx, int mx, double *f, int func_id)
And the aimed python function should be:
value_list = module.eval(point_matrix, func_id)
Here, eval will call a benchmark function and return their values. func_id is the id of function eval going to call, nx is the dimension of the function, and mx is the number of points which will be evaluated.
Actually, I did not clearly understand how SWIG pass values between typemaps (like, temp$argnum, why always using $argnum?). But by looking into the wrap code, I finished the typemap.i file:
%module cec17
%{
#include "cec17.h"
%}
%typemap(in) (double *x, int nx, int mx) (int count){
if (PyList_Check($input)) {
$3 = PyList_Size($input);
$2 = PyList_Size(PyList_GetItem($input, 0));
count = $3;
int i = 0, j = 0;
$1 = (double *) malloc($2*$3*sizeof(double));
for (i = 0; i < $3; i++){
for (j = 0; j < $2; j++){
PyObject *o = PyList_GetItem(PyList_GetItem($input, i), j);
if (PyFloat_Check(o))
$1[i*$2+j] = PyFloat_AsDouble(o);
else {
PyErr_SetString(PyExc_TypeError, "list must contrain strings");
free($1);
return NULL;
}
}
}
} else {
PyErr_SetString(PyExc_TypeError, "not a list");
return NULL;
}
}
%typemap(freearg) double *x {
free((void *) $1);
}
%typemap(in, numinputs=0) double *f (double temp) {
$1 = &temp;
}
%typemap(argout) double *f {
int i = 0;
int s = count1;
printf("pass arg %d", s);
$result = PyList_New(0);
for (i = 0; i < s; i++){
PyList_Append($result, PyFloat_FromDouble($1[i]));
}
}
void eval(double *x, int nx, int mx, double *f, int func_num);
However, strange things happened then. Usually, I test 30 dimensional functions. When evaluating less than 10 points (mx < 10), the module works fine. When evaluating more points, an error occurs:
[1] 13616 segmentation fault (core dumped) python test.py
I'm quite sure the problem is not in the c code, because the only place where 'mx' occurs is in the 'for-loop' line in which are evaluations of each point.
I also tried to read the wrap code and debug, but I just can't find where the problem is. Following is a part of the wrap code generated by SWIG, and I added a 'printf' line. Even this string is not printed before the error.
#ifdef __cplusplus
extern "C" {
#endif
SWIGINTERN PyObject *_wrap_eval(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
double *arg1 = (double *) 0 ;
int arg2 ;
int arg3 ;
double *arg4 = (double *) 0 ;
int arg5 ;
int count1 ;
double temp4 ;
int val5 ;
int ecode5 = 0 ;
PyObject * obj0 = 0 ;
PyObject * obj1 = 0 ;
printf("check point 0");
{
arg4 = &temp4;
}
if (!PyArg_ParseTuple(args,(char *)"OO:eval",&obj0,&obj1)) SWIG_fail;
{
if (PyList_Check(obj0)) {
arg3 = PyList_Size(obj0);
arg2 = PyList_Size(PyList_GetItem(obj0, 0));
count1 = arg3;
int i = 0, j = 0;
arg1 = (double *) malloc(arg2*arg3*sizeof(double));
for (i = 0; i < arg3; i++){
for (j = 0; j < arg2; j++){
PyObject *o = PyList_GetItem(PyList_GetItem(obj0, i), j);
if (PyFloat_Check(o))
arg1[i*arg2+j] = PyFloat_AsDouble(o);
else {
PyErr_SetString(PyExc_TypeError, "list must contrain strings");
free(arg1);
return NULL;
}
}
}
} else {
PyErr_SetString(PyExc_TypeError, "not a list");
return NULL;
}
}
ecode5 = SWIG_AsVal_int(obj1, &val5);
if (!SWIG_IsOK(ecode5)) {
SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "eval" "', argument " "5"" of type '" "int""'");
}
arg5 = (int)(val5);
eval(arg1,arg2,arg3,arg4,arg5);
resultobj = SWIG_Py_Void();
{
int i = 0;
int s = count1;
resultobj = PyList_New(0);
for (i = 0; i < s; i++){
PyList_Append(resultobj, PyFloat_FromDouble(arg4[i]));
}
}
return resultobj;
fail:
return NULL;
}
The problem seems a little tedious. Maybe you could just show me how to write the proper typemap.i code.

I'm not sure what your evaluation function is supposed to do, so I took a guess and implemented a wrapper for it. I took value_list = module.eval(point_matrix, func_id) to mean you want to return a list of result of evaluating some function against each row of data points, and came up with the following. Things I changed:
The typemaps replace the first four parameters with a Python list of lists of numbers.
space for the results in f was malloced.
To accept other numeric types except float, PyFloat_AsDouble was called on each value, and PyErr_Occurred was called to see if it failed to convert.
The freearg typemap now frees both allocations.
The argout typemap now handles the f output parameter correctly.
I added a sample eval implementation.
%module cec17
%typemap(in) (double *x, int nx, int mx, double* f) %{
if (PyList_Check($input)) {
$3 = PyList_Size($input);
$2 = PyList_Size(PyList_GetItem($input, 0));
$1 = malloc($2 * $3 * sizeof(double));
$4 = malloc($3 * sizeof(double));
for (int i = 0; i < $3; i++) {
for (int j = 0; j < $2; j++) {
PyObject *o = PyList_GetItem(PyList_GetItem($input, i), j);
double tmp = PyFloat_AsDouble(o);
if(PyErr_Occurred())
SWIG_fail;
$1[i * $2 + j] = PyFloat_AsDouble(o);
}
}
} else {
PyErr_SetString(PyExc_TypeError, "not a list");
return NULL;
}
%}
%typemap(freearg) (double *x, int nx, int mx, double* f) %{
free($1);
free($4);
%}
%typemap(argout) (double *x, int nx, int mx, double* f) (PyObject* tmp) %{
tmp = PyList_New($3);
for (int i = 0; i < $3; i++) {
PyList_SET_ITEM(tmp, i, PyFloat_FromDouble($4[i]));
}
$result = SWIG_Python_AppendOutput($result, tmp);
%}
%inline %{
void eval(double *x, int nx, int mx, double *f, int func_num)
{
for(int i = 0; i < mx; ++i) {
f[i] = 0.0;
for(int j = 0; j < nx; ++j)
f[i] += x[i*nx+j];
}
}
%}
Output:
>>> import cec17
>>> cec17.eval([[1,2,3],[4,5,6]],99)
[6.0, 15.0]
Error checking could be improved. For example, checking for sequences instead of lists. Only the outer list is checked that it actually is a list, so if [1,2,3] was the first parameter instead of nested lists, it won't behave properly. There is no check that all the sublists are the same size, either.
Hope this helps. Let me know if anything is unclear.

Python ctypes cannot get long string value returned from .so file

I am learning C and trying to import a .so into my python file for higher performance by using a python package ctypes. So everything going well until I had a hard time when trying to get a string returned from .so file.
C code:
char *convert_to_16(char *characters, int n){
char sub_buffer[3];
char code[3];
char *buffer = (char*)malloc(sizeof(characters) * 2);
for(int i=0; i < n; i++){
strncpy(code, characters+i, 1);
sprintf(sub_buffer, "%x", *code);
strncat(buffer, sub_buffer, 2);
}
return buffer;
}
// main() only for test
int main(){
char param[] = "ABCDEFGHTUIKLL";
printf("%s\n", param);
int length = strlen(param);
printf("%s\n", convert_to_16(param, length));
}
It runs well with output:
41424344454647485455494b4c4c
Python code :
c_convert_to_16 = ctypes.CDLL('./convert_to_16.so').convert_to_16
c_convert_to_16.restype = ctypes.c_char_p
a_string = "ABCDEFGHTUIKLL"
new_16base_string = c_convert_to_16(a_string, len(a_string))
print new_16base_string
It runs but only returns two characters:
41
I read the official doc and set restype as ctypes.c_char_p, and try to set it to other values. But it seems it's the only option, just oddly only two characters were returned.
Is it the problem of my ctypes configuration or my C wasn't written correctly?
Many thanks.

I don't know much about ctypes in python but you should create your string like that c_char_p("ABCDEFGHTUIKLL").
And maybe tell what argument take your function c_convert_to_16.argtypes = [c_char_p, c_size_t]
This will fix your undefined behavior in C:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
char *convert_to_16(char const *array, size_t const len);
char *convert_to_16(char const *array, size_t const len) {
size_t const len_buffer = len * 2 + 1;
char *buffer = malloc(len_buffer);
if (buffer == NULL) {
return NULL;
}
size_t used = 0;
for (size_t i = 0; i < len; i++) {
if (len_buffer < used || len_buffer - used < 3) {
free(buffer);
return NULL;
}
int ret = snprintf(buffer + used, 3, "%.2x", (unsigned char)array[i]);
if (ret != 2) {
free(buffer);
return NULL;
}
used += 2;
}
return buffer;
}
int main(void) {
char const param[] = "ABCDEFGHTUIKLL";
printf("%s\n", param);
char *ret = convert_to_16(param, sizeof param - 1);
if (ret != NULL) {
printf("%s\n", ret);
}
free(ret);
}

Python ctypes : OSError undefined symbol when loading library

In Ubuntu 14.04, I wrote a C file called hash.c:
/* hash.c: hash table with linear probing */
typedef struct {
void *key;
void *value;
} ht_entry;
typedef struct {
ht_entry *table;
int len;
int num_entries;
int (*hash_fn)(void *key);
int (*key_cmp)(void *k1, void *k2);
} hashtable;
and compiled it with
gcc -shared hash.c -o test.so -fPIC
Afterwards, I tried to load test.so in a Python script (for testing), but I got the following error: "OSError: .../test.so: undefined symbol: hash_fn"
hash_fn is a function pointer in the hashtable struct. It is referenced a number of times by functions later in the file.
I do not understand why this error is happening. I have Googled but all other cases either concern C++ or includes. In my case I just have 1 C file that includes only stdio and stdlib.
here is the FULL code.
When I comment out all but hash_create and print_info, it loads succesfully. When I uncomment find(), it the error happens.
(print_info is just for testing that ctypes works)
/* hash.c: hash table with linear probing */
#include <stdio.h>
#include <stdlib.h>
typedef struct {
void *key;
void *value;
} ht_entry;
typedef struct {
ht_entry *table;
int len;
int num_entries;
int (*hash_fn)(void *key);
int (*key_cmp)(void *k1, void *k2);
} hashtable;
static void close_gap(hashtable *ht, int i);
static int find(hashtable *ht, void *key);
hashtable* hash_create(int len, int (*hash_fn)(void*), int (*key_cmp)(void*, void*))
{
hashtable* ht = (hashtable*) malloc(sizeof(hashtable));
ht->len = len;
ht->table = calloc(len, sizeof(ht_entry));
ht->hash_fn = hash_fn;
ht->key_cmp = key_cmp;
ht->table[0].key = 2;
ht->table[0].value = 3;
return ht;
}
void print_info(hashtable *ht)
{
printf("%d, %d, %d\n", ht->len, ht->table[0].key, ht->table[0].value);
}
void* hash_retrieve(hashtable* ht, void *key)
{
int i = find(ht, key);
if(i < 0) {
return NULL;
}
return ht->table[i].value;
}
void hash_insert(hashtable* ht, void *key, void *value)
{
if(ht->num_entries == ht->len) {
return;
}
int i = hash_fn(key) % ht->len;
while(ht->table[i].key != NULL) {
i = (i + i) % ht->len;
}
ht->table[i].key = key;
ht->table[i].value = value;
}
void hash_remove(hashtable *ht, void *key)
{
int i = find(ht, key);
if(i < 0) {
return;
}
ht->table[i].key = 0;
ht->table[i].value = 0;
close_gap(ht, i);
}
static int find(hashtable *ht, void *key)
{
int i = hash_fn(key) % ht->len;
int num_checked = 0;
while(ht->table[i].key && num_checked != ht->len) {
if(!ht->key_cmp(ht->table[i].key, key)) {
return i;
}
num_checked++;
i = (i + i) % ht->len;
}
return -1;
}
static void close_gap(hashtable *ht, int i)
{
int j = (i + 1) % ht->len;
while(ht->table[j].key) {
int loc = ht->hash_fn(ht->table[j].key);
if((j > i && (loc <= i || loc > j)) || (j < i && (loc <= i && loc > j))) {
ht->table[i] = ht->table[j];
ht->table[j].key = 0;
ht->table[j].value = 0;
close_gap(ht, j);
return;
}
}
}

When I use your compilation line I get five warnings. There are several problems here. First you are trying to assign an int to void * in several places. That raises a warning, and it would crash at runtime because you are passing 2 and 3 as addresses.
Second, you are calling hash_fn in a couple of places instead of ht->hash_fn. That causes the linker error, but you should consider my other changes, otherwise it will crash at runtime with a SIGSEGV:
/* hash.c: hash table with linear probing */
#include <stdio.h>
#include <stdlib.h>
typedef struct {
void *key;
void *value;
} ht_entry;
typedef struct {
ht_entry *table;
int len;
int num_entries;
int (*hash_fn)(void *key);
int (*key_cmp)(void *k1, void *k2);
} hashtable;
static void close_gap(hashtable *ht, int i);
static int find(hashtable *ht, void *key);
hashtable* hash_create(int len, int (*hash_fn)(void*), int (*key_cmp)(void*, void*))
{
hashtable* ht = (hashtable*) malloc(sizeof(hashtable));
ht->len = len;
ht->table = calloc(len, sizeof(ht_entry));
ht->hash_fn = hash_fn;
ht->key_cmp = key_cmp;
// <<< Code changed here
/*
ht->table[0].key = 2;
ht->table[0].value = 3;
*/
{
int *p = malloc(sizeof(int));
*p = 2;
ht->table[0].key = p;
p = malloc(sizeof(int));
*p = 3;
ht->table[0].value = p;
}
// end of code change
return ht;
}
void print_info(hashtable *ht)
{
// <<<< Code changed
printf("%d, %d, %d\n", ht->len,
*(int *)ht->table[0].key, *(int *)ht->table[0].value);
}
void* hash_retrieve(hashtable* ht, void *key)
{
int i = find(ht, key);
if(i < 0) {
return NULL;
}
return ht->table[i].value;
}
void hash_insert(hashtable* ht, void *key, void *value)
{
if(ht->num_entries == ht->len) {
return;
}
// <<< Code changed
int i = ht->hash_fn(key) % ht->len;
while(ht->table[i].key != NULL) {
i = (i + i) % ht->len;
}
ht->table[i].key = key;
ht->table[i].value = value;
}
void hash_remove(hashtable *ht, void *key)
{
int i = find(ht, key);
if(i < 0) {
return;
ht->table[i].key = 0;
ht->table[i].value = 0;
close_gap(ht, i);
}
static int find(hashtable *ht, void *key)
{
// <<< Code changed
int i = ht->hash_fn(key) % ht->len;
int num_checked = 0;
while(ht->table[i].key && num_checked != ht->len) {
if(!ht->key_cmp(ht->table[i].key, key)) {
return i;
}
num_checked++;
i = (i + i) % ht->len;
}
return -1;
}
static void close_gap(hashtable *ht, int i)
{
int j = (i + 1) % ht->len;
while(ht->table[j].key) {
int loc = ht->hash_fn(ht->table[j].key);
if((j > i && (loc <= i || loc > j)) || (j < i && (loc <= i && loc > j))) {
ht->table[i] = ht->table[j];
ht->table[j].key = 0;
ht->table[j].value = 0;
close_gap(ht, j);
return;
}
}
}
I only coded around the errors and warnings, I did not check the logic. You will see that I have used malloc to allocate memory for key and value. Obviously you will need memory management on these two (i.e. free()).