We Keep Coding

Cannot assign contents of list attribute in Pybind11 defined class

I have a sparse matrix implementation in C++, and I used pybind11 to expose it to python. Here is the problem:
>>> D1 = phc.SparseMatrix(3, [[0],[1],[2]])
>>> D1.cData
[[0], [1], [2]]
>>> D1.cData[1] = [1,2]
>>> D1.cData
[[0], [1], [2]] #Should be [[0], [1,2], [2]]
In python, I cannot change the contents of the SparseMatrix.cData attribute with the assignment operator. I can change the entire list with D1.cData = [[1],[2],[3]]. This behavior is bewildering to me. D1.cData is just a list, so I would expect that the above code would work.
I suspect it has something to do with my pybind11 code since this behavior is not present in python-defined custom classes. But I have no idea what is wrong (I am a novice programmer). Here is the source code info:
Python Bindings
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
namespace py = pybind11;
#include <SparseMatrix.h>
namespace phc = ph_computation;
using SparseMatrix = phc::SparseMatrix;
using Column = phc::Column;
using CData = phc::CData;
PYBIND11_MODULE(ph_computations, m)
{
m.doc() = "ph_computations python bindings";
using namespace pybind11::literals;
m.def("add_cols", &phc::add_cols);//begin SparseMatrix.h
py::class_<SparseMatrix>(m, "SparseMatrix")
.def(py::init<size_t, CData>())
.def(py::init<std::string>())
.def_readwrite("n_rows", &SparseMatrix::n_rows)
.def_readwrite("n_cols", &SparseMatrix::n_cols)
.def_readwrite("cData", &SparseMatrix::cData)
.def("__neq__", &SparseMatrix::operator!=)
.def("__eq__", &SparseMatrix::operator==)
.def("__add__", &SparseMatrix::operator+)
.def("__mul__", &SparseMatrix::operator*)
.def("transpose", &SparseMatrix::transpose)
.def("__str__", &SparseMatrix::print)
.def("save", &SparseMatrix::save)
;
m.def("identity", &phc::make_identity);
m.def("matching_pivots", &phc::matching_pivots);//end SparseMatrix.h
}
SparseMatrix.h
#pragma once
#include <iostream>
#include <sstream>
#include <fstream>
#include <string>
#include <iterator>
#include <algorithm>
#include <vector>
#include <stdexcept>
namespace ph_computation{
using Int = int;
using Column = std::vector<Int>;//a Column is represented by a vector of indices
using CData = std::vector<Column>;//a matrix is represented by a vector of Columns
//Add columns in Z2
Column add_cols(const Column& c1, const Column& c2);
struct SparseMatrix
{
size_t n_rows{0};
size_t n_cols{0};
CData cData;
SparseMatrix()=default;
SparseMatrix(size_t n_rows_, CData cData_):
n_rows(n_rows_), n_cols(cData_.size()), cData(cData_){}
SparseMatrix(std::string path);
bool operator!=(const SparseMatrix &other) const;
bool operator==(const SparseMatrix &other) const;
SparseMatrix operator+(const SparseMatrix &other) const;
SparseMatrix operator*(const SparseMatrix &other) const;
void transpose();
void print() const;
void save(std::string path);
};
SparseMatrix make_identity(size_t n_cols_);
bool matching_pivots(const SparseMatrix& a, const SparseMatrix& b);
}
SparseMatrix.cpp (you probably don't need this)
#include <SparseMatrix.h>
namespace ph_computation {
Column add_cols(const Column& c1, const Column& c2){
Column c3;
int idx1{0};
int idx2{0};
while(idx1 < c1.size() && idx2 < c2.size()){
if(c1[idx1] < c2[idx2]){
c3.push_back(c1[idx1]);
++idx1;
}
else if(c1[idx1] > c2[idx2]){
c3.push_back(c2[idx2]);
++idx2;
}
else {
++idx1;
++idx2;
}
}
if (idx1 < c1.size()){
c3.insert(c3.end(), std::next(c1.begin(), idx1), c1.end());
}
else if (idx2 < c2.size()){
c3.insert(c3.end(), std::next(c2.begin(), idx2), c2.end());
}
return c3;
}
SparseMatrix make_identity(size_t n_cols_){
CData cData_(n_cols_);
for (int j = 0; j < n_cols_; ++j){
cData_[j] = {j};
}
return SparseMatrix(n_cols_, cData_);
}
SparseMatrix::SparseMatrix(std::string path){
std::fstream f_in;
f_in.open(path, std::ios::in);
if(f_in.is_open()){
std::string n_rows_line;
std::getline(f_in, n_rows_line); //first line of file contains number of rows
n_rows = std::stoi(n_rows_line);
std::string n_cols_line;
std::getline(f_in, n_cols_line); //second line of file contains number of cols
n_cols = std::stoi(n_cols_line);
CData cData_(n_cols);
cData = cData_;
std::string line;
int j = 0;
int nnz, data;
while (std::getline(f_in, line)){
std::stringstream line_str = std::stringstream(line);
while (line_str >> nnz){
Column col_j(nnz);
for (int i =0; i < nnz; ++i){
line_str >> data;
col_j[i] = data;
}
cData[j] = col_j;
}
++j;
}
f_in.close();
}
else{
throw std::runtime_error("File did not open.");
}
}
bool SparseMatrix::operator!=(const SparseMatrix &other) const{
if (n_rows != other.n_rows || cData != other.cData){
return true;
}
return false;
}
bool SparseMatrix::operator==(const SparseMatrix &other) const{
return !(*this != other);
}
SparseMatrix SparseMatrix::operator+(const SparseMatrix &other) const{
if (n_rows != other.n_rows || n_cols != other.n_cols){
throw std::invalid_argument("Matrices must have same dimension to add.");
}
CData ans_cData;
for (int j = 0; j < n_cols; ++j){
ans_cData.push_back(add_cols(cData[j], other.cData[j]));
}
return SparseMatrix(n_rows, ans_cData);
}
SparseMatrix SparseMatrix::operator*(const SparseMatrix &other) const{
if(n_cols != other.n_rows){
throw std::invalid_argument("Matrices must have compatible dimensions.");
}
size_t ans_rows = n_rows;
CData ans_cData(other.n_cols);
SparseMatrix ans(ans_rows, ans_cData);
for(int j =0; j<ans.n_cols; ++j){
for(int idx : other.cData[j]){
ans.cData[j] = add_cols(ans.cData[j], cData[idx]);
}
}
return ans;
}
void SparseMatrix::transpose(){
CData cData_T(n_rows);
for(int j =0; j<n_cols; ++j){
if(!cData[j].empty()){
for(int x: cData[j]){
cData_T[x].push_back(j);
}
}
}
cData = cData_T;
n_rows = n_cols;
n_cols = cData.size();
}
void SparseMatrix::print() const{
for (int i = 0; i < n_rows; ++i){
for (int j = 0; j < n_cols; ++j){
if (cData[j].empty())
{std::cout << " 0";}
else if (std::binary_search(cData[j].begin(), cData[j].end(), i))//Assumes row indices
{std::cout << " 1";} //are ordered
else
{std::cout << " 0";}
if (n_cols-1 == j)
{std::cout << " \n";}
}
}
}
void SparseMatrix::save(std::string path){
std::fstream f_out;
f_out.open(path, std::ios::out);
if(f_out.is_open()){
f_out << n_rows << "\n";
f_out << n_cols << "\n";
for(int j = 0; j < n_cols; ++j){
int col_j_sz = cData[j].size();
f_out << col_j_sz;
for(int i = 0; i < col_j_sz; ++i){
f_out << " " << cData[j][i];
}
f_out << "\n";
}
f_out.close();
}
else{
throw std::runtime_error("File did not open.");
}
}
bool matching_pivots(const SparseMatrix& a, const SparseMatrix& b){
if(a.n_rows != b.n_rows || a.n_cols != b.n_cols){
throw std::invalid_argument("Input matrices must have the same size.");
}
for (int j = 0; j<a.n_cols; ++j){
bool a_j_empty = a.cData[j].empty();
bool b_j_empty = b.cData[j].empty();
if (a_j_empty != b_j_empty){
return false;
}
else if (!a_j_empty){
if(a.cData[j].back() != b.cData[j].back()){
return false;
}
}
}
return true;
}
} // namespace ph_computation

I found the answer in the pybind11 documentation here: https://pybind11.readthedocs.io/en/stable/advanced/cast/stl.html?highlight=opaque#making-opaque-types
Apparently, STL container data members can be overwritten in their entirety in python, but modification of the data member through list methods does not work. I don't really understand it, but the link above answers the question.

How to return an array in C into python?

I have been working on a selection sort with C extension on python, which aimed to intake a list in python, sort using C code and return a sorted list in python. Sounds simple, but I just could not get the value of the sorted list correct in python, as I would get a value of 1 when I was trying to print the sorted list.
Here is my code in C:
#include <Python.h>
int selectionSort(int array[], int N){
int i, j, min_element;
for (i = 0; i < N-1; i++) {
min_element = i;
for (j = i+1; j < N; j++)
if (array[j] < array[min_element])
min_element = j;
int temp = array[min_element];
array[min_element] = array[i];
array[i] = temp;
}
return *array;
}
static PyObject* selectSort(PyObject *self, PyObject *args)
{
PyObject* list;
int *array, N;
if (!PyArg_ParseTuple(args, "O", &list))
return NULL;
N = PyObject_Length(list);
if (N < 0)
return NULL;
array = (int *) malloc(sizeof(int *) * N);
if (array == NULL)
return NULL;
for (int index = 0; index < N; index++) {
PyObject *item;
item = PyList_GetItem(list, index);
if (!PyFloat_Check(item))
array[index] = 0;
array[index] = PyFloat_AsDouble(item);
}
return Py_BuildValue("i", selectionSort(array, N));
}
static PyMethodDef myMethods[] = {
{ "selectSort", selectSort, METH_VARARGS, "..." },
{ NULL, NULL, 0, NULL }
};
static struct PyModuleDef myModule = {
PyModuleDef_HEAD_INIT,
"myModule",
"Test Module",
-1,
myMethods
};
PyMODINIT_FUNC PyInit_myModule(void)
{
return PyModule_Create(&myModule);
}
Here is the command line I executed:
>>> import myModule
>>> unsortedList = [1, 4, 3, 90, 22, 34, 32]
>>> sortedList = myModule.selectSort(unsortedList)
>>> print(sortedList)
1
Anyone has any ideas on this? I would really appreciate it! Thanks!

Just change return type from int to int* and return with return array, not return *array.

Python ctypes cannot get long string value returned from .so file

I am learning C and trying to import a .so into my python file for higher performance by using a python package ctypes. So everything going well until I had a hard time when trying to get a string returned from .so file.
C code:
char *convert_to_16(char *characters, int n){
char sub_buffer[3];
char code[3];
char *buffer = (char*)malloc(sizeof(characters) * 2);
for(int i=0; i < n; i++){
strncpy(code, characters+i, 1);
sprintf(sub_buffer, "%x", *code);
strncat(buffer, sub_buffer, 2);
}
return buffer;
}
// main() only for test
int main(){
char param[] = "ABCDEFGHTUIKLL";
printf("%s\n", param);
int length = strlen(param);
printf("%s\n", convert_to_16(param, length));
}
It runs well with output:
41424344454647485455494b4c4c
Python code :
c_convert_to_16 = ctypes.CDLL('./convert_to_16.so').convert_to_16
c_convert_to_16.restype = ctypes.c_char_p
a_string = "ABCDEFGHTUIKLL"
new_16base_string = c_convert_to_16(a_string, len(a_string))
print new_16base_string
It runs but only returns two characters:
41
I read the official doc and set restype as ctypes.c_char_p, and try to set it to other values. But it seems it's the only option, just oddly only two characters were returned.
Is it the problem of my ctypes configuration or my C wasn't written correctly?
Many thanks.

I don't know much about ctypes in python but you should create your string like that c_char_p("ABCDEFGHTUIKLL").
And maybe tell what argument take your function c_convert_to_16.argtypes = [c_char_p, c_size_t]
This will fix your undefined behavior in C:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
char *convert_to_16(char const *array, size_t const len);
char *convert_to_16(char const *array, size_t const len) {
size_t const len_buffer = len * 2 + 1;
char *buffer = malloc(len_buffer);
if (buffer == NULL) {
return NULL;
}
size_t used = 0;
for (size_t i = 0; i < len; i++) {
if (len_buffer < used || len_buffer - used < 3) {
free(buffer);
return NULL;
}
int ret = snprintf(buffer + used, 3, "%.2x", (unsigned char)array[i]);
if (ret != 2) {
free(buffer);
return NULL;
}
used += 2;
}
return buffer;
}
int main(void) {
char const param[] = "ABCDEFGHTUIKLL";
printf("%s\n", param);
char *ret = convert_to_16(param, sizeof param - 1);
if (ret != NULL) {
printf("%s\n", ret);
}
free(ret);
}

Python ctypes : OSError undefined symbol when loading library

In Ubuntu 14.04, I wrote a C file called hash.c:
/* hash.c: hash table with linear probing */
typedef struct {
void *key;
void *value;
} ht_entry;
typedef struct {
ht_entry *table;
int len;
int num_entries;
int (*hash_fn)(void *key);
int (*key_cmp)(void *k1, void *k2);
} hashtable;
and compiled it with
gcc -shared hash.c -o test.so -fPIC
Afterwards, I tried to load test.so in a Python script (for testing), but I got the following error: "OSError: .../test.so: undefined symbol: hash_fn"
hash_fn is a function pointer in the hashtable struct. It is referenced a number of times by functions later in the file.
I do not understand why this error is happening. I have Googled but all other cases either concern C++ or includes. In my case I just have 1 C file that includes only stdio and stdlib.
here is the FULL code.
When I comment out all but hash_create and print_info, it loads succesfully. When I uncomment find(), it the error happens.
(print_info is just for testing that ctypes works)
/* hash.c: hash table with linear probing */
#include <stdio.h>
#include <stdlib.h>
typedef struct {
void *key;
void *value;
} ht_entry;
typedef struct {
ht_entry *table;
int len;
int num_entries;
int (*hash_fn)(void *key);
int (*key_cmp)(void *k1, void *k2);
} hashtable;
static void close_gap(hashtable *ht, int i);
static int find(hashtable *ht, void *key);
hashtable* hash_create(int len, int (*hash_fn)(void*), int (*key_cmp)(void*, void*))
{
hashtable* ht = (hashtable*) malloc(sizeof(hashtable));
ht->len = len;
ht->table = calloc(len, sizeof(ht_entry));
ht->hash_fn = hash_fn;
ht->key_cmp = key_cmp;
ht->table[0].key = 2;
ht->table[0].value = 3;
return ht;
}
void print_info(hashtable *ht)
{
printf("%d, %d, %d\n", ht->len, ht->table[0].key, ht->table[0].value);
}
void* hash_retrieve(hashtable* ht, void *key)
{
int i = find(ht, key);
if(i < 0) {
return NULL;
}
return ht->table[i].value;
}
void hash_insert(hashtable* ht, void *key, void *value)
{
if(ht->num_entries == ht->len) {
return;
}
int i = hash_fn(key) % ht->len;
while(ht->table[i].key != NULL) {
i = (i + i) % ht->len;
}
ht->table[i].key = key;
ht->table[i].value = value;
}
void hash_remove(hashtable *ht, void *key)
{
int i = find(ht, key);
if(i < 0) {
return;
}
ht->table[i].key = 0;
ht->table[i].value = 0;
close_gap(ht, i);
}
static int find(hashtable *ht, void *key)
{
int i = hash_fn(key) % ht->len;
int num_checked = 0;
while(ht->table[i].key && num_checked != ht->len) {
if(!ht->key_cmp(ht->table[i].key, key)) {
return i;
}
num_checked++;
i = (i + i) % ht->len;
}
return -1;
}
static void close_gap(hashtable *ht, int i)
{
int j = (i + 1) % ht->len;
while(ht->table[j].key) {
int loc = ht->hash_fn(ht->table[j].key);
if((j > i && (loc <= i || loc > j)) || (j < i && (loc <= i && loc > j))) {
ht->table[i] = ht->table[j];
ht->table[j].key = 0;
ht->table[j].value = 0;
close_gap(ht, j);
return;
}
}
}

When I use your compilation line I get five warnings. There are several problems here. First you are trying to assign an int to void * in several places. That raises a warning, and it would crash at runtime because you are passing 2 and 3 as addresses.
Second, you are calling hash_fn in a couple of places instead of ht->hash_fn. That causes the linker error, but you should consider my other changes, otherwise it will crash at runtime with a SIGSEGV:
/* hash.c: hash table with linear probing */
#include <stdio.h>
#include <stdlib.h>
typedef struct {
void *key;
void *value;
} ht_entry;
typedef struct {
ht_entry *table;
int len;
int num_entries;
int (*hash_fn)(void *key);
int (*key_cmp)(void *k1, void *k2);
} hashtable;
static void close_gap(hashtable *ht, int i);
static int find(hashtable *ht, void *key);
hashtable* hash_create(int len, int (*hash_fn)(void*), int (*key_cmp)(void*, void*))
{
hashtable* ht = (hashtable*) malloc(sizeof(hashtable));
ht->len = len;
ht->table = calloc(len, sizeof(ht_entry));
ht->hash_fn = hash_fn;
ht->key_cmp = key_cmp;
// <<< Code changed here
/*
ht->table[0].key = 2;
ht->table[0].value = 3;
*/
{
int *p = malloc(sizeof(int));
*p = 2;
ht->table[0].key = p;
p = malloc(sizeof(int));
*p = 3;
ht->table[0].value = p;
}
// end of code change
return ht;
}
void print_info(hashtable *ht)
{
// <<<< Code changed
printf("%d, %d, %d\n", ht->len,
*(int *)ht->table[0].key, *(int *)ht->table[0].value);
}
void* hash_retrieve(hashtable* ht, void *key)
{
int i = find(ht, key);
if(i < 0) {
return NULL;
}
return ht->table[i].value;
}
void hash_insert(hashtable* ht, void *key, void *value)
{
if(ht->num_entries == ht->len) {
return;
}
// <<< Code changed
int i = ht->hash_fn(key) % ht->len;
while(ht->table[i].key != NULL) {
i = (i + i) % ht->len;
}
ht->table[i].key = key;
ht->table[i].value = value;
}
void hash_remove(hashtable *ht, void *key)
{
int i = find(ht, key);
if(i < 0) {
return;
ht->table[i].key = 0;
ht->table[i].value = 0;
close_gap(ht, i);
}
static int find(hashtable *ht, void *key)
{
// <<< Code changed
int i = ht->hash_fn(key) % ht->len;
int num_checked = 0;
while(ht->table[i].key && num_checked != ht->len) {
if(!ht->key_cmp(ht->table[i].key, key)) {
return i;
}
num_checked++;
i = (i + i) % ht->len;
}
return -1;
}
static void close_gap(hashtable *ht, int i)
{
int j = (i + 1) % ht->len;
while(ht->table[j].key) {
int loc = ht->hash_fn(ht->table[j].key);
if((j > i && (loc <= i || loc > j)) || (j < i && (loc <= i && loc > j))) {
ht->table[i] = ht->table[j];
ht->table[j].key = 0;
ht->table[j].value = 0;
close_gap(ht, j);
return;
}
}
}
I only coded around the errors and warnings, I did not check the logic. You will see that I have used malloc to allocate memory for key and value. Obviously you will need memory management on these two (i.e. free()).

extending global c variable array with swig and python

I have a global variable array in c that I'd like to pull into python. And I'm having difficulties with varout typemap:
/* example.c */
int foo[] = {0, 1};
And here is the very vanilla interface:
/* example.i */
%module example
%{
extern int foo[2];
%}
%typemap(varout) int foo[] {
int i;
//$1, $1_dim0, $1_dim1
$result = PyList_New($1_dim0);
for (i = 0; i < $1_dim0; i++) {
PyObject *o = PyInt_FromLong((double) $1[i]);
PyList_SetItem($result,i,o);
}
}
%include "example.c"
When I try to build it with the following SConstruct:
import distutils.sysconfig
env = Environment(SWIGFLAGS='-python -shadow -Wall'.split(),
CPPPATH=[distutils.sysconfig.get_python_inc()],
SHLIBPREFIX="")
env.SharedLibrary('_example.so', ['example.c', 'example.i'])
$1_dim0 special variable is not populated, resulting in the following non-compilable code in example_wrap.c:
SWIGINTERN PyObject *Swig_var_foo_get(void) {
PyObject *pyobj = 0;
{
int i;
//foo, , foo_dim1
pyobj = PyList_New();
for (i = 0; i < ; i++) {
PyObject *o = PyInt_FromLong((double) foo[i]);
PyList_SetItem(pyobj,i,o);
}
}
return pyobj;
}
So clearly the typemap match has happened, but dimensionality of array is missing. What am I missing? Hard coding the dimension does works.
In general, is there any way to extend global cvar variables with swig?
$ swig -version
SWIG Version 2.0.4
Compiled with g++ [i686-pc-linux-gnu]
Configured options: +pcre
Please see http://www.swig.org for reporting bugs and further information

You're almost there with your varout typemap. You need to make two minor changes:
You need to add the size ANY to the int foo[] typemap:
%typemap(varout) int foo[ANY] {
int i;
//$1, $1_dim0, $1_dim1
$result = PyList_New($1_dim0);
for (i = 0; i < $1_dim0; i++) {
PyObject *o = PyInt_FromLong((double) $1[i]);
PyList_SetItem($result,i,o);
}
}
This makes sure your typemap is a match for arrays of (any) known size, not just equivalent to int *foo.
You need to modify example.c to make the size of foo clearer. It's legal and correct C as it stands but tricky to deduce the size of the array unless you happen to be a complete C compiler. Changing it to:
int foo[2] = {0, 1};
is sufficient to make sure that it matches the varout typemap.
With those two changes the generated code works as you'd hope:
SWIGINTERN PyObject *Swig_var_foo_get(void) {
PyObject *pyobj = 0;
{
int i;
//foo, 2, foo_dim1
pyobj = PyList_New(2);
for (i = 0; i < 2; i++) {
PyObject *o = PyInt_FromLong((double) foo[i]);
PyList_SetItem(pyobj,i,o);
}
}
return pyobj;
}
is what gets generated on my machine with those changes.

For those like me who ponders what to do with arrays of non-simple types -- here is one way to do it:
The non-simple type:
typedef struct {
int a;
float b;
} Foo;
and a global array:
extern Foo *foov[40];
%typemap(varout) Foo *foov[ANY] {
int i;
$result = PyList_New($1_dim0);
for (i = 0; i < $1_dim0; i++) {
PyObject *o = SWIG_NewPointerObj($1[i], SWIGTYPE_p_Foo, 0);
PyList_SetItem($result, i, o);
}
}
Just shared this since it took me forever to find out, and this article helped. Just needed to find out how to allocate the SWIG version of my non-simple type -- found that buried here:
http://www.swig.org/Doc2.0/Python.html#Python_nn64