I want to reproduce "Conv1D" results of pytorch in C code.
I tried to implement "Conv1D" using three methods (C code, Python, Pytorch), but the results are different. Only seven fraction digits are reasonable. Assuming there are multiple layers of conv1d in the structure, the fraction digits accuracy will gradually decrease.
According to everyone's recommend, I tried to change the C code type of input data to double but the result is still incorrect.
Have I done something wrong?
For example:
The output of Pytorch: 0.2380688339471817017
The output of Python: 0.2380688637495040894
The output of C code (float): 0.2380688637
The output of C code (double): 0.238068885344539680
Here is my current implementation
Input:
input dim. = 80, output dim. = 128, kernel size = 5
Pytorch: Conv1D_input.npy, Conv1D_weight.npy
Python: Conv1D_input.npy, Conv1D_weight.npy (same as Pytorch)
C code: Conv1D_input.txt, Conv1D_weight.txt (convert from Pytorch, IEEE 754 single precision)
Pytorch
import torch
import numpy as np
from torch import nn
from torch.autograd import Variable
import torch.nn.functional as F
import argparse
import sys
import io
import time
import os
class RNN(nn.Module):
def __init__(self, input_size, hidden_size):
super(RNN, self).__init__()
self.input_size = input_size
self.hidden_size = hidden_size
self.c1 = nn.Conv1d(input_size, hidden_size, kernel_size = 5, bias=False)
self.c1.weight = torch.nn.Parameter(torch.Tensor(np.load("CONV1D_WEIGHT.npy")))
def forward(self, inputs):
c = self.c1(inputs)
return c
input_size = 80
hidden_size = 128
kernel_size = 5
rnn = RNN(input_size, hidden_size)
inputs = torch.nn.Parameter(torch.Tensor(np.load("CONV1D_IN.npy")))
print("inputs", inputs)
outputs = rnn(inputs)
sub_np456 = outputs[0].cpu().detach().numpy()
np.savetxt("Pytorch_CONV1D_OUTPUT.txt", sub_np456)
print('outputs', outputs)
Python
import struct
import numpy as np
if __name__ == "__main__":
row = 80
col = 327
count = 0
res_out_dim = 128
in_dim = 80
kernel_size = 5
filter = np.zeros((80, 5), dtype = np.float32)
featureMaps = np.zeros((128, 323), dtype = np.float32)
spectrum = np.load("CONV1D_INPUT.npy")
weight = np.load("CONV1D_WEIGHT.npy")
spectrum_2d = spectrum.reshape(80, 327)
for i in range(res_out_dim):
for j in range(in_dim):
for k in range(kernel_size):
filter[j][k] = weight[i][j][k]
while count < (col-kernel_size+1):
for j in range(in_dim):
for k in range(count, kernel_size+count):
featureMaps[i][count] = featureMaps[i][count] + spectrum_2d[j][k]*filter[j][k-count]
count = count + 1
count = 0
np.savetxt("Python_CONV1D_OUTPUT.txt", featureMaps)
C code (float)
#include<stdio.h>
#include<stdlib.h>
#include<math.h>
#include<time.h>
const char CONV1D_WEIGHT[] = "CONV1D_WEIGHT.txt";
const char CONV1D_INPUT[] = "CONV1D_INPUT.txt";
void parameterFree(float **matrix, int row)
{
int i = 0;
for(i=0; i<row; i++)
free(matrix[i]);
free(matrix);
}
float** createMatrix_2D(int row, int col)
{
int i = 0;
float **matrix = NULL;
matrix = (float**)malloc(sizeof(float*) * row);
if(matrix == NULL)
printf("Matrix2D malloc failed\n");
for(i=0; i<row; i++)
{
matrix[i] = (float*)malloc(sizeof(float) * col);
if(matrix[i] == NULL)
printf("Matrix2D malloc failed\n");
}
return matrix;
}
float** conv_1D(const char weightFile[], float **source, int *row, int *col, int in_dim, int res_out_dim, int kernel_size)
{
float **filter = createMatrix_2D(in_dim, kernel_size);
float **featureMaps = createMatrix_2D(res_out_dim, *col-kernel_size+1);
int i = 0, j = 0, k = 0, count = 0;
char str[10];
float data = 0.0;
FILE *fp = fopen(weightFile, "r");
if(fp == NULL)
printf("Resnet file open failed\n");
else
{
/*initial featureMaps*/
for(i=0; i<res_out_dim; i++)
{
for(j=0; j<*col-kernel_size+1; j++)
{
featureMaps[i][j] = 0.0;
}
}
/*next filter*/
for(i=0; i<res_out_dim; i++)
{
/*read filter*/
for(j=0; j<in_dim; j++)
{
for(k=0; k<kernel_size; k++)
{
fscanf(fp, "%s", str);
sscanf(str, "%x", &data);
filter[j][k] = data;
}
}
/* (part of source * filter) */
while(count < *col-kernel_size+1)
{
for(j=0; j<in_dim; j++)
{
for(k=count; k<kernel_size+count; k++)
{
featureMaps[i][count] += source[j][k]*filter[j][k-count];
}
}
count++;
}
count = 0;
}
fclose(fp);
}
parameterFree(source, *row);
parameterFree(filter, in_dim);
*row = res_out_dim;
*col = *col-kernel_size+1;
return featureMaps;
}
int main()
{
int row = 80;
int col = 327;
int in_dim = 80;
int res_out_dim = 128;
int kernel_size = 5;
int i, j;
float data;
char str[10];
float **input = createMatrix_2D(row, col);
FILE *fp = fopen(CONV1D_INPUT, "r");
FILE *fp2 = fopen("C code_CONV1D_OUTPUT.txt", "w");
if(fp == NULL)
printf("File open failed\n");
else
{
for(i=0; i<row; i++)
{
for(j=0; j<col; j++)
{
fscanf(fp, "%s", str);
sscanf(str, "%x", &data);
input[i][j] = data;
}
}
}
float **CONV1D_ANS = conv_1D(CONV1D_WEIGHT, input, &row, &col, in_dim, res_out_dim, kernel_size);
for(i=0; i<row; i++)
{
for(j=0; j<col; j++)
{
fprintf(fp2, "[%.12f] ", CONV1D_ANS[i][j]);
}
fprintf(fp2, "\n");
}
return 0;
}
C code (double)
#include<stdio.h>
#include<stdlib.h>
#include<math.h>
#include<time.h>
const char CONV1D_WEIGHT[] = "CONV1D_WEIGHT.txt";
const char CONV1D_INPUT[] = "CONV1D_INPUT.txt";
void parameterFree(double **matrix, int row)
{
int i = 0;
for(i=0; i<row; i++)
free(matrix[i]);
free(matrix);
}
double** createMatrix_2D(int row, int col)
{
int i = 0;
double **matrix = NULL;
matrix = (double**)malloc(sizeof(double*) * row);
if(matrix == NULL)
printf("Matrix2D malloc failed\n");
for(i=0; i<row; i++)
{
matrix[i] = (double*)malloc(sizeof(double) * col);
if(matrix[i] == NULL)
printf("Matrix2D malloc failed\n");
}
return matrix;
}
double** conv_1D(const char weightFile[], double **source, int *row, int *col, int in_dim, int res_out_dim, int kernel_size)
{
double **filter = createMatrix_2D(in_dim, kernel_size);
double **featureMaps = createMatrix_2D(res_out_dim, *col-kernel_size+1);
int i = 0, j = 0, k = 0, count = 0;
char str[10];
float data = 0.0;
FILE *fp = fopen(weightFile, "r");
if(fp == NULL)
printf("Resnet file open failed\n");
else
{
/*initial featureMaps*/
for(i=0; i<res_out_dim; i++)
{
for(j=0; j<*col-kernel_size+1; j++)
{
featureMaps[i][j] = 0.0;
}
}
/*next filter*/
for(i=0; i<res_out_dim; i++)
{
/*read filter*/
for(j=0; j<in_dim; j++)
{
for(k=0; k<kernel_size; k++)
{
fscanf(fp, "%s", str);
sscanf(str, "%x", &data);
filter[j][k] = (double)data;
}
}
/* (part of source * filter) */
while(count < *col-kernel_size+1)
{
for(j=0; j<in_dim; j++)
{
for(k=count; k<kernel_size+count; k++)
{
featureMaps[i][count] += source[j][k]*filter[j][k-count];
}
}
count++;
}
count = 0;
}
fclose(fp);
}
parameterFree(source, *row);
parameterFree(filter, in_dim);
*row = res_out_dim;
*col = *col-kernel_size+1;
return featureMaps;
}
int main()
{
int row = 80;
int col = 327;
int in_dim = 80;
int res_out_dim = 128;
int kernel_size = 5;
int i, j;
float data;
char str[10];
double **input = createMatrix_2D(row, col);
FILE *fp = fopen(CONV1D_INPUT, "r");
FILE *fp2 = fopen("C code_CONV1D_OUTPUT.txt", "w");
if(fp == NULL)
printf("File open failed\n");
else
{
for(i=0; i<row; i++)
{
for(j=0; j<col; j++)
{
fscanf(fp, "%s", str);
sscanf(str, "%x", &data);
input[i][j] = (double)data;
}
}
}
double **CONV1D_ANS = conv_1D(CONV1D_WEIGHT, input, &row, &col, in_dim, res_out_dim, kernel_size);
for(i=0; i<row; i++)
{
for(j=0; j<col; j++)
{
fprintf(fp2, "[%.18f] ", CONV1D_ANS[i][j]);
}
fprintf(fp2, "\n");
}
return 0;
}
Floating point numbers are not precise (by design). Depending on in which order operations are performed, the results might vary. Even worse, some formulas are straight numerical unstable, whereas another one for the same analytical expression can be stable.
Compilers often rearange statements as an optimization measure. Convolution is an operation which contains notoriously many operations and loops. So unless you directly compare executed bytecode, this speculation is kind of pointless.
Related
I have been working on a selection sort with C extension on python, which aimed to intake a list in python, sort using C code and return a sorted list in python. Sounds simple, but I just could not get the value of the sorted list correct in python, as I would get a value of 1 when I was trying to print the sorted list.
Here is my code in C:
#include <Python.h>
int selectionSort(int array[], int N){
int i, j, min_element;
for (i = 0; i < N-1; i++) {
min_element = i;
for (j = i+1; j < N; j++)
if (array[j] < array[min_element])
min_element = j;
int temp = array[min_element];
array[min_element] = array[i];
array[i] = temp;
}
return *array;
}
static PyObject* selectSort(PyObject *self, PyObject *args)
{
PyObject* list;
int *array, N;
if (!PyArg_ParseTuple(args, "O", &list))
return NULL;
N = PyObject_Length(list);
if (N < 0)
return NULL;
array = (int *) malloc(sizeof(int *) * N);
if (array == NULL)
return NULL;
for (int index = 0; index < N; index++) {
PyObject *item;
item = PyList_GetItem(list, index);
if (!PyFloat_Check(item))
array[index] = 0;
array[index] = PyFloat_AsDouble(item);
}
return Py_BuildValue("i", selectionSort(array, N));
}
static PyMethodDef myMethods[] = {
{ "selectSort", selectSort, METH_VARARGS, "..." },
{ NULL, NULL, 0, NULL }
};
static struct PyModuleDef myModule = {
PyModuleDef_HEAD_INIT,
"myModule",
"Test Module",
-1,
myMethods
};
PyMODINIT_FUNC PyInit_myModule(void)
{
return PyModule_Create(&myModule);
}
Here is the command line I executed:
>>> import myModule
>>> unsortedList = [1, 4, 3, 90, 22, 34, 32]
>>> sortedList = myModule.selectSort(unsortedList)
>>> print(sortedList)
1
Anyone has any ideas on this? I would really appreciate it! Thanks!
Just change return type from int to int* and return with return array, not return *array.
I have a program written in python with OpenCV. I want to add a feature which is otsu thresholding with mask. So, I get a code from here written in c++. I tried to convert it as python, but it's too slow (because of python). Finally, I make up my mind to use c++ with python. I try to embedding, and I find pyopencv_to() function. But, I can't use it because of PyArray_Check(). When program entered this function, die immediately. It doesn't give any error message. I guess it may be segmentation fault. Many stack overflow's answers says that "use import_array()". But it doesn't work for me.
Here is my code.
convert.cpp
#include <Python.h>
#include "numpy/ndarrayobject.h"
#include "opencv2/core/core.hpp"
#include "convert.hpp"
static PyObject* opencv_error = 0;
static int failmsg(const char *fmt, ...)
{
char str[1000];
va_list ap;
va_start(ap, fmt);
vsnprintf(str, sizeof(str), fmt, ap);
va_end(ap);
PyErr_SetString(PyExc_TypeError, str);
return 0;
}
class PyAllowThreads
{
public:
PyAllowThreads() : _state(PyEval_SaveThread()) {}
~PyAllowThreads()
{
PyEval_RestoreThread(_state);
}
private:
PyThreadState* _state;
};
class PyEnsureGIL
{
public:
PyEnsureGIL() : _state(PyGILState_Ensure()) {}
~PyEnsureGIL()
{
PyGILState_Release(_state);
}
private:
PyGILState_STATE _state;
};
#define ERRWRAP2(expr) \
try \
{ \
PyAllowThreads allowThreads; \
expr; \
} \
catch (const cv::Exception &e) \
{ \
PyErr_SetString(opencv_error, e.what()); \
return 0; \
}
using namespace cv;
static PyObject* failmsgp(const char *fmt, ...)
{
char str[1000];
va_list ap;
va_start(ap, fmt);
vsnprintf(str, sizeof(str), fmt, ap);
va_end(ap);
PyErr_SetString(PyExc_TypeError, str);
return 0;
}
static size_t REFCOUNT_OFFSET = (size_t)&(((PyObject*)0)->ob_refcnt) +
(0x12345678 != *(const size_t*)"\x78\x56\x34\x12\0\0\0\0\0")*sizeof(int);
static inline PyObject* pyObjectFromRefcount(const int* refcount)
{
return (PyObject*)((size_t)refcount - REFCOUNT_OFFSET);
}
static inline int* refcountFromPyObject(const PyObject* obj)
{
return (int*)((size_t)obj + REFCOUNT_OFFSET);
}
class NumpyAllocator : public MatAllocator
{
public:
NumpyAllocator() {}
~NumpyAllocator() {}
void allocate(int dims, const int* sizes, int type, int*& refcount,
uchar*& datastart, uchar*& data, size_t* step)
{
PyEnsureGIL gil;
int depth = CV_MAT_DEPTH(type);
int cn = CV_MAT_CN(type);
const int f = (int)(sizeof(size_t)/8);
int typenum = depth == CV_8U ? NPY_UBYTE : depth == CV_8S ? NPY_BYTE :
depth == CV_16U ? NPY_USHORT : depth == CV_16S ? NPY_SHORT :
depth == CV_32S ? NPY_INT : depth == CV_32F ? NPY_FLOAT :
depth == CV_64F ? NPY_DOUBLE : f*NPY_ULONGLONG + (f^1)*NPY_UINT;
int i;
npy_intp _sizes[CV_MAX_DIM+1];
for( i = 0; i < dims; i++ )
_sizes[i] = sizes[i];
if( cn > 1 )
{
/*if( _sizes[dims-1] == 1 )
_sizes[dims-1] = cn;
else*/
_sizes[dims++] = cn;
}
PyObject* o = PyArray_SimpleNew(dims, _sizes, typenum);
if(!o)
CV_Error_(CV_StsError, ("The numpy array of typenum=%d, ndims=%d can not be created", typenum, dims));
refcount = refcountFromPyObject(o);
npy_intp* _strides = PyArray_STRIDES(o);
for( i = 0; i < dims - (cn > 1); i++ )
step[i] = (size_t)_strides[i];
datastart = data = (uchar*)PyArray_DATA(o);
}
void deallocate(int* refcount, uchar*, uchar*)
{
PyEnsureGIL gil;
if( !refcount )
return;
PyObject* o = pyObjectFromRefcount(refcount);
Py_INCREF(o);
Py_DECREF(o);
}
};
NumpyAllocator g_numpyAllocator;
enum { ARG_NONE = 0, ARG_MAT = 1, ARG_SCALAR = 2 };
int init_numpy() {
import_array();
return 0;
}
const static int numpy_initialized = init_numpy();
int pyopencv_to(const PyObject* o, Mat& m, const char* name, bool allowND)
{
if(!o || o == Py_None)
{
if( !m.data )
m.allocator = &g_numpyAllocator;
return true;
}
if( !PyArray_Check(o) ) // this line makes error without message
{
failmsg("%s is not a numpy array", name);
return false;
}
// NPY_LONG (64 bit) is converted to CV_32S (32 bit)
int typenum = PyArray_TYPE(o);
int type = typenum == NPY_UBYTE ? CV_8U : typenum == NPY_BYTE ? CV_8S :
typenum == NPY_USHORT ? CV_16U : typenum == NPY_SHORT ? CV_16S :
typenum == NPY_INT || typenum == NPY_LONG ? CV_32S :
typenum == NPY_FLOAT ? CV_32F :
typenum == NPY_DOUBLE ? CV_64F : -1;
if( type < 0 )
{
failmsg("%s data type = %d is not supported", name, typenum);
return false;
}
int ndims = PyArray_NDIM(o);
if(ndims >= CV_MAX_DIM)
{
failmsg("%s dimensionality (=%d) is too high", name, ndims);
return false;
}
int size[CV_MAX_DIM+1];
size_t step[CV_MAX_DIM+1], elemsize = CV_ELEM_SIZE1(type);
const npy_intp* _sizes = PyArray_DIMS(o);
const npy_intp* _strides = PyArray_STRIDES(o);
bool transposed = false;
for(int i = 0; i < ndims; i++)
{
size[i] = (int)_sizes[i];
step[i] = (size_t)_strides[i];
}
if( ndims == 0 || step[ndims-1] > elemsize ) {
size[ndims] = 1;
step[ndims] = elemsize;
ndims++;
}
if( ndims >= 2 && step[0] < step[1] )
{
std::swap(size[0], size[1]);
std::swap(step[0], step[1]);
transposed = true;
}
if( ndims == 3 && size[2] <= CV_CN_MAX && step[1] == elemsize*size[2] )
{
ndims--;
type |= CV_MAKETYPE(0, size[2]);
}
if( ndims > 2 && !allowND )
{
failmsg("%s has more than 2 dimensions", name);
return false;
}
m = cv::Mat(ndims, size, type, PyArray_DATA(o), step);
if( m.data )
{
m.u->refcount = *refcountFromPyObject(o);
m.addref(); // protect the original numpy array from deallocation
// (since Mat destructor will decrement the reference counter)
};
m.allocator = &g_numpyAllocator;
if( transposed )
{
cv::Mat tmp;
tmp.allocator = &g_numpyAllocator;
transpose(m, tmp);
m = tmp;
}
return true;
}
PyObject* pyopencv_from(const Mat& m)
{
if( !m.data )
Py_RETURN_NONE;
Mat temp, *p = (Mat*)&m;
if(!(p->u->refcount) || p->allocator != &g_numpyAllocator)
{
temp.allocator = &g_numpyAllocator;
ERRWRAP2(m.copyTo(temp));
p = &temp;
}
p->addref();
return pyObjectFromRefcount(&(p->u->refcount));
}
threshold.cpp
#include <Python.h>
#include "opencv2/opencv.hpp"
#include "convert.hpp"
#include "numpy/ndarrayobject.h"
using namespace std;
using namespace cv;
double otsu_8u_with_mask(const Mat1b src, const Mat1b& mask)
{
const int N = 256;
int M = 0;
int i, j, h[N] = { 0 };
for (i = 0; i < src.rows; i++)
{
const uchar* psrc = src.ptr(i);
const uchar* pmask = mask.ptr(i);
for (j = 0; j < src.cols; j++)
{
if (pmask[j])
{
h[psrc[j]]++;
++M;
}
}
}
double mu = 0, scale = 1. / (M);
for (i = 0; i < N; i++)
mu += i * (double)h[i];
mu *= scale;
double mu1 = 0, q1 = 0;
double max_sigma = 0, max_val = 0;
for (i = 0; i < N; i++)
{
double p_i, q2, mu2, sigma;
p_i = h[i] * scale;
mu1 *= q1;
q1 += p_i;
q2 = 1. - q1;
if (std::min(q1, q2) < FLT_EPSILON || std::max(q1, q2) > 1. - FLT_EPSILON)
continue;
mu1 = (mu1 + i * p_i) / q1;
mu2 = (mu - q1 * mu1) / q2;
sigma = q1 * q2*(mu1 - mu2)*(mu1 - mu2);
if (sigma > max_sigma)
{
max_sigma = sigma;
max_val = i;
}
}
return max_val;
}
static PyObject * otsu_with_mask(PyObject *self, PyObject * args) {
PyObject pySrc, pyMask;
Mat src, mask;
import_array();
if (!PyArg_ParseTuple(args, "OO", &pySrc, &pyMask))
return NULL;
pyopencv_to(&pySrc, src, "source");
pyopencv_to(&pyMask, mask, "mask");
double thresh = otsu_8u_with_mask(src, mask);
return Py_BuildValue("i", thresh);
}
static PyMethodDef ThresholdMethods[] = {
{"otsu_with_mask", otsu_with_mask, METH_VARARGS, "Otsu thresholding with mask."},
{ NULL, NULL, 0, NULL}
};
static struct PyModuleDef thresholdModule = {
PyModuleDef_HEAD_INIT,
"customThreshold",
"Thresholding module.",
-1,
ThresholdMethods
};
PyMODINIT_FUNC PyInit_customThreshold(void) {
return PyModule_Create(&thresholdModule);
}
convert.hpp
#ifndef __CONVERT_HPP__
#define __CONVERT_HPP__
#include <Python.h>
#include "opencv2/opencv.hpp"
using namespace cv;
int pyopencv_to(const PyObject* o, Mat& m, const char* name = "<unknown>", bool allowND=true);
PyObject* pyopencv_from(const Mat& m);
#endif
Why do you choose to use C++ with Python wrap to do this simple task? I think you could achieve the same result easily using Python only...?
I assume you want to use adaptive thresholding method in OpenCV.
First of all, you can compute the adaptive threshold value of the input gray image. The value can be computed by the following function:
def compute_otsu_value(im_gray):
hist = cv2.calcHist([im_gray], [0], None, [256], [0, 256])
hist_norm = hist.ravel() / hist.max()
cum_sum_mat = hist_norm.cumsum()
fn_min = np.inf
thresh = -1
for i in xrange(1, 256):
p1, p2 = np.hsplit(hist_norm, [i])
q1, q2 = cum_sum_mat[i], cum_sum_mat[255] - cum_sum_mat[i]
if q1 == 0 or q2 == 0:
continue
b1, b2 = np.hsplit(np.arange(256), [i])
m1, m2 = np.sum(p1 * b1) / q1, np.sum(p2 * b2) / q2
v1, v2 = np.sum(((b1-m1)**2)*p1)/q1, np.sum(((b2-m2)**2)*p2)/q2
fn = v1 * q1 + v2 * q2
if fn < fn_min:
fn_min = fn
thresh = i
return thresh
Finally, in the main() function, you can load the input image as a gray image, and get the threshold image accordingly.
im_gray = cv2.imread("input.png", 0)
otsu_value = comput_otsu_values(im_gray)
im_th = cv2.threshold(im_gray, otsu_value, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)
I am using SWIG to build a Python module for some functions' evaluation based on their C code.
The main function I need is defined as follow:
void eval(double *x, int nx, int mx, double *f, int func_id)
And the aimed python function should be:
value_list = module.eval(point_matrix, func_id)
Here, eval will call a benchmark function and return their values. func_id is the id of function eval going to call, nx is the dimension of the function, and mx is the number of points which will be evaluated.
Actually, I did not clearly understand how SWIG pass values between typemaps (like, temp$argnum, why always using $argnum?). But by looking into the wrap code, I finished the typemap.i file:
%module cec17
%{
#include "cec17.h"
%}
%typemap(in) (double *x, int nx, int mx) (int count){
if (PyList_Check($input)) {
$3 = PyList_Size($input);
$2 = PyList_Size(PyList_GetItem($input, 0));
count = $3;
int i = 0, j = 0;
$1 = (double *) malloc($2*$3*sizeof(double));
for (i = 0; i < $3; i++){
for (j = 0; j < $2; j++){
PyObject *o = PyList_GetItem(PyList_GetItem($input, i), j);
if (PyFloat_Check(o))
$1[i*$2+j] = PyFloat_AsDouble(o);
else {
PyErr_SetString(PyExc_TypeError, "list must contrain strings");
free($1);
return NULL;
}
}
}
} else {
PyErr_SetString(PyExc_TypeError, "not a list");
return NULL;
}
}
%typemap(freearg) double *x {
free((void *) $1);
}
%typemap(in, numinputs=0) double *f (double temp) {
$1 = &temp;
}
%typemap(argout) double *f {
int i = 0;
int s = count1;
printf("pass arg %d", s);
$result = PyList_New(0);
for (i = 0; i < s; i++){
PyList_Append($result, PyFloat_FromDouble($1[i]));
}
}
void eval(double *x, int nx, int mx, double *f, int func_num);
However, strange things happened then. Usually, I test 30 dimensional functions. When evaluating less than 10 points (mx < 10), the module works fine. When evaluating more points, an error occurs:
[1] 13616 segmentation fault (core dumped) python test.py
I'm quite sure the problem is not in the c code, because the only place where 'mx' occurs is in the 'for-loop' line in which are evaluations of each point.
I also tried to read the wrap code and debug, but I just can't find where the problem is. Following is a part of the wrap code generated by SWIG, and I added a 'printf' line. Even this string is not printed before the error.
#ifdef __cplusplus
extern "C" {
#endif
SWIGINTERN PyObject *_wrap_eval(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
double *arg1 = (double *) 0 ;
int arg2 ;
int arg3 ;
double *arg4 = (double *) 0 ;
int arg5 ;
int count1 ;
double temp4 ;
int val5 ;
int ecode5 = 0 ;
PyObject * obj0 = 0 ;
PyObject * obj1 = 0 ;
printf("check point 0");
{
arg4 = &temp4;
}
if (!PyArg_ParseTuple(args,(char *)"OO:eval",&obj0,&obj1)) SWIG_fail;
{
if (PyList_Check(obj0)) {
arg3 = PyList_Size(obj0);
arg2 = PyList_Size(PyList_GetItem(obj0, 0));
count1 = arg3;
int i = 0, j = 0;
arg1 = (double *) malloc(arg2*arg3*sizeof(double));
for (i = 0; i < arg3; i++){
for (j = 0; j < arg2; j++){
PyObject *o = PyList_GetItem(PyList_GetItem(obj0, i), j);
if (PyFloat_Check(o))
arg1[i*arg2+j] = PyFloat_AsDouble(o);
else {
PyErr_SetString(PyExc_TypeError, "list must contrain strings");
free(arg1);
return NULL;
}
}
}
} else {
PyErr_SetString(PyExc_TypeError, "not a list");
return NULL;
}
}
ecode5 = SWIG_AsVal_int(obj1, &val5);
if (!SWIG_IsOK(ecode5)) {
SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "eval" "', argument " "5"" of type '" "int""'");
}
arg5 = (int)(val5);
eval(arg1,arg2,arg3,arg4,arg5);
resultobj = SWIG_Py_Void();
{
int i = 0;
int s = count1;
resultobj = PyList_New(0);
for (i = 0; i < s; i++){
PyList_Append(resultobj, PyFloat_FromDouble(arg4[i]));
}
}
return resultobj;
fail:
return NULL;
}
The problem seems a little tedious. Maybe you could just show me how to write the proper typemap.i code.
I'm not sure what your evaluation function is supposed to do, so I took a guess and implemented a wrapper for it. I took value_list = module.eval(point_matrix, func_id) to mean you want to return a list of result of evaluating some function against each row of data points, and came up with the following. Things I changed:
The typemaps replace the first four parameters with a Python list of lists of numbers.
space for the results in f was malloced.
To accept other numeric types except float, PyFloat_AsDouble was called on each value, and PyErr_Occurred was called to see if it failed to convert.
The freearg typemap now frees both allocations.
The argout typemap now handles the f output parameter correctly.
I added a sample eval implementation.
%module cec17
%typemap(in) (double *x, int nx, int mx, double* f) %{
if (PyList_Check($input)) {
$3 = PyList_Size($input);
$2 = PyList_Size(PyList_GetItem($input, 0));
$1 = malloc($2 * $3 * sizeof(double));
$4 = malloc($3 * sizeof(double));
for (int i = 0; i < $3; i++) {
for (int j = 0; j < $2; j++) {
PyObject *o = PyList_GetItem(PyList_GetItem($input, i), j);
double tmp = PyFloat_AsDouble(o);
if(PyErr_Occurred())
SWIG_fail;
$1[i * $2 + j] = PyFloat_AsDouble(o);
}
}
} else {
PyErr_SetString(PyExc_TypeError, "not a list");
return NULL;
}
%}
%typemap(freearg) (double *x, int nx, int mx, double* f) %{
free($1);
free($4);
%}
%typemap(argout) (double *x, int nx, int mx, double* f) (PyObject* tmp) %{
tmp = PyList_New($3);
for (int i = 0; i < $3; i++) {
PyList_SET_ITEM(tmp, i, PyFloat_FromDouble($4[i]));
}
$result = SWIG_Python_AppendOutput($result, tmp);
%}
%inline %{
void eval(double *x, int nx, int mx, double *f, int func_num)
{
for(int i = 0; i < mx; ++i) {
f[i] = 0.0;
for(int j = 0; j < nx; ++j)
f[i] += x[i*nx+j];
}
}
%}
Output:
>>> import cec17
>>> cec17.eval([[1,2,3],[4,5,6]],99)
[6.0, 15.0]
Error checking could be improved. For example, checking for sequences instead of lists. Only the outer list is checked that it actually is a list, so if [1,2,3] was the first parameter instead of nested lists, it won't behave properly. There is no check that all the sublists are the same size, either.
Hope this helps. Let me know if anything is unclear.
In Ubuntu 14.04, I wrote a C file called hash.c:
/* hash.c: hash table with linear probing */
typedef struct {
void *key;
void *value;
} ht_entry;
typedef struct {
ht_entry *table;
int len;
int num_entries;
int (*hash_fn)(void *key);
int (*key_cmp)(void *k1, void *k2);
} hashtable;
and compiled it with
gcc -shared hash.c -o test.so -fPIC
Afterwards, I tried to load test.so in a Python script (for testing), but I got the following error: "OSError: .../test.so: undefined symbol: hash_fn"
hash_fn is a function pointer in the hashtable struct. It is referenced a number of times by functions later in the file.
I do not understand why this error is happening. I have Googled but all other cases either concern C++ or includes. In my case I just have 1 C file that includes only stdio and stdlib.
here is the FULL code.
When I comment out all but hash_create and print_info, it loads succesfully. When I uncomment find(), it the error happens.
(print_info is just for testing that ctypes works)
/* hash.c: hash table with linear probing */
#include <stdio.h>
#include <stdlib.h>
typedef struct {
void *key;
void *value;
} ht_entry;
typedef struct {
ht_entry *table;
int len;
int num_entries;
int (*hash_fn)(void *key);
int (*key_cmp)(void *k1, void *k2);
} hashtable;
static void close_gap(hashtable *ht, int i);
static int find(hashtable *ht, void *key);
hashtable* hash_create(int len, int (*hash_fn)(void*), int (*key_cmp)(void*, void*))
{
hashtable* ht = (hashtable*) malloc(sizeof(hashtable));
ht->len = len;
ht->table = calloc(len, sizeof(ht_entry));
ht->hash_fn = hash_fn;
ht->key_cmp = key_cmp;
ht->table[0].key = 2;
ht->table[0].value = 3;
return ht;
}
void print_info(hashtable *ht)
{
printf("%d, %d, %d\n", ht->len, ht->table[0].key, ht->table[0].value);
}
void* hash_retrieve(hashtable* ht, void *key)
{
int i = find(ht, key);
if(i < 0) {
return NULL;
}
return ht->table[i].value;
}
void hash_insert(hashtable* ht, void *key, void *value)
{
if(ht->num_entries == ht->len) {
return;
}
int i = hash_fn(key) % ht->len;
while(ht->table[i].key != NULL) {
i = (i + i) % ht->len;
}
ht->table[i].key = key;
ht->table[i].value = value;
}
void hash_remove(hashtable *ht, void *key)
{
int i = find(ht, key);
if(i < 0) {
return;
}
ht->table[i].key = 0;
ht->table[i].value = 0;
close_gap(ht, i);
}
static int find(hashtable *ht, void *key)
{
int i = hash_fn(key) % ht->len;
int num_checked = 0;
while(ht->table[i].key && num_checked != ht->len) {
if(!ht->key_cmp(ht->table[i].key, key)) {
return i;
}
num_checked++;
i = (i + i) % ht->len;
}
return -1;
}
static void close_gap(hashtable *ht, int i)
{
int j = (i + 1) % ht->len;
while(ht->table[j].key) {
int loc = ht->hash_fn(ht->table[j].key);
if((j > i && (loc <= i || loc > j)) || (j < i && (loc <= i && loc > j))) {
ht->table[i] = ht->table[j];
ht->table[j].key = 0;
ht->table[j].value = 0;
close_gap(ht, j);
return;
}
}
}
When I use your compilation line I get five warnings. There are several problems here. First you are trying to assign an int to void * in several places. That raises a warning, and it would crash at runtime because you are passing 2 and 3 as addresses.
Second, you are calling hash_fn in a couple of places instead of ht->hash_fn. That causes the linker error, but you should consider my other changes, otherwise it will crash at runtime with a SIGSEGV:
/* hash.c: hash table with linear probing */
#include <stdio.h>
#include <stdlib.h>
typedef struct {
void *key;
void *value;
} ht_entry;
typedef struct {
ht_entry *table;
int len;
int num_entries;
int (*hash_fn)(void *key);
int (*key_cmp)(void *k1, void *k2);
} hashtable;
static void close_gap(hashtable *ht, int i);
static int find(hashtable *ht, void *key);
hashtable* hash_create(int len, int (*hash_fn)(void*), int (*key_cmp)(void*, void*))
{
hashtable* ht = (hashtable*) malloc(sizeof(hashtable));
ht->len = len;
ht->table = calloc(len, sizeof(ht_entry));
ht->hash_fn = hash_fn;
ht->key_cmp = key_cmp;
// <<< Code changed here
/*
ht->table[0].key = 2;
ht->table[0].value = 3;
*/
{
int *p = malloc(sizeof(int));
*p = 2;
ht->table[0].key = p;
p = malloc(sizeof(int));
*p = 3;
ht->table[0].value = p;
}
// end of code change
return ht;
}
void print_info(hashtable *ht)
{
// <<<< Code changed
printf("%d, %d, %d\n", ht->len,
*(int *)ht->table[0].key, *(int *)ht->table[0].value);
}
void* hash_retrieve(hashtable* ht, void *key)
{
int i = find(ht, key);
if(i < 0) {
return NULL;
}
return ht->table[i].value;
}
void hash_insert(hashtable* ht, void *key, void *value)
{
if(ht->num_entries == ht->len) {
return;
}
// <<< Code changed
int i = ht->hash_fn(key) % ht->len;
while(ht->table[i].key != NULL) {
i = (i + i) % ht->len;
}
ht->table[i].key = key;
ht->table[i].value = value;
}
void hash_remove(hashtable *ht, void *key)
{
int i = find(ht, key);
if(i < 0) {
return;
ht->table[i].key = 0;
ht->table[i].value = 0;
close_gap(ht, i);
}
static int find(hashtable *ht, void *key)
{
// <<< Code changed
int i = ht->hash_fn(key) % ht->len;
int num_checked = 0;
while(ht->table[i].key && num_checked != ht->len) {
if(!ht->key_cmp(ht->table[i].key, key)) {
return i;
}
num_checked++;
i = (i + i) % ht->len;
}
return -1;
}
static void close_gap(hashtable *ht, int i)
{
int j = (i + 1) % ht->len;
while(ht->table[j].key) {
int loc = ht->hash_fn(ht->table[j].key);
if((j > i && (loc <= i || loc > j)) || (j < i && (loc <= i && loc > j))) {
ht->table[i] = ht->table[j];
ht->table[j].key = 0;
ht->table[j].value = 0;
close_gap(ht, j);
return;
}
}
}
I only coded around the errors and warnings, I did not check the logic. You will see that I have used malloc to allocate memory for key and value. Obviously you will need memory management on these two (i.e. free()).
#include <stdio.h>
#include <Python/Python.h>
#include <string.h>
char *baseN(int num, char *LETTERS);
int myHash(char *s, char *LETTERS);
int indexOfString(char *s, char c);
char *lstrip(char *s, char strp);
void removeFirst(char *s);
static PyObject *ex_baseN(PyObject *self, PyObject *args) {
int num;
char *LETTERS;
if (!PyArg_ParseTuple(args, "is", &num, &LETTERS)) {
Py_RETURN_NONE;
}
char *result = baseN(num, LETTERS);
PyObject *retval = (PyObject *) Py_BuildValue("s", result);
return retval;
}
static PyObject *ex_myHash(PyObject *self, PyObject *args) {
char *s;
char *LETTERS;
if (!PyArg_ParseTuple(args, "ss", &s, &LETTERS)) {
Py_RETURN_NONE;
}
int result = myHash(s, LETTERS);
PyObject *retval = (PyObject *) Py_BuildValue("i", result);
return retval;
}
static PyMethodDef foo_methods[] = {
{"myHash", (PyCFunction) ex_myHash, METH_VARARGS},
{"baseN", (PyCFunction) ex_baseN, METH_VARARGS},
{NULL, NULL, 0, NULL}
};
PyMODINIT_FUNC initmyEx() {
Py_InitModule3("myEx", foo_methods, "My first extension module.");
}
char *baseN(int num, char *LETTERS) {
int len = strlen(LETTERS);
if (num == 0) {
char *result = (char *) malloc(sizeof(char));
sprintf(result, "%c", LETTERS[0]);
return result;
}
char *s = baseN(num / len, LETTERS);
lstrip(s, LETTERS[0]);
char *result = (char *) malloc(sizeof(char) * (strlen(s) + 1));
int result_len = strlen(s) + 1;
for (int i = 0; i < result_len; i++) {
if (i < result_len - 1) {
result[i] = s[i];
} else {
result[i] = LETTERS[num % len];
}
}
return result;
}
void removeFirst(char *s) {
int len = strlen(s);
for (int i = 0; i < len; i++) {
if (i < len - 1) {
s[i] = s[i + 1];
} else {
s[i] = '\0';
}
}
}
char *lstrip(char *s, char strp) {
int len = strlen(s);
if (len >= 0) {
if (s[0] == strp) {
removeFirst(s);
}
}
return s;
}
int myHash(char *s, char *LETTERS) {
int h = 7;
int len = strlen(s);
for (int i = 0; i < len; i++) {
int index = indexOfString(LETTERS, s[i]);
h = 37 * h + index;
}
return h;
}
int indexOfString(char *s, char c) {
int len = strlen(s);
for (int i = 0; i < len; i++) {
if (s[i] == c) {
return i;
}
}
return -1;
}
int main(){
char * result = baseN(10119, "abcdefg");
printf("%s\n",result);
return 0;
}
I wrote a python extension of the above, but after compiling, and running in the Python interpreter ipython, the result is:
In [4]: myEx.myHash('asdfg','ascfwdzxfxcg')
Out[4]: 485465319
In [5]: myEx.baseN(1000,'asdfghj')
Abort trap: 6
The function named baseN does not work, why????
There is a buffer overflow in your code:
char *baseN(int num, char *LETTERS) {
int len = strlen(LETTERS);
if (num == 0) {
char *result = (char *) malloc(sizeof(char)); // 1 char allocated
sprintf(result, "%c", LETTERS[0]); // 2 chars written including NUL char
return result;
}
The current sprintf needs:
char *result = malloc(2);