I have a program written in python with OpenCV. I want to add a feature which is otsu thresholding with mask. So, I get a code from here written in c++. I tried to convert it as python, but it's too slow (because of python). Finally, I make up my mind to use c++ with python. I try to embedding, and I find pyopencv_to() function. But, I can't use it because of PyArray_Check(). When program entered this function, die immediately. It doesn't give any error message. I guess it may be segmentation fault. Many stack overflow's answers says that "use import_array()". But it doesn't work for me.
Here is my code.
convert.cpp
#include <Python.h>
#include "numpy/ndarrayobject.h"
#include "opencv2/core/core.hpp"
#include "convert.hpp"
static PyObject* opencv_error = 0;
static int failmsg(const char *fmt, ...)
{
char str[1000];
va_list ap;
va_start(ap, fmt);
vsnprintf(str, sizeof(str), fmt, ap);
va_end(ap);
PyErr_SetString(PyExc_TypeError, str);
return 0;
}
class PyAllowThreads
{
public:
PyAllowThreads() : _state(PyEval_SaveThread()) {}
~PyAllowThreads()
{
PyEval_RestoreThread(_state);
}
private:
PyThreadState* _state;
};
class PyEnsureGIL
{
public:
PyEnsureGIL() : _state(PyGILState_Ensure()) {}
~PyEnsureGIL()
{
PyGILState_Release(_state);
}
private:
PyGILState_STATE _state;
};
#define ERRWRAP2(expr) \
try \
{ \
PyAllowThreads allowThreads; \
expr; \
} \
catch (const cv::Exception &e) \
{ \
PyErr_SetString(opencv_error, e.what()); \
return 0; \
}
using namespace cv;
static PyObject* failmsgp(const char *fmt, ...)
{
char str[1000];
va_list ap;
va_start(ap, fmt);
vsnprintf(str, sizeof(str), fmt, ap);
va_end(ap);
PyErr_SetString(PyExc_TypeError, str);
return 0;
}
static size_t REFCOUNT_OFFSET = (size_t)&(((PyObject*)0)->ob_refcnt) +
(0x12345678 != *(const size_t*)"\x78\x56\x34\x12\0\0\0\0\0")*sizeof(int);
static inline PyObject* pyObjectFromRefcount(const int* refcount)
{
return (PyObject*)((size_t)refcount - REFCOUNT_OFFSET);
}
static inline int* refcountFromPyObject(const PyObject* obj)
{
return (int*)((size_t)obj + REFCOUNT_OFFSET);
}
class NumpyAllocator : public MatAllocator
{
public:
NumpyAllocator() {}
~NumpyAllocator() {}
void allocate(int dims, const int* sizes, int type, int*& refcount,
uchar*& datastart, uchar*& data, size_t* step)
{
PyEnsureGIL gil;
int depth = CV_MAT_DEPTH(type);
int cn = CV_MAT_CN(type);
const int f = (int)(sizeof(size_t)/8);
int typenum = depth == CV_8U ? NPY_UBYTE : depth == CV_8S ? NPY_BYTE :
depth == CV_16U ? NPY_USHORT : depth == CV_16S ? NPY_SHORT :
depth == CV_32S ? NPY_INT : depth == CV_32F ? NPY_FLOAT :
depth == CV_64F ? NPY_DOUBLE : f*NPY_ULONGLONG + (f^1)*NPY_UINT;
int i;
npy_intp _sizes[CV_MAX_DIM+1];
for( i = 0; i < dims; i++ )
_sizes[i] = sizes[i];
if( cn > 1 )
{
/*if( _sizes[dims-1] == 1 )
_sizes[dims-1] = cn;
else*/
_sizes[dims++] = cn;
}
PyObject* o = PyArray_SimpleNew(dims, _sizes, typenum);
if(!o)
CV_Error_(CV_StsError, ("The numpy array of typenum=%d, ndims=%d can not be created", typenum, dims));
refcount = refcountFromPyObject(o);
npy_intp* _strides = PyArray_STRIDES(o);
for( i = 0; i < dims - (cn > 1); i++ )
step[i] = (size_t)_strides[i];
datastart = data = (uchar*)PyArray_DATA(o);
}
void deallocate(int* refcount, uchar*, uchar*)
{
PyEnsureGIL gil;
if( !refcount )
return;
PyObject* o = pyObjectFromRefcount(refcount);
Py_INCREF(o);
Py_DECREF(o);
}
};
NumpyAllocator g_numpyAllocator;
enum { ARG_NONE = 0, ARG_MAT = 1, ARG_SCALAR = 2 };
int init_numpy() {
import_array();
return 0;
}
const static int numpy_initialized = init_numpy();
int pyopencv_to(const PyObject* o, Mat& m, const char* name, bool allowND)
{
if(!o || o == Py_None)
{
if( !m.data )
m.allocator = &g_numpyAllocator;
return true;
}
if( !PyArray_Check(o) ) // this line makes error without message
{
failmsg("%s is not a numpy array", name);
return false;
}
// NPY_LONG (64 bit) is converted to CV_32S (32 bit)
int typenum = PyArray_TYPE(o);
int type = typenum == NPY_UBYTE ? CV_8U : typenum == NPY_BYTE ? CV_8S :
typenum == NPY_USHORT ? CV_16U : typenum == NPY_SHORT ? CV_16S :
typenum == NPY_INT || typenum == NPY_LONG ? CV_32S :
typenum == NPY_FLOAT ? CV_32F :
typenum == NPY_DOUBLE ? CV_64F : -1;
if( type < 0 )
{
failmsg("%s data type = %d is not supported", name, typenum);
return false;
}
int ndims = PyArray_NDIM(o);
if(ndims >= CV_MAX_DIM)
{
failmsg("%s dimensionality (=%d) is too high", name, ndims);
return false;
}
int size[CV_MAX_DIM+1];
size_t step[CV_MAX_DIM+1], elemsize = CV_ELEM_SIZE1(type);
const npy_intp* _sizes = PyArray_DIMS(o);
const npy_intp* _strides = PyArray_STRIDES(o);
bool transposed = false;
for(int i = 0; i < ndims; i++)
{
size[i] = (int)_sizes[i];
step[i] = (size_t)_strides[i];
}
if( ndims == 0 || step[ndims-1] > elemsize ) {
size[ndims] = 1;
step[ndims] = elemsize;
ndims++;
}
if( ndims >= 2 && step[0] < step[1] )
{
std::swap(size[0], size[1]);
std::swap(step[0], step[1]);
transposed = true;
}
if( ndims == 3 && size[2] <= CV_CN_MAX && step[1] == elemsize*size[2] )
{
ndims--;
type |= CV_MAKETYPE(0, size[2]);
}
if( ndims > 2 && !allowND )
{
failmsg("%s has more than 2 dimensions", name);
return false;
}
m = cv::Mat(ndims, size, type, PyArray_DATA(o), step);
if( m.data )
{
m.u->refcount = *refcountFromPyObject(o);
m.addref(); // protect the original numpy array from deallocation
// (since Mat destructor will decrement the reference counter)
};
m.allocator = &g_numpyAllocator;
if( transposed )
{
cv::Mat tmp;
tmp.allocator = &g_numpyAllocator;
transpose(m, tmp);
m = tmp;
}
return true;
}
PyObject* pyopencv_from(const Mat& m)
{
if( !m.data )
Py_RETURN_NONE;
Mat temp, *p = (Mat*)&m;
if(!(p->u->refcount) || p->allocator != &g_numpyAllocator)
{
temp.allocator = &g_numpyAllocator;
ERRWRAP2(m.copyTo(temp));
p = &temp;
}
p->addref();
return pyObjectFromRefcount(&(p->u->refcount));
}
threshold.cpp
#include <Python.h>
#include "opencv2/opencv.hpp"
#include "convert.hpp"
#include "numpy/ndarrayobject.h"
using namespace std;
using namespace cv;
double otsu_8u_with_mask(const Mat1b src, const Mat1b& mask)
{
const int N = 256;
int M = 0;
int i, j, h[N] = { 0 };
for (i = 0; i < src.rows; i++)
{
const uchar* psrc = src.ptr(i);
const uchar* pmask = mask.ptr(i);
for (j = 0; j < src.cols; j++)
{
if (pmask[j])
{
h[psrc[j]]++;
++M;
}
}
}
double mu = 0, scale = 1. / (M);
for (i = 0; i < N; i++)
mu += i * (double)h[i];
mu *= scale;
double mu1 = 0, q1 = 0;
double max_sigma = 0, max_val = 0;
for (i = 0; i < N; i++)
{
double p_i, q2, mu2, sigma;
p_i = h[i] * scale;
mu1 *= q1;
q1 += p_i;
q2 = 1. - q1;
if (std::min(q1, q2) < FLT_EPSILON || std::max(q1, q2) > 1. - FLT_EPSILON)
continue;
mu1 = (mu1 + i * p_i) / q1;
mu2 = (mu - q1 * mu1) / q2;
sigma = q1 * q2*(mu1 - mu2)*(mu1 - mu2);
if (sigma > max_sigma)
{
max_sigma = sigma;
max_val = i;
}
}
return max_val;
}
static PyObject * otsu_with_mask(PyObject *self, PyObject * args) {
PyObject pySrc, pyMask;
Mat src, mask;
import_array();
if (!PyArg_ParseTuple(args, "OO", &pySrc, &pyMask))
return NULL;
pyopencv_to(&pySrc, src, "source");
pyopencv_to(&pyMask, mask, "mask");
double thresh = otsu_8u_with_mask(src, mask);
return Py_BuildValue("i", thresh);
}
static PyMethodDef ThresholdMethods[] = {
{"otsu_with_mask", otsu_with_mask, METH_VARARGS, "Otsu thresholding with mask."},
{ NULL, NULL, 0, NULL}
};
static struct PyModuleDef thresholdModule = {
PyModuleDef_HEAD_INIT,
"customThreshold",
"Thresholding module.",
-1,
ThresholdMethods
};
PyMODINIT_FUNC PyInit_customThreshold(void) {
return PyModule_Create(&thresholdModule);
}
convert.hpp
#ifndef __CONVERT_HPP__
#define __CONVERT_HPP__
#include <Python.h>
#include "opencv2/opencv.hpp"
using namespace cv;
int pyopencv_to(const PyObject* o, Mat& m, const char* name = "<unknown>", bool allowND=true);
PyObject* pyopencv_from(const Mat& m);
#endif
Why do you choose to use C++ with Python wrap to do this simple task? I think you could achieve the same result easily using Python only...?
I assume you want to use adaptive thresholding method in OpenCV.
First of all, you can compute the adaptive threshold value of the input gray image. The value can be computed by the following function:
def compute_otsu_value(im_gray):
hist = cv2.calcHist([im_gray], [0], None, [256], [0, 256])
hist_norm = hist.ravel() / hist.max()
cum_sum_mat = hist_norm.cumsum()
fn_min = np.inf
thresh = -1
for i in xrange(1, 256):
p1, p2 = np.hsplit(hist_norm, [i])
q1, q2 = cum_sum_mat[i], cum_sum_mat[255] - cum_sum_mat[i]
if q1 == 0 or q2 == 0:
continue
b1, b2 = np.hsplit(np.arange(256), [i])
m1, m2 = np.sum(p1 * b1) / q1, np.sum(p2 * b2) / q2
v1, v2 = np.sum(((b1-m1)**2)*p1)/q1, np.sum(((b2-m2)**2)*p2)/q2
fn = v1 * q1 + v2 * q2
if fn < fn_min:
fn_min = fn
thresh = i
return thresh
Finally, in the main() function, you can load the input image as a gray image, and get the threshold image accordingly.
im_gray = cv2.imread("input.png", 0)
otsu_value = comput_otsu_values(im_gray)
im_th = cv2.threshold(im_gray, otsu_value, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)
Related
I have a sparse matrix implementation in C++, and I used pybind11 to expose it to python. Here is the problem:
>>> D1 = phc.SparseMatrix(3, [[0],[1],[2]])
>>> D1.cData
[[0], [1], [2]]
>>> D1.cData[1] = [1,2]
>>> D1.cData
[[0], [1], [2]] #Should be [[0], [1,2], [2]]
In python, I cannot change the contents of the SparseMatrix.cData attribute with the assignment operator. I can change the entire list with D1.cData = [[1],[2],[3]]. This behavior is bewildering to me. D1.cData is just a list, so I would expect that the above code would work.
I suspect it has something to do with my pybind11 code since this behavior is not present in python-defined custom classes. But I have no idea what is wrong (I am a novice programmer). Here is the source code info:
Python Bindings
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
namespace py = pybind11;
#include <SparseMatrix.h>
namespace phc = ph_computation;
using SparseMatrix = phc::SparseMatrix;
using Column = phc::Column;
using CData = phc::CData;
PYBIND11_MODULE(ph_computations, m)
{
m.doc() = "ph_computations python bindings";
using namespace pybind11::literals;
m.def("add_cols", &phc::add_cols);//begin SparseMatrix.h
py::class_<SparseMatrix>(m, "SparseMatrix")
.def(py::init<size_t, CData>())
.def(py::init<std::string>())
.def_readwrite("n_rows", &SparseMatrix::n_rows)
.def_readwrite("n_cols", &SparseMatrix::n_cols)
.def_readwrite("cData", &SparseMatrix::cData)
.def("__neq__", &SparseMatrix::operator!=)
.def("__eq__", &SparseMatrix::operator==)
.def("__add__", &SparseMatrix::operator+)
.def("__mul__", &SparseMatrix::operator*)
.def("transpose", &SparseMatrix::transpose)
.def("__str__", &SparseMatrix::print)
.def("save", &SparseMatrix::save)
;
m.def("identity", &phc::make_identity);
m.def("matching_pivots", &phc::matching_pivots);//end SparseMatrix.h
}
SparseMatrix.h
#pragma once
#include <iostream>
#include <sstream>
#include <fstream>
#include <string>
#include <iterator>
#include <algorithm>
#include <vector>
#include <stdexcept>
namespace ph_computation{
using Int = int;
using Column = std::vector<Int>;//a Column is represented by a vector of indices
using CData = std::vector<Column>;//a matrix is represented by a vector of Columns
//Add columns in Z2
Column add_cols(const Column& c1, const Column& c2);
struct SparseMatrix
{
size_t n_rows{0};
size_t n_cols{0};
CData cData;
SparseMatrix()=default;
SparseMatrix(size_t n_rows_, CData cData_):
n_rows(n_rows_), n_cols(cData_.size()), cData(cData_){}
SparseMatrix(std::string path);
bool operator!=(const SparseMatrix &other) const;
bool operator==(const SparseMatrix &other) const;
SparseMatrix operator+(const SparseMatrix &other) const;
SparseMatrix operator*(const SparseMatrix &other) const;
void transpose();
void print() const;
void save(std::string path);
};
SparseMatrix make_identity(size_t n_cols_);
bool matching_pivots(const SparseMatrix& a, const SparseMatrix& b);
}
SparseMatrix.cpp (you probably don't need this)
#include <SparseMatrix.h>
namespace ph_computation {
Column add_cols(const Column& c1, const Column& c2){
Column c3;
int idx1{0};
int idx2{0};
while(idx1 < c1.size() && idx2 < c2.size()){
if(c1[idx1] < c2[idx2]){
c3.push_back(c1[idx1]);
++idx1;
}
else if(c1[idx1] > c2[idx2]){
c3.push_back(c2[idx2]);
++idx2;
}
else {
++idx1;
++idx2;
}
}
if (idx1 < c1.size()){
c3.insert(c3.end(), std::next(c1.begin(), idx1), c1.end());
}
else if (idx2 < c2.size()){
c3.insert(c3.end(), std::next(c2.begin(), idx2), c2.end());
}
return c3;
}
SparseMatrix make_identity(size_t n_cols_){
CData cData_(n_cols_);
for (int j = 0; j < n_cols_; ++j){
cData_[j] = {j};
}
return SparseMatrix(n_cols_, cData_);
}
SparseMatrix::SparseMatrix(std::string path){
std::fstream f_in;
f_in.open(path, std::ios::in);
if(f_in.is_open()){
std::string n_rows_line;
std::getline(f_in, n_rows_line); //first line of file contains number of rows
n_rows = std::stoi(n_rows_line);
std::string n_cols_line;
std::getline(f_in, n_cols_line); //second line of file contains number of cols
n_cols = std::stoi(n_cols_line);
CData cData_(n_cols);
cData = cData_;
std::string line;
int j = 0;
int nnz, data;
while (std::getline(f_in, line)){
std::stringstream line_str = std::stringstream(line);
while (line_str >> nnz){
Column col_j(nnz);
for (int i =0; i < nnz; ++i){
line_str >> data;
col_j[i] = data;
}
cData[j] = col_j;
}
++j;
}
f_in.close();
}
else{
throw std::runtime_error("File did not open.");
}
}
bool SparseMatrix::operator!=(const SparseMatrix &other) const{
if (n_rows != other.n_rows || cData != other.cData){
return true;
}
return false;
}
bool SparseMatrix::operator==(const SparseMatrix &other) const{
return !(*this != other);
}
SparseMatrix SparseMatrix::operator+(const SparseMatrix &other) const{
if (n_rows != other.n_rows || n_cols != other.n_cols){
throw std::invalid_argument("Matrices must have same dimension to add.");
}
CData ans_cData;
for (int j = 0; j < n_cols; ++j){
ans_cData.push_back(add_cols(cData[j], other.cData[j]));
}
return SparseMatrix(n_rows, ans_cData);
}
SparseMatrix SparseMatrix::operator*(const SparseMatrix &other) const{
if(n_cols != other.n_rows){
throw std::invalid_argument("Matrices must have compatible dimensions.");
}
size_t ans_rows = n_rows;
CData ans_cData(other.n_cols);
SparseMatrix ans(ans_rows, ans_cData);
for(int j =0; j<ans.n_cols; ++j){
for(int idx : other.cData[j]){
ans.cData[j] = add_cols(ans.cData[j], cData[idx]);
}
}
return ans;
}
void SparseMatrix::transpose(){
CData cData_T(n_rows);
for(int j =0; j<n_cols; ++j){
if(!cData[j].empty()){
for(int x: cData[j]){
cData_T[x].push_back(j);
}
}
}
cData = cData_T;
n_rows = n_cols;
n_cols = cData.size();
}
void SparseMatrix::print() const{
for (int i = 0; i < n_rows; ++i){
for (int j = 0; j < n_cols; ++j){
if (cData[j].empty())
{std::cout << " 0";}
else if (std::binary_search(cData[j].begin(), cData[j].end(), i))//Assumes row indices
{std::cout << " 1";} //are ordered
else
{std::cout << " 0";}
if (n_cols-1 == j)
{std::cout << " \n";}
}
}
}
void SparseMatrix::save(std::string path){
std::fstream f_out;
f_out.open(path, std::ios::out);
if(f_out.is_open()){
f_out << n_rows << "\n";
f_out << n_cols << "\n";
for(int j = 0; j < n_cols; ++j){
int col_j_sz = cData[j].size();
f_out << col_j_sz;
for(int i = 0; i < col_j_sz; ++i){
f_out << " " << cData[j][i];
}
f_out << "\n";
}
f_out.close();
}
else{
throw std::runtime_error("File did not open.");
}
}
bool matching_pivots(const SparseMatrix& a, const SparseMatrix& b){
if(a.n_rows != b.n_rows || a.n_cols != b.n_cols){
throw std::invalid_argument("Input matrices must have the same size.");
}
for (int j = 0; j<a.n_cols; ++j){
bool a_j_empty = a.cData[j].empty();
bool b_j_empty = b.cData[j].empty();
if (a_j_empty != b_j_empty){
return false;
}
else if (!a_j_empty){
if(a.cData[j].back() != b.cData[j].back()){
return false;
}
}
}
return true;
}
} // namespace ph_computation
I found the answer in the pybind11 documentation here: https://pybind11.readthedocs.io/en/stable/advanced/cast/stl.html?highlight=opaque#making-opaque-types
Apparently, STL container data members can be overwritten in their entirety in python, but modification of the data member through list methods does not work. I don't really understand it, but the link above answers the question.
I have already looked for instructions to do this. I have used import_Array(). I am using Python 3.10 I have the Windows Visual Studio.
this is my code:
PyObject* switch_rut(PyObject* self, PyObject* args) {
PyArrayObject* inp1;
PyArrayObject* inp2;
PyObject* outp;
double swv;
double* res;
if (!PyArg_ParseTuple(args, "O!O!d:switcher", &PyArray_Type, &inp1, &PyArray_Type, &inp2, &swv)) {
return NULL;
}
if (inp1->nd != 1 || inp1->descr->type_num != PyArray_DOUBLE || inp2->nd != 1 || inp2->descr->type_num != PyArray_DOUBLE || ) {
PyErr_SetString(PyExc_ValueError,
"arrays must be one-dimensional and of type float");
return NULL;
}
if (inp1->dimensions[0] != inp2->dimensions[0]) {
PyErr_SetString(PyExc_ValueError,
"arrays must be of equal length");
return NULL;
}
int* cnt = new int[1];
int nd = 1;
cnt[0] = inp1->dimensions[0];
int acnt = inp1->dimensions[0];
res = new double[acnt];
double ipar1, ipar2, iparr;
for (int i = 0; i < acnt; i++) {
ipar1 = *(double*)(inp1->data + i * inp1->strides[0]);
ipar2 = *(double*)(inp2->data + i * inp2->strides[0]);
iparr = ipar2 * swv + ipar1 * (1 - swv);
res[i] = iparr;
}
outp = PyArray_SimpleNewFromData(nd, cnt, NPY_DOUBLE,reinterpret_cast<void*> res);
return outp;
}
What I was talking about was how to embed Python in C or C++ with Numpy arrays
and how to return a value as a Numpy array.
This is what I used.
PyArray_New(&PyArray_Type, nd, bcnt, NPY_DOUBLE, NULL, (void*) res, 0, NPY_ARRAY_CARRAY, NULL);
I want to reproduce "Conv1D" results of pytorch in C code.
I tried to implement "Conv1D" using three methods (C code, Python, Pytorch), but the results are different. Only seven fraction digits are reasonable. Assuming there are multiple layers of conv1d in the structure, the fraction digits accuracy will gradually decrease.
According to everyone's recommend, I tried to change the C code type of input data to double but the result is still incorrect.
Have I done something wrong?
For example:
The output of Pytorch: 0.2380688339471817017
The output of Python: 0.2380688637495040894
The output of C code (float): 0.2380688637
The output of C code (double): 0.238068885344539680
Here is my current implementation
Input:
input dim. = 80, output dim. = 128, kernel size = 5
Pytorch: Conv1D_input.npy, Conv1D_weight.npy
Python: Conv1D_input.npy, Conv1D_weight.npy (same as Pytorch)
C code: Conv1D_input.txt, Conv1D_weight.txt (convert from Pytorch, IEEE 754 single precision)
Pytorch
import torch
import numpy as np
from torch import nn
from torch.autograd import Variable
import torch.nn.functional as F
import argparse
import sys
import io
import time
import os
class RNN(nn.Module):
def __init__(self, input_size, hidden_size):
super(RNN, self).__init__()
self.input_size = input_size
self.hidden_size = hidden_size
self.c1 = nn.Conv1d(input_size, hidden_size, kernel_size = 5, bias=False)
self.c1.weight = torch.nn.Parameter(torch.Tensor(np.load("CONV1D_WEIGHT.npy")))
def forward(self, inputs):
c = self.c1(inputs)
return c
input_size = 80
hidden_size = 128
kernel_size = 5
rnn = RNN(input_size, hidden_size)
inputs = torch.nn.Parameter(torch.Tensor(np.load("CONV1D_IN.npy")))
print("inputs", inputs)
outputs = rnn(inputs)
sub_np456 = outputs[0].cpu().detach().numpy()
np.savetxt("Pytorch_CONV1D_OUTPUT.txt", sub_np456)
print('outputs', outputs)
Python
import struct
import numpy as np
if __name__ == "__main__":
row = 80
col = 327
count = 0
res_out_dim = 128
in_dim = 80
kernel_size = 5
filter = np.zeros((80, 5), dtype = np.float32)
featureMaps = np.zeros((128, 323), dtype = np.float32)
spectrum = np.load("CONV1D_INPUT.npy")
weight = np.load("CONV1D_WEIGHT.npy")
spectrum_2d = spectrum.reshape(80, 327)
for i in range(res_out_dim):
for j in range(in_dim):
for k in range(kernel_size):
filter[j][k] = weight[i][j][k]
while count < (col-kernel_size+1):
for j in range(in_dim):
for k in range(count, kernel_size+count):
featureMaps[i][count] = featureMaps[i][count] + spectrum_2d[j][k]*filter[j][k-count]
count = count + 1
count = 0
np.savetxt("Python_CONV1D_OUTPUT.txt", featureMaps)
C code (float)
#include<stdio.h>
#include<stdlib.h>
#include<math.h>
#include<time.h>
const char CONV1D_WEIGHT[] = "CONV1D_WEIGHT.txt";
const char CONV1D_INPUT[] = "CONV1D_INPUT.txt";
void parameterFree(float **matrix, int row)
{
int i = 0;
for(i=0; i<row; i++)
free(matrix[i]);
free(matrix);
}
float** createMatrix_2D(int row, int col)
{
int i = 0;
float **matrix = NULL;
matrix = (float**)malloc(sizeof(float*) * row);
if(matrix == NULL)
printf("Matrix2D malloc failed\n");
for(i=0; i<row; i++)
{
matrix[i] = (float*)malloc(sizeof(float) * col);
if(matrix[i] == NULL)
printf("Matrix2D malloc failed\n");
}
return matrix;
}
float** conv_1D(const char weightFile[], float **source, int *row, int *col, int in_dim, int res_out_dim, int kernel_size)
{
float **filter = createMatrix_2D(in_dim, kernel_size);
float **featureMaps = createMatrix_2D(res_out_dim, *col-kernel_size+1);
int i = 0, j = 0, k = 0, count = 0;
char str[10];
float data = 0.0;
FILE *fp = fopen(weightFile, "r");
if(fp == NULL)
printf("Resnet file open failed\n");
else
{
/*initial featureMaps*/
for(i=0; i<res_out_dim; i++)
{
for(j=0; j<*col-kernel_size+1; j++)
{
featureMaps[i][j] = 0.0;
}
}
/*next filter*/
for(i=0; i<res_out_dim; i++)
{
/*read filter*/
for(j=0; j<in_dim; j++)
{
for(k=0; k<kernel_size; k++)
{
fscanf(fp, "%s", str);
sscanf(str, "%x", &data);
filter[j][k] = data;
}
}
/* (part of source * filter) */
while(count < *col-kernel_size+1)
{
for(j=0; j<in_dim; j++)
{
for(k=count; k<kernel_size+count; k++)
{
featureMaps[i][count] += source[j][k]*filter[j][k-count];
}
}
count++;
}
count = 0;
}
fclose(fp);
}
parameterFree(source, *row);
parameterFree(filter, in_dim);
*row = res_out_dim;
*col = *col-kernel_size+1;
return featureMaps;
}
int main()
{
int row = 80;
int col = 327;
int in_dim = 80;
int res_out_dim = 128;
int kernel_size = 5;
int i, j;
float data;
char str[10];
float **input = createMatrix_2D(row, col);
FILE *fp = fopen(CONV1D_INPUT, "r");
FILE *fp2 = fopen("C code_CONV1D_OUTPUT.txt", "w");
if(fp == NULL)
printf("File open failed\n");
else
{
for(i=0; i<row; i++)
{
for(j=0; j<col; j++)
{
fscanf(fp, "%s", str);
sscanf(str, "%x", &data);
input[i][j] = data;
}
}
}
float **CONV1D_ANS = conv_1D(CONV1D_WEIGHT, input, &row, &col, in_dim, res_out_dim, kernel_size);
for(i=0; i<row; i++)
{
for(j=0; j<col; j++)
{
fprintf(fp2, "[%.12f] ", CONV1D_ANS[i][j]);
}
fprintf(fp2, "\n");
}
return 0;
}
C code (double)
#include<stdio.h>
#include<stdlib.h>
#include<math.h>
#include<time.h>
const char CONV1D_WEIGHT[] = "CONV1D_WEIGHT.txt";
const char CONV1D_INPUT[] = "CONV1D_INPUT.txt";
void parameterFree(double **matrix, int row)
{
int i = 0;
for(i=0; i<row; i++)
free(matrix[i]);
free(matrix);
}
double** createMatrix_2D(int row, int col)
{
int i = 0;
double **matrix = NULL;
matrix = (double**)malloc(sizeof(double*) * row);
if(matrix == NULL)
printf("Matrix2D malloc failed\n");
for(i=0; i<row; i++)
{
matrix[i] = (double*)malloc(sizeof(double) * col);
if(matrix[i] == NULL)
printf("Matrix2D malloc failed\n");
}
return matrix;
}
double** conv_1D(const char weightFile[], double **source, int *row, int *col, int in_dim, int res_out_dim, int kernel_size)
{
double **filter = createMatrix_2D(in_dim, kernel_size);
double **featureMaps = createMatrix_2D(res_out_dim, *col-kernel_size+1);
int i = 0, j = 0, k = 0, count = 0;
char str[10];
float data = 0.0;
FILE *fp = fopen(weightFile, "r");
if(fp == NULL)
printf("Resnet file open failed\n");
else
{
/*initial featureMaps*/
for(i=0; i<res_out_dim; i++)
{
for(j=0; j<*col-kernel_size+1; j++)
{
featureMaps[i][j] = 0.0;
}
}
/*next filter*/
for(i=0; i<res_out_dim; i++)
{
/*read filter*/
for(j=0; j<in_dim; j++)
{
for(k=0; k<kernel_size; k++)
{
fscanf(fp, "%s", str);
sscanf(str, "%x", &data);
filter[j][k] = (double)data;
}
}
/* (part of source * filter) */
while(count < *col-kernel_size+1)
{
for(j=0; j<in_dim; j++)
{
for(k=count; k<kernel_size+count; k++)
{
featureMaps[i][count] += source[j][k]*filter[j][k-count];
}
}
count++;
}
count = 0;
}
fclose(fp);
}
parameterFree(source, *row);
parameterFree(filter, in_dim);
*row = res_out_dim;
*col = *col-kernel_size+1;
return featureMaps;
}
int main()
{
int row = 80;
int col = 327;
int in_dim = 80;
int res_out_dim = 128;
int kernel_size = 5;
int i, j;
float data;
char str[10];
double **input = createMatrix_2D(row, col);
FILE *fp = fopen(CONV1D_INPUT, "r");
FILE *fp2 = fopen("C code_CONV1D_OUTPUT.txt", "w");
if(fp == NULL)
printf("File open failed\n");
else
{
for(i=0; i<row; i++)
{
for(j=0; j<col; j++)
{
fscanf(fp, "%s", str);
sscanf(str, "%x", &data);
input[i][j] = (double)data;
}
}
}
double **CONV1D_ANS = conv_1D(CONV1D_WEIGHT, input, &row, &col, in_dim, res_out_dim, kernel_size);
for(i=0; i<row; i++)
{
for(j=0; j<col; j++)
{
fprintf(fp2, "[%.18f] ", CONV1D_ANS[i][j]);
}
fprintf(fp2, "\n");
}
return 0;
}
Floating point numbers are not precise (by design). Depending on in which order operations are performed, the results might vary. Even worse, some formulas are straight numerical unstable, whereas another one for the same analytical expression can be stable.
Compilers often rearange statements as an optimization measure. Convolution is an operation which contains notoriously many operations and loops. So unless you directly compare executed bytecode, this speculation is kind of pointless.
In Ubuntu 14.04, I wrote a C file called hash.c:
/* hash.c: hash table with linear probing */
typedef struct {
void *key;
void *value;
} ht_entry;
typedef struct {
ht_entry *table;
int len;
int num_entries;
int (*hash_fn)(void *key);
int (*key_cmp)(void *k1, void *k2);
} hashtable;
and compiled it with
gcc -shared hash.c -o test.so -fPIC
Afterwards, I tried to load test.so in a Python script (for testing), but I got the following error: "OSError: .../test.so: undefined symbol: hash_fn"
hash_fn is a function pointer in the hashtable struct. It is referenced a number of times by functions later in the file.
I do not understand why this error is happening. I have Googled but all other cases either concern C++ or includes. In my case I just have 1 C file that includes only stdio and stdlib.
here is the FULL code.
When I comment out all but hash_create and print_info, it loads succesfully. When I uncomment find(), it the error happens.
(print_info is just for testing that ctypes works)
/* hash.c: hash table with linear probing */
#include <stdio.h>
#include <stdlib.h>
typedef struct {
void *key;
void *value;
} ht_entry;
typedef struct {
ht_entry *table;
int len;
int num_entries;
int (*hash_fn)(void *key);
int (*key_cmp)(void *k1, void *k2);
} hashtable;
static void close_gap(hashtable *ht, int i);
static int find(hashtable *ht, void *key);
hashtable* hash_create(int len, int (*hash_fn)(void*), int (*key_cmp)(void*, void*))
{
hashtable* ht = (hashtable*) malloc(sizeof(hashtable));
ht->len = len;
ht->table = calloc(len, sizeof(ht_entry));
ht->hash_fn = hash_fn;
ht->key_cmp = key_cmp;
ht->table[0].key = 2;
ht->table[0].value = 3;
return ht;
}
void print_info(hashtable *ht)
{
printf("%d, %d, %d\n", ht->len, ht->table[0].key, ht->table[0].value);
}
void* hash_retrieve(hashtable* ht, void *key)
{
int i = find(ht, key);
if(i < 0) {
return NULL;
}
return ht->table[i].value;
}
void hash_insert(hashtable* ht, void *key, void *value)
{
if(ht->num_entries == ht->len) {
return;
}
int i = hash_fn(key) % ht->len;
while(ht->table[i].key != NULL) {
i = (i + i) % ht->len;
}
ht->table[i].key = key;
ht->table[i].value = value;
}
void hash_remove(hashtable *ht, void *key)
{
int i = find(ht, key);
if(i < 0) {
return;
}
ht->table[i].key = 0;
ht->table[i].value = 0;
close_gap(ht, i);
}
static int find(hashtable *ht, void *key)
{
int i = hash_fn(key) % ht->len;
int num_checked = 0;
while(ht->table[i].key && num_checked != ht->len) {
if(!ht->key_cmp(ht->table[i].key, key)) {
return i;
}
num_checked++;
i = (i + i) % ht->len;
}
return -1;
}
static void close_gap(hashtable *ht, int i)
{
int j = (i + 1) % ht->len;
while(ht->table[j].key) {
int loc = ht->hash_fn(ht->table[j].key);
if((j > i && (loc <= i || loc > j)) || (j < i && (loc <= i && loc > j))) {
ht->table[i] = ht->table[j];
ht->table[j].key = 0;
ht->table[j].value = 0;
close_gap(ht, j);
return;
}
}
}
When I use your compilation line I get five warnings. There are several problems here. First you are trying to assign an int to void * in several places. That raises a warning, and it would crash at runtime because you are passing 2 and 3 as addresses.
Second, you are calling hash_fn in a couple of places instead of ht->hash_fn. That causes the linker error, but you should consider my other changes, otherwise it will crash at runtime with a SIGSEGV:
/* hash.c: hash table with linear probing */
#include <stdio.h>
#include <stdlib.h>
typedef struct {
void *key;
void *value;
} ht_entry;
typedef struct {
ht_entry *table;
int len;
int num_entries;
int (*hash_fn)(void *key);
int (*key_cmp)(void *k1, void *k2);
} hashtable;
static void close_gap(hashtable *ht, int i);
static int find(hashtable *ht, void *key);
hashtable* hash_create(int len, int (*hash_fn)(void*), int (*key_cmp)(void*, void*))
{
hashtable* ht = (hashtable*) malloc(sizeof(hashtable));
ht->len = len;
ht->table = calloc(len, sizeof(ht_entry));
ht->hash_fn = hash_fn;
ht->key_cmp = key_cmp;
// <<< Code changed here
/*
ht->table[0].key = 2;
ht->table[0].value = 3;
*/
{
int *p = malloc(sizeof(int));
*p = 2;
ht->table[0].key = p;
p = malloc(sizeof(int));
*p = 3;
ht->table[0].value = p;
}
// end of code change
return ht;
}
void print_info(hashtable *ht)
{
// <<<< Code changed
printf("%d, %d, %d\n", ht->len,
*(int *)ht->table[0].key, *(int *)ht->table[0].value);
}
void* hash_retrieve(hashtable* ht, void *key)
{
int i = find(ht, key);
if(i < 0) {
return NULL;
}
return ht->table[i].value;
}
void hash_insert(hashtable* ht, void *key, void *value)
{
if(ht->num_entries == ht->len) {
return;
}
// <<< Code changed
int i = ht->hash_fn(key) % ht->len;
while(ht->table[i].key != NULL) {
i = (i + i) % ht->len;
}
ht->table[i].key = key;
ht->table[i].value = value;
}
void hash_remove(hashtable *ht, void *key)
{
int i = find(ht, key);
if(i < 0) {
return;
ht->table[i].key = 0;
ht->table[i].value = 0;
close_gap(ht, i);
}
static int find(hashtable *ht, void *key)
{
// <<< Code changed
int i = ht->hash_fn(key) % ht->len;
int num_checked = 0;
while(ht->table[i].key && num_checked != ht->len) {
if(!ht->key_cmp(ht->table[i].key, key)) {
return i;
}
num_checked++;
i = (i + i) % ht->len;
}
return -1;
}
static void close_gap(hashtable *ht, int i)
{
int j = (i + 1) % ht->len;
while(ht->table[j].key) {
int loc = ht->hash_fn(ht->table[j].key);
if((j > i && (loc <= i || loc > j)) || (j < i && (loc <= i && loc > j))) {
ht->table[i] = ht->table[j];
ht->table[j].key = 0;
ht->table[j].value = 0;
close_gap(ht, j);
return;
}
}
}
I only coded around the errors and warnings, I did not check the logic. You will see that I have used malloc to allocate memory for key and value. Obviously you will need memory management on these two (i.e. free()).
#include <stdio.h>
#include <Python/Python.h>
#include <string.h>
char *baseN(int num, char *LETTERS);
int myHash(char *s, char *LETTERS);
int indexOfString(char *s, char c);
char *lstrip(char *s, char strp);
void removeFirst(char *s);
static PyObject *ex_baseN(PyObject *self, PyObject *args) {
int num;
char *LETTERS;
if (!PyArg_ParseTuple(args, "is", &num, &LETTERS)) {
Py_RETURN_NONE;
}
char *result = baseN(num, LETTERS);
PyObject *retval = (PyObject *) Py_BuildValue("s", result);
return retval;
}
static PyObject *ex_myHash(PyObject *self, PyObject *args) {
char *s;
char *LETTERS;
if (!PyArg_ParseTuple(args, "ss", &s, &LETTERS)) {
Py_RETURN_NONE;
}
int result = myHash(s, LETTERS);
PyObject *retval = (PyObject *) Py_BuildValue("i", result);
return retval;
}
static PyMethodDef foo_methods[] = {
{"myHash", (PyCFunction) ex_myHash, METH_VARARGS},
{"baseN", (PyCFunction) ex_baseN, METH_VARARGS},
{NULL, NULL, 0, NULL}
};
PyMODINIT_FUNC initmyEx() {
Py_InitModule3("myEx", foo_methods, "My first extension module.");
}
char *baseN(int num, char *LETTERS) {
int len = strlen(LETTERS);
if (num == 0) {
char *result = (char *) malloc(sizeof(char));
sprintf(result, "%c", LETTERS[0]);
return result;
}
char *s = baseN(num / len, LETTERS);
lstrip(s, LETTERS[0]);
char *result = (char *) malloc(sizeof(char) * (strlen(s) + 1));
int result_len = strlen(s) + 1;
for (int i = 0; i < result_len; i++) {
if (i < result_len - 1) {
result[i] = s[i];
} else {
result[i] = LETTERS[num % len];
}
}
return result;
}
void removeFirst(char *s) {
int len = strlen(s);
for (int i = 0; i < len; i++) {
if (i < len - 1) {
s[i] = s[i + 1];
} else {
s[i] = '\0';
}
}
}
char *lstrip(char *s, char strp) {
int len = strlen(s);
if (len >= 0) {
if (s[0] == strp) {
removeFirst(s);
}
}
return s;
}
int myHash(char *s, char *LETTERS) {
int h = 7;
int len = strlen(s);
for (int i = 0; i < len; i++) {
int index = indexOfString(LETTERS, s[i]);
h = 37 * h + index;
}
return h;
}
int indexOfString(char *s, char c) {
int len = strlen(s);
for (int i = 0; i < len; i++) {
if (s[i] == c) {
return i;
}
}
return -1;
}
int main(){
char * result = baseN(10119, "abcdefg");
printf("%s\n",result);
return 0;
}
I wrote a python extension of the above, but after compiling, and running in the Python interpreter ipython, the result is:
In [4]: myEx.myHash('asdfg','ascfwdzxfxcg')
Out[4]: 485465319
In [5]: myEx.baseN(1000,'asdfghj')
Abort trap: 6
The function named baseN does not work, why????
There is a buffer overflow in your code:
char *baseN(int num, char *LETTERS) {
int len = strlen(LETTERS);
if (num == 0) {
char *result = (char *) malloc(sizeof(char)); // 1 char allocated
sprintf(result, "%c", LETTERS[0]); // 2 chars written including NUL char
return result;
}
The current sprintf needs:
char *result = malloc(2);