I am a python programmer. My girlfriend is taking a C class. This frustrates me, something so simple I can't find online nor I can figure out. Let's cut to the chase. I have a simple Python program that I need help trying to translate to C.
lst = input("Enter a list of numbers with a space in between each number\n")
newList = lst.split(" ")
#selection sort has been pre defined
x = newList.selectSort()
print(x)
Sorry this was done on my phone.
Her assignment isn't just this. It's adding multiple functions that work together. I just need to know how this works in order to pull the full program together.
First of all, you have to define the number of item in the list then you can input them.
Then, you have to store them in an array and do the sorting process manually.
I've done the sorting process without defining a function. If you want to use a function, just pass the array and return the sorted array.
#include <stdio.h>
int main()
{
int n, c, d, position, swap;
printf("Enter number of elements\n");
scanf("%d", &n);
int array[n];
printf("Enter %d integers\n", n);
for ( c = 0 ; c < n ; c++ )
scanf("%d", &array[c]);
for ( c = 0 ; c < ( n - 1 ) ; c++ )
{
position = c;
for ( d = c + 1 ; d < n ; d++ )
{
if ( array[position] > array[d] )
position = d;
}
if ( position != c )
{
swap = array[c];
array[c] = array[position];
array[position] = swap;
}
}
printf("Sorted list in ascending order:\n");
for ( c = 0 ; c < n ; c++ )
printf("%d\n", array[c]);
return 0;
}
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <string.h>
// Macro for sorting
#define sort(name, data_set, len, comparator, inverse) \
name##_sort(data_set, len, comparator, inverse)
#define SORT_DEFINE(name, data_type) \
\
/* Sort data set
#data_set data set to sort
#len length of data set
#comparator comparator to compare two elements, return positive value when first element is bigger
#inverse whether the result should be inversed
*/\
void name##_sort(data_type *data_set, int len, int (*comparator)(data_type, data_type), bool inverse) \
{ \
int i; \
int j; \
bool change = true; \
int ret; \
data_type tmp; \
\
for (i = 0; change && i < len - 1; i++) \
{ \
change = false; \
for (j = 0; j < len - 1 - i; j++) \
{ \
ret = comparator(data_set[j], data_set[j + 1]); \
if ((!inverse && ret > 0) || (inverse && ret < 0)) \
{ \
change = true; \
tmp = data_set[j]; \
data_set[j] = data_set[j + 1]; \
data_set[j + 1] = tmp; \
} \
} \
} \
}
/* Split string
#content origin string content
#delim delimiter for splitting
#psize pointer pointing at the variable to store token size
#return tokens after splitting
*/
const char **split(char *content, const char *delim, int *psize)
{
char *token;
const char **tokens;
int capacity;
int size = 0;
token = strtok(content, delim);
if (!token)
{
return NULL;
}
// Initialize tokens
tokens = malloc(sizeof(char *) * 64);
if (!tokens)
{
exit(-1);
}
capacity = 64;
tokens[size++] = token;
while ((token = strtok(NULL, delim)))
{
if (size >= capacity)
{
tokens = realloc(tokens, sizeof(char *) * capacity * 2);
if (!tokens)
{
exit(-1);
}
capacity *= 2;
}
tokens[size++] = token;
}
*psize = size;
return tokens;
}
// Define sort function for data_type = const char *
SORT_DEFINE(str, const char *);
// Define sort function for data_type = int
SORT_DEFINE(int, int)
int intcmp(int v1, int v2)
{
return v1 - v2;
}
int main(int argc, char *argv[])
{
char buff[128];
const char **tokens;
int size;
int i;
int *ints;
// Get input from stdin
fgets(buff, 128, stdin);
// Split string
tokens = split(buff, " \t\n", &size);
ints = malloc(sizeof(int) * size);
// Sort strings [min -> max]
sort(str, tokens, size, strcmp, false);
// Print strings and transfer them to integers
for (i = 0; i < size; i++)
{
printf("[%02d]: <%s>\n", i, tokens[i]);
ints[i] = atoi(tokens[i]);
}
// Sort integers [max -> min]
sort(int, ints, size, intcmp, true);
// Print integers
for (i = 0; i < size; i++)
{
printf("[%02d]: <%d>\n", i, ints[i]);
}
free(ints);
free(tokens);
return 0;
}
Use macro SORT_DEFINE(), sort(), and function split() to do your own job. The main() function is just a demo to show how to use them.
Related
cpp code snippet where i am exporting the function encrypt which takes argument like below
#include <iostream>
using namespace std;
string Function(char* str)
{
string temp = str;
int length = temp.length();
int disp = length,i=0;
char printChar;
char *store = (char *)malloc(sizeof(char)*100);
while(disp > 0) {
printChar = *(str + length - disp);
if ((printChar >= 65) && (printChar <= 90)) {
if (printChar+disp > 90) {
printChar = ((printChar+disp) % 90) + 64;
store[i] = printChar;
}else{
printChar += disp;
store[i] = printChar;
};
}
else if ((printChar >= 97) && (printChar <= 122)) {
if (printChar+disp > 122) {
printChar = ((printChar+disp) % 122) + 96;
store[i] = printChar;
}else{
printChar += disp;
store[i] = printChar;
};
}
else {
store[i] = printChar;
};
disp -= 1;
i += 1;
};
return store;
}
// The part where i am exporting
extern "C" {
string encrypt(char* str) // this is the functionName(in this case encrypt) which i am giving when exporting
{
return Function(str);
}
}
What should be my python code to pass a string in this exported function somewhat like:
userString = input()
result = encrypt(userstring) // this is function created in cpp
print(result)
I don't know how to use ctypes properly. So please can someone help with this question?
I am writing some code that takes binary data from Python, Pipes it to C++, does some processing on the data, (in this case calculating a mutual information metric) and then pipes the results back to python. While testing I have found that everything works fine if the data I send is a set of 2 arrays with dimensions less than 1500 X 1500, but if I send 2 arrays that are 2K X 2K I get back a lot of corrupted nonsense.
I currently believe the algorithmic portion of the code is fine because it provides the expected answers during testing with small (<=1500 X1500) arrays. That leads me to believe that this is an issue with either the stdin or stdout piping. That maybe I’m passing some intrinsic limit somewhere.
The Python Code and C++ code are below.
Python Code:
import subprocess
import struct
import sys
import numpy as np
#set up the variables needed
bytesPerDouble = 8
sizeX = 2000
sizeY = 2000
offset = sizeX*sizeY
totalBytesPerArray = sizeX*sizeY*bytesPerDouble
totalBytes = totalBytesPerArray*2 #the 2 is because we pass 2 different versions of the 2D array
#setup the testing data array
a = np.zeros(sizeX*sizeY*2, dtype='d')
for i in range(sizeX):
for j in range(sizeY):
a[j+i*sizeY] = i
a[j+i*sizeY+offset] = i
if i % 10 == 0:
a[j+i*sizeY+offset] = j
data = a.tobytes('C')
strTotalBytes = str(totalBytes)
strLineBytes = str(sizeY*bytesPerDouble)
#communicate with c++ code
print("starting C++ code")
command = "C:\Python27\PythonPipes.exe"
proc = subprocess.Popen([command, strTotalBytes, strLineBytes, str(sizeY), str(sizeX)], stdin=subprocess.PIPE,stderr=subprocess.PIPE,stdout=subprocess.PIPE)
ByteBuffer = (data)
proc.stdin.write(ByteBuffer)
print("Reading results back from C++")
for i in range(sizeX):
returnvalues = proc.stdout.read(sizeY*bytesPerDouble)
a = buffer(returnvalues)
b = struct.unpack_from(str(sizeY)+'d', a)
print str(b) + " " + str(i)
print('done')
C++ Code:
Main function:
int main(int argc, char **argv) {
int count = 0;
long totalbytes = stoi(argv[argc-4], nullptr,10); //bytes being transfered
long bytechunk = stoi(argv[argc - 3], nullptr, 10); //bytes being transfered at a time
long height = stoi(argv[argc-2], nullptr, 10); //bytes being transfered at a time
long width = stoi(argv[argc-1], nullptr, 10); //bytes being transfered at a time
long offset = totalbytes / sizeof(double) / 2;
data = new double[totalbytes/sizeof(double)];
int columnindex = 0;
//read in data from pipe
while (count<totalbytes) {
fread(&(data[columnindex]), 1, bytechunk, stdin);
columnindex += bytechunk / sizeof(double);
count += bytechunk;
}
//calculate the data transform
MutualInformation MI = MutualInformation();
MI.Initialize(data, height, width, offset);
MI.calcMI();
count = 0;
//*
//write out data to pipe
columnindex = 0;
while (count<totalbytes/2) {
fwrite(&(MI.getOutput()[columnindex]), 1, bytechunk, stdout);
fflush(stdout);
count += bytechunk;
columnindex += bytechunk/sizeof(double);
}
//*/
delete [] data;
return 0;
}
and in case you need it the actual processing code:
double MutualInformation::calcMI(){
double rvalue = 0.0;
std::map<int, map<int, double>> lHistXY = map<int, map<int, double>>();
std::map<int, double> lHistX = map<int, double>();
std::map<int, double> lHistY = map<int, double>();
typedef std::map<int, std::map<int, double>>::iterator HistXY_iter;
typedef std::map<int, double>::iterator HistY_iter;
//calculate Entropys and MI
double MI = 0.0;
double Hx = 0.0;
double Hy = 0.0;
double Px = 0.0;
double Py = 0.0;
double Pxy = 0.0;
//scan through the image
int ip = 0;
int jp = 0;
int chipsize = 3;
//setup zero array
double * zeros = new double[this->mHeight];
for (int j = 0; j < this->mHeight; j++){
zeros[j] = 0.0;
}
//zero out Output array
for (int i = 0; i < this->mWidth; i++){
memcpy(&(this->mOutput[i*this->mHeight]), zeros, this->mHeight*8);
}
double index = 0.0;
for (int ioutter = chipsize; ioutter < (this->mWidth - chipsize); ioutter++){
//write out processing status
//index = (double)ioutter;
//fwrite(&index, 8, 1, stdout);
//fflush(stdout);
//*
for (int j = chipsize; j < (this->mHeight - chipsize); j++){
//clear the histograms
lHistX.clear();
lHistY.clear();
lHistXY.clear();
//chip out a section of the image
for (int k = -chipsize; k <= chipsize; k++){
for (int l = -chipsize; l <= chipsize; l++){
ip = ioutter + k;
jp = j + l;
//update X histogram
if (lHistX.count(int(this->mData[ip*this->mHeight + jp]))){
lHistX[int(this->mData[ip*this->mHeight + jp])] += 1.0;
}else{
lHistX[int(this->mData[ip*this->mHeight + jp])] = 1.0;
}
//update Y histogram
if (lHistY.count(int(this->mData[ip*this->mHeight + jp+this->mOffset]))){
lHistY[int(this->mData[ip*this->mHeight + jp+this->mOffset])] += 1.0;
}
else{
lHistY[int(this->mData[ip*this->mHeight + jp+this->mOffset])] = 1.0;
}
//update X and Y Histogram
if (lHistXY.count(int(this->mData[ip*this->mHeight + jp]))){
//X Key exists check if Y key exists
if (lHistXY[int(this->mData[ip*this->mHeight + jp])].count(int(this->mData[ip*this->mHeight + jp + this->mOffset]))){
//X & Y keys exist
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] += 1;
}else{
//X exist but Y doesn't
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] = 1;
}
}else{
//X Key Didn't exist
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] = 1;
};
}
}
//calculate PMI, Hx, Hy
// iterator->first = key
// iterator->second = value
MI = 0.0;
Hx = 0.0;
Hy = 0.0;
for (HistXY_iter Hist2D_iter = lHistXY.begin(); Hist2D_iter != lHistXY.end(); Hist2D_iter++) {
Px = lHistX[Hist2D_iter->first] / ((double) this->mOffset);
Hx -= Px*log(Px);
for (HistY_iter HistY_iter = Hist2D_iter->second.begin(); HistY_iter != Hist2D_iter->second.end(); HistY_iter++) {
Py = lHistY[HistY_iter->first] / ((double) this->mOffset);
Hy -= Py*log(Py);
Pxy = HistY_iter->second / ((double) this->mOffset);
MI += Pxy*log(Pxy / Py / Px);
}
}
//normalize PMI to max(Hx,Hy) so that the PMI value runs from 0 to 1
if (Hx >= Hy && Hx > 0.0){
MI /= Hx;
}else if(Hy > Hx && Hy > 0.0){
MI /= Hy;
}
else{
MI = 0.0;
}
//write PMI to data output array
if (MI < 1.1){
this->mOutput[ioutter*this->mHeight + j] = MI;
}
else{
this->mOutput[ioutter*this->mHeight + j] = 0.0;
}
}
}
return rvalue;
}
with arrays that return something that makes sense I get output bounded between 0 and 1 like this:
(0.0, 0.0, 0.0, 0.7160627908692593, 0.6376472316395495, 0.5728801401524277,...
with the 2Kx2K or higher arrays I get nonesense like this (even though the code clamps the values between 0 and 1):
(-2.2491400820412374e+228, -2.2491400820412374e+228, -2.2491400820412374e+228, -2.2491400820412374e+228, -2.2491400820412374e+228,...
I would like to know why this code is corrupting the data set after it is assigned between 0.0 and 1, and whether or not it is a piping issue, a stdin/stdout issue, a buffer issue of some sort, or a coding issue I am simply not seeing.
Update I tried passing the data in smaller chunks using the code that Chris suggested with no luck. also of note is that I added a catch for ferror on stdout and it never got tripped so I am pretty sure that the bytes are at least making it to stdout. Is it possible that something else is writing to stdout somehow? maybe an extra byte making its way into stdout while my program is running? I find this doubtful as the errors are appearing consistently on the 4th fwrite read in the 10th entry.
Per Craig's request here is the full C++ code (the full Python Code is already posted): it is sitting in 3 files:
main.cpp
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <iostream>
#include "./MutualInformation.h"
double * data;
using namespace std;
void
xxwrite(unsigned char *buf, size_t wlen, FILE *fo)
{
size_t xlen;
for (; wlen > 0; wlen -= xlen, buf += xlen) {
xlen = wlen;
if (xlen > 1024)
xlen = 1024;
xlen = fwrite(buf, 1, xlen, fo);
fflush(fo);
}
}
int main(int argc, char **argv) {
int count = 0;
long totalbytes = stoi(argv[argc-4], nullptr,10); //bytes being transfered
long bytechunk = stoi(argv[argc - 3], nullptr, 10); //bytes being transfered at a time
long height = stoi(argv[argc-2], nullptr, 10); //bytes being transfered at a time
long width = stoi(argv[argc-1], nullptr, 10); //bytes being transfered at a time
long offset = totalbytes / sizeof(double) / 2;
data = new double[totalbytes/sizeof(double)];
int columnindex = 0;
//read in data from pipe
while (count<totalbytes) {
fread(&(data[columnindex]), 1, bytechunk, stdin);
columnindex += bytechunk / sizeof(double);
count += bytechunk;
}
//calculate the data transform
MutualInformation MI = MutualInformation();
MI.Initialize(data, height, width, offset);
MI.calcMI();
count = 0;
columnindex = 0;
while (count<totalbytes/2) {
xxwrite((unsigned char*)&(MI.getOutput()[columnindex]), bytechunk, stdout);
count += bytechunk;
columnindex += bytechunk/sizeof(double);
}
delete [] data;
return 0;
}
MutualInformation.h
#include <map>
using namespace std;
class MutualInformation
{
private:
double * mData;
double * mOutput;
long mHeight;
long mWidth;
long mOffset;
public:
MutualInformation();
~MutualInformation();
bool Initialize(double * data, long Height, long Width, long Offset);
const double * getOutput();
double calcMI();
};
MutualInformation.cpp
#include "MutualInformation.h"
MutualInformation::MutualInformation()
{
this->mData = nullptr;
this->mOutput = nullptr;
this->mHeight = 0;
this->mWidth = 0;
}
MutualInformation::~MutualInformation()
{
delete[] this->mOutput;
}
bool MutualInformation::Initialize(double * data, long Height, long Width, long Offset){
bool rvalue = false;
this->mData = data;
this->mHeight = Height;
this->mWidth = Width;
this->mOffset = Offset;
//allocate output data
this->mOutput = new double[this->mHeight*this->mWidth];
return rvalue;
}
const double * MutualInformation::getOutput(){
return this->mOutput;
}
double MutualInformation::calcMI(){
double rvalue = 0.0;
std::map<int, map<int, double>> lHistXY = map<int, map<int, double>>();
std::map<int, double> lHistX = map<int, double>();
std::map<int, double> lHistY = map<int, double>();
typedef std::map<int, std::map<int, double>>::iterator HistXY_iter;
typedef std::map<int, double>::iterator HistY_iter;
//calculate Entropys and MI
double MI = 0.0;
double Hx = 0.0;
double Hy = 0.0;
double Px = 0.0;
double Py = 0.0;
double Pxy = 0.0;
//scan through the image
int ip = 0;
int jp = 0;
int chipsize = 3;
//setup zero array
double * zeros = new double[this->mHeight];
for (int j = 0; j < this->mHeight; j++){
zeros[j] = 0.0;
}
//zero out Output array
for (int i = 0; i < this->mWidth; i++){
memcpy(&(this->mOutput[i*this->mHeight]), zeros, this->mHeight*8);
}
double index = 0.0;
for (int ioutter = chipsize; ioutter < (this->mWidth - chipsize); ioutter++){
for (int j = chipsize; j < (this->mHeight - chipsize); j++){
//clear the histograms
lHistX.clear();
lHistY.clear();
lHistXY.clear();
//chip out a section of the image
for (int k = -chipsize; k <= chipsize; k++){
for (int l = -chipsize; l <= chipsize; l++){
ip = ioutter + k;
jp = j + l;
//update X histogram
if (lHistX.count(int(this->mData[ip*this->mHeight + jp]))){
lHistX[int(this->mData[ip*this->mHeight + jp])] += 1.0;
}else{
lHistX[int(this->mData[ip*this->mHeight + jp])] = 1.0;
}
//update Y histogram
if (lHistY.count(int(this->mData[ip*this->mHeight + jp+this->mOffset]))){
lHistY[int(this->mData[ip*this->mHeight + jp+this->mOffset])] += 1.0;
}
else{
lHistY[int(this->mData[ip*this->mHeight + jp+this->mOffset])] = 1.0;
}
//update X and Y Histogram
if (lHistXY.count(int(this->mData[ip*this->mHeight + jp]))){
//X Key exists check if Y key exists
if (lHistXY[int(this->mData[ip*this->mHeight + jp])].count(int(this->mData[ip*this->mHeight + jp + this->mOffset]))){
//X & Y keys exist
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] += 1;
}else{
//X exist but Y doesn't
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] = 1;
}
}else{
//X Key Didn't exist
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] = 1;
};
}
}
//calculate PMI, Hx, Hy
// iterator->first = key
// iterator->second = value
MI = 0.0;
Hx = 0.0;
Hy = 0.0;
for (HistXY_iter Hist2D_iter = lHistXY.begin(); Hist2D_iter != lHistXY.end(); Hist2D_iter++) {
Px = lHistX[Hist2D_iter->first] / ((double) this->mOffset);
Hx -= Px*log(Px);
for (HistY_iter HistY_iter = Hist2D_iter->second.begin(); HistY_iter != Hist2D_iter->second.end(); HistY_iter++) {
Py = lHistY[HistY_iter->first] / ((double) this->mOffset);
Hy -= Py*log(Py);
Pxy = HistY_iter->second / ((double) this->mOffset);
MI += Pxy*log(Pxy / Py / Px);
}
}
//normalize PMI to max(Hx,Hy) so that the PMI value runs from 0 to 1
if (Hx >= Hy && Hx > 0.0){
MI /= Hx;
}else if(Hy > Hx && Hy > 0.0){
MI /= Hy;
}
else{
MI = 0.0;
}
//write PMI to data output array
if (MI < 1.1){
this->mOutput[ioutter*this->mHeight + j] = MI;
}
else{
this->mOutput[ioutter*this->mHeight + j] = 0.0;
//cout << "problem with output";
}
}
}
//*/
return rvalue;
}
SOLVED By 6502
6502's answer below solved my problem. I needed to explicitly tell Windows to use a binary mode for stdin / stdout. to do that I had to include 2 new header files in my main cpp file.
#include <fcntl.h>
#include <io.h>
add the following lines of code (modified away from 6502's POSIX versions because Visual Studio complained) to the beginning of my main function
_setmode(_fileno(stdout), O_BINARY);
_setmode(_fileno(stdin), O_BINARY);
and then add these lines to my Python code:
import os, msvcrt
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
The problem is that stdin/stdout in windows are opened in text mode, not in binary mode and therefore will mess up when the character 13 (\r) is sent.
You can set for example binary mode in Python with
import os, msvcrt
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
and in C++ with
_setmode(fileno(stdout), O_BINARY);
_setmode(fileno(stdin), O_BINARY);
See https://msdn.microsoft.com/en-us/library/tw4k6df8.aspx
Your C++ fwrite code does not account for getting a "short" transfer.
Here's a slight tweak:
//write out data to pipe
columnindex = 0;
while (count < totalbytes / 2) {
wlen = fwrite(&(MI.getOutput()[columnindex]), 1, bytechunk, stdout);
fflush(stdout);
count += wlen;
columnindex += wlen / sizeof(double);
}
Note: You still need to be careful as this would still have issues if wlen returns and it's not a multiple of sizeof(double). For example, if bytechunk were 16 and wlen came back with 14, you'd need an additional fwrite with length 2 before continuing the loop. A generalization of this is just to treat the entire data matrix as a giant byte buffer and loop on that.
Actually, you'll get about the same efficiency with many much smaller transfers that are capped by a fixed (i.e. "known safe amount") of [say] 1024 bytes. This works because the output is a byte stream.
Here's a slightly more general solution that I've often used:
void
xxwrite(void *buf,size_t wlen,FILE *fo)
{
size_t xlen;
for (; wlen > 0; wlen -= xlen, buf += xlen) {
xlen = wlen;
if (xlen > 1024)
xlen = 1024;
xlen = fwrite(buf,1,xlen,fo);
fflush(fo);
}
}
//write out data to pipe
columnindex = 0;
while (count < totalbytes / 2) {
xxwrite(&(MI.getOutput()[columnindex]), bytechunk, stdout);
count += bytechunk;
columnindex += bytechunk / sizeof(double);
}
UPDATE:
I've downloaded all your code and run it. I've got good news and bad news: The code runs fine here, even for a matrix size above 3000. I ran it both using xxwrite and without and the results were the same.
Using my limited python skills, I added some pretty print to your python script (e.g. some line wrap) and had it check every value for range and annotate any bad values. There were none found by the script. Also, visual inspection of the values turned up nothing [this was true before the pretty print, so it hasn't introduced anything]. Just lots of zeros and then blocks in the 0.9 range.
The only difference I can see is that I'm using gcc [and, of course, python] on linux. But, from your script it seems your using Windows [based on the C:\... path for your C++ executable. This shouldn't matter for this application, but I mention it anyway.
So, pipes work here. One thing you might try is to direct the C++ output to a file. Then, have the script read back from the file (i.e. no pipe) and see if that makes a difference. I tend to think not, but ...
Also, I don't know what compiler and python implementation you're using under Windows. Whenever I have to do this, I usually have Cygwin installed as it gives one of the closest implementations of linux/Unix-like environment (i.e. pipes are more likely to work as advertised).
Anyway, here's the modified script. Also note that I added os.getenv to grab alternate matrix sizes and an alternate place for the C++ executable, so that it would work for both of us with minimal pain
#!/usr/bin/python
import subprocess
import struct
import sys
import os
import numpy as np
val = os.getenv("MTX","2000")
sizeX = int(val)
sizeY = sizeX
print "sizeX=%d sizeY=%d" % (sizeX,sizeY)
#set up the variables needed
bytesPerDouble = 8
offset = sizeX*sizeY
totalBytesPerArray = sizeX*sizeY*bytesPerDouble
totalBytes = totalBytesPerArray*2 #the 2 is because we pass 2 different versions of the 2D array
#setup the testing data array
a = np.zeros(sizeX*sizeY*2, dtype='d')
for i in range(sizeX):
for j in range(sizeY):
a[j+i*sizeY] = i
a[j+i*sizeY+offset] = i
if i % 10 == 0:
a[j+i*sizeY+offset] = j
data = a.tobytes('C')
strTotalBytes = str(totalBytes)
strLineBytes = str(sizeY*bytesPerDouble)
#communicate with c++ code
print("starting C++ code")
command = os.getenv("CPGM",None);
if command is None:
command = "C:\Python27\PythonPipes.exe"
proc = subprocess.Popen([command, strTotalBytes, strLineBytes, str(sizeY), str(sizeX)], stdin=subprocess.PIPE,stderr=subprocess.PIPE,stdout=subprocess.PIPE)
ByteBuffer = (data)
proc.stdin.write(ByteBuffer)
def prt(i,b):
hangflg = 0
per = 8
for j in range(0,len(b)):
if ((j % per) == 0):
print("[%d,%d]" % (i,j)),
q = b[j]
print(q),
hangflg = 1
if (q < 0.0) or (q > 1.0):
print("=WTF"),
if ((j % per) == (per - 1)):
print("")
hangflg = 0
if (hangflg):
print("")
print("Reading results back from C++")
for i in range(sizeX):
returnvalues = proc.stdout.read(sizeY*bytesPerDouble)
a = buffer(returnvalues)
b = struct.unpack_from(str(sizeY)+'d', a)
prt(i,b)
###print str(b) + " " + str(i)
###print str(i) + ": " + str(b)
print('done')
I know this question has been asked before, but I don't find a useful solution. The full error is :
Executing: "/home/mint/Documents/test_sensor/cycl_test/top_block.py"
Using Volk machine: avx_64_mmx_orc
Traceback (most recent call last):
File "/home/mint/Documents/test_sensor/cycl_test/top_block.py", line 87, in <module>
tb = top_block()
File "/home/mint/Documents/test_sensor/cycl_test/top_block.py", line 61, in __init__
self.cycl_MME_cpp_0 = cycl.MME_cpp(1000, 16, 0.1)
AttributeError: 'module' object has no attribute 'MME_cpp'
I have found some possible reasons from previous questions:
mismatching of function parameter(including referencing function in .cc and .h)
swig problem
callbacks in .xml file
nm -C -u libgnuradio-.so
I've checked the first 3 reasons, but I'm not sure how to find out the undefined symblos using the last methods. There is part of the outcome:
U gsl_linalg_SV_decomp
U gsl_matrix_alloc
U gsl_matrix_set
U gsl_matrix_set_zero
U gsl_vector_alloc
U gsl_vector_get
Does this means all the gsl functions are undefined? If so how can I deal with it?
Except for the four reasons,is there any other hints about my questions?
I appreciate your help.
addendum:
1.MME_cpp_impl.cc
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <gnuradio/io_signature.h>
#include "MME_cpp_impl.h"
#include <stdio.h>
#include <math.h>
#include <gsl/gsl_matrix.h>
#include <gsl/gsl_vector.h>
#include <gsl/gsl_linalg.h>
namespace gr {
namespace cycl {
MME_cpp::sptr
MME_cpp::make(int Ns, int M, float Pfa)
{
return gnuradio::get_initial_sptr
(new MME_cpp_impl(Ns, M, Pfa));
}
/*
* The private constructor
*/
MME_cpp_impl::MME_cpp_impl(int Ns, int M, float Pfa)
: gr::sync_block("MME_cpp",
gr::io_signature::make(1, 1, sizeof(float)),
gr::io_signature::make(1, 1, sizeof(float))),
d_Ns(Ns), d_M(M), d_Pfa(Pfa)
{}
/*
* Our virtual destructor.
*/
MME_cpp_impl::~MME_cpp_impl()
{
}
/*This function gives the CDF value for the pfa in input*/
float
TracyWidom (float p){
float pd, tw;
tw = 0.45;
pd = 1 - p;
if (pd >= 0.01 && pd <= 0.05){
tw = 18*(pd - (17/75)); printf("a - %f\n", tw);
}else if (pd >= 0.05 && pd <= 0.1){
tw = 8*(pd - (179/400)); printf("b - %f\n", tw);
}else if (pd >= 0.1 && pd <= 0.3){
tw = (87/20)*(pd - (643/870)); printf("c - %f\n", tw);
}else if (pd >= 0.3 && pd <= 0.5){
tw = (16/5)*(pd - (287/320)); printf("d - %f\n", tw);
}else if (pd >= 0.5 && pd <= 0.7){
tw = (17/5)*(pd - (297/340)); printf("e - %f\n", tw);
}else if (pd >= 0.7 && pd <= 0.9){
tw = (5.2)*(pd - (0.813)); printf("f - %f\n", tw);
}else if (pd >= 0.9 && pd <= 0.95){
tw = (53/5)*(pd - (909/1060)); printf("g - %f\n", tw);
}else if (pd >= 0.95 && pd <= 1){
tw = 26*(pd - (593/650)); printf("h - %f\n", tw);
}else{
printf ("wrong pfa value: it must be between 0 and 1\n");
printf ("the pfa value standard is 0.1\n");
tw = (53/5)*(0.9 - (909/1060));
}
return tw;
}
/*this function calculates the threshold for the test
the inputs are:
ns = numbers of samples/L;
L = length of the correlation function;
pfa = probability of false alarm*/
float
gamma (int ns, int L, float tw){
float A, B, C, ratio1, ratio2, g;
A = sqrt(ns)+sqrt(L*ns);
B = sqrt(ns)-sqrt(L*ns);
C = ns*ns*L;
ratio1 = pow(A,2)/pow(B,2);
ratio2 = 1+( pow(A,-0.667) / pow(C,0.167) )*tw;
g = ratio1*ratio2;
return g;
}
/*This function makes the detection test*/
float
test (float r, float t){
float decision;
if(r > -1){
if(r <= t){
decision = 0;
}
else{
decision = 1;
}
}
return decision;}
//-------------end functions-----------
int
MME_cpp_impl::work(int noutput_items,
gr_vector_const_void_star &input_items,
gr_vector_void_star &output_items)
{
const float *in = (const float *) input_items[0];
float *out = (float *) output_items[0];
//float cov;
//int ninputs = input_items.size ();
//for(int i=0; i<noutput_items*Ns; ++i){
// cov = compute_convariance(in[i]);
//int lenght = floor(d_samples / d_L) * d_L;
float vett[d_Ns];
int lenght = floor(d_Ns / d_M) ;
int p, j, k=0;
float thr, lmax, lmin, ratio=-1, TW, mem=0;
// char c[1];
gsl_matrix * hankel = gsl_matrix_alloc (lenght,d_M);
gsl_matrix * V = gsl_matrix_alloc (d_M,d_M);
gsl_vector * S = gsl_vector_alloc (d_M);
gsl_vector * temp = gsl_vector_alloc (d_M);
//FILE *story;
gsl_matrix_set_zero (hankel);
TW = TracyWidom (d_Pfa);
thr = gamma(lenght, d_M, TW); //calculate the threshold
for (int i = 0; i < noutput_items; i++){
vett[k]=in[i];
k++;
if (k >= lenght*d_M){
k = 0;
// story = fopen("filestory.txt", "a");
for( p=0; p<lenght; p++ ){
for( j=0; j<d_M; j++ ){
gsl_matrix_set (hankel, p, j, vett[p+j]);
}
}
gsl_linalg_SV_decomp (hankel, V, S, temp);//using SDV to calculate eigenvalue
lmax = gsl_vector_get(S, 0);//maximal eigenvalue
lmin = gsl_vector_get(S, (d_M -1));//minimal eigenvalue
ratio = lmax/lmin;
mem = test(ratio, thr);
//fprintf(story, "%f - ratio=%f - soglia=%f\n ", mem, ratio, thr);
//fclose(story);
}
out[i] = mem;
}
// Tell runtime system how many output items we produced.
return noutput_items;
}
} /* namespace cycl /
} / namespace gr */
2.MME_cpp_impl.h
#ifndef INCLUDED_CYCL_MME_CPP_IMPL_H
#define INCLUDED_CYCL_MME_CPP_IMPL_H
#include <cycl/MME_cpp.h>
namespace gr {
namespace cycl {
class MME_cpp_impl : public MME_cpp
{
private:
int d_Ns, d_M;
float d_Pfa;
public:
MME_cpp_impl(int Ns, int M, float Pfa);
~MME_cpp_impl();
int Ns() const { return d_Ns; }
void set_Ns(int Ns) { d_Ns = Ns; }
int M() const { return d_M; }
void set_M(int M) { d_M = M; }
float Pfa() const { return d_Pfa; }
void set_Pfa(float Pfa) { d_Pfa = Pfa; }
// Where all the action really happens
int work(int noutput_items,
gr_vector_const_void_star &input_items,
gr_vector_void_star &output_items);
//float compute_covariance(float & d, int i, int j);
friend float TracyWidom (float p);
friend float gamma (int ns, int L, float tw);
friend float test (float r, float t);
};
float TracyWidom (float p);
float gamma (int ns, int L, float tw);
float test (float r, float t);
} // namespace cycl
} // namespace gr
#endif /* INCLUDED_CYCL_MME_CPP_IMPL_H */
3.MME_cpp.h
#ifndef INCLUDED_CYCL_MME_CPP_H
#define INCLUDED_CYCL_MME_CPP_H
#include <cycl/api.h>
#include <gnuradio/sync_block.h>
namespace gr {
namespace cycl {
/*!
* \brief <+description of block+>
* \ingroup cycl
*
*/
class CYCL_API MME_cpp : virtual public gr::sync_block
{
public:
typedef boost::shared_ptr<MME_cpp> sptr;
/*!
* \brief Return a shared_ptr to a new instance of cycl::MME_cpp.
*
* To avoid accidental use of raw pointers, cycl::MME_cpp's
* constructor is in a private implementation
* class. cycl::MME_cpp::make is the public interface for
* creating new instances.
*/
static sptr make(int Ns, int M, float Pfa);
virtual int Ns() const = 0;
virtual void set_Ns(int Ns) = 0;
virtual int M() const = 0;
virtual void set_M(int M) = 0;
virtual float Pfa() const = 0;
virtual void set_Pfa(float Pfa) = 0;
};
} // namespace cycl
} // namespace gr
#endif /* INCLUDED_CYCL_MME_CPP_H */
4.cycl_MME_cpp.xml
<?xml version="1.0"?>
<block>
<name>MME_cpp</name>
<key>cycl_MME_cpp</key>
<category>cycl</category>
<import>import cycl</import>
<make>cycl.MME_cpp($Ns, $M, $Pfa)</make>
<callback>set_Ns($Ns)</callback>
<callback>set_M($M)</callback>
<callback>set_Pfa($Pfa)</callback>
<param>
<name>Number of samples</name>
<key>Ns</key>
<type>int</type>
</param>
<param>
<name>Length of correlation function</name>
<key>M</key>
<type>int</type>
</param>
<param>
<name>false probability</name>
<key>Pfa</key>
<type>float</type>
</param>
<sink>
<name>in</name>
<type>float</type>
</sink>
<source>
<name>out</name>
<type>float</type>
</source>
</block>
I had the same problem while developing a block. I noticed that the CMake script complained that the swig library is missing. Although this was not a fatal error, it does not work without it.
After installing swig I could use my module: sudo apt-get install swig
This is likely due to the path that your .so is installed by default. I recently ran into this issue, and found that /usr/local/lib was not in my /etc/ld.so.conf by default (Arch Linux). After updating /etc/ld.so.conf, I ran:
sudo ldconfig
and the issue was resolved.
I'm trying to create Python/Cython wrapper for C++ library that uses cv::Mat class from OpenCV. In official Python wrapper all functions take NumPy's ndarray instead of cv::Mat, which is quite convenient. But in my own wrapper, how do I do such conversion? That is, how do I create cv::Mat from np.ndarray?
As suggested by kyamagu, you can use OpenCV's official python wrapper code, especially the pyopencv_to and pyopencv_from.
I've been struggling as you did with all the dependencies and the generated header files. Nevertheless, it is possible to reduce the complexity of this by "cleaning" the cv2.cpp as lightalchemist did here in order to keep only what is necessary. You will need to adapt it to your need and to the version of OpenCV you're using but its basically the same code that I used.
#include <Python.h>
#include "numpy/ndarrayobject.h"
#include "opencv2/core/core.hpp"
static PyObject* opencv_error = 0;
static int failmsg(const char *fmt, ...)
{
char str[1000];
va_list ap;
va_start(ap, fmt);
vsnprintf(str, sizeof(str), fmt, ap);
va_end(ap);
PyErr_SetString(PyExc_TypeError, str);
return 0;
}
class PyAllowThreads
{
public:
PyAllowThreads() : _state(PyEval_SaveThread()) {}
~PyAllowThreads()
{
PyEval_RestoreThread(_state);
}
private:
PyThreadState* _state;
};
class PyEnsureGIL
{
public:
PyEnsureGIL() : _state(PyGILState_Ensure()) {}
~PyEnsureGIL()
{
PyGILState_Release(_state);
}
private:
PyGILState_STATE _state;
};
#define ERRWRAP2(expr) \
try \
{ \
PyAllowThreads allowThreads; \
expr; \
} \
catch (const cv::Exception &e) \
{ \
PyErr_SetString(opencv_error, e.what()); \
return 0; \
}
using namespace cv;
static PyObject* failmsgp(const char *fmt, ...)
{
char str[1000];
va_list ap;
va_start(ap, fmt);
vsnprintf(str, sizeof(str), fmt, ap);
va_end(ap);
PyErr_SetString(PyExc_TypeError, str);
return 0;
}
static size_t REFCOUNT_OFFSET = (size_t)&(((PyObject*)0)->ob_refcnt) +
(0x12345678 != *(const size_t*)"\x78\x56\x34\x12\0\0\0\0\0")*sizeof(int);
static inline PyObject* pyObjectFromRefcount(const int* refcount)
{
return (PyObject*)((size_t)refcount - REFCOUNT_OFFSET);
}
static inline int* refcountFromPyObject(const PyObject* obj)
{
return (int*)((size_t)obj + REFCOUNT_OFFSET);
}
class NumpyAllocator : public MatAllocator
{
public:
NumpyAllocator() {}
~NumpyAllocator() {}
void allocate(int dims, const int* sizes, int type, int*& refcount,
uchar*& datastart, uchar*& data, size_t* step)
{
PyEnsureGIL gil;
int depth = CV_MAT_DEPTH(type);
int cn = CV_MAT_CN(type);
const int f = (int)(sizeof(size_t)/8);
int typenum = depth == CV_8U ? NPY_UBYTE : depth == CV_8S ? NPY_BYTE :
depth == CV_16U ? NPY_USHORT : depth == CV_16S ? NPY_SHORT :
depth == CV_32S ? NPY_INT : depth == CV_32F ? NPY_FLOAT :
depth == CV_64F ? NPY_DOUBLE : f*NPY_ULONGLONG + (f^1)*NPY_UINT;
int i;
npy_intp _sizes[CV_MAX_DIM+1];
for( i = 0; i < dims; i++ )
_sizes[i] = sizes[i];
if( cn > 1 )
{
/*if( _sizes[dims-1] == 1 )
_sizes[dims-1] = cn;
else*/
_sizes[dims++] = cn;
}
PyObject* o = PyArray_SimpleNew(dims, _sizes, typenum);
if(!o)
CV_Error_(CV_StsError, ("The numpy array of typenum=%d, ndims=%d can not be created", typenum, dims));
refcount = refcountFromPyObject(o);
npy_intp* _strides = PyArray_STRIDES(o);
for( i = 0; i < dims - (cn > 1); i++ )
step[i] = (size_t)_strides[i];
datastart = data = (uchar*)PyArray_DATA(o);
}
void deallocate(int* refcount, uchar*, uchar*)
{
PyEnsureGIL gil;
if( !refcount )
return;
PyObject* o = pyObjectFromRefcount(refcount);
Py_INCREF(o);
Py_DECREF(o);
}
};
NumpyAllocator g_numpyAllocator;
enum { ARG_NONE = 0, ARG_MAT = 1, ARG_SCALAR = 2 };
static int pyopencv_to(const PyObject* o, Mat& m, const char* name = "<unknown>", bool allowND=true)
{
if(!o || o == Py_None)
{
if( !m.data )
m.allocator = &g_numpyAllocator;
return true;
}
if( PyInt_Check(o) )
{
double v[] = {PyInt_AsLong((PyObject*)o), 0., 0., 0.};
m = Mat(4, 1, CV_64F, v).clone();
return true;
}
if( PyFloat_Check(o) )
{
double v[] = {PyFloat_AsDouble((PyObject*)o), 0., 0., 0.};
m = Mat(4, 1, CV_64F, v).clone();
return true;
}
if( PyTuple_Check(o) )
{
int i, sz = (int)PyTuple_Size((PyObject*)o);
m = Mat(sz, 1, CV_64F);
for( i = 0; i < sz; i++ )
{
PyObject* oi = PyTuple_GET_ITEM(o, i);
if( PyInt_Check(oi) )
m.at<double>(i) = (double)PyInt_AsLong(oi);
else if( PyFloat_Check(oi) )
m.at<double>(i) = (double)PyFloat_AsDouble(oi);
else
{
failmsg("%s is not a numerical tuple", name);
m.release();
return false;
}
}
return true;
}
if( !PyArray_Check(o) )
{
failmsg("%s is not a numpy array, neither a scalar", name);
return false;
}
bool needcopy = false, needcast = false;
int typenum = PyArray_TYPE(o), new_typenum = typenum;
int type = typenum == NPY_UBYTE ? CV_8U :
typenum == NPY_BYTE ? CV_8S :
typenum == NPY_USHORT ? CV_16U :
typenum == NPY_SHORT ? CV_16S :
typenum == NPY_INT ? CV_32S :
typenum == NPY_INT32 ? CV_32S :
typenum == NPY_FLOAT ? CV_32F :
typenum == NPY_DOUBLE ? CV_64F : -1;
if( type < 0 )
{
if( typenum == NPY_INT64 || typenum == NPY_UINT64 || type == NPY_LONG )
{
needcopy = needcast = true;
new_typenum = NPY_INT;
type = CV_32S;
}
else
{
failmsg("%s data type = %d is not supported", name, typenum);
return false;
}
}
int ndims = PyArray_NDIM(o);
if(ndims >= CV_MAX_DIM)
{
failmsg("%s dimensionality (=%d) is too high", name, ndims);
return false;
}
int size[CV_MAX_DIM+1];
size_t step[CV_MAX_DIM+1], elemsize = CV_ELEM_SIZE1(type);
const npy_intp* _sizes = PyArray_DIMS(o);
const npy_intp* _strides = PyArray_STRIDES(o);
bool ismultichannel = ndims == 3 && _sizes[2] <= CV_CN_MAX;
for( int i = ndims-1; i >= 0 && !needcopy; i-- )
{
// these checks handle cases of
// a) multi-dimensional (ndims > 2) arrays, as well as simpler 1- and 2-dimensional cases
// b) transposed arrays, where _strides[] elements go in non-descending order
// c) flipped arrays, where some of _strides[] elements are negative
if( (i == ndims-1 && (size_t)_strides[i] != elemsize) ||
(i < ndims-1 && _strides[i] < _strides[i+1]) )
needcopy = true;
}
if( ismultichannel && _strides[1] != (npy_intp)elemsize*_sizes[2] )
needcopy = true;
if (needcopy)
{
if( needcast )
o = (PyObject*)PyArray_Cast((PyArrayObject*)o, new_typenum);
else
o = (PyObject*)PyArray_GETCONTIGUOUS((PyArrayObject*)o);
_strides = PyArray_STRIDES(o);
}
for(int i = 0; i < ndims; i++)
{
size[i] = (int)_sizes[i];
step[i] = (size_t)_strides[i];
}
// handle degenerate case
if( ndims == 0) {
size[ndims] = 1;
step[ndims] = elemsize;
ndims++;
}
if( ismultichannel )
{
ndims--;
type |= CV_MAKETYPE(0, size[2]);
}
if( ndims > 2 && !allowND )
{
failmsg("%s has more than 2 dimensions", name);
return false;
}
m = Mat(ndims, size, type, PyArray_DATA(o), step);
if( m.data )
{
m.refcount = refcountFromPyObject(o);
if (!needcopy)
{
m.addref(); // protect the original numpy array from deallocation
// (since Mat destructor will decrement the reference counter)
}
};
m.allocator = &g_numpyAllocator;
return true;
}
static PyObject* pyopencv_from(const Mat& m)
{
if( !m.data )
Py_RETURN_NONE;
Mat temp, *p = (Mat*)&m;
if(!p->refcount || p->allocator != &g_numpyAllocator)
{
temp.allocator = &g_numpyAllocator;
ERRWRAP2(m.copyTo(temp));
p = &temp;
}
p->addref();
return pyObjectFromRefcount(p->refcount);
}
Once you have a cleaned up cv2.cpp file, here is some Cython code that takes care of the conversion. Notice the definition and the call to the import_array() function (it's a NumPy function defined in a header included somewhere in cv2.cpp), this is necessary to define some macros used by pyopencv_to, if you don't call it you will get segmentation faults as lightalchemist pointed out.
from cpython.ref cimport PyObject
# Declares OpenCV's cv::Mat class
cdef extern from "opencv2/core/core.hpp":
cdef cppclass Mat:
pass
# Declares the official wrapper conversion functions + NumPy's import_array() function
cdef extern from "cv2.cpp":
void import_array()
PyObject* pyopencv_from(const _Mat&)
int pyopencv_to(PyObject*, _Mat&)
# Function to be called at initialization
cdef void init():
import_array()
# Python to C++ conversion
cdef Mat nparrayToMat(object array):
cdef Mat mat
cdef PyObject* pyobject = <PyObject*> array
pyopencv_to(pyobject, mat)
return <Mat> mat
# C++ to Python conversion
cdef object matToNparray(Mat mat):
return <object> pyopencv_from(mat)
Note: somehow I got an error with NumPy 1.8.0 on Fedora 20 while compiling due to a strange return statement in the import_array macro, I had to manually remove it to make it work but I can't find this return statement in the NumPy's 1.8.0 GitHub source code
It turns out that there's no simple way to convert (any) np.ndarray into corresponding cv::Mat. Basically, one needs to do only 2 things:
Create empty cv::Mat of corresponding size and type.
Copy data.
However, devil hides in details. Both ndarray and Mat may hold quite varying data formats. For instance, data in NumPy arrays may be in C or in Fortran order, array object may own its data or keep a view to another array, channels may go in a different order (RGB in NumPy vs. BGR in OpenCV), etc.
So instead of trying to solve generic problem I decided to stay with simple code that fits my needs and may be easily modified by anyone interested.
Following code in Cython works with float32/CV_32FC1 images with default byte order:
cdef void array2mat(np.ndarray arr, Mat& mat):
cdef int r = arr.shape[0]
cdef int c = arr.shape[1]
cdef int mat_type = CV_32FC1 # or CV_64FC1, or CV_8UC3, or whatever
mat.create(r, c, mat_type)
cdef unsigned int px_size = 4 # 8 for single-channel double image or
# 1*3 for three-channel uint8 image
memcpy(mat.data, arr.data, r*c*px_size)
To use this code in Cython one also needs to declare some types and constants, e.g. like this:
import numpy as np
# Cython makes it simple to import NumPy
cimport numpy as np
# OpenCV's matrix class
cdef extern from "opencv2/opencv.hpp" namespace "cv":
cdef cppclass Mat:
Mat() except +
Mat(int, int, int, void*) except +
void create(int, int, int)
void* data
int type() const
int cols
int rows
int channels()
Mat clone() const
# some OpenCV matrix types
cdef extern from "opencv2/opencv.hpp":
cdef int CV_8UC3
cdef int CV_8UC1
cdef int CV_32FC1
cdef int CV_64FC1
Opposite conversion (from cv::Mat to np.ndarray) may be achieved in a similar way.
Bonus: there's also nice blog post describing same kind of conversion for RGB/BGR images.
I guess you can directly use or take some logic from the converter from the official python wrapper. There isn't much documentation for this module, but maybe the output of the wrapper generator is helpful to understand how to use it.
If it helps, I wrote a wrapper that does exactly this. It's a convenience library that registers a boost::python converter to implicitly convert between OpenCV's popular cv::Mat datatype and NumPy's popular np.array() datatype. This allows a developer to go back and forth between their OpenCV C++ API and Python API written using NumPy with relative ease, avoiding the need to write additional wrappers that handle PyObjects being passed around or returned.
Take a look:
https://github.com/spillai/numpy-opencv-converter
Based on tlorieul's answer, here is the code I used for building a Python/C++ module:
https://gist.github.com/des0ps/88f1332319867a678a74bdbc0e7401c2
This has been tested with Python3 and OpenCV3.
I'm trying to translate the following Python code into C++:
import struct
import binascii
inputstring = ("0000003F" "0000803F" "AD10753F" "00000080")
num_vals = 4
for i in range(num_vals):
rawhex = inputstring[i*8:(i*8)+8]
# <f for little endian float
val = struct.unpack("<f", binascii.unhexlify(rawhex))[0]
print val
# Output:
# 0.5
# 1.0
# 0.957285702229
# -0.0
So it reads 32-bit worth of the hex-encoded string, turns it into a byte-array with the unhexlify method, and interprets it as a little-endian float value.
The following almost works, but the code is kind of crappy (and the last 00000080 parses incorrectly):
#include <sstream>
#include <iostream>
int main()
{
// The hex-encoded string, and number of values are loaded from a file.
// The num_vals might be wrong, so some basic error checking is needed.
std::string inputstring = "0000003F" "0000803F" "AD10753F" "00000080";
int num_vals = 4;
std::istringstream ss(inputstring);
for(unsigned int i = 0; i < num_vals; ++i)
{
char rawhex[8];
// The ifdef is wrong. It is not the way to detect endianness (it's
// always defined)
#ifdef BIG_ENDIAN
rawhex[6] = ss.get();
rawhex[7] = ss.get();
rawhex[4] = ss.get();
rawhex[5] = ss.get();
rawhex[2] = ss.get();
rawhex[3] = ss.get();
rawhex[0] = ss.get();
rawhex[1] = ss.get();
#else
rawhex[0] = ss.get();
rawhex[1] = ss.get();
rawhex[2] = ss.get();
rawhex[3] = ss.get();
rawhex[4] = ss.get();
rawhex[5] = ss.get();
rawhex[6] = ss.get();
rawhex[7] = ss.get();
#endif
if(ss.good())
{
std::stringstream convert;
convert << std::hex << rawhex;
int32_t val;
convert >> val;
std::cerr << (*(float*)(&val)) << "\n";
}
else
{
std::ostringstream os;
os << "Not enough values in LUT data. Found " << i;
os << ". Expected " << num_vals;
std::cerr << os.str() << std::endl;
throw std::exception();
}
}
}
(compiles on OS X 10.7/gcc-4.2.1, with a simple g++ blah.cpp)
Particularly, I'd like to get rid of the BIG_ENDIAN macro stuff, as I'm sure there is a nicer way to do this, as this post discusses.
Few other random details - I can't use Boost (too large a dependency for the project). The string will usually contain between 1536 (83*3) and 98304 float values (323*3), at most 786432 (643*3)
(edit2: added another value, 00000080 == -0.0)
The following is your updated code modified to remove the #ifdef BIG_ENDIAN block. It uses a read technique that should be host byte order independent. It does this by reading the hex bytes (which are little endian in your source string) into a big endian string format compatible with the iostream std::hex operator. Once in this format it should not matter what the host byte order is.
Additionally, it fixes a bug in that rawhex needs to be zero terminated to be inserted into convert without trailing garbage in some cases.
I do not have a big endian system to test on, so please verify on your platform. This was compiled and tested under Cygwin.
#include <sstream>
#include <iostream>
int main()
{
// The hex-encoded string, and number of values are loaded from a file.
// The num_vals might be wrong, so some basic error checking is needed.
std::string inputstring = "0000003F0000803FAD10753F00000080";
int num_vals = 4;
std::istringstream ss(inputstring);
size_t const k_DataSize = sizeof(float);
size_t const k_HexOctetLen = 2;
for (uint32_t i = 0; i < num_vals; ++i)
{
char rawhex[k_DataSize * k_HexOctetLen + 1];
// read little endian string into memory array
for (uint32_t j=k_DataSize; (j > 0) && ss.good(); --j)
{
ss.read(rawhex + ((j-1) * k_HexOctetLen), k_HexOctetLen);
}
// terminate the string (needed for safe conversion)
rawhex[k_DataSize * k_HexOctetLen] = 0;
if (ss.good())
{
std::stringstream convert;
convert << std::hex << rawhex;
uint32_t val;
convert >> val;
std::cerr << (*(float*)(&val)) << "\n";
}
else
{
std::ostringstream os;
os << "Not enough values in LUT data. Found " << i;
os << ". Expected " << num_vals;
std::cerr << os.str() << std::endl;
throw std::exception();
}
}
}
I think the whole istringstring business is an overkill. It's much easier to parse this yourself one digit at a time.
First, create a function to convert a hex digit into an integer:
signed char htod(char c)
{
c = tolower(c);
if(isdigit(c))
return c - '0';
if(c >= 'a' && c <= 'f')
return c - 'a' + 10;
return -1;
}
Then simply convert the string into an integer. The code below doesn't check for errors and assumes big endianness -- but you should be able to fill in the details.
unsigned long t = 0;
for(int i = 0; i < s.length(); ++i)
t |= (t << 4) & htod(s[i]);
Then your float is
float f = * (float *) &t;
This is what we ended up with, OpenColorIO/src/core/FileFormatIridasLook.cpp
(Amardeep's answer with the unsigned uint32_t fix would likely work also)
// convert hex ascii to int
// return true on success, false on failure
bool hexasciitoint(char& ival, char character)
{
if(character>=48 && character<=57) // [0-9]
{
ival = static_cast<char>(character-48);
return true;
}
else if(character>=65 && character<=70) // [A-F]
{
ival = static_cast<char>(10+character-65);
return true;
}
else if(character>=97 && character<=102) // [a-f]
{
ival = static_cast<char>(10+character-97);
return true;
}
ival = 0;
return false;
}
// convert array of 8 hex ascii to f32
// The input hexascii is required to be a little-endian representation
// as used in the iridas file format
// "AD10753F" -> 0.9572857022285461f on ALL architectures
bool hexasciitofloat(float& fval, const char * ascii)
{
// Convert all ASCII numbers to their numerical representations
char asciinums[8];
for(unsigned int i=0; i<8; ++i)
{
if(!hexasciitoint(asciinums[i], ascii[i]))
{
return false;
}
}
unsigned char * fvalbytes = reinterpret_cast<unsigned char *>(&fval);
#if OCIO_LITTLE_ENDIAN
// Since incoming values are little endian, and we're on little endian
// preserve the byte order
fvalbytes[0] = (unsigned char) (asciinums[1] | (asciinums[0] << 4));
fvalbytes[1] = (unsigned char) (asciinums[3] | (asciinums[2] << 4));
fvalbytes[2] = (unsigned char) (asciinums[5] | (asciinums[4] << 4));
fvalbytes[3] = (unsigned char) (asciinums[7] | (asciinums[6] << 4));
#else
// Since incoming values are little endian, and we're on big endian
// flip the byte order
fvalbytes[3] = (unsigned char) (asciinums[1] | (asciinums[0] << 4));
fvalbytes[2] = (unsigned char) (asciinums[3] | (asciinums[2] << 4));
fvalbytes[1] = (unsigned char) (asciinums[5] | (asciinums[4] << 4));
fvalbytes[0] = (unsigned char) (asciinums[7] | (asciinums[6] << 4));
#endif
return true;
}