Related
Like I said I think I found a working strategy tested it in Tradingview worked pretty well actually but I just can't find any python code that calculates DiNapoli Stochastic Indicator.
I have the math formula, some pinescript and mq4 code if anyone can help me with this I think he/she will be the first in the entire internet .
Pinescript Code:
//#author LazyBear
study("DiNapoli Preferred Stochastic Oscillator [LazyBear]", shorttitle="DPSTOCH_LB" , overlay=false)
fk =input(8, title="Fast K")
sk =input(3, title="Slow K")
sd =input(3, title="Slow D")
min_ = lowest(low, fk)
max_ = highest(high, fk)
fast = (close - min_)/(max_ - min_)*100
r = nz(r[1]) + (fast - nz(r[1]))/sk
s = nz(s[1]) + (r - nz(s[1]))/sd
ob=hline(70, title="OBLevel"), os=hline(30, title="OSLevel"), fill(ob,os, gray)
plot(r, color=blue, title="Dinapoli Stoch"), plot(s, color=red, title="Signal")
Lua Code
function Init()
indicator:name("Bigger timeframe Dinapoli Preferred Stochastic");
indicator:description("");
indicator:requiredSource(core.Bar);
indicator:type(core.Oscillator);
indicator.parameters:addGroup("Calculation");
indicator.parameters:addGroup("Calculation");
indicator.parameters:addInteger("K","Number of periods for %K", "", 10, 2, 1000);
indicator.parameters:addInteger("SD", "%D slowing periods", "", 5, 2, 1000);
indicator.parameters:addInteger("D", "Number of periods for %D", "", 5, 2, 1000);
indicator.parameters:addString("BS", "Time frame to calculate stochastic", "", "D1");
indicator.parameters:setFlag("BS", core.FLAG_PERIODS);
indicator.parameters:addGroup("Display");
indicator.parameters:addColor("K_color", "Color of K", "Color of K", core.rgb(0, 255, 0));
indicator.parameters:addColor("D_color", "Color of D", "Color of D", core.rgb(255, 0, 0));
indicator.parameters:addInteger("Kwidth", "K Line Width", "", 1, 1, 5);
indicator.parameters:addInteger("Kstyle", "K Line Style", "", core.LINE_SOLID);
indicator.parameters:setFlag("Kstyle", core.FLAG_LEVEL_STYLE);
indicator.parameters:addInteger("Dwidth", "D Line Width", "", 1, 1, 5);
indicator.parameters:addInteger("Dstyle", "D Line Style", "", core.LINE_SOLID);
indicator.parameters:setFlag("Dstyle", core.FLAG_LEVEL_STYLE);
end
local source; -- the source
local bf_data = nil; -- the high/low data
local k;
local d;
local sd;
local BS;
local bf_length; -- length of the bigger frame in seconds
local dates; -- candle dates
local host;
local Stochastic;
local SK, SD;
local day_offset;
local week_offset;
local extent;
function Prepare(nameOnly)
source = instance.source;
host = core.host;
day_offset = host:execute("getTradingDayOffset");
week_offset = host:execute("getTradingWeekOffset");
BS = instance.parameters.BS;
k = instance.parameters.K;
sd = instance.parameters.SD;
d = instance.parameters.D;
assert(core.indicators:findIndicator("DINAPOLI PREFERRED STOCHASTIC") ~= nil, "Please, download and install DINAPOLI PREFERRED STOCHASTIC.LUA indicator");
extent = ( k + sd + d) * 2;
local s, e, s1, e1;
s, e = core.getcandle(source:barSize(), core.now(), 0, 0);
s1, e1 = core.getcandle(BS, core.now(), 0, 0);
assert ((e - s) < (e1 - s1), "The chosen time frame must be bigger than the chart time frame!");
bf_length = math.floor((e1 - s1) * 86400 + 0.5);
local name = profile:id() .. "(" .. source:name() .. "," .. BS .. "," .. k .. "," .. sd .. "," .. d .. ")";
instance:name(name);
if nameOnly then
return;
end
SK = instance:addStream("K", core.Line, name .. ".K", "K", instance.parameters.K_color, 0);
SK:setWidth(instance.parameters.Kwidth);
SK:setStyle(instance.parameters.Kstyle);
SD = instance:addStream("D", core.Line, name .. ".D", "D", instance.parameters.D_color, 0);
SD:setWidth(instance.parameters.Dwidth);
SD:setStyle(instance.parameters.Dstyle);
SK:addLevel(20);
SK:addLevel(50);
SK:addLevel(80);
SK:setPrecision(math.max(2, instance.source:getPrecision()));
SD:setPrecision(math.max(2, instance.source:getPrecision()));
end
local loading = false;
local loadingFrom, loadingTo;
local pday = nil;
-- the function which is called to calculate the period
function Update(period, mode)
-- get date and time of the hi/lo candle in the reference data
local bf_candle;
bf_candle = core.getcandle(BS, source:date(period), day_offset, week_offset);
-- if data for the specific candle are still loading
-- then do nothing
if loading and bf_candle >= loadingFrom and (loadingTo == 0 or bf_candle <= loadingTo) then
return ;
end
-- if the period is before the source start
-- the do nothing
if period < source:first() then
return ;
end
-- if data is not loaded yet at all
-- load the data
if bf_data == nil then
-- there is no data at all, load initial data
local to, t;
local from;
if (source:isAlive()) then
-- if the source is subscribed for updates
-- then subscribe the current collection as well
to = 0;
else
-- else load up to the last currently available date
t, to = core.getcandle(BS, source:date(period), day_offset, week_offset);
end
from = core.getcandle(BS, source:date(source:first()), day_offset, week_offset);
SK:setBookmark(1, period);
-- shift so the bigger frame data is able to provide us with the stoch data at the first period
from = math.floor(from * 86400 - (bf_length * extent) + 0.5) / 86400;
local nontrading, nontradingend;
nontrading, nontradingend = core.isnontrading(from, day_offset);
if nontrading then
-- if it is non-trading, shift for two days to skip the non-trading periods
from = math.floor((from - 2) * 86400 - (bf_length * extent) + 0.5) / 86400;
end
loading = true;
loadingFrom = from;
loadingTo = to;
bf_data = host:execute("getHistory", 1, source:instrument(), BS, loadingFrom, to, source:isBid());
Stochastic = core.indicators:create("DINAPOLI PREFERRED STOCHASTIC", bf_data, k, sd, d);
return ;
end
-- check whether the requested candle is before
-- the reference collection start
if (bf_candle < bf_data:date(0)) then
SK:setBookmark(1, period);
if loading then
return ;
end
-- shift so the bigger frame data is able to provide us with the stoch data at the first period
from = math.floor(bf_candle * 86400 - (bf_length * extent) + 0.5) / 86400;
local nontrading, nontradingend;
nontrading, nontradingend = core.isnontrading(from, day_offset);
if nontrading then
-- if it is non-trading, shift for two days to skip the non-trading periods
from = math.floor((from - 2) * 86400 - (bf_length * extent) + 0.5) / 86400;
end
loading = true;
loadingFrom = from;
loadingTo = bf_data:date(0);
host:execute("extendHistory", 1, bf_data, loadingFrom, loadingTo);
return ;
end
-- check whether the requested candle is after
-- the reference collection end
if (not(source:isAlive()) and bf_candle > bf_data:date(bf_data:size() - 1)) then
SK:setBookmark(1, period);
if loading then
return ;
end
loading = true;
loadingFrom = bf_data:date(bf_data:size() - 1);
loadingTo = bf_candle;
host:execute("extendHistory", 1, bf_data, loadingFrom, loadingTo);
return ;
end
Stochastic:update(mode);
local p;
p = core.findDate (bf_data, bf_candle, true);
if p == -1 then
return ;
end
if Stochastic:getStream(0):hasData(p) then
SK[period] = Stochastic:getStream(0)[p];
end
if Stochastic:getStream(1):hasData(p) then
SD[period] = Stochastic:getStream(1)[p];
end
end
-- the function is called when the async operation is finished
function AsyncOperationFinished(cookie)
local period;
pday = nil;
period = SK:getBookmark(1);
if (period < 0) then
period = 0;
end
loading = false;
instance:updateFrom(period);
end
Addendum: https://ninjatrader.com/support/forum/forum/ninjascript-file-sharing/ninjascript-file-sharing-discussion/3545-dinapoli-stochastic/page2
I am writing some code that takes binary data from Python, Pipes it to C++, does some processing on the data, (in this case calculating a mutual information metric) and then pipes the results back to python. While testing I have found that everything works fine if the data I send is a set of 2 arrays with dimensions less than 1500 X 1500, but if I send 2 arrays that are 2K X 2K I get back a lot of corrupted nonsense.
I currently believe the algorithmic portion of the code is fine because it provides the expected answers during testing with small (<=1500 X1500) arrays. That leads me to believe that this is an issue with either the stdin or stdout piping. That maybe I’m passing some intrinsic limit somewhere.
The Python Code and C++ code are below.
Python Code:
import subprocess
import struct
import sys
import numpy as np
#set up the variables needed
bytesPerDouble = 8
sizeX = 2000
sizeY = 2000
offset = sizeX*sizeY
totalBytesPerArray = sizeX*sizeY*bytesPerDouble
totalBytes = totalBytesPerArray*2 #the 2 is because we pass 2 different versions of the 2D array
#setup the testing data array
a = np.zeros(sizeX*sizeY*2, dtype='d')
for i in range(sizeX):
for j in range(sizeY):
a[j+i*sizeY] = i
a[j+i*sizeY+offset] = i
if i % 10 == 0:
a[j+i*sizeY+offset] = j
data = a.tobytes('C')
strTotalBytes = str(totalBytes)
strLineBytes = str(sizeY*bytesPerDouble)
#communicate with c++ code
print("starting C++ code")
command = "C:\Python27\PythonPipes.exe"
proc = subprocess.Popen([command, strTotalBytes, strLineBytes, str(sizeY), str(sizeX)], stdin=subprocess.PIPE,stderr=subprocess.PIPE,stdout=subprocess.PIPE)
ByteBuffer = (data)
proc.stdin.write(ByteBuffer)
print("Reading results back from C++")
for i in range(sizeX):
returnvalues = proc.stdout.read(sizeY*bytesPerDouble)
a = buffer(returnvalues)
b = struct.unpack_from(str(sizeY)+'d', a)
print str(b) + " " + str(i)
print('done')
C++ Code:
Main function:
int main(int argc, char **argv) {
int count = 0;
long totalbytes = stoi(argv[argc-4], nullptr,10); //bytes being transfered
long bytechunk = stoi(argv[argc - 3], nullptr, 10); //bytes being transfered at a time
long height = stoi(argv[argc-2], nullptr, 10); //bytes being transfered at a time
long width = stoi(argv[argc-1], nullptr, 10); //bytes being transfered at a time
long offset = totalbytes / sizeof(double) / 2;
data = new double[totalbytes/sizeof(double)];
int columnindex = 0;
//read in data from pipe
while (count<totalbytes) {
fread(&(data[columnindex]), 1, bytechunk, stdin);
columnindex += bytechunk / sizeof(double);
count += bytechunk;
}
//calculate the data transform
MutualInformation MI = MutualInformation();
MI.Initialize(data, height, width, offset);
MI.calcMI();
count = 0;
//*
//write out data to pipe
columnindex = 0;
while (count<totalbytes/2) {
fwrite(&(MI.getOutput()[columnindex]), 1, bytechunk, stdout);
fflush(stdout);
count += bytechunk;
columnindex += bytechunk/sizeof(double);
}
//*/
delete [] data;
return 0;
}
and in case you need it the actual processing code:
double MutualInformation::calcMI(){
double rvalue = 0.0;
std::map<int, map<int, double>> lHistXY = map<int, map<int, double>>();
std::map<int, double> lHistX = map<int, double>();
std::map<int, double> lHistY = map<int, double>();
typedef std::map<int, std::map<int, double>>::iterator HistXY_iter;
typedef std::map<int, double>::iterator HistY_iter;
//calculate Entropys and MI
double MI = 0.0;
double Hx = 0.0;
double Hy = 0.0;
double Px = 0.0;
double Py = 0.0;
double Pxy = 0.0;
//scan through the image
int ip = 0;
int jp = 0;
int chipsize = 3;
//setup zero array
double * zeros = new double[this->mHeight];
for (int j = 0; j < this->mHeight; j++){
zeros[j] = 0.0;
}
//zero out Output array
for (int i = 0; i < this->mWidth; i++){
memcpy(&(this->mOutput[i*this->mHeight]), zeros, this->mHeight*8);
}
double index = 0.0;
for (int ioutter = chipsize; ioutter < (this->mWidth - chipsize); ioutter++){
//write out processing status
//index = (double)ioutter;
//fwrite(&index, 8, 1, stdout);
//fflush(stdout);
//*
for (int j = chipsize; j < (this->mHeight - chipsize); j++){
//clear the histograms
lHistX.clear();
lHistY.clear();
lHistXY.clear();
//chip out a section of the image
for (int k = -chipsize; k <= chipsize; k++){
for (int l = -chipsize; l <= chipsize; l++){
ip = ioutter + k;
jp = j + l;
//update X histogram
if (lHistX.count(int(this->mData[ip*this->mHeight + jp]))){
lHistX[int(this->mData[ip*this->mHeight + jp])] += 1.0;
}else{
lHistX[int(this->mData[ip*this->mHeight + jp])] = 1.0;
}
//update Y histogram
if (lHistY.count(int(this->mData[ip*this->mHeight + jp+this->mOffset]))){
lHistY[int(this->mData[ip*this->mHeight + jp+this->mOffset])] += 1.0;
}
else{
lHistY[int(this->mData[ip*this->mHeight + jp+this->mOffset])] = 1.0;
}
//update X and Y Histogram
if (lHistXY.count(int(this->mData[ip*this->mHeight + jp]))){
//X Key exists check if Y key exists
if (lHistXY[int(this->mData[ip*this->mHeight + jp])].count(int(this->mData[ip*this->mHeight + jp + this->mOffset]))){
//X & Y keys exist
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] += 1;
}else{
//X exist but Y doesn't
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] = 1;
}
}else{
//X Key Didn't exist
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] = 1;
};
}
}
//calculate PMI, Hx, Hy
// iterator->first = key
// iterator->second = value
MI = 0.0;
Hx = 0.0;
Hy = 0.0;
for (HistXY_iter Hist2D_iter = lHistXY.begin(); Hist2D_iter != lHistXY.end(); Hist2D_iter++) {
Px = lHistX[Hist2D_iter->first] / ((double) this->mOffset);
Hx -= Px*log(Px);
for (HistY_iter HistY_iter = Hist2D_iter->second.begin(); HistY_iter != Hist2D_iter->second.end(); HistY_iter++) {
Py = lHistY[HistY_iter->first] / ((double) this->mOffset);
Hy -= Py*log(Py);
Pxy = HistY_iter->second / ((double) this->mOffset);
MI += Pxy*log(Pxy / Py / Px);
}
}
//normalize PMI to max(Hx,Hy) so that the PMI value runs from 0 to 1
if (Hx >= Hy && Hx > 0.0){
MI /= Hx;
}else if(Hy > Hx && Hy > 0.0){
MI /= Hy;
}
else{
MI = 0.0;
}
//write PMI to data output array
if (MI < 1.1){
this->mOutput[ioutter*this->mHeight + j] = MI;
}
else{
this->mOutput[ioutter*this->mHeight + j] = 0.0;
}
}
}
return rvalue;
}
with arrays that return something that makes sense I get output bounded between 0 and 1 like this:
(0.0, 0.0, 0.0, 0.7160627908692593, 0.6376472316395495, 0.5728801401524277,...
with the 2Kx2K or higher arrays I get nonesense like this (even though the code clamps the values between 0 and 1):
(-2.2491400820412374e+228, -2.2491400820412374e+228, -2.2491400820412374e+228, -2.2491400820412374e+228, -2.2491400820412374e+228,...
I would like to know why this code is corrupting the data set after it is assigned between 0.0 and 1, and whether or not it is a piping issue, a stdin/stdout issue, a buffer issue of some sort, or a coding issue I am simply not seeing.
Update I tried passing the data in smaller chunks using the code that Chris suggested with no luck. also of note is that I added a catch for ferror on stdout and it never got tripped so I am pretty sure that the bytes are at least making it to stdout. Is it possible that something else is writing to stdout somehow? maybe an extra byte making its way into stdout while my program is running? I find this doubtful as the errors are appearing consistently on the 4th fwrite read in the 10th entry.
Per Craig's request here is the full C++ code (the full Python Code is already posted): it is sitting in 3 files:
main.cpp
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <iostream>
#include "./MutualInformation.h"
double * data;
using namespace std;
void
xxwrite(unsigned char *buf, size_t wlen, FILE *fo)
{
size_t xlen;
for (; wlen > 0; wlen -= xlen, buf += xlen) {
xlen = wlen;
if (xlen > 1024)
xlen = 1024;
xlen = fwrite(buf, 1, xlen, fo);
fflush(fo);
}
}
int main(int argc, char **argv) {
int count = 0;
long totalbytes = stoi(argv[argc-4], nullptr,10); //bytes being transfered
long bytechunk = stoi(argv[argc - 3], nullptr, 10); //bytes being transfered at a time
long height = stoi(argv[argc-2], nullptr, 10); //bytes being transfered at a time
long width = stoi(argv[argc-1], nullptr, 10); //bytes being transfered at a time
long offset = totalbytes / sizeof(double) / 2;
data = new double[totalbytes/sizeof(double)];
int columnindex = 0;
//read in data from pipe
while (count<totalbytes) {
fread(&(data[columnindex]), 1, bytechunk, stdin);
columnindex += bytechunk / sizeof(double);
count += bytechunk;
}
//calculate the data transform
MutualInformation MI = MutualInformation();
MI.Initialize(data, height, width, offset);
MI.calcMI();
count = 0;
columnindex = 0;
while (count<totalbytes/2) {
xxwrite((unsigned char*)&(MI.getOutput()[columnindex]), bytechunk, stdout);
count += bytechunk;
columnindex += bytechunk/sizeof(double);
}
delete [] data;
return 0;
}
MutualInformation.h
#include <map>
using namespace std;
class MutualInformation
{
private:
double * mData;
double * mOutput;
long mHeight;
long mWidth;
long mOffset;
public:
MutualInformation();
~MutualInformation();
bool Initialize(double * data, long Height, long Width, long Offset);
const double * getOutput();
double calcMI();
};
MutualInformation.cpp
#include "MutualInformation.h"
MutualInformation::MutualInformation()
{
this->mData = nullptr;
this->mOutput = nullptr;
this->mHeight = 0;
this->mWidth = 0;
}
MutualInformation::~MutualInformation()
{
delete[] this->mOutput;
}
bool MutualInformation::Initialize(double * data, long Height, long Width, long Offset){
bool rvalue = false;
this->mData = data;
this->mHeight = Height;
this->mWidth = Width;
this->mOffset = Offset;
//allocate output data
this->mOutput = new double[this->mHeight*this->mWidth];
return rvalue;
}
const double * MutualInformation::getOutput(){
return this->mOutput;
}
double MutualInformation::calcMI(){
double rvalue = 0.0;
std::map<int, map<int, double>> lHistXY = map<int, map<int, double>>();
std::map<int, double> lHistX = map<int, double>();
std::map<int, double> lHistY = map<int, double>();
typedef std::map<int, std::map<int, double>>::iterator HistXY_iter;
typedef std::map<int, double>::iterator HistY_iter;
//calculate Entropys and MI
double MI = 0.0;
double Hx = 0.0;
double Hy = 0.0;
double Px = 0.0;
double Py = 0.0;
double Pxy = 0.0;
//scan through the image
int ip = 0;
int jp = 0;
int chipsize = 3;
//setup zero array
double * zeros = new double[this->mHeight];
for (int j = 0; j < this->mHeight; j++){
zeros[j] = 0.0;
}
//zero out Output array
for (int i = 0; i < this->mWidth; i++){
memcpy(&(this->mOutput[i*this->mHeight]), zeros, this->mHeight*8);
}
double index = 0.0;
for (int ioutter = chipsize; ioutter < (this->mWidth - chipsize); ioutter++){
for (int j = chipsize; j < (this->mHeight - chipsize); j++){
//clear the histograms
lHistX.clear();
lHistY.clear();
lHistXY.clear();
//chip out a section of the image
for (int k = -chipsize; k <= chipsize; k++){
for (int l = -chipsize; l <= chipsize; l++){
ip = ioutter + k;
jp = j + l;
//update X histogram
if (lHistX.count(int(this->mData[ip*this->mHeight + jp]))){
lHistX[int(this->mData[ip*this->mHeight + jp])] += 1.0;
}else{
lHistX[int(this->mData[ip*this->mHeight + jp])] = 1.0;
}
//update Y histogram
if (lHistY.count(int(this->mData[ip*this->mHeight + jp+this->mOffset]))){
lHistY[int(this->mData[ip*this->mHeight + jp+this->mOffset])] += 1.0;
}
else{
lHistY[int(this->mData[ip*this->mHeight + jp+this->mOffset])] = 1.0;
}
//update X and Y Histogram
if (lHistXY.count(int(this->mData[ip*this->mHeight + jp]))){
//X Key exists check if Y key exists
if (lHistXY[int(this->mData[ip*this->mHeight + jp])].count(int(this->mData[ip*this->mHeight + jp + this->mOffset]))){
//X & Y keys exist
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] += 1;
}else{
//X exist but Y doesn't
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] = 1;
}
}else{
//X Key Didn't exist
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] = 1;
};
}
}
//calculate PMI, Hx, Hy
// iterator->first = key
// iterator->second = value
MI = 0.0;
Hx = 0.0;
Hy = 0.0;
for (HistXY_iter Hist2D_iter = lHistXY.begin(); Hist2D_iter != lHistXY.end(); Hist2D_iter++) {
Px = lHistX[Hist2D_iter->first] / ((double) this->mOffset);
Hx -= Px*log(Px);
for (HistY_iter HistY_iter = Hist2D_iter->second.begin(); HistY_iter != Hist2D_iter->second.end(); HistY_iter++) {
Py = lHistY[HistY_iter->first] / ((double) this->mOffset);
Hy -= Py*log(Py);
Pxy = HistY_iter->second / ((double) this->mOffset);
MI += Pxy*log(Pxy / Py / Px);
}
}
//normalize PMI to max(Hx,Hy) so that the PMI value runs from 0 to 1
if (Hx >= Hy && Hx > 0.0){
MI /= Hx;
}else if(Hy > Hx && Hy > 0.0){
MI /= Hy;
}
else{
MI = 0.0;
}
//write PMI to data output array
if (MI < 1.1){
this->mOutput[ioutter*this->mHeight + j] = MI;
}
else{
this->mOutput[ioutter*this->mHeight + j] = 0.0;
//cout << "problem with output";
}
}
}
//*/
return rvalue;
}
SOLVED By 6502
6502's answer below solved my problem. I needed to explicitly tell Windows to use a binary mode for stdin / stdout. to do that I had to include 2 new header files in my main cpp file.
#include <fcntl.h>
#include <io.h>
add the following lines of code (modified away from 6502's POSIX versions because Visual Studio complained) to the beginning of my main function
_setmode(_fileno(stdout), O_BINARY);
_setmode(_fileno(stdin), O_BINARY);
and then add these lines to my Python code:
import os, msvcrt
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
The problem is that stdin/stdout in windows are opened in text mode, not in binary mode and therefore will mess up when the character 13 (\r) is sent.
You can set for example binary mode in Python with
import os, msvcrt
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
and in C++ with
_setmode(fileno(stdout), O_BINARY);
_setmode(fileno(stdin), O_BINARY);
See https://msdn.microsoft.com/en-us/library/tw4k6df8.aspx
Your C++ fwrite code does not account for getting a "short" transfer.
Here's a slight tweak:
//write out data to pipe
columnindex = 0;
while (count < totalbytes / 2) {
wlen = fwrite(&(MI.getOutput()[columnindex]), 1, bytechunk, stdout);
fflush(stdout);
count += wlen;
columnindex += wlen / sizeof(double);
}
Note: You still need to be careful as this would still have issues if wlen returns and it's not a multiple of sizeof(double). For example, if bytechunk were 16 and wlen came back with 14, you'd need an additional fwrite with length 2 before continuing the loop. A generalization of this is just to treat the entire data matrix as a giant byte buffer and loop on that.
Actually, you'll get about the same efficiency with many much smaller transfers that are capped by a fixed (i.e. "known safe amount") of [say] 1024 bytes. This works because the output is a byte stream.
Here's a slightly more general solution that I've often used:
void
xxwrite(void *buf,size_t wlen,FILE *fo)
{
size_t xlen;
for (; wlen > 0; wlen -= xlen, buf += xlen) {
xlen = wlen;
if (xlen > 1024)
xlen = 1024;
xlen = fwrite(buf,1,xlen,fo);
fflush(fo);
}
}
//write out data to pipe
columnindex = 0;
while (count < totalbytes / 2) {
xxwrite(&(MI.getOutput()[columnindex]), bytechunk, stdout);
count += bytechunk;
columnindex += bytechunk / sizeof(double);
}
UPDATE:
I've downloaded all your code and run it. I've got good news and bad news: The code runs fine here, even for a matrix size above 3000. I ran it both using xxwrite and without and the results were the same.
Using my limited python skills, I added some pretty print to your python script (e.g. some line wrap) and had it check every value for range and annotate any bad values. There were none found by the script. Also, visual inspection of the values turned up nothing [this was true before the pretty print, so it hasn't introduced anything]. Just lots of zeros and then blocks in the 0.9 range.
The only difference I can see is that I'm using gcc [and, of course, python] on linux. But, from your script it seems your using Windows [based on the C:\... path for your C++ executable. This shouldn't matter for this application, but I mention it anyway.
So, pipes work here. One thing you might try is to direct the C++ output to a file. Then, have the script read back from the file (i.e. no pipe) and see if that makes a difference. I tend to think not, but ...
Also, I don't know what compiler and python implementation you're using under Windows. Whenever I have to do this, I usually have Cygwin installed as it gives one of the closest implementations of linux/Unix-like environment (i.e. pipes are more likely to work as advertised).
Anyway, here's the modified script. Also note that I added os.getenv to grab alternate matrix sizes and an alternate place for the C++ executable, so that it would work for both of us with minimal pain
#!/usr/bin/python
import subprocess
import struct
import sys
import os
import numpy as np
val = os.getenv("MTX","2000")
sizeX = int(val)
sizeY = sizeX
print "sizeX=%d sizeY=%d" % (sizeX,sizeY)
#set up the variables needed
bytesPerDouble = 8
offset = sizeX*sizeY
totalBytesPerArray = sizeX*sizeY*bytesPerDouble
totalBytes = totalBytesPerArray*2 #the 2 is because we pass 2 different versions of the 2D array
#setup the testing data array
a = np.zeros(sizeX*sizeY*2, dtype='d')
for i in range(sizeX):
for j in range(sizeY):
a[j+i*sizeY] = i
a[j+i*sizeY+offset] = i
if i % 10 == 0:
a[j+i*sizeY+offset] = j
data = a.tobytes('C')
strTotalBytes = str(totalBytes)
strLineBytes = str(sizeY*bytesPerDouble)
#communicate with c++ code
print("starting C++ code")
command = os.getenv("CPGM",None);
if command is None:
command = "C:\Python27\PythonPipes.exe"
proc = subprocess.Popen([command, strTotalBytes, strLineBytes, str(sizeY), str(sizeX)], stdin=subprocess.PIPE,stderr=subprocess.PIPE,stdout=subprocess.PIPE)
ByteBuffer = (data)
proc.stdin.write(ByteBuffer)
def prt(i,b):
hangflg = 0
per = 8
for j in range(0,len(b)):
if ((j % per) == 0):
print("[%d,%d]" % (i,j)),
q = b[j]
print(q),
hangflg = 1
if (q < 0.0) or (q > 1.0):
print("=WTF"),
if ((j % per) == (per - 1)):
print("")
hangflg = 0
if (hangflg):
print("")
print("Reading results back from C++")
for i in range(sizeX):
returnvalues = proc.stdout.read(sizeY*bytesPerDouble)
a = buffer(returnvalues)
b = struct.unpack_from(str(sizeY)+'d', a)
prt(i,b)
###print str(b) + " " + str(i)
###print str(i) + ": " + str(b)
print('done')
We would really appreciate any kind of help, because we are driving crazy with our program making it faster using C language.
The values obtained don't change, always are 0,0,0,0
Here is the code, running in Linux:
from scipy import weave
pasa = 0
coorX = -11.8
coorY = -7.9
INC=0.01296
##def weave_update():
code="""
int i,j, pasa;
double coorX, coorY,INC;
for (i=0; i < 1296;i++){
yminf = coorY + INC*(i);
ymaxf = yminf + INC;
for (j=0; j < 1936;j++){
xminc = coorX + INC*(j);
xmaxc = xminc + INC;
pasa = 1;
break;
}
if (pasa == 1){
break;
}
}
"""
weave.inline(code,['yminf','xminc','xmaxc','ymaxf'],type_converters=weave.converters.blitz,compiler='gcc')
print yminf,xminc,xmaxc,ymaxf
Looks like two issues. First, you need to pass in all of the variables that the C code needs access to from python. So, your inline call needs to be:
weave.inline(code, ['coorX','coorY','INC'])
Secondly, you need to return the values you want from the weave code, because modifying them in C doesn't affect their value in Python. Here's one way to do it:
py::tuple ret(4);
ret[0] = yminf;
ret[1] = xminc;
ret[2] = xmaxc;
ret[3] = ymaxf;
return_val = ret;
With these modifications, the following file seems to work correctly:
from scipy import weave
coorX = -11.8
coorY = -7.9
INC = 0.01296
code="""
int i,j, pasa = 0;
double yminf,xminc,xmaxc,ymaxf;
for (i=0; i < 1296;i++){
yminf = coorY + INC*(i);
ymaxf = yminf + INC;
for (j=0; j < 1936;j++){
xminc = coorX + INC*(j);
xmaxc = xminc + INC;
pasa = 1;
break;
}
if (pasa == 1){
break;
}
}
py::tuple ret(4);
ret[0] = yminf;
ret[1] = xminc;
ret[2] = xmaxc;
ret[3] = ymaxf;
return_val = ret;
"""
yminf,xminc,xmaxc,ymaxf = weave.inline(code,['coorX','coorY','INC'])
print yminf,xminc,xmaxc,ymaxf
I'm pretty green with Python so hopefully I can phrase this question correctly.
The overall problem involves calling a C routine from Python. I can get pretty close by kluging together a few related SO questions/answers, but I cannot seem to make things come together quite right. There are two aspects: first is calling the C routine with pointers and the second is the use of a callback function.
Background
Rubner provides an Earth Movers Distance (EMD) routine written in C [ EMD C code location ] He also provides two example C programs that call the EMD routine. I am trying to develop a Python routine as an alternative to, for example, example2.c that will call the EMD routine. (Yes, I'm familiar with the OpenCV implementation of EMD.)
For convenience here is the header file for the emd.c code I'd like call from python:
/* DEFINITIONS */
#define MAX_SIG_SIZE 100
#define MAX_ITERATIONS 500
#define INFINITY 1e20
#define EPSILON 1e-6
/*****************************************************************************/
/* feature_t SHOULD BE MODIFIED BY THE USER TO REFLECT THE FEATURE TYPE */
typedef int feature_t;
/* typedef struct { int X,Y,Z; } feature_t;*/
/*typedef struct { int X; } feature_t; */
/*****************************************************************************/
typedef struct
{
int n; /* Number of features in the signature */
feature_t *Features; /* Pointer to the features vector */
float *Weights; /* Pointer to the weights of the features */
} signature_t;
typedef struct
{
int from; /* Feature number in signature 1 */
int to; /* Feature number in signature 2 */
float amount; /* Amount of flow from "from" to "to" */
} flow_t;
float emd(signature_t *Signature1, signature_t *Signature2,
float (*func)(feature_t *, feature_t *),
flow_t *Flow, int *FlowSize);
#endif
Finally, here is the python codes that I've kluged together so far. I think (but am not sure) that I've gotten the structures setup correctly. (Note that this is a simplified version of the possible feature structures in the Rubner emd.c code . I would eventually like to get the whole thing working, but I'm starting out simple for now.) The first problem I'm having is somewhere in the argtypes for the calling function. I've tried a few variations, but the examples available on the web are pretty slim and I've hit a wall.
import ctypes
MAX_FEATURE_SIZE = 30
ARRAYFE = ctypes.c_int*MAX_FEATURE_SIZE
ARRAYWE= ctypes.c_float*MAX_FEATURE_SIZE
ARRAYFL = ctypes.c_float*(2*MAX_FEATURE_SIZE-1)
flowSize = ctypes.c_int
emdlib = ctypes.CDLL('emdlib.dylib')
ctypes.CMPFUNC = ctypes.CFUNCTYPE(ctypes.c_float, ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int))
def py_dist_func(f1,f2):
print "dist: ", f1, f2
return(abs(f1-f2))
dist = ctypes.CMPFUNC(py_dist_func)
n = ctypes.c_int
flowSize = ctypes.c_int
class flow_t(ctypes.Structure):
_fields_ = [("from", ctypes.c_int),
("to", ctypes.c_int),
("amount", ctypes.c_float)]
class signature_t(ctypes.Structure):
_fields_ = [("N", n),("feature", ARRAYFE),
("weight", ARRAYWE)]
# emdlib.emd.argtypes = [ctypes.POINTER(signature_t), ctypes.POINTER(signature_t), ctypes.POINTER(ctypes.c_float), ctypes.POINTER(flow_t), ctypes.POINTER(ctypes.c_int)]
# emdlib.emd.argtypes = [ctypes.POINTER(signature_t), ctypes.POINTER(signature_t), ctypes.CMPFUNC(py_dist_func), ctypes.POINTER(flow_t), ctypes.POINTER(ctypes.c_int)]
emdlib.emd.argtypes = [ctypes.POINTER(signature_t), ctypes.POINTER(signature_t), ctypes.c_float, ctypes.POINTER(flow_t), ctypes.POINTER(ctypes.c_int)]
# emd.restype = ctypes.c_float
emdlib.emd.restype = flow_t
signature1=signature_t()
signature2=signature_t()
feature1 = ARRAYFE
feature2 = ARRAYFE
weight1 =ARRAYWE
weight2 = ARRAYWE
feature1 = [0,1,2,3]
feature2 = [0,3]
weight1 = [1,1,1,1]
weight2 = [1,1]
#signature1= [4,feature1, weight1]
#signature2 = [2, feature2, weight2]
# sample: arr = (ctypes.c_int * len(pyarr))(*pyarr)
signature1.N = len(feature1)
signature1.feature = (ctypes.c_int * MAX_FEATURE_SIZE)(*feature1)
signature2.feature = (ctypes.c_int * MAX_FEATURE_SIZE)(*feature2)
signature1.weight = (ctypes.c_float * MAX_FEATURE_SIZE)(*weight1)
signature2.weight = (ctypes.c_float * MAX_FEATURE_SIZE)(*weight2)
e = emdlib.emd(ctypes.byref(signature1), ctypes.byref(signature2), dist, ctypes.POINTER(flow_t), flowSize)
print "EMD= ", e
print "flowSize", flowSize
Any suggestions on where I've gone wrong would be greatly appreciated.
The second problem I'm sure I'll run into is with the argtypes for the returning pointers; any suggestions here would be appreciated as well.
Thanks in advance.
-------------- Updated (working) CODE
import ctypes
import math
import itertools
MAX_FEATURE_SIZE = 25
FEATURE_t = ctypes.c_int
FEATURE_ptr = ctypes.POINTER(FEATURE_t)
WEIGHT_t = ctypes.c_float
WEIGHT_ptr = ctypes.POINTER(WEIGHT_t)
COUNT_t = ctypes.c_int
COUNT_ptr = ctypes.POINTER(COUNT_t)
class FLOW_t(ctypes.Structure):
_fields_ = [("frm", ctypes.c_int),
("to", ctypes.c_int),
("amount", ctypes.c_float)]
# Note that ctypes.POINTER is compatible with a ctypes array declared
# as TYPE * array_len. This is equivalent to the way we can say 'char
# *foo = "ABCDEF"' in C.
class SIGNATURE_t(ctypes.Structure):
_fields_ = [("N", COUNT_t ),
("feature", FEATURE_ptr),
("weight", WEIGHT_ptr)]
FLOW_ARRAY_t = FLOW_t * (2*MAX_FEATURE_SIZE - 1)
CMPFUNC_t = ctypes.CFUNCTYPE(ctypes.c_float, FEATURE_ptr, FEATURE_ptr)
SIGNATURE_ptr = ctypes.POINTER(SIGNATURE_t)
FLOW_ptr = ctypes.POINTER(FLOW_t)
# Convenience function - keeps us from having to remember all the types and parameters later on
def make_signature(features, weights):
sig = SIGNATURE_t()
sig.N = len(features)
sig.feature = (len(features) * FEATURE_t)(*features)
sig.weight = (len(weights) * WEIGHT_t)(*weights)
return sig
# We want to pass into C a custom distance function from Python
def py_dist_func(f1,f2):
# print "f1, f2: %d, %d" % ( f1[0], f2[0] )
d= distance(f1[0],f2[0])
return d
# set this up as a holder for distance function between any two n-D points
def distance(p0,p1):
return(math.fabs(p0-p1))
dist_callback = CMPFUNC_t(py_dist_func)
#print "Importing emdlib"
emdlib = ctypes.CDLL('emdlib.dylib')
#print "Setting argtypes"
emdlib.emd.argtypes = [ SIGNATURE_ptr,
SIGNATURE_ptr,
CMPFUNC_t,
FLOW_ptr,
COUNT_ptr ]
#print "Setting restype"
emdlib.emd.restype = ctypes.c_float
feature1 = [0, 1,2,3,4,5,6,7,8]
feature2 = [0, 1,2,3,4,5,6,7,8]
weight1 = [0.275,0.296,0.002,0.131,0.208,0.048,0.058,0.098,0.455]
weight2 = [0.285,0.421,0.028,0.021,0.240,0.166,0.023,0.054,0.469]
#print "Creating signatures"
signature1 = make_signature(feature1, weight1)
signature2 = make_signature(feature2, weight2)
flow_array = FLOW_ARRAY_t()
flow_size = COUNT_t()
#print "Calling EMD"
e = emdlib.emd(ctypes.byref(signature1),
ctypes.byref(signature2),
dist_callback,
flow_array,
ctypes.byref(flow_size))
print "EMD= ", e
print "Number of FlowS", flow_size.value
print "Flow"
print "from to amount"
totalFlow=0.0
for i in range(0,flow_size.value):
# print "Flow from %d to %d amount :%f" %(flow_array[i].frm, flow_array[i].to, flow_array[i].amount)
print " %d %d %f" %(flow_array[i].frm, flow_array[i].to, flow_array[i].amount)
totalFlow=totalFlow+flow_array[i].amount
#
# now adjust EMD to account for different signature masses and make it a metric
alpha=1.0
mass1=sum(weight1)
mass2=sum(weight2)
fList=[feature1,feature2]
max_distance= 0.0
for p0, p1 in list(itertools.product(*fList)):
# print p0,p1, distance(p0,p1), max_distance
max_distance = max(max_distance, distance(p0, p1))
print "\nMax distance= %f" % max_distance
print "Total Source = %f" % mass1
print "Total Demand = %f" % mass2
print "Total Flow= %f\n " % totalFlow
print "Alpha = %f\n" %alpha
# emdHat = e*totalFlow+math.sqrt((mass1-mass2)*(mass1-mass2))*alpha*max_distance
emdHat = e*totalFlow+math.fabs((mass1-mass2))*alpha*max_distance
print "Corrected Earth Movers Distance \n"
print "emdHat = %f\n" % emdHat;
Through various arcane methods and the valuable comments, I finally got a piece of code working. As I mentioned in the comments, I'm not sure what the etiquette is, but I've seen enough similar questions asked that I thought it would be useful to post the final bit of code. It's not pretty, and if you find it useful enough to clean it up, I'd appreciate a link to a more elegant implementation.
I have a Delphi library that is exposing a results with a procedure like this:
procedure Script_GetFindedList(List : Pointer; out Len : Cardinal); stdcall;
var X : TArray<Cardinal>;
begin
TScriptMethod.Create(SCGetFindedList).SendExecMethod.Free;
NamedPipe.WaitForReply(Pipe_WaitDelay);
if not ResultReady then ExitProcess(0);
SetLength(X,FuncResultStream.Size div 4);
FuncResultStream.Read(X[0],FuncResultStream.Size);
Len := Length(X) * 4;
if Assigned(List) then
Move(X[0],PByteArray(List)^[0],Len);
end;
And I am able to call it from normal delphi Code like this:
function TFindEngine.GetFindedList : TArray<Cardinal>;
var BufLen : Cardinal;
begin
Script_GetFindedList(nil, BufLen);
if BufLen = 0 then Exit;
SetLength(Result,BufLen div 4);
Script_GetFindedList(PByteArray(Result), BufLen);
end;
I would like to wrap the code in Python using the ctypes library and I have some code like this:
from ctypes import *
my_dll = windll.Script
def GetFindedList():
my_dll.Script_GetFindedList.argtypes = [POINTER(c_uint), POINTER(c_uint)]
my_dll.Script_GetFindedList.restype = None
BufLen = c_uint()
my_dll.Script_GetFindedList(None, byref(BufLen))
if BufLen.value > 0:
print("BufLen.value : {}".format(BufLen.value))
##################################################################
# alternate solution that just leaks memory while doing nothind
# buf = array('I', range(BufLen.value))
# addr, count = buf.buffer_info()
# Result = cast(addr, POINTER( (c_uint * BufLen.value) ))
Result = (c_uint * BufLen.value)()
print("Result before: {}".format(list(Result)))
my_dll.Script_GetFindedList(byref(Result), byref(BufLen))
print("Result after: {}".format(list(Result)))
return Result
else:
return []
But this is not working: I just get the correct BufLen.value but then, with the second call to dll I am not able to populate my array. I did many similar tries, but with no luck. Is there someone that can advise me?
Thank you.
I'd call it like this:
from ctypes import *
my_dll = windll.Script
my_dll.Script_GetFindedList.restype = None
size = c_uint()
my_dll.Script_GetFindedList(None, byref(size))
result = (c_uint*(size.value//4))()
my_dll.Script_GetFindedList(result, byref(size))
result = list(result)
This function would be much better if you return buffer length rather than size.
I tested this using the following code:
Delphi
library TestDLL;
procedure Script_GetFindedList(List : Pointer; out Len : Cardinal); stdcall;
var
X: TArray<Cardinal>;
begin
X := TArray<Cardinal>.Create(1, 2, 3, 4, 5);
Len := Length(X) * 4;
if Assigned(List) then
Move(Pointer(X)^, List^, Len);
end;
exports
Script_GetFindedList;
begin
end.
Python
from ctypes import *
my_dll = WinDLL(r'full/path/to/TestDLL.dll')
my_dll.Script_GetFindedList.restype = None
size = c_uint()
my_dll.Script_GetFindedList(None, byref(size))
result = (c_uint*(size.value//4))()
my_dll.Script_GetFindedList(result, byref(size))
result = list(result)
print result
Output
[1L, 2L, 3L, 4L, 5L]