We Keep Coding

python script slowing down and almost stops after a couple of seconds

I am trying to make a resource monitor with an arduino and it is working great for almost 20 seconds before it almost stops running.
When it slows down, it takes almost 5 seconds between updates.
I have tried to comment out everything with psutil and giving it a permanent value.
and have tried the same with GPUtil.
Here is the python code
import serial.tools.list_ports
import serial
import psutil
import GPUtil
import time
import serial
ports = list(serial.tools.list_ports.comports())
baud = 9600
for p in ports:
if "Arduino" in p[1]:
port = p[0]
ser=serial.Serial(port, baud, timeout=1)
try:
while True:
cpuUsage = psutil.cpu_percent()
ramUsage = psutil.virtual_memory()
cpuUsage = str(cpuUsage)
GPUs = GPUtil.getGPUs()
gpuUsage = GPUs[0].load
gpuUsage = str(gpuUsage)
gpuUsage = gpuUsage[2:]
ramUsage = str(ramUsage.percent)
toSend = cpuUsage + "," + gpuUsage + ","+ ramUsage
print (toSend)
ser.write(toSend.encode())
#print("20.5".encode())
#line = ser.readline()[:-2]
#line.decode()
#print ("Read : " , line);
time.sleep(0.1)
except:
print ("error")
Here is the Arduino code
#include <FastLED.h>
#define NUM_LEDS 15
#define DATA_PIN 6
float Cpu = 40;
float Gpu = 99;
float Ram = 60;
String Cpu_Read;
String Gpu_Read;
String Ram_Read;
int RXLED = 17;
String StrNumbers = "";
int numbers;
int Received = 0;
int Separrator = 0;
String Text = "";
CRGB leds[NUM_LEDS];
void setup() {
// put your setup code here, to run once:
FastLED.addLeds<WS2812B, DATA_PIN, GRB>(leds, NUM_LEDS);
FastLED.setBrightness(5);
Serial.begin(115200);
pinMode(LED_BUILTIN_TX,INPUT);
pinMode(LED_BUILTIN_RX,INPUT);
}
void loop() {
// put your main code here, to run repeatedly:
char inByte = ' ';
StrNumbers = "";
Text = "";
Separrator = 0;
Cpu_Read = "";
Gpu_Read = "";
Ram_Read = "";
Received = 0;
pinMode(LED_BUILTIN_TX,INPUT);
while(Serial.available() > 0){ // only send data back if data has been sent
pinMode(LED_BUILTIN_TX,OUTPUT);
inByte = Serial.read(); // read the incoming data
if (inByte == ','){
Separrator += 1;
}
else if (Separrator == 0){
Cpu_Read += (char) inByte;
}
else if (Separrator == 1){
Gpu_Read += (char) inByte;
}
else if (Separrator == 2){
Ram_Read += (char) inByte;
Serial.print("Ram : ");
Serial.println(Ram_Read);
}
else{
Serial.println("Error");
}
/*
inByte = Serial.read(); // read the incoming data
if (isDigit(inByte)) { // tests if inByte is a digit
StrNumbers += (char) inByte;
Serial.println(inByte); // send the data back in a new line so that it is not all one long line
//Serial.println(numbers); // send the data back in a new line so that it is not all one long line
}
else if (inByte == ".")
{
abortLoop = 1;
}
else{
Text +=(char) inByte;
}
*/
Received = 1;
}
if (Received == 1){
Cpu = Cpu_Read.toInt();
Gpu = Gpu_Read.toInt();
Ram = Ram_Read.toInt();
UpdateMonitor(Cpu ,Gpu ,Ram);
}
Text.trim();
if (StrNumbers != ""){
numbers = StrNumbers.toInt();
Serial.println(numbers);
if (numbers > 100) numbers = 100;
UpdateMonitor(Cpu,numbers,Ram);
}
if(Text!= ""){
//Serial.println(Text); // send the data back in a new line so that it is not all one long line
}
if (Text == "ResourceMonitor"){
Serial.println("Yes");
}
else if (Text != ""){
Serial.println(Text);
numbers = Text.toInt();
if (numbers > 100) numbers = 100;
UpdateMonitor(Cpu, numbers, Ram);
}
}
void UpdateMonitor(int cpu,int gpu, int ram){
int Cpu_Usage = map(cpu, 0, 100, 0, 5);
int Gpu_Usage = map(gpu, 0, 100, 0, 5);
int Ram_Usage = map(ram, 0, 100, 0, 5);
FastLED.clear();
for(int led = 0; led < Ram_Usage; led++) {
leds[led] = CRGB::Blue;
}
for(int led = 0; led < Gpu_Usage; led++) {
leds[9 - led] = CRGB::Green;
}
for(int led = 0; led < Cpu_Usage; led++) {
leds[led+10] = CRGB::Red;
}
FastLED.show();
}

it is fixed now not sure what fixet it but it works atleast :)
thansk to all that have tried to helt

How to import Motor on Micropython into a DFRobot Quad Motor Shield

I am attempting to import two motors onto my pyboard in micropython that are connected to my DFRobot DC Quad Motor Shield. Example code from them is written in Arduino and I cannot translate. Attached is my previous code that did not work and the motor shield over view with its speeds and connections.
Any help is appreciated!
Code Previously Written
Board Overview
Here is what I have
i2c = machine.I2C(scl=machine.Pin('Y9'), sda=machine.Pin('Y10'))
motors = motor.DCMotors(i2c)
MOTOR1 = 3
MOTOR2 = 4
#Initiate Communication from Sonar sensor
sensor_front = adafruit_hcsr04.HCSR04(trigger_pin=board.Pin('X3'), echo_pin=board.Pin('X4'))
sensor_back = adafruit_hcsr04.HCSR04(trigger_pin=board.Pin('X6'), echo_pin=board.Pin('X7'))
#Create minimum distance For Ultrasonic sensor
min_distance = sensor.distance(70)
button = pyb.switch()
def autonomy():
#no_problem = True
while (True):
if (button()):
dist_front = sensor1.distance(15)
dist_back = sensor2.distance(15)
if dist_front > min_distance:
print('Nothing can stop me!')
motors.speed(MOTOR1, 1500)
motors.speed(MOTOR2,-1500)
Here is the arduino code
/*!
* #file QuadMotorDriverShield.ino
* #brief QuadMotorDriverShield.ino Motor control program
*
* Every 2 seconds to control motor positive inversion
*
* #author linfeng(490289303#qq.com)
* #version V1.0
* #date 2016-4-5
*/
const int E1 = 3; ///<Motor1 Speed
const int E2 = 11;///<Motor2 Speed
const int E3 = 5; ///<Motor3 Speed
const int E4 = 6; ///<Motor4 Speed
const int M1 = 4; ///<Motor1 Direction
const int M2 = 12;///<Motor2 Direction
const int M3 = 8; ///<Motor3 Direction
const int M4 = 7; ///<Motor4 Direction
void M1_advance(char Speed) ///<Motor1 Advance
{
digitalWrite(M1,LOW);
analogWrite(E1,Speed);
}
void M2_advance(char Speed) ///<Motor2 Advance
{
digitalWrite(M2,HIGH);
analogWrite(E2,Speed);
}
void M3_advance(char Speed) ///<Motor3 Advance
{
digitalWrite(M3,LOW);
analogWrite(E3,Speed);
}
void M4_advance(char Speed) ///<Motor4 Advance
{
digitalWrite(M4,HIGH);
analogWrite(E4,Speed);
}
void M1_back(char Speed) ///<Motor1 Back off
{
digitalWrite(M1,HIGH);
analogWrite(E1,Speed);
}
void M2_back(char Speed) ///<Motor2 Back off
{
digitalWrite(M2,LOW);
analogWrite(E2,Speed);
}
void M3_back(char Speed) ///<Motor3 Back off
{
digitalWrite(M3,HIGH);
analogWrite(E3,Speed);
}
void M4_back(char Speed) ///<Motor4 Back off
{
digitalWrite(M4,LOW);
analogWrite(E4,Speed);
}
void setup() {
for(int i=3;i<9;i++)
pinMode(i,OUTPUT);
for(int i=11;i<13;i++)
pinMode(i,OUTPUT);
}
void loop() {
M1_advance(100);
M2_advance(100);
M3_advance(100);
M4_advance(100);
delay(2000); ///<Delay 2S
M1_back(100);
M2_back(100);
M3_back(100);
M4_back(100);
delay(2000); ///<Delay 2S
}

Here's attempt one. Consider using fuses until we're certain this is how to connect a DFRobot Motor HAT to a pyboard.
The connection point on the motor hat is the green digital bus.
import pyb
# don't use timers 2,3,5,6
'''
connect:
pyboard Motor Hat
X1 4
X3 3
GND GND
'''
FREQ = 50
backward = pyb.Pin('X1' ,pyb.Pin.OUT_PP)
speed = pyb.Timer(9, freq=FREQ).channel(1, pyb.Timer.PWM,\
pin = pyb.Pin('X3'))
while True:
backward.high()
for percent in range(100):
speed.pulse_width_percent(percent + 1)
pyb.delay(10)
speed.pulse_width_percent(0)
backward.low()
for percent in range(100):
speed.pulse_width_percent(percent + 1)
pyb.delay(10)
speed.pulse_width_percent(0)
I've guessed FREQ to be 50Hz it's that for many servos and i'm guessing arduino uses this as a default value for PWM (pulse width modulation).
Connect a motor to connector M1 and a battery to the motor hat's power socket. Don't connect to v+ on the pyboard. Power your pyboard with it's own power source. You motor hat probably creates a lot of electrical noise.
The motor should go backwards speeding up then suddenly stopping before going forwards getting faster then suddenly stop.
good luck

Triggering data acquisition in Arduino with Python

I would like to trigger data acquisition on my Arduino UNO with connected accelerometer (MPU 6050) by using Python.
The idea is that when the command is given in Python, the Arduino would start saving data on its SRAM and when a certain number of measurements would be saved, the data would be sent in a package back to Python.
This is my current Arduino code:
#include<Wire.h>
#define MPU6050_DLPF_94HZ MPU6050_DLPF_CFG_2
const int MPU_addr_1 = 0x68; // I2C address of the first MPU-6050
const int baudrate = 19200;
int16_t AcX1; // definition of variables
const int len = 200; // Buffer size
float analogDataArray[len];
int count = 0;
void setup() {
Wire.begin();
Wire.beginTransmission(MPU_addr_1);
Wire.write(0x6B); // PWR_MGMT_1 register
Wire.write(0); // set to zero (wakes up the MPU-6050)
Wire.endTransmission(true);
Serial.begin(baudrate);
}
void loop() {
Wire.beginTransmission(MPU_addr_1);
Wire.write(0x3B); // starting with register 0x3B (ACCEL_XOUT_H)
Wire.endTransmission(false);
Wire.requestFrom(MPU_addr_1, 8, true); // request a total of 14 registers
float AcX1 = Wire.read() << 8 | Wire.read(); // 0x3B (ACCEL_XOUT_H) & 0x3C (ACCEL_XOUT_L)
if (Serial.available())
{
if (Serial.read() == 'Y') {
analogDataArray[count] = AcX1;
count = count + 1;
if (count >= len) {
for (int i; i <= len; i = i + 1) {
Serial.println(analogDataArray[i] / 16384);
}
}
count = 0;
}
}
delay(5);
}
and this is my Python code:
import serial
arduinoData = serial.Serial('COM3', 19200)
com = input('Press "Y":' )
arduinoData.write(bytes(com, 'utf-8'))
vec = []
run = True
while run is True:
while (arduinoData.inWaiting() == 0):
pass
arduinoString = arduinoData.readline()
vec.append(float(arduinoString))
if len(vec) >= 100:
run = False
print(vec)
I've managed to get it working for 1 measurement but as soon as I defined an array inside Arduino to save multiple measurements, the code doesn't work. I'm sure that it is close to working, but I can't find the detail that is stopping me from that.
Thank you for any provided help.
Kind regards,
L

I got it working, the problem was in my Arduino code, as expected.
#include<Wire.h>
#define MPU6050_DLPF_94HZ MPU6050_DLPF_CFG_2
const int MPU_addr_1 = 0x68; // I2C address of the first MPU-6050
const int baudrate = 9600;
int16_t AcX1; // definition of variables
const int len = 200; // Buffer size
float analogDataArray[len];
int count = 0;
int ans;
void setup() {
Wire.begin();
Wire.beginTransmission(MPU_addr_1);
Wire.write(0x6B); // PWR_MGMT_1 register
Wire.write(0); // set to zero (wakes up the MPU-6050)
Wire.endTransmission(true);
Serial.begin(baudrate);
}
void loop() {
Wire.beginTransmission(MPU_addr_1);
Wire.write(0x3B); // starting with register 0x3B (ACCEL_XOUT_H)
Wire.endTransmission(false);
Wire.requestFrom(MPU_addr_1, 8, true); // request a total of 14 registers
float AcX1 = Wire.read() << 8 | Wire.read(); // 0x3B (ACCEL_XOUT_H) & 0x3C (ACCEL_XOUT_L)
if (Serial.available() && Serial.read() == 'Y') {
ans = 1;
}
if (ans == 1) {
analogDataArray[count] = AcX1;
count = count + 1;
if (count >= len) {
for (int i; i <= len; i = i + 1) {
Serial.println(analogDataArray[i] / 16384);
}
ans = 0;
}
}
delay(5);
}

Data corruption Piping between C++ and Python

I am writing some code that takes binary data from Python, Pipes it to C++, does some processing on the data, (in this case calculating a mutual information metric) and then pipes the results back to python. While testing I have found that everything works fine if the data I send is a set of 2 arrays with dimensions less than 1500 X 1500, but if I send 2 arrays that are 2K X 2K I get back a lot of corrupted nonsense.
I currently believe the algorithmic portion of the code is fine because it provides the expected answers during testing with small (<=1500 X1500) arrays. That leads me to believe that this is an issue with either the stdin or stdout piping. That maybe I’m passing some intrinsic limit somewhere.
The Python Code and C++ code are below.
Python Code:
import subprocess
import struct
import sys
import numpy as np
#set up the variables needed
bytesPerDouble = 8
sizeX = 2000
sizeY = 2000
offset = sizeX*sizeY
totalBytesPerArray = sizeX*sizeY*bytesPerDouble
totalBytes = totalBytesPerArray*2 #the 2 is because we pass 2 different versions of the 2D array
#setup the testing data array
a = np.zeros(sizeX*sizeY*2, dtype='d')
for i in range(sizeX):
for j in range(sizeY):
a[j+i*sizeY] = i
a[j+i*sizeY+offset] = i
if i % 10 == 0:
a[j+i*sizeY+offset] = j
data = a.tobytes('C')
strTotalBytes = str(totalBytes)
strLineBytes = str(sizeY*bytesPerDouble)
#communicate with c++ code
print("starting C++ code")
command = "C:\Python27\PythonPipes.exe"
proc = subprocess.Popen([command, strTotalBytes, strLineBytes, str(sizeY), str(sizeX)], stdin=subprocess.PIPE,stderr=subprocess.PIPE,stdout=subprocess.PIPE)
ByteBuffer = (data)
proc.stdin.write(ByteBuffer)
print("Reading results back from C++")
for i in range(sizeX):
returnvalues = proc.stdout.read(sizeY*bytesPerDouble)
a = buffer(returnvalues)
b = struct.unpack_from(str(sizeY)+'d', a)
print str(b) + " " + str(i)
print('done')
C++ Code:
Main function:
int main(int argc, char **argv) {
int count = 0;
long totalbytes = stoi(argv[argc-4], nullptr,10); //bytes being transfered
long bytechunk = stoi(argv[argc - 3], nullptr, 10); //bytes being transfered at a time
long height = stoi(argv[argc-2], nullptr, 10); //bytes being transfered at a time
long width = stoi(argv[argc-1], nullptr, 10); //bytes being transfered at a time
long offset = totalbytes / sizeof(double) / 2;
data = new double[totalbytes/sizeof(double)];
int columnindex = 0;
//read in data from pipe
while (count<totalbytes) {
fread(&(data[columnindex]), 1, bytechunk, stdin);
columnindex += bytechunk / sizeof(double);
count += bytechunk;
}
//calculate the data transform
MutualInformation MI = MutualInformation();
MI.Initialize(data, height, width, offset);
MI.calcMI();
count = 0;
//*
//write out data to pipe
columnindex = 0;
while (count<totalbytes/2) {
fwrite(&(MI.getOutput()[columnindex]), 1, bytechunk, stdout);
fflush(stdout);
count += bytechunk;
columnindex += bytechunk/sizeof(double);
}
//*/
delete [] data;
return 0;
}
and in case you need it the actual processing code:
double MutualInformation::calcMI(){
double rvalue = 0.0;
std::map<int, map<int, double>> lHistXY = map<int, map<int, double>>();
std::map<int, double> lHistX = map<int, double>();
std::map<int, double> lHistY = map<int, double>();
typedef std::map<int, std::map<int, double>>::iterator HistXY_iter;
typedef std::map<int, double>::iterator HistY_iter;
//calculate Entropys and MI
double MI = 0.0;
double Hx = 0.0;
double Hy = 0.0;
double Px = 0.0;
double Py = 0.0;
double Pxy = 0.0;
//scan through the image
int ip = 0;
int jp = 0;
int chipsize = 3;
//setup zero array
double * zeros = new double[this->mHeight];
for (int j = 0; j < this->mHeight; j++){
zeros[j] = 0.0;
}
//zero out Output array
for (int i = 0; i < this->mWidth; i++){
memcpy(&(this->mOutput[i*this->mHeight]), zeros, this->mHeight*8);
}
double index = 0.0;
for (int ioutter = chipsize; ioutter < (this->mWidth - chipsize); ioutter++){
//write out processing status
//index = (double)ioutter;
//fwrite(&index, 8, 1, stdout);
//fflush(stdout);
//*
for (int j = chipsize; j < (this->mHeight - chipsize); j++){
//clear the histograms
lHistX.clear();
lHistY.clear();
lHistXY.clear();
//chip out a section of the image
for (int k = -chipsize; k <= chipsize; k++){
for (int l = -chipsize; l <= chipsize; l++){
ip = ioutter + k;
jp = j + l;
//update X histogram
if (lHistX.count(int(this->mData[ip*this->mHeight + jp]))){
lHistX[int(this->mData[ip*this->mHeight + jp])] += 1.0;
}else{
lHistX[int(this->mData[ip*this->mHeight + jp])] = 1.0;
}
//update Y histogram
if (lHistY.count(int(this->mData[ip*this->mHeight + jp+this->mOffset]))){
lHistY[int(this->mData[ip*this->mHeight + jp+this->mOffset])] += 1.0;
}
else{
lHistY[int(this->mData[ip*this->mHeight + jp+this->mOffset])] = 1.0;
}
//update X and Y Histogram
if (lHistXY.count(int(this->mData[ip*this->mHeight + jp]))){
//X Key exists check if Y key exists
if (lHistXY[int(this->mData[ip*this->mHeight + jp])].count(int(this->mData[ip*this->mHeight + jp + this->mOffset]))){
//X & Y keys exist
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] += 1;
}else{
//X exist but Y doesn't
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] = 1;
}
}else{
//X Key Didn't exist
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] = 1;
};
}
}
//calculate PMI, Hx, Hy
// iterator->first = key
// iterator->second = value
MI = 0.0;
Hx = 0.0;
Hy = 0.0;
for (HistXY_iter Hist2D_iter = lHistXY.begin(); Hist2D_iter != lHistXY.end(); Hist2D_iter++) {
Px = lHistX[Hist2D_iter->first] / ((double) this->mOffset);
Hx -= Px*log(Px);
for (HistY_iter HistY_iter = Hist2D_iter->second.begin(); HistY_iter != Hist2D_iter->second.end(); HistY_iter++) {
Py = lHistY[HistY_iter->first] / ((double) this->mOffset);
Hy -= Py*log(Py);
Pxy = HistY_iter->second / ((double) this->mOffset);
MI += Pxy*log(Pxy / Py / Px);
}
}
//normalize PMI to max(Hx,Hy) so that the PMI value runs from 0 to 1
if (Hx >= Hy && Hx > 0.0){
MI /= Hx;
}else if(Hy > Hx && Hy > 0.0){
MI /= Hy;
}
else{
MI = 0.0;
}
//write PMI to data output array
if (MI < 1.1){
this->mOutput[ioutter*this->mHeight + j] = MI;
}
else{
this->mOutput[ioutter*this->mHeight + j] = 0.0;
}
}
}
return rvalue;
}
with arrays that return something that makes sense I get output bounded between 0 and 1 like this:
(0.0, 0.0, 0.0, 0.7160627908692593, 0.6376472316395495, 0.5728801401524277,...
with the 2Kx2K or higher arrays I get nonesense like this (even though the code clamps the values between 0 and 1):
(-2.2491400820412374e+228, -2.2491400820412374e+228, -2.2491400820412374e+228, -2.2491400820412374e+228, -2.2491400820412374e+228,...
I would like to know why this code is corrupting the data set after it is assigned between 0.0 and 1, and whether or not it is a piping issue, a stdin/stdout issue, a buffer issue of some sort, or a coding issue I am simply not seeing.
Update I tried passing the data in smaller chunks using the code that Chris suggested with no luck. also of note is that I added a catch for ferror on stdout and it never got tripped so I am pretty sure that the bytes are at least making it to stdout. Is it possible that something else is writing to stdout somehow? maybe an extra byte making its way into stdout while my program is running? I find this doubtful as the errors are appearing consistently on the 4th fwrite read in the 10th entry.
Per Craig's request here is the full C++ code (the full Python Code is already posted): it is sitting in 3 files:
main.cpp
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <iostream>
#include "./MutualInformation.h"
double * data;
using namespace std;
void
xxwrite(unsigned char *buf, size_t wlen, FILE *fo)
{
size_t xlen;
for (; wlen > 0; wlen -= xlen, buf += xlen) {
xlen = wlen;
if (xlen > 1024)
xlen = 1024;
xlen = fwrite(buf, 1, xlen, fo);
fflush(fo);
}
}
int main(int argc, char **argv) {
int count = 0;
long totalbytes = stoi(argv[argc-4], nullptr,10); //bytes being transfered
long bytechunk = stoi(argv[argc - 3], nullptr, 10); //bytes being transfered at a time
long height = stoi(argv[argc-2], nullptr, 10); //bytes being transfered at a time
long width = stoi(argv[argc-1], nullptr, 10); //bytes being transfered at a time
long offset = totalbytes / sizeof(double) / 2;
data = new double[totalbytes/sizeof(double)];
int columnindex = 0;
//read in data from pipe
while (count<totalbytes) {
fread(&(data[columnindex]), 1, bytechunk, stdin);
columnindex += bytechunk / sizeof(double);
count += bytechunk;
}
//calculate the data transform
MutualInformation MI = MutualInformation();
MI.Initialize(data, height, width, offset);
MI.calcMI();
count = 0;
columnindex = 0;
while (count<totalbytes/2) {
xxwrite((unsigned char*)&(MI.getOutput()[columnindex]), bytechunk, stdout);
count += bytechunk;
columnindex += bytechunk/sizeof(double);
}
delete [] data;
return 0;
}
MutualInformation.h
#include <map>
using namespace std;
class MutualInformation
{
private:
double * mData;
double * mOutput;
long mHeight;
long mWidth;
long mOffset;
public:
MutualInformation();
~MutualInformation();
bool Initialize(double * data, long Height, long Width, long Offset);
const double * getOutput();
double calcMI();
};
MutualInformation.cpp
#include "MutualInformation.h"
MutualInformation::MutualInformation()
{
this->mData = nullptr;
this->mOutput = nullptr;
this->mHeight = 0;
this->mWidth = 0;
}
MutualInformation::~MutualInformation()
{
delete[] this->mOutput;
}
bool MutualInformation::Initialize(double * data, long Height, long Width, long Offset){
bool rvalue = false;
this->mData = data;
this->mHeight = Height;
this->mWidth = Width;
this->mOffset = Offset;
//allocate output data
this->mOutput = new double[this->mHeight*this->mWidth];
return rvalue;
}
const double * MutualInformation::getOutput(){
return this->mOutput;
}
double MutualInformation::calcMI(){
double rvalue = 0.0;
std::map<int, map<int, double>> lHistXY = map<int, map<int, double>>();
std::map<int, double> lHistX = map<int, double>();
std::map<int, double> lHistY = map<int, double>();
typedef std::map<int, std::map<int, double>>::iterator HistXY_iter;
typedef std::map<int, double>::iterator HistY_iter;
//calculate Entropys and MI
double MI = 0.0;
double Hx = 0.0;
double Hy = 0.0;
double Px = 0.0;
double Py = 0.0;
double Pxy = 0.0;
//scan through the image
int ip = 0;
int jp = 0;
int chipsize = 3;
//setup zero array
double * zeros = new double[this->mHeight];
for (int j = 0; j < this->mHeight; j++){
zeros[j] = 0.0;
}
//zero out Output array
for (int i = 0; i < this->mWidth; i++){
memcpy(&(this->mOutput[i*this->mHeight]), zeros, this->mHeight*8);
}
double index = 0.0;
for (int ioutter = chipsize; ioutter < (this->mWidth - chipsize); ioutter++){
for (int j = chipsize; j < (this->mHeight - chipsize); j++){
//clear the histograms
lHistX.clear();
lHistY.clear();
lHistXY.clear();
//chip out a section of the image
for (int k = -chipsize; k <= chipsize; k++){
for (int l = -chipsize; l <= chipsize; l++){
ip = ioutter + k;
jp = j + l;
//update X histogram
if (lHistX.count(int(this->mData[ip*this->mHeight + jp]))){
lHistX[int(this->mData[ip*this->mHeight + jp])] += 1.0;
}else{
lHistX[int(this->mData[ip*this->mHeight + jp])] = 1.0;
}
//update Y histogram
if (lHistY.count(int(this->mData[ip*this->mHeight + jp+this->mOffset]))){
lHistY[int(this->mData[ip*this->mHeight + jp+this->mOffset])] += 1.0;
}
else{
lHistY[int(this->mData[ip*this->mHeight + jp+this->mOffset])] = 1.0;
}
//update X and Y Histogram
if (lHistXY.count(int(this->mData[ip*this->mHeight + jp]))){
//X Key exists check if Y key exists
if (lHistXY[int(this->mData[ip*this->mHeight + jp])].count(int(this->mData[ip*this->mHeight + jp + this->mOffset]))){
//X & Y keys exist
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] += 1;
}else{
//X exist but Y doesn't
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] = 1;
}
}else{
//X Key Didn't exist
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] = 1;
};
}
}
//calculate PMI, Hx, Hy
// iterator->first = key
// iterator->second = value
MI = 0.0;
Hx = 0.0;
Hy = 0.0;
for (HistXY_iter Hist2D_iter = lHistXY.begin(); Hist2D_iter != lHistXY.end(); Hist2D_iter++) {
Px = lHistX[Hist2D_iter->first] / ((double) this->mOffset);
Hx -= Px*log(Px);
for (HistY_iter HistY_iter = Hist2D_iter->second.begin(); HistY_iter != Hist2D_iter->second.end(); HistY_iter++) {
Py = lHistY[HistY_iter->first] / ((double) this->mOffset);
Hy -= Py*log(Py);
Pxy = HistY_iter->second / ((double) this->mOffset);
MI += Pxy*log(Pxy / Py / Px);
}
}
//normalize PMI to max(Hx,Hy) so that the PMI value runs from 0 to 1
if (Hx >= Hy && Hx > 0.0){
MI /= Hx;
}else if(Hy > Hx && Hy > 0.0){
MI /= Hy;
}
else{
MI = 0.0;
}
//write PMI to data output array
if (MI < 1.1){
this->mOutput[ioutter*this->mHeight + j] = MI;
}
else{
this->mOutput[ioutter*this->mHeight + j] = 0.0;
//cout << "problem with output";
}
}
}
//*/
return rvalue;
}
SOLVED By 6502
6502's answer below solved my problem. I needed to explicitly tell Windows to use a binary mode for stdin / stdout. to do that I had to include 2 new header files in my main cpp file.
#include <fcntl.h>
#include <io.h>
add the following lines of code (modified away from 6502's POSIX versions because Visual Studio complained) to the beginning of my main function
_setmode(_fileno(stdout), O_BINARY);
_setmode(_fileno(stdin), O_BINARY);
and then add these lines to my Python code:
import os, msvcrt
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)

The problem is that stdin/stdout in windows are opened in text mode, not in binary mode and therefore will mess up when the character 13 (\r) is sent.
You can set for example binary mode in Python with
import os, msvcrt
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
and in C++ with
_setmode(fileno(stdout), O_BINARY);
_setmode(fileno(stdin), O_BINARY);
See https://msdn.microsoft.com/en-us/library/tw4k6df8.aspx

Your C++ fwrite code does not account for getting a "short" transfer.
Here's a slight tweak:
//write out data to pipe
columnindex = 0;
while (count < totalbytes / 2) {
wlen = fwrite(&(MI.getOutput()[columnindex]), 1, bytechunk, stdout);
fflush(stdout);
count += wlen;
columnindex += wlen / sizeof(double);
}
Note: You still need to be careful as this would still have issues if wlen returns and it's not a multiple of sizeof(double). For example, if bytechunk were 16 and wlen came back with 14, you'd need an additional fwrite with length 2 before continuing the loop. A generalization of this is just to treat the entire data matrix as a giant byte buffer and loop on that.
Actually, you'll get about the same efficiency with many much smaller transfers that are capped by a fixed (i.e. "known safe amount") of [say] 1024 bytes. This works because the output is a byte stream.
Here's a slightly more general solution that I've often used:
void
xxwrite(void *buf,size_t wlen,FILE *fo)
{
size_t xlen;
for (; wlen > 0; wlen -= xlen, buf += xlen) {
xlen = wlen;
if (xlen > 1024)
xlen = 1024;
xlen = fwrite(buf,1,xlen,fo);
fflush(fo);
}
}
//write out data to pipe
columnindex = 0;
while (count < totalbytes / 2) {
xxwrite(&(MI.getOutput()[columnindex]), bytechunk, stdout);
count += bytechunk;
columnindex += bytechunk / sizeof(double);
}
UPDATE:
I've downloaded all your code and run it. I've got good news and bad news: The code runs fine here, even for a matrix size above 3000. I ran it both using xxwrite and without and the results were the same.
Using my limited python skills, I added some pretty print to your python script (e.g. some line wrap) and had it check every value for range and annotate any bad values. There were none found by the script. Also, visual inspection of the values turned up nothing [this was true before the pretty print, so it hasn't introduced anything]. Just lots of zeros and then blocks in the 0.9 range.
The only difference I can see is that I'm using gcc [and, of course, python] on linux. But, from your script it seems your using Windows [based on the C:\... path for your C++ executable. This shouldn't matter for this application, but I mention it anyway.
So, pipes work here. One thing you might try is to direct the C++ output to a file. Then, have the script read back from the file (i.e. no pipe) and see if that makes a difference. I tend to think not, but ...
Also, I don't know what compiler and python implementation you're using under Windows. Whenever I have to do this, I usually have Cygwin installed as it gives one of the closest implementations of linux/Unix-like environment (i.e. pipes are more likely to work as advertised).
Anyway, here's the modified script. Also note that I added os.getenv to grab alternate matrix sizes and an alternate place for the C++ executable, so that it would work for both of us with minimal pain
#!/usr/bin/python
import subprocess
import struct
import sys
import os
import numpy as np
val = os.getenv("MTX","2000")
sizeX = int(val)
sizeY = sizeX
print "sizeX=%d sizeY=%d" % (sizeX,sizeY)
#set up the variables needed
bytesPerDouble = 8
offset = sizeX*sizeY
totalBytesPerArray = sizeX*sizeY*bytesPerDouble
totalBytes = totalBytesPerArray*2 #the 2 is because we pass 2 different versions of the 2D array
#setup the testing data array
a = np.zeros(sizeX*sizeY*2, dtype='d')
for i in range(sizeX):
for j in range(sizeY):
a[j+i*sizeY] = i
a[j+i*sizeY+offset] = i
if i % 10 == 0:
a[j+i*sizeY+offset] = j
data = a.tobytes('C')
strTotalBytes = str(totalBytes)
strLineBytes = str(sizeY*bytesPerDouble)
#communicate with c++ code
print("starting C++ code")
command = os.getenv("CPGM",None);
if command is None:
command = "C:\Python27\PythonPipes.exe"
proc = subprocess.Popen([command, strTotalBytes, strLineBytes, str(sizeY), str(sizeX)], stdin=subprocess.PIPE,stderr=subprocess.PIPE,stdout=subprocess.PIPE)
ByteBuffer = (data)
proc.stdin.write(ByteBuffer)
def prt(i,b):
hangflg = 0
per = 8
for j in range(0,len(b)):
if ((j % per) == 0):
print("[%d,%d]" % (i,j)),
q = b[j]
print(q),
hangflg = 1
if (q < 0.0) or (q > 1.0):
print("=WTF"),
if ((j % per) == (per - 1)):
print("")
hangflg = 0
if (hangflg):
print("")
print("Reading results back from C++")
for i in range(sizeX):
returnvalues = proc.stdout.read(sizeY*bytesPerDouble)
a = buffer(returnvalues)
b = struct.unpack_from(str(sizeY)+'d', a)
prt(i,b)
###print str(b) + " " + str(i)
###print str(i) + ": " + str(b)
print('done')

Python PySerial FlushInput not working

I am new to programming and serial communication, so forgive me for likely messing up something simple.
The script I am writing is reading temperature data sent via serial port from an Arduino. When it first loads up, the Arduino sketch sends a "menu" with options for reading, writing, and clearing EEPROM.
When I load the python script, sometimes it works well enough. Other times, it just doesn't read anything. And other times, it floods my terminal with temp reading data for about 5 seconds, then it sends the menu and acts normally.
I have added the flushinput and time delays, but to no avail.
Another quick question: if I'm in the while loop collecting data from serial, which is without end, how would I be able to enter a command to pause and switch to another part of the program without interrupting the data stream? For example, if I'm collecting data from the temp sensor and using serial to do real-time plotting of temp vs time, but I want to be able to interrupt this at anytime to stop writing and switch to reading (saving data to .txt file). How could this be done?
Any help would be immensely appreciated.
Here is the Python code:
import serial
import time
import numpy as np
import matplotlib.pyplot as plt
ser = serial.Serial('/dev/tty.usbmodem1421', 9600, timeout = 5)
ser.flushInput()
ser.flushOutput()
time.sleep(0.1)
print ("Connected to serial port: {}".format(ser.name))
def Menu():
data_left = 1
while (data_left != 0):
print(ser.readline())
time.sleep(.1)
data_left = ser.inWaiting()
return
def RealTimePlot(y, count, k, minPlot, maxPlot):
//stuff
def WriteData(mode, plotData = 'n'):
//stuff
def ReadData(mode):
//stuff
def main():
launch = 0
plotData = ProgramIntro()
time.sleep(1)
Menu()
mode = raw_input("> ")
while (1):
if ((mode == "end") or (mode == 'exit')) :
ser.close()
exit()
elif (mode == 'm'):
ser.write(mode)
time.sleep(1)
Menu()
mode = raw_input("> ")
if (mode == 'm'):
print("Already at menu. Would you like to:")
print("[w]rite")
print("[r]ead")
print("[c]lear")
elif (mode == 'w'):
count = 0
WriteData(mode, plotData)
#mode = PauseProgram()
elif (mode == 'r'):
ReadData(mode)
#mode = PauseProgram()
elif (mode == 'c'):
ser.write(mode)
time.sleep(1)
while ser.inWaiting() > 0:
out = ser.readline()
print(out)
time.sleep(0.1)
mode = 'm'
print("Goodbye!")
#Closes the connection
ser.close()
And Arduino Code:
// References
// https://learn.sparkfun.com/tutorials/sik-experiment-guide-for-arduino---v32/experiment-7-reading-a-temperature-sensor
// https://www.arduino.cc/en/Tutorial/EEPROMWrite
// https://www.arduino.cc/en/Tutorial/EEPROMRead
//
// The following may be useful to read on storing EEPROM data:
// http://www.vernier.com/engineering/arduino/store-data-eeprom/
//
#include <ctype.h>
#include <stdlib.h>
#include <math.h>
#include <EEPROM.h>
// Define pins
const int writePin = 5;
const int readPin = 9;
const int pausePin = 7;
const int collectDataBtn = 11;
const int RTPin = 3;
const int sensorPin = A0; // Temperature Pin
char RTData;
int j = 0;
// SETUP of Serial port and PINS for LED
void setup() {
Serial.begin(9600);
pinMode(writePin, OUTPUT);
pinMode(RTPin, OUTPUT);
pinMode(readPin, OUTPUT);
pinMode(pausePin, OUTPUT);
pinMode(sensorPin, INPUT);
pinMode(collectDataBtn, INPUT);
while(Serial.available())
Serial.read();
}
char mode = 'm';
char prevMode = 'm';
int k = 0;
// MAIN LOOP
void loop() {
while (j == 0) {
if (k == 0) {
Serial.print("Would you like to plot real-time data? (y/n) ");
k++;
}
if (Serial.available() > 0) {
RTData = Serial.read();
if (RTData == 'y') {
digitalWrite(RTPin, HIGH);
j++;
}
else {
j++;
}
}
return;
}
switch (mode) {
// MENU mode
case 'm':
case 'M':
// Set LEDs to indicate menu mode
digitalWrite(writePin, HIGH);
digitalWrite(readPin, HIGH);
digitalWrite(pausePin, LOW);
// Print menu to serial port
PrintMenu();
while (mode == 'm') {
if (Serial.available() > 0) {
prevMode = mode;
mode = Serial.read();
}
}
break;
// WRITE mode
case 'w':
case 'W':
Serial.println("Entering WRITE mode...");
mode = WriteData(sensorPin, collectDataBtn);
break;
// READ mode
case 'r':
case 'R':
// Sets LEDs to indicate READ mode
digitalWrite(writePin, LOW);
digitalWrite(readPin, HIGH);
digitalWrite(pausePin, LOW);
Serial.println("Entering READ mode...");
mode = ReadData(sensorPin, prevMode);
break;
// CLEAR eeprom mode
case 'c':
case 'C':
Serial.println("Clearing EEPROM...");
ClearEEPROM();
Serial.println("EEPROM cleared!");
Serial.println("Returning to menu...\n\n");
mode = 'm';
break;
// PAUSE mode
default:
// Sets LEDs to indicate PAUSE mode
digitalWrite(writePin, LOW);
digitalWrite(readPin, LOW);
digitalWrite(pausePin, HIGH);
mode = 'p';
mode = PauseMode(mode);
break;
}
} // END void loop
void ClearEEPROM(void) {
for (int i = 0 ; i < EEPROM.length() ; i++) {
if ((i % 100 == 0) || (i % 50 == 0)) {
digitalWrite(writePin, HIGH);
digitalWrite(readPin, LOW);
digitalWrite(pausePin, LOW);
}
else {
digitalWrite(writePin, LOW);
digitalWrite(readPin, LOW);
digitalWrite(pausePin, HIGH);
}
EEPROM.write(i, 0);
}
return;
}
char CheckModeChange(void) {
if (Serial.available() > 0) {
// read the incoming char:
prevMode = mode;
mode = Serial.read();
}
return mode;
}
void PrintMenu(void) {
Serial.println("\n");
Serial.println(" ** Menu ** ");
Serial.println("------------------");
Serial.println("| [W]rite Data |");
Serial.println("| [R]ead Data |");
Serial.println("| [C]lear Data |");
Serial.println("------------------");
Serial.println("Enter [m]enu to return to menu at any point,");
Serial.println("or press any other key to pause during any point of the program.");
return;
}
char PauseMode(char mode){
Serial.println("Program Paused. Choose an option from the following to continue: ");
Serial.println("[m]enu");
Serial.println("[w]rite");
Serial.println("[r]ead");
Serial.println("[c]lear \n");
while (mode == 'p') {
if (Serial.available() > 0) {
mode = Serial.read();
}
}
return mode;
}
char WriteData(int sensorPin, int collectDataBtn) {
// Declarations
int cnt, voltVal;
int addr = 0; // Initializes address at 0 for EEPROM
int beginData = 0;
float voltage, degreesC;
char mode = 'w';
// Collect data when button pushed. This will be replaced by z-axis acceleration threshold
// of rocket.
Serial.println("Waiting for launch to begin...");
while (beginData == 0) {
if (cnt > 20000) {
cnt = 0;
}
else if (cnt < 10000) {
digitalWrite(writePin, LOW);
digitalWrite(readPin, LOW);
digitalWrite(pausePin, LOW);
}
else {
digitalWrite(writePin, HIGH);
digitalWrite(readPin, LOW);
digitalWrite(pausePin, LOW);
}
cnt++;
if (digitalRead(collectDataBtn) == HIGH) {
beginData = 1;
}
prevMode = mode;
mode = CheckModeChange();
if (prevMode != mode) {
beginData = 1;
}
}
digitalWrite(writePin, HIGH);
digitalWrite(readPin, LOW);
digitalWrite(pausePin, LOW);
// Write Data loop
while (mode == 'w') {
// First we'll measure the voltage at the analog pin. Normally
// we'd use analogRead(), which returns a number from 0 to 1023.
// Here we've written a function (further down) called
// getVoltage() that returns the true voltage (0 to 5 Volts)
// present on an analog input pin.
voltage = getVoltage(sensorPin);
// Now we'll convert the voltage to degrees Celsius.
// This formula comes from the temperature sensor datasheet:
// CONVERT IN PYTHON SCRIPT
// degreesC = (voltage - 0.5) * 100.0;
//Serial.print("Address[");
//Serial.print(addr);
//Serial.print("]: \t");
Serial.println(voltage);
/***
Write the value to the appropriate byte of the EEPROM.
these values will remain there when the board is
turned off.
***/
// Convert for storage to EEPROM
voltVal = (voltage * 10000) / 4;
// Write to EEPROM
EEPROM.write(addr, voltVal);
/***
Advance to the next address, when at the end restart at the beginning.
Larger AVR processors have larger EEPROM sizes, E.g:
- Arduno Duemilanove: 512b EEPROM storage.
- Arduino Uno: 1kb EEPROM storage.
- Arduino Mega: 4kb EEPROM storage.
Rather than hard-coding the length, you should use the pre-provided length function.
This will make your code portable to all AVR processors.
***/
addr = addr + 1;
/***
As the EEPROM sizes are powers of two, wrapping (preventing overflow) of an
EEPROM address is also doable by a bitwise and of the length - 1.
++addr &= EEPROM.length() - 1;
***/
// Check for mode change
mode = CheckModeChange();
delay(100);
}
return mode;
}
char ReadData(int Pin, char prevMode) {
// Declarations
byte value;
float voltageVal, voltage, degreesC;
int addr = 0;
char mode = 'r';
// Checks previous mode. If previous mode was write, now that we are reading
// we should reset the EEPROM address to 0 so we can read from the beginning.
if ( (prevMode == 'w') || (prevMode == 'W') ) {
addr = 0;
}
while (mode == 'r') {
value = EEPROM.read(addr);
voltageVal = value;
voltage = voltageVal * 4 / 10000;
//Serial.print("Voltage: ");
//Serial.print("\t");
//Serial.println(voltage);
degreesC = (voltage - 0.5) * 100.0;
//Serial.print("Degrees C: ");
//Serial.print("\t");
//Serial.print("Address[");
//Serial.print(addr);
//Serial.print("]: \t");
Serial.println(degreesC);
/***
Advance to the next address, when at the end restart at the beginning.
Larger AVR processors have larger EEPROM sizes, E.g:
- Arduno Duemilanove: 512b EEPROM storage.
- Arduino Uno: 1kb EEPROM storage.
- Arduino Mega: 4kb EEPROM storage.
Rather than hard-coding the length, you should use the pre-provided length function.
This will make your code portable to all AVR processors.
***/
addr = addr + 1;
if (addr == EEPROM.length()) {
addr = 0;
}
// Check for mode change
mode = CheckModeChange();
delay(100);
}
return mode;
}
float getVoltage(int pin) {
// This function has one input parameter, the analog pin number
// to read. You might notice that this function does not have
// "void" in front of it; this is because it returns a floating-
// point value, which is the true voltage on that pin (0 to 5V).
return (analogRead(pin) * 0.004882814);
// This equation converts the 0 to 1023 value that analogRead()
// returns, into a 0.0 to 5.0 value that is the true voltage
// being read at that pin.
}