We Keep Coding

python script slowing down and almost stops after a couple of seconds

I am trying to make a resource monitor with an arduino and it is working great for almost 20 seconds before it almost stops running.
When it slows down, it takes almost 5 seconds between updates.
I have tried to comment out everything with psutil and giving it a permanent value.
and have tried the same with GPUtil.
Here is the python code
import serial.tools.list_ports
import serial
import psutil
import GPUtil
import time
import serial
ports = list(serial.tools.list_ports.comports())
baud = 9600
for p in ports:
if "Arduino" in p[1]:
port = p[0]
ser=serial.Serial(port, baud, timeout=1)
try:
while True:
cpuUsage = psutil.cpu_percent()
ramUsage = psutil.virtual_memory()
cpuUsage = str(cpuUsage)
GPUs = GPUtil.getGPUs()
gpuUsage = GPUs[0].load
gpuUsage = str(gpuUsage)
gpuUsage = gpuUsage[2:]
ramUsage = str(ramUsage.percent)
toSend = cpuUsage + "," + gpuUsage + ","+ ramUsage
print (toSend)
ser.write(toSend.encode())
#print("20.5".encode())
#line = ser.readline()[:-2]
#line.decode()
#print ("Read : " , line);
time.sleep(0.1)
except:
print ("error")
Here is the Arduino code
#include <FastLED.h>
#define NUM_LEDS 15
#define DATA_PIN 6
float Cpu = 40;
float Gpu = 99;
float Ram = 60;
String Cpu_Read;
String Gpu_Read;
String Ram_Read;
int RXLED = 17;
String StrNumbers = "";
int numbers;
int Received = 0;
int Separrator = 0;
String Text = "";
CRGB leds[NUM_LEDS];
void setup() {
// put your setup code here, to run once:
FastLED.addLeds<WS2812B, DATA_PIN, GRB>(leds, NUM_LEDS);
FastLED.setBrightness(5);
Serial.begin(115200);
pinMode(LED_BUILTIN_TX,INPUT);
pinMode(LED_BUILTIN_RX,INPUT);
}
void loop() {
// put your main code here, to run repeatedly:
char inByte = ' ';
StrNumbers = "";
Text = "";
Separrator = 0;
Cpu_Read = "";
Gpu_Read = "";
Ram_Read = "";
Received = 0;
pinMode(LED_BUILTIN_TX,INPUT);
while(Serial.available() > 0){ // only send data back if data has been sent
pinMode(LED_BUILTIN_TX,OUTPUT);
inByte = Serial.read(); // read the incoming data
if (inByte == ','){
Separrator += 1;
}
else if (Separrator == 0){
Cpu_Read += (char) inByte;
}
else if (Separrator == 1){
Gpu_Read += (char) inByte;
}
else if (Separrator == 2){
Ram_Read += (char) inByte;
Serial.print("Ram : ");
Serial.println(Ram_Read);
}
else{
Serial.println("Error");
}
/*
inByte = Serial.read(); // read the incoming data
if (isDigit(inByte)) { // tests if inByte is a digit
StrNumbers += (char) inByte;
Serial.println(inByte); // send the data back in a new line so that it is not all one long line
//Serial.println(numbers); // send the data back in a new line so that it is not all one long line
}
else if (inByte == ".")
{
abortLoop = 1;
}
else{
Text +=(char) inByte;
}
*/
Received = 1;
}
if (Received == 1){
Cpu = Cpu_Read.toInt();
Gpu = Gpu_Read.toInt();
Ram = Ram_Read.toInt();
UpdateMonitor(Cpu ,Gpu ,Ram);
}
Text.trim();
if (StrNumbers != ""){
numbers = StrNumbers.toInt();
Serial.println(numbers);
if (numbers > 100) numbers = 100;
UpdateMonitor(Cpu,numbers,Ram);
}
if(Text!= ""){
//Serial.println(Text); // send the data back in a new line so that it is not all one long line
}
if (Text == "ResourceMonitor"){
Serial.println("Yes");
}
else if (Text != ""){
Serial.println(Text);
numbers = Text.toInt();
if (numbers > 100) numbers = 100;
UpdateMonitor(Cpu, numbers, Ram);
}
}
void UpdateMonitor(int cpu,int gpu, int ram){
int Cpu_Usage = map(cpu, 0, 100, 0, 5);
int Gpu_Usage = map(gpu, 0, 100, 0, 5);
int Ram_Usage = map(ram, 0, 100, 0, 5);
FastLED.clear();
for(int led = 0; led < Ram_Usage; led++) {
leds[led] = CRGB::Blue;
}
for(int led = 0; led < Gpu_Usage; led++) {
leds[9 - led] = CRGB::Green;
}
for(int led = 0; led < Cpu_Usage; led++) {
leds[led+10] = CRGB::Red;
}
FastLED.show();
}

it is fixed now not sure what fixet it but it works atleast :)
thansk to all that have tried to helt

UART connection between STM32 and Raspberry Pi 3

i'm trying to establish an UART connection between STM32F4 and Raspberry Pi 3 to send motion sensor data.
STM32 C code:
/* Includes ------------------------------------------------------------------*/
#include "stm32f4_discovery.h"
/* Private typedef -----------------------------------------------------------*/
/* Private define ------------------------------------------------------------*/
/* Private macro -------------------------------------------------------------*/
/* Private variables ---------------------------------------------------------*/
int8_t polje[4];
/* Private function prototypes -----------------------------------------------*/
/* Private functions ---------------------------------------------------------*/
void delay(uint32_t delay) {
while(delay--);
}
// write to SPI1
void SPI1_Write(int8_t data)
{
// short delay
volatile int d = 500;
while(d--);
while(SPI_I2S_GetFlagStatus(SPI1, SPI_I2S_FLAG_TXE) == RESET){}
SPI_I2S_SendData(SPI1, data);
while (SPI_I2S_GetFlagStatus(SPI1, SPI_I2S_FLAG_RXNE) == RESET){}
SPI_I2S_ReceiveData(SPI1);
}
// read from SPI1
int8_t SPI1_Read()
{
// short delay
volatile int d = 500;
while(d--);
while(SPI_I2S_GetFlagStatus(SPI1, SPI_I2S_FLAG_TXE) == RESET){}
SPI_I2S_SendData(SPI1, 0x00);
while(SPI_I2S_GetFlagStatus(SPI1, SPI_I2S_FLAG_RXNE) == RESET){}
return SPI_I2S_ReceiveData(SPI1);
}
void initSPI1(void)
{
// RCC
RCC_APB2PeriphClockCmd(RCC_APB2Periph_SPI1, ENABLE);
RCC_AHB1PeriphClockCmd(RCC_AHB1Periph_GPIOA, ENABLE);
// PA5, PA6, PA7 for MISO, MOSI, and SCLK
GPIO_InitTypeDef GPIO_InitStructure;
GPIO_InitStructure.GPIO_Pin = GPIO_Pin_5 | GPIO_Pin_6 | GPIO_Pin_7;
GPIO_InitStructure.GPIO_Mode = GPIO_Mode_AF;
GPIO_InitStructure.GPIO_OType = GPIO_OType_PP;
GPIO_InitStructure.GPIO_Speed = GPIO_Speed_2MHz;
GPIO_InitStructure.GPIO_PuPd = GPIO_PuPd_DOWN;
GPIO_Init(GPIOA, &GPIO_InitStructure);
GPIO_PinAFConfig(GPIOA, GPIO_PinSource5, GPIO_AF_SPI1);
GPIO_PinAFConfig(GPIOA, GPIO_PinSource6, GPIO_AF_SPI1);
GPIO_PinAFConfig(GPIOA, GPIO_PinSource7, GPIO_AF_SPI1);
// SPI1 INIT
SPI_InitTypeDef spi;
spi.SPI_Direction = SPI_Direction_2Lines_FullDuplex;
spi.SPI_Mode = SPI_Mode_Master;
spi.SPI_DataSize = SPI_DataSize_8b;
spi.SPI_CPOL = SPI_CPOL_Low;
spi.SPI_CPHA = SPI_CPHA_1Edge;
spi.SPI_NSS = SPI_NSS_Soft;
spi.SPI_BaudRatePrescaler = SPI_BaudRatePrescaler_4;
spi.SPI_FirstBit = SPI_FirstBit_MSB;
spi.SPI_CRCPolynomial = 7;
SPI_Init(SPI1, &spi);
// SPI1 ENABLE
SPI_Cmd(SPI1, ENABLE);
}
void initMotionSensor()
{
// RCC
RCC_AHB1PeriphClockCmd(RCC_AHB1Periph_GPIOE, ENABLE);
// PE3 for slave select
GPIO_ResetBits(GPIOE, GPIO_Pin_3);
// configure and start sensor
GPIO_InitTypeDef ss;
ss.GPIO_Pin = GPIO_Pin_3;
ss.GPIO_Mode = GPIO_Mode_OUT;
ss.GPIO_OType = GPIO_OType_PP;
ss.GPIO_PuPd = GPIO_PuPd_NOPULL;
ss.GPIO_Speed = GPIO_Speed_2MHz;
GPIO_Init(GPIOE, &ss);
GPIO_SetBits(GPIOE, GPIO_Pin_3);
// Aktiviram slave
//delay(500);
GPIO_ResetBits(GPIOE, GPIO_Pin_3);
// Zapisem inicializacijo senzorja
// Zapisem 0x47 na naslov 0x20
SPI1_Write(0x20);
SPI1_Write(0x47);
GPIO_SetBits(GPIOE, GPIO_Pin_3);
//delay(500);
}
void initUSART1()
{
// RCC
RCC_AHB1PeriphClockCmd(RCC_AHB1Periph_GPIOA, ENABLE);
RCC_AHB1PeriphClockCmd(RCC_AHB1Periph_GPIOD, ENABLE);
RCC_AHB1PeriphClockCmd(RCC_AHB1Periph_GPIOB, ENABLE);
RCC_APB2PeriphClockCmd(RCC_APB2Periph_USART1, ENABLE);
// USART1 init
USART_InitTypeDef USART_InitStruct;
USART_InitStruct.USART_BaudRate = 9600;
USART_InitStruct.USART_WordLength = USART_WordLength_8b;
USART_InitStruct.USART_StopBits = USART_StopBits_1;
USART_InitStruct.USART_Parity = USART_Parity_No;
USART_InitStruct.USART_HardwareFlowControl = USART_HardwareFlowControl_None;
USART_InitStruct.USART_Mode = USART_Mode_Tx | USART_Mode_Rx;
USART_Init(USART1, &USART_InitStruct);
// USART1 enable
USART_Cmd(USART1, ENABLE);
// PB6 and PB7 for USART1 Tx and Rx
GPIO_InitTypeDef usart_dev;
usart_dev.GPIO_Pin = GPIO_Pin_6 | GPIO_Pin_7;
usart_dev.GPIO_Mode = GPIO_Mode_AF;
usart_dev.GPIO_OType = GPIO_OType_PP;
usart_dev.GPIO_Speed = GPIO_Speed_50MHz;
usart_dev.GPIO_PuPd = GPIO_PuPd_UP;
GPIO_Init(GPIOB, &usart_dev);
GPIO_PinAFConfig(GPIOB, GPIO_PinSource6,GPIO_AF_USART1);
GPIO_PinAFConfig(GPIOB, GPIO_PinSource7,GPIO_AF_USART1);
}
void initDMA2()
{
// RCC
RCC_AHB1PeriphClockCmd(RCC_AHB1Periph_DMA2, ENABLE);
// init DMA2
DMA_InitTypeDef DMA_InitStructure;
DMA_InitStructure.DMA_Channel = DMA_Channel_4;
DMA_InitStructure.DMA_DIR = DMA_DIR_MemoryToPeripheral;
DMA_InitStructure.DMA_Memory0BaseAddr = (uint32_t) polje;
DMA_InitStructure.DMA_BufferSize = 4;
DMA_InitStructure.DMA_PeripheralBaseAddr = (uint32_t) &(USART1 -> DR);
DMA_InitStructure.DMA_MemoryInc = DMA_MemoryInc_Enable;
DMA_InitStructure.DMA_PeripheralInc = DMA_PeripheralInc_Disable;
DMA_InitStructure.DMA_MemoryDataSize = DMA_MemoryDataSize_Byte;
DMA_InitStructure.DMA_PeripheralDataSize = DMA_PeripheralDataSize_Byte;
DMA_InitStructure.DMA_Mode = DMA_Mode_Circular;
DMA_InitStructure.DMA_Priority = DMA_Priority_Medium;
DMA_InitStructure.DMA_FIFOMode = DMA_FIFOMode_Disable;
DMA_InitStructure.DMA_FIFOThreshold = DMA_FIFOThreshold_1QuarterFull;
DMA_InitStructure.DMA_MemoryBurst = DMA_MemoryBurst_Single;
DMA_InitStructure.DMA_PeripheralBurst = DMA_PeripheralBurst_Single;
DMA_Init(DMA2_Stream7, &DMA_InitStructure);
// enable USART1 DMA Tx
USART_DMACmd(USART1, USART_DMAReq_Tx, ENABLE);
// enable DMA2 stream
DMA_Cmd(DMA2_Stream7, ENABLE);
}
/**
* #brief Main program
* #param None
* #retval None
*/
int main(void)
{
RCC_AHB1PeriphClockCmd(RCC_AHB1Periph_GPIOD, ENABLE);
// led init
GPIO_InitTypeDef leds;
leds.GPIO_Pin = GPIO_Pin_12 | GPIO_Pin_13 | GPIO_Pin_14 | GPIO_Pin_15;
leds.GPIO_Mode = GPIO_Mode_OUT;
leds.GPIO_OType = GPIO_OType_PP;
leds.GPIO_PuPd = GPIO_PuPd_NOPULL;
leds.GPIO_Speed = GPIO_Speed_2MHz;
GPIO_Init(GPIOD, &leds);
// SPI1
initSPI1();
// LIS motion sensor
initMotionSensor();
// USART1
initUSART1();
// DMA2
initDMA2();
while (1)
{
// read x1
GPIO_ResetBits(GPIOE, GPIO_Pin_3);
SPI1_Write(0x29 | 0x80);
volatile int8_t x1 = SPI1_Read();
GPIO_SetBits(GPIOE, GPIO_Pin_3);
delay(500);
// read y1
GPIO_ResetBits(GPIOE, GPIO_Pin_3);
SPI1_Write(0x2B | 0x80);
volatile int8_t y1 = SPI1_Read();
GPIO_SetBits(GPIOE, GPIO_Pin_3);
if (x1 < -5) {
// Vklopi ledico
GPIO_SetBits(GPIOD, GPIO_Pin_12);
delay(500);
GPIO_ResetBits(GPIOD, GPIO_Pin_12);
}
if (x1 > 5) {
// Vklopi drugo ledico
GPIO_SetBits(GPIOD, GPIO_Pin_14);
delay(500);
GPIO_ResetBits(GPIOD, GPIO_Pin_14);
}
if (y1 < -5) {
// Vklopi ledico
GPIO_SetBits(GPIOD, GPIO_Pin_15);
delay(500);
GPIO_ResetBits(GPIOD, GPIO_Pin_15);
}
if (y1 > 5) {
// Vklopi drugo ledico
GPIO_SetBits(GPIOD, GPIO_Pin_13);
delay(500);
GPIO_ResetBits(GPIOD, GPIO_Pin_13);
}
delay(500);
// read x
GPIO_ResetBits(GPIOE, GPIO_Pin_3);
SPI1_Write(0x29 | 0x80);
volatile int8_t x2 = SPI1_Read();
GPIO_SetBits(GPIOE, GPIO_Pin_3);
delay(500);
// read y
GPIO_ResetBits(GPIOE, GPIO_Pin_3);
SPI1_Write(0x2B | 0x80);
volatile int8_t y2 = SPI1_Read();
GPIO_SetBits(GPIOE, GPIO_Pin_3);
delay(500);
polje[0] = x1;
polje[1] = x2;
polje[2] = y1;
polje[3] = y2;
delay(500);
}
}
Raspberry Pi code:
import time
import ctypes
import serial
import sys
from binascii import hexlify
ser = serial.Serial(
port='/dev/ttyS0',
baudrate=9600,
parity=serial.PARITY_NONE,
stopbits=serial.STOPBITS_ONE,
bytesize=serial.EIGHTBITS,
timeout=1,
dsrdtr=False
)
counter = 0
if (ser.isOpen() == False):
ser.open()
#Flush before receiving or sending any data
ser.flushInput()
ser.flushOutput()
while 1:
x = ser.read()
print (hexlify(x))
time.sleep(0.3)
I already enabled the serial port interface on Raspberry Pi and disabled the terminal over serial option. I'm sure that the code on STM32 works, because I tried it with another STM32 as a receiver. I connected 2 GND pins together and STM32's Tx (PB6 in my case) to RasPi Rx (GPIO15). As an output i'm getting some numbers, but they don't change when I accelerate STM32 (the difference is seen on livewatch). I'm getting strange output on RasPi even if unplug the Tx - Rx connection.
Does anyone have an idea what could be wrong?
Thank you in advance!

Not sure it may be the cause, but when I was doing the same work I used /dev/ttyAMA0 on Raspberry Pi 3. So check UART on this side of your system.
I disable Bluetooth so PIN 14/15 can now operate as UART TX/RX. This lead to that /dev/ttyAMA0 now start operates as UART port, and not /dev/ttyS0.
See more on official site.
This is my sample code for Raspberry (simplified):
import serial
ser = serial.Serial()
ser.port = '/dev/ttyAMA0'
ser.baudrate = 115200
ser.timeout = 60 # 1 min
ser.open()
msg = ''
while True:
char = ser.read(1) # 1 byte
msg = msg+char.decode('utf-8')
if char == b'\r':
break
And there for STM32F031K6:
usart.c:
...
huart1.Instance = USART1;
huart1.Init.BaudRate = 115200;
huart1.Init.WordLength = UART_WORDLENGTH_8B;
huart1.Init.StopBits = UART_STOPBITS_1;
huart1.Init.Parity = UART_PARITY_NONE;
huart1.Init.Mode = UART_MODE_TX_RX;
huart1.Init.HwFlowCtl = UART_HWCONTROL_NONE;
huart1.Init.OverSampling = UART_OVERSAMPLING_16;
huart1.Init.OneBitSampling = UART_ONE_BIT_SAMPLE_DISABLE;
huart1.AdvancedInit.AdvFeatureInit = UART_ADVFEATURE_NO_INIT;
...
main.c:
...
MX_USART1_UART_Init();
uint8_t str[STR_SIZE];
str_len = sprintf(str, "some message\r");
HAL_UART_Transmit(&huart1, str, (uint32_t)str_len, 0xFFFF);
...

Using a named (fifo) pipe to transport arrays (images) between python and c++

I need to send an array (representing an image) through a named FIFO pipe from a python process to a c++ process, and then back the other way (on a Linux system).
The below code works great when using named pipes between two Python processes. It uses numpy's tostring() and fromstring() functions:
Send frames over named pipe (Python)
import cv2
import numpy as np
from time import sleep
##########################################################
FIFO_Images = "./../pipes/images.fifo"
videoName = "./../../videos/videoName.avi"
delim = "break"
##########################################################
def sendImage(h, w, d, pixelarray):
imageString = pixelarray.tostring()
with open(FIFO_Images, "w") as f:
f.write(str(h)+ delim + str(w)+ delim + str(d) + delim + imageString)
sleep(.01)
return
##########################################################
cap = cv2.VideoCapture(videoName)
while(cap.isOpened()):
ret, frame_rgb = cap.read()
h, w, d = frame_rgb.shape
sendImage(h, w, d, frame_rgb)
cap.release()
cv2.destroyAllWindows()
Read frames over named pipe (Python)
import cv2
import numpy as np
##########################################################
FIFO_Images = "./../pipes/images.fifo"
delim = "break"
##########################################################
def getFrame():
with open(FIFO_Images, "r") as f:
data = f.read().split(delim)
#parse incoming string, which has format (height, width, depth, imageData)
h=int(data[0])
w=int(data[1])
d=int(data[2])
imageString = data[3]
#convert array string into numpy array
array = np.fromstring(imageString, dtype=np.uint8)
#reshape numpy array into the required dimensions
frame = array.reshape((h,w,d))
return frame
##########################################################
while(True):
frame = getFrame()
cv2.imshow('frame', frame)
cv2.waitKey(1) & 0xFF
However, I couldn't figure out how to read the entire image from the pipe on the cpp side, since it takes "\n" as a delimiter for the read automatically.
My workaround was to do a base64 encoding on the "tostring()" image, then send that over the pipe. This works, but the base64 decoding on the other slide is much too slow for real-time applications (~0.2 seconds per frame). Code:
Send base64-encoded images over named pipe (Python)
import cv2
import numpy as np
from time import time
from time import sleep
import base64
##########################################################
FIFO_Images = "./../pipes/images.fifo"
videoName = "./../../videos/videoName.avi"
delim = ";;"
##########################################################
def sendImage(h, w, d, pixelarray):
flat = pixelarray.flatten()
imageString = base64.b64encode(pixelarray.tostring())
fullString = str(h)+ delim + str(w)+ delim + str(d)+ delim + imageString + delim + "\n"
with open(FIFO_Images, "w") as f:
f.write(fullString)
return
##########################################################
cap = cv2.VideoCapture(videoName)
count = 0
while(cap.isOpened()):
ret, frame_rgb = cap.read()
h, w, d = frame_rgb.shape
frame_gbr = cv2.cvtColor(frame_rgb, cv2.COLOR_RGB2BGR)
sendImage(h, w, d, frame_rgb)
cap.release()
cv2.destroyAllWindows()
Read base64-encoded images over named pipe (C++)
#include "opencv2/opencv.hpp"
#include <iostream>
#include <fstream>
#include <string>
#include <vector>
#include <unistd.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <linux/stat.h>
#include <ctime>
using namespace std;
using namespace cv;
#define FIFO_FILE "./../../../pipes/images.fifo"
#define MAX_BUF 10000000
FILE *fp;
char readbuf[MAX_BUF + 1]; //add 1 to the expected size to accomodate the mysterious "extra byte", which I think signals the end of the line.
/************************BASE64 Decoding*********************************************/
std::string base64_encode(unsigned char const* , unsigned int len);
std::string base64_decode(std::string const& s);
static const std::string base64_chars =
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz"
"0123456789+/";
static inline bool is_base64(unsigned char c) {
return (isalnum(c) || (c == '+') || (c == '/'));
}
std::string base64_encode(unsigned char const* bytes_to_encode, unsigned int in_len) {
std::string ret;
int i = 0;
int j = 0;
unsigned char char_array_3[3];
unsigned char char_array_4[4];
while (in_len--) {
char_array_3[i++] = *(bytes_to_encode++);
if (i == 3) {
char_array_4[0] = (char_array_3[0] & 0xfc) >> 2;
char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4);
char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6);
char_array_4[3] = char_array_3[2] & 0x3f;
for(i = 0; (i <4) ; i++)
ret += base64_chars[char_array_4[i]];
i = 0;
}
}
if (i)
{
for(j = i; j < 3; j++)
char_array_3[j] = '\0';
char_array_4[0] = (char_array_3[0] & 0xfc) >> 2;
char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4);
char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6);
char_array_4[3] = char_array_3[2] & 0x3f;
for (j = 0; (j < i + 1); j++)
ret += base64_chars[char_array_4[j]];
while((i++ < 3))
ret += '=';
}
return ret;
}
std::string base64_decode(std::string const& encoded_string) {
int in_len = encoded_string.size();
int i = 0;
int j = 0;
int in_ = 0;
unsigned char char_array_4[4], char_array_3[3];
std::string ret;
while (in_len-- && ( encoded_string[in_] != '=') && is_base64(encoded_string[in_])) {
char_array_4[i++] = encoded_string[in_]; in_++;
if (i ==4) {
for (i = 0; i <4; i++)
char_array_4[i] = base64_chars.find(char_array_4[i]);
char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);
char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
for (i = 0; (i < 3); i++)
ret += char_array_3[i];
i = 0;
}
}
if (i) {
for (j = i; j <4; j++)
char_array_4[j] = 0;
for (j = 0; j <4; j++)
char_array_4[j] = base64_chars.find(char_array_4[j]);
char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);
char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
for (j = 0; (j < i - 1); j++) ret += char_array_3[j];
}
return ret;
}
/*********************************************************************/
int stringToInt(string str)
{
int num;
if (!(istringstream(str) >> num)) num = 0;
return num;
}
/*********************************************************************/
bool timerOn = 0;
clock_t timerStart;
void Timer(string process)
{
if (!timerOn)
{
timerStart = clock();
timerOn = true;
}
else if (timerOn)
{
double duration = (clock() - timerStart) / (double) CLOCKS_PER_SEC;
cout << "Time to complete: ";
printf("%.2f", duration);
cout << ": " << process << endl;
timerOn = false;
}
}
/*********************************************************************/
void getFrame()
{
string fullString;
string delimiter = ";;";
size_t pos = 0;
string token;
int h;
int w;
int d;
string imgString;
int fifo;
bool cont(true);
/***************************
Read from the pipe
www.tldp.org/LDP/lpg/node18.html
***************************/
Timer("Read from pipe");
fp = fopen(FIFO_FILE, "r");
fgets(readbuf, MAX_BUF + 1, fp); // Stops when MAX_BUF characters are read, the newline character ("\n") is read, or the EOF (end of file) is reached
string line(readbuf);
fclose(fp);
Timer("Read from pipe");
//////parse the string into components
Timer("Parse string");
int counter = 0;
while ((pos = line.find(delimiter)) != string::npos)
{
token = line.substr(0,pos);
if (counter == 0)
{
h = stringToInt(token);
}
else if (counter == 1)
{
w = stringToInt(token);
}
else if (counter == 2)
{
d = stringToInt(token);
}
else if (counter == 3)
{
imgString = token;
//cout << imgString[0] << endl;
}
else
{
cout << "ERROR: Too many paramaters passed" << endl;
return;
}
line.erase(0, pos + delimiter.length());
counter ++;
}
if (counter == 3)
{
imgString = token;
}
if (counter < 3)
{
cout << "ERROR: Not enough paramaters passed: " << counter << endl;
//return;
}
Timer("Parse string");
/***************************
Convert from Base64
***************************/
Timer("Decode Base64");
std::string decoded = base64_decode(imgString);
Timer("Decode Base64");
/***************************
Convert to vector of ints
***************************/
Timer("Convert to vector of ints");
std::vector<uchar> imgVector;
for (int i = 0; i < decoded.length(); i = i+1) // + 4)
{
int temp = (char(decoded[i]));
imgVector.push_back(temp);
}
Timer("Convert to vector of ints");
//////convert the vector into a matrix
Mat frame = Mat(imgVector).reshape(d, h);
namedWindow("Frame", WINDOW_AUTOSIZE);
imshow("Frame", frame);
waitKey(1);
}
int main()
{
/* Create the FIFO if it does not exist */
umask(0);
mknod(FIFO_FILE, S_IFIFO|0666, 0);
while(1)
{
getFrame();
}
return 0;
}
There must be a more efficient way to accomplish this. Can anyone make a recommendation? While I'm happy to hear suggestions for other methods to accomplish this, I am constrained to using named pipes for now.

This is overcomplicated. If you need to send binary data, send their length first, then newline (\n), and then the data (raw, no base64). Receive it on the other side by readling a line, parsing the number and then just reading a block of data of given length.
Example - writing binary data to a FIFO (or file) in Python:
#!/usr/bin/env python3
import os
fifo_name = 'fifo'
def main():
data = b'blob\n\x00 123'
try:
os.mkfifo(fifo_name)
except FileExistsError:
pass
with open(fifo_name, 'wb') as f:
# b for binary mode
f.write('{}\n'.format(len(data)).encode())
f.write(data)
if __name__ == '__main__':
main()
Reading binary data from FIFO in C++:
#include <iostream>
#include <fstream>
#include <string>
#include <vector>
#include <sys/stat.h>
int main(int argc, char *argv[]) {
const char *fifo_name = "fifo";
mknod(fifo_name, S_IFIFO | 0666, 0);
std::ifstream f(fifo_name);
std::string line;
getline(f, line);
auto data_size = std::stoi(line);
std::cout << "Size: " << data_size << std::endl;
std::string data;
{
std::vector<char> buf(data_size);
f.read(buf.data(), data_size);
// write to vector data is valid since C++11
data.assign(buf.data(), buf.size());
}
if (!f.good()) {
std::cerr << "Read failed" << std::endl;
}
std::cout << "Data size: " << data.size() << " content: " << data << std::endl;
}

Data corruption Piping between C++ and Python

I am writing some code that takes binary data from Python, Pipes it to C++, does some processing on the data, (in this case calculating a mutual information metric) and then pipes the results back to python. While testing I have found that everything works fine if the data I send is a set of 2 arrays with dimensions less than 1500 X 1500, but if I send 2 arrays that are 2K X 2K I get back a lot of corrupted nonsense.
I currently believe the algorithmic portion of the code is fine because it provides the expected answers during testing with small (<=1500 X1500) arrays. That leads me to believe that this is an issue with either the stdin or stdout piping. That maybe I’m passing some intrinsic limit somewhere.
The Python Code and C++ code are below.
Python Code:
import subprocess
import struct
import sys
import numpy as np
#set up the variables needed
bytesPerDouble = 8
sizeX = 2000
sizeY = 2000
offset = sizeX*sizeY
totalBytesPerArray = sizeX*sizeY*bytesPerDouble
totalBytes = totalBytesPerArray*2 #the 2 is because we pass 2 different versions of the 2D array
#setup the testing data array
a = np.zeros(sizeX*sizeY*2, dtype='d')
for i in range(sizeX):
for j in range(sizeY):
a[j+i*sizeY] = i
a[j+i*sizeY+offset] = i
if i % 10 == 0:
a[j+i*sizeY+offset] = j
data = a.tobytes('C')
strTotalBytes = str(totalBytes)
strLineBytes = str(sizeY*bytesPerDouble)
#communicate with c++ code
print("starting C++ code")
command = "C:\Python27\PythonPipes.exe"
proc = subprocess.Popen([command, strTotalBytes, strLineBytes, str(sizeY), str(sizeX)], stdin=subprocess.PIPE,stderr=subprocess.PIPE,stdout=subprocess.PIPE)
ByteBuffer = (data)
proc.stdin.write(ByteBuffer)
print("Reading results back from C++")
for i in range(sizeX):
returnvalues = proc.stdout.read(sizeY*bytesPerDouble)
a = buffer(returnvalues)
b = struct.unpack_from(str(sizeY)+'d', a)
print str(b) + " " + str(i)
print('done')
C++ Code:
Main function:
int main(int argc, char **argv) {
int count = 0;
long totalbytes = stoi(argv[argc-4], nullptr,10); //bytes being transfered
long bytechunk = stoi(argv[argc - 3], nullptr, 10); //bytes being transfered at a time
long height = stoi(argv[argc-2], nullptr, 10); //bytes being transfered at a time
long width = stoi(argv[argc-1], nullptr, 10); //bytes being transfered at a time
long offset = totalbytes / sizeof(double) / 2;
data = new double[totalbytes/sizeof(double)];
int columnindex = 0;
//read in data from pipe
while (count<totalbytes) {
fread(&(data[columnindex]), 1, bytechunk, stdin);
columnindex += bytechunk / sizeof(double);
count += bytechunk;
}
//calculate the data transform
MutualInformation MI = MutualInformation();
MI.Initialize(data, height, width, offset);
MI.calcMI();
count = 0;
//*
//write out data to pipe
columnindex = 0;
while (count<totalbytes/2) {
fwrite(&(MI.getOutput()[columnindex]), 1, bytechunk, stdout);
fflush(stdout);
count += bytechunk;
columnindex += bytechunk/sizeof(double);
}
//*/
delete [] data;
return 0;
}
and in case you need it the actual processing code:
double MutualInformation::calcMI(){
double rvalue = 0.0;
std::map<int, map<int, double>> lHistXY = map<int, map<int, double>>();
std::map<int, double> lHistX = map<int, double>();
std::map<int, double> lHistY = map<int, double>();
typedef std::map<int, std::map<int, double>>::iterator HistXY_iter;
typedef std::map<int, double>::iterator HistY_iter;
//calculate Entropys and MI
double MI = 0.0;
double Hx = 0.0;
double Hy = 0.0;
double Px = 0.0;
double Py = 0.0;
double Pxy = 0.0;
//scan through the image
int ip = 0;
int jp = 0;
int chipsize = 3;
//setup zero array
double * zeros = new double[this->mHeight];
for (int j = 0; j < this->mHeight; j++){
zeros[j] = 0.0;
}
//zero out Output array
for (int i = 0; i < this->mWidth; i++){
memcpy(&(this->mOutput[i*this->mHeight]), zeros, this->mHeight*8);
}
double index = 0.0;
for (int ioutter = chipsize; ioutter < (this->mWidth - chipsize); ioutter++){
//write out processing status
//index = (double)ioutter;
//fwrite(&index, 8, 1, stdout);
//fflush(stdout);
//*
for (int j = chipsize; j < (this->mHeight - chipsize); j++){
//clear the histograms
lHistX.clear();
lHistY.clear();
lHistXY.clear();
//chip out a section of the image
for (int k = -chipsize; k <= chipsize; k++){
for (int l = -chipsize; l <= chipsize; l++){
ip = ioutter + k;
jp = j + l;
//update X histogram
if (lHistX.count(int(this->mData[ip*this->mHeight + jp]))){
lHistX[int(this->mData[ip*this->mHeight + jp])] += 1.0;
}else{
lHistX[int(this->mData[ip*this->mHeight + jp])] = 1.0;
}
//update Y histogram
if (lHistY.count(int(this->mData[ip*this->mHeight + jp+this->mOffset]))){
lHistY[int(this->mData[ip*this->mHeight + jp+this->mOffset])] += 1.0;
}
else{
lHistY[int(this->mData[ip*this->mHeight + jp+this->mOffset])] = 1.0;
}
//update X and Y Histogram
if (lHistXY.count(int(this->mData[ip*this->mHeight + jp]))){
//X Key exists check if Y key exists
if (lHistXY[int(this->mData[ip*this->mHeight + jp])].count(int(this->mData[ip*this->mHeight + jp + this->mOffset]))){
//X & Y keys exist
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] += 1;
}else{
//X exist but Y doesn't
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] = 1;
}
}else{
//X Key Didn't exist
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] = 1;
};
}
}
//calculate PMI, Hx, Hy
// iterator->first = key
// iterator->second = value
MI = 0.0;
Hx = 0.0;
Hy = 0.0;
for (HistXY_iter Hist2D_iter = lHistXY.begin(); Hist2D_iter != lHistXY.end(); Hist2D_iter++) {
Px = lHistX[Hist2D_iter->first] / ((double) this->mOffset);
Hx -= Px*log(Px);
for (HistY_iter HistY_iter = Hist2D_iter->second.begin(); HistY_iter != Hist2D_iter->second.end(); HistY_iter++) {
Py = lHistY[HistY_iter->first] / ((double) this->mOffset);
Hy -= Py*log(Py);
Pxy = HistY_iter->second / ((double) this->mOffset);
MI += Pxy*log(Pxy / Py / Px);
}
}
//normalize PMI to max(Hx,Hy) so that the PMI value runs from 0 to 1
if (Hx >= Hy && Hx > 0.0){
MI /= Hx;
}else if(Hy > Hx && Hy > 0.0){
MI /= Hy;
}
else{
MI = 0.0;
}
//write PMI to data output array
if (MI < 1.1){
this->mOutput[ioutter*this->mHeight + j] = MI;
}
else{
this->mOutput[ioutter*this->mHeight + j] = 0.0;
}
}
}
return rvalue;
}
with arrays that return something that makes sense I get output bounded between 0 and 1 like this:
(0.0, 0.0, 0.0, 0.7160627908692593, 0.6376472316395495, 0.5728801401524277,...
with the 2Kx2K or higher arrays I get nonesense like this (even though the code clamps the values between 0 and 1):
(-2.2491400820412374e+228, -2.2491400820412374e+228, -2.2491400820412374e+228, -2.2491400820412374e+228, -2.2491400820412374e+228,...
I would like to know why this code is corrupting the data set after it is assigned between 0.0 and 1, and whether or not it is a piping issue, a stdin/stdout issue, a buffer issue of some sort, or a coding issue I am simply not seeing.
Update I tried passing the data in smaller chunks using the code that Chris suggested with no luck. also of note is that I added a catch for ferror on stdout and it never got tripped so I am pretty sure that the bytes are at least making it to stdout. Is it possible that something else is writing to stdout somehow? maybe an extra byte making its way into stdout while my program is running? I find this doubtful as the errors are appearing consistently on the 4th fwrite read in the 10th entry.
Per Craig's request here is the full C++ code (the full Python Code is already posted): it is sitting in 3 files:
main.cpp
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <iostream>
#include "./MutualInformation.h"
double * data;
using namespace std;
void
xxwrite(unsigned char *buf, size_t wlen, FILE *fo)
{
size_t xlen;
for (; wlen > 0; wlen -= xlen, buf += xlen) {
xlen = wlen;
if (xlen > 1024)
xlen = 1024;
xlen = fwrite(buf, 1, xlen, fo);
fflush(fo);
}
}
int main(int argc, char **argv) {
int count = 0;
long totalbytes = stoi(argv[argc-4], nullptr,10); //bytes being transfered
long bytechunk = stoi(argv[argc - 3], nullptr, 10); //bytes being transfered at a time
long height = stoi(argv[argc-2], nullptr, 10); //bytes being transfered at a time
long width = stoi(argv[argc-1], nullptr, 10); //bytes being transfered at a time
long offset = totalbytes / sizeof(double) / 2;
data = new double[totalbytes/sizeof(double)];
int columnindex = 0;
//read in data from pipe
while (count<totalbytes) {
fread(&(data[columnindex]), 1, bytechunk, stdin);
columnindex += bytechunk / sizeof(double);
count += bytechunk;
}
//calculate the data transform
MutualInformation MI = MutualInformation();
MI.Initialize(data, height, width, offset);
MI.calcMI();
count = 0;
columnindex = 0;
while (count<totalbytes/2) {
xxwrite((unsigned char*)&(MI.getOutput()[columnindex]), bytechunk, stdout);
count += bytechunk;
columnindex += bytechunk/sizeof(double);
}
delete [] data;
return 0;
}
MutualInformation.h
#include <map>
using namespace std;
class MutualInformation
{
private:
double * mData;
double * mOutput;
long mHeight;
long mWidth;
long mOffset;
public:
MutualInformation();
~MutualInformation();
bool Initialize(double * data, long Height, long Width, long Offset);
const double * getOutput();
double calcMI();
};
MutualInformation.cpp
#include "MutualInformation.h"
MutualInformation::MutualInformation()
{
this->mData = nullptr;
this->mOutput = nullptr;
this->mHeight = 0;
this->mWidth = 0;
}
MutualInformation::~MutualInformation()
{
delete[] this->mOutput;
}
bool MutualInformation::Initialize(double * data, long Height, long Width, long Offset){
bool rvalue = false;
this->mData = data;
this->mHeight = Height;
this->mWidth = Width;
this->mOffset = Offset;
//allocate output data
this->mOutput = new double[this->mHeight*this->mWidth];
return rvalue;
}
const double * MutualInformation::getOutput(){
return this->mOutput;
}
double MutualInformation::calcMI(){
double rvalue = 0.0;
std::map<int, map<int, double>> lHistXY = map<int, map<int, double>>();
std::map<int, double> lHistX = map<int, double>();
std::map<int, double> lHistY = map<int, double>();
typedef std::map<int, std::map<int, double>>::iterator HistXY_iter;
typedef std::map<int, double>::iterator HistY_iter;
//calculate Entropys and MI
double MI = 0.0;
double Hx = 0.0;
double Hy = 0.0;
double Px = 0.0;
double Py = 0.0;
double Pxy = 0.0;
//scan through the image
int ip = 0;
int jp = 0;
int chipsize = 3;
//setup zero array
double * zeros = new double[this->mHeight];
for (int j = 0; j < this->mHeight; j++){
zeros[j] = 0.0;
}
//zero out Output array
for (int i = 0; i < this->mWidth; i++){
memcpy(&(this->mOutput[i*this->mHeight]), zeros, this->mHeight*8);
}
double index = 0.0;
for (int ioutter = chipsize; ioutter < (this->mWidth - chipsize); ioutter++){
for (int j = chipsize; j < (this->mHeight - chipsize); j++){
//clear the histograms
lHistX.clear();
lHistY.clear();
lHistXY.clear();
//chip out a section of the image
for (int k = -chipsize; k <= chipsize; k++){
for (int l = -chipsize; l <= chipsize; l++){
ip = ioutter + k;
jp = j + l;
//update X histogram
if (lHistX.count(int(this->mData[ip*this->mHeight + jp]))){
lHistX[int(this->mData[ip*this->mHeight + jp])] += 1.0;
}else{
lHistX[int(this->mData[ip*this->mHeight + jp])] = 1.0;
}
//update Y histogram
if (lHistY.count(int(this->mData[ip*this->mHeight + jp+this->mOffset]))){
lHistY[int(this->mData[ip*this->mHeight + jp+this->mOffset])] += 1.0;
}
else{
lHistY[int(this->mData[ip*this->mHeight + jp+this->mOffset])] = 1.0;
}
//update X and Y Histogram
if (lHistXY.count(int(this->mData[ip*this->mHeight + jp]))){
//X Key exists check if Y key exists
if (lHistXY[int(this->mData[ip*this->mHeight + jp])].count(int(this->mData[ip*this->mHeight + jp + this->mOffset]))){
//X & Y keys exist
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] += 1;
}else{
//X exist but Y doesn't
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] = 1;
}
}else{
//X Key Didn't exist
lHistXY[int(this->mData[ip*this->mHeight + jp])][int(this->mData[ip*this->mHeight + jp + this->mOffset])] = 1;
};
}
}
//calculate PMI, Hx, Hy
// iterator->first = key
// iterator->second = value
MI = 0.0;
Hx = 0.0;
Hy = 0.0;
for (HistXY_iter Hist2D_iter = lHistXY.begin(); Hist2D_iter != lHistXY.end(); Hist2D_iter++) {
Px = lHistX[Hist2D_iter->first] / ((double) this->mOffset);
Hx -= Px*log(Px);
for (HistY_iter HistY_iter = Hist2D_iter->second.begin(); HistY_iter != Hist2D_iter->second.end(); HistY_iter++) {
Py = lHistY[HistY_iter->first] / ((double) this->mOffset);
Hy -= Py*log(Py);
Pxy = HistY_iter->second / ((double) this->mOffset);
MI += Pxy*log(Pxy / Py / Px);
}
}
//normalize PMI to max(Hx,Hy) so that the PMI value runs from 0 to 1
if (Hx >= Hy && Hx > 0.0){
MI /= Hx;
}else if(Hy > Hx && Hy > 0.0){
MI /= Hy;
}
else{
MI = 0.0;
}
//write PMI to data output array
if (MI < 1.1){
this->mOutput[ioutter*this->mHeight + j] = MI;
}
else{
this->mOutput[ioutter*this->mHeight + j] = 0.0;
//cout << "problem with output";
}
}
}
//*/
return rvalue;
}
SOLVED By 6502
6502's answer below solved my problem. I needed to explicitly tell Windows to use a binary mode for stdin / stdout. to do that I had to include 2 new header files in my main cpp file.
#include <fcntl.h>
#include <io.h>
add the following lines of code (modified away from 6502's POSIX versions because Visual Studio complained) to the beginning of my main function
_setmode(_fileno(stdout), O_BINARY);
_setmode(_fileno(stdin), O_BINARY);
and then add these lines to my Python code:
import os, msvcrt
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)

The problem is that stdin/stdout in windows are opened in text mode, not in binary mode and therefore will mess up when the character 13 (\r) is sent.
You can set for example binary mode in Python with
import os, msvcrt
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
and in C++ with
_setmode(fileno(stdout), O_BINARY);
_setmode(fileno(stdin), O_BINARY);
See https://msdn.microsoft.com/en-us/library/tw4k6df8.aspx

Your C++ fwrite code does not account for getting a "short" transfer.
Here's a slight tweak:
//write out data to pipe
columnindex = 0;
while (count < totalbytes / 2) {
wlen = fwrite(&(MI.getOutput()[columnindex]), 1, bytechunk, stdout);
fflush(stdout);
count += wlen;
columnindex += wlen / sizeof(double);
}
Note: You still need to be careful as this would still have issues if wlen returns and it's not a multiple of sizeof(double). For example, if bytechunk were 16 and wlen came back with 14, you'd need an additional fwrite with length 2 before continuing the loop. A generalization of this is just to treat the entire data matrix as a giant byte buffer and loop on that.
Actually, you'll get about the same efficiency with many much smaller transfers that are capped by a fixed (i.e. "known safe amount") of [say] 1024 bytes. This works because the output is a byte stream.
Here's a slightly more general solution that I've often used:
void
xxwrite(void *buf,size_t wlen,FILE *fo)
{
size_t xlen;
for (; wlen > 0; wlen -= xlen, buf += xlen) {
xlen = wlen;
if (xlen > 1024)
xlen = 1024;
xlen = fwrite(buf,1,xlen,fo);
fflush(fo);
}
}
//write out data to pipe
columnindex = 0;
while (count < totalbytes / 2) {
xxwrite(&(MI.getOutput()[columnindex]), bytechunk, stdout);
count += bytechunk;
columnindex += bytechunk / sizeof(double);
}
UPDATE:
I've downloaded all your code and run it. I've got good news and bad news: The code runs fine here, even for a matrix size above 3000. I ran it both using xxwrite and without and the results were the same.
Using my limited python skills, I added some pretty print to your python script (e.g. some line wrap) and had it check every value for range and annotate any bad values. There were none found by the script. Also, visual inspection of the values turned up nothing [this was true before the pretty print, so it hasn't introduced anything]. Just lots of zeros and then blocks in the 0.9 range.
The only difference I can see is that I'm using gcc [and, of course, python] on linux. But, from your script it seems your using Windows [based on the C:\... path for your C++ executable. This shouldn't matter for this application, but I mention it anyway.
So, pipes work here. One thing you might try is to direct the C++ output to a file. Then, have the script read back from the file (i.e. no pipe) and see if that makes a difference. I tend to think not, but ...
Also, I don't know what compiler and python implementation you're using under Windows. Whenever I have to do this, I usually have Cygwin installed as it gives one of the closest implementations of linux/Unix-like environment (i.e. pipes are more likely to work as advertised).
Anyway, here's the modified script. Also note that I added os.getenv to grab alternate matrix sizes and an alternate place for the C++ executable, so that it would work for both of us with minimal pain
#!/usr/bin/python
import subprocess
import struct
import sys
import os
import numpy as np
val = os.getenv("MTX","2000")
sizeX = int(val)
sizeY = sizeX
print "sizeX=%d sizeY=%d" % (sizeX,sizeY)
#set up the variables needed
bytesPerDouble = 8
offset = sizeX*sizeY
totalBytesPerArray = sizeX*sizeY*bytesPerDouble
totalBytes = totalBytesPerArray*2 #the 2 is because we pass 2 different versions of the 2D array
#setup the testing data array
a = np.zeros(sizeX*sizeY*2, dtype='d')
for i in range(sizeX):
for j in range(sizeY):
a[j+i*sizeY] = i
a[j+i*sizeY+offset] = i
if i % 10 == 0:
a[j+i*sizeY+offset] = j
data = a.tobytes('C')
strTotalBytes = str(totalBytes)
strLineBytes = str(sizeY*bytesPerDouble)
#communicate with c++ code
print("starting C++ code")
command = os.getenv("CPGM",None);
if command is None:
command = "C:\Python27\PythonPipes.exe"
proc = subprocess.Popen([command, strTotalBytes, strLineBytes, str(sizeY), str(sizeX)], stdin=subprocess.PIPE,stderr=subprocess.PIPE,stdout=subprocess.PIPE)
ByteBuffer = (data)
proc.stdin.write(ByteBuffer)
def prt(i,b):
hangflg = 0
per = 8
for j in range(0,len(b)):
if ((j % per) == 0):
print("[%d,%d]" % (i,j)),
q = b[j]
print(q),
hangflg = 1
if (q < 0.0) or (q > 1.0):
print("=WTF"),
if ((j % per) == (per - 1)):
print("")
hangflg = 0
if (hangflg):
print("")
print("Reading results back from C++")
for i in range(sizeX):
returnvalues = proc.stdout.read(sizeY*bytesPerDouble)
a = buffer(returnvalues)
b = struct.unpack_from(str(sizeY)+'d', a)
prt(i,b)
###print str(b) + " " + str(i)
###print str(i) + ": " + str(b)
print('done')

Python PySerial FlushInput not working

I am new to programming and serial communication, so forgive me for likely messing up something simple.
The script I am writing is reading temperature data sent via serial port from an Arduino. When it first loads up, the Arduino sketch sends a "menu" with options for reading, writing, and clearing EEPROM.
When I load the python script, sometimes it works well enough. Other times, it just doesn't read anything. And other times, it floods my terminal with temp reading data for about 5 seconds, then it sends the menu and acts normally.
I have added the flushinput and time delays, but to no avail.
Another quick question: if I'm in the while loop collecting data from serial, which is without end, how would I be able to enter a command to pause and switch to another part of the program without interrupting the data stream? For example, if I'm collecting data from the temp sensor and using serial to do real-time plotting of temp vs time, but I want to be able to interrupt this at anytime to stop writing and switch to reading (saving data to .txt file). How could this be done?
Any help would be immensely appreciated.
Here is the Python code:
import serial
import time
import numpy as np
import matplotlib.pyplot as plt
ser = serial.Serial('/dev/tty.usbmodem1421', 9600, timeout = 5)
ser.flushInput()
ser.flushOutput()
time.sleep(0.1)
print ("Connected to serial port: {}".format(ser.name))
def Menu():
data_left = 1
while (data_left != 0):
print(ser.readline())
time.sleep(.1)
data_left = ser.inWaiting()
return
def RealTimePlot(y, count, k, minPlot, maxPlot):
//stuff
def WriteData(mode, plotData = 'n'):
//stuff
def ReadData(mode):
//stuff
def main():
launch = 0
plotData = ProgramIntro()
time.sleep(1)
Menu()
mode = raw_input("> ")
while (1):
if ((mode == "end") or (mode == 'exit')) :
ser.close()
exit()
elif (mode == 'm'):
ser.write(mode)
time.sleep(1)
Menu()
mode = raw_input("> ")
if (mode == 'm'):
print("Already at menu. Would you like to:")
print("[w]rite")
print("[r]ead")
print("[c]lear")
elif (mode == 'w'):
count = 0
WriteData(mode, plotData)
#mode = PauseProgram()
elif (mode == 'r'):
ReadData(mode)
#mode = PauseProgram()
elif (mode == 'c'):
ser.write(mode)
time.sleep(1)
while ser.inWaiting() > 0:
out = ser.readline()
print(out)
time.sleep(0.1)
mode = 'm'
print("Goodbye!")
#Closes the connection
ser.close()
And Arduino Code:
// References
// https://learn.sparkfun.com/tutorials/sik-experiment-guide-for-arduino---v32/experiment-7-reading-a-temperature-sensor
// https://www.arduino.cc/en/Tutorial/EEPROMWrite
// https://www.arduino.cc/en/Tutorial/EEPROMRead
//
// The following may be useful to read on storing EEPROM data:
// http://www.vernier.com/engineering/arduino/store-data-eeprom/
//
#include <ctype.h>
#include <stdlib.h>
#include <math.h>
#include <EEPROM.h>
// Define pins
const int writePin = 5;
const int readPin = 9;
const int pausePin = 7;
const int collectDataBtn = 11;
const int RTPin = 3;
const int sensorPin = A0; // Temperature Pin
char RTData;
int j = 0;
// SETUP of Serial port and PINS for LED
void setup() {
Serial.begin(9600);
pinMode(writePin, OUTPUT);
pinMode(RTPin, OUTPUT);
pinMode(readPin, OUTPUT);
pinMode(pausePin, OUTPUT);
pinMode(sensorPin, INPUT);
pinMode(collectDataBtn, INPUT);
while(Serial.available())
Serial.read();
}
char mode = 'm';
char prevMode = 'm';
int k = 0;
// MAIN LOOP
void loop() {
while (j == 0) {
if (k == 0) {
Serial.print("Would you like to plot real-time data? (y/n) ");
k++;
}
if (Serial.available() > 0) {
RTData = Serial.read();
if (RTData == 'y') {
digitalWrite(RTPin, HIGH);
j++;
}
else {
j++;
}
}
return;
}
switch (mode) {
// MENU mode
case 'm':
case 'M':
// Set LEDs to indicate menu mode
digitalWrite(writePin, HIGH);
digitalWrite(readPin, HIGH);
digitalWrite(pausePin, LOW);
// Print menu to serial port
PrintMenu();
while (mode == 'm') {
if (Serial.available() > 0) {
prevMode = mode;
mode = Serial.read();
}
}
break;
// WRITE mode
case 'w':
case 'W':
Serial.println("Entering WRITE mode...");
mode = WriteData(sensorPin, collectDataBtn);
break;
// READ mode
case 'r':
case 'R':
// Sets LEDs to indicate READ mode
digitalWrite(writePin, LOW);
digitalWrite(readPin, HIGH);
digitalWrite(pausePin, LOW);
Serial.println("Entering READ mode...");
mode = ReadData(sensorPin, prevMode);
break;
// CLEAR eeprom mode
case 'c':
case 'C':
Serial.println("Clearing EEPROM...");
ClearEEPROM();
Serial.println("EEPROM cleared!");
Serial.println("Returning to menu...\n\n");
mode = 'm';
break;
// PAUSE mode
default:
// Sets LEDs to indicate PAUSE mode
digitalWrite(writePin, LOW);
digitalWrite(readPin, LOW);
digitalWrite(pausePin, HIGH);
mode = 'p';
mode = PauseMode(mode);
break;
}
} // END void loop
void ClearEEPROM(void) {
for (int i = 0 ; i < EEPROM.length() ; i++) {
if ((i % 100 == 0) || (i % 50 == 0)) {
digitalWrite(writePin, HIGH);
digitalWrite(readPin, LOW);
digitalWrite(pausePin, LOW);
}
else {
digitalWrite(writePin, LOW);
digitalWrite(readPin, LOW);
digitalWrite(pausePin, HIGH);
}
EEPROM.write(i, 0);
}
return;
}
char CheckModeChange(void) {
if (Serial.available() > 0) {
// read the incoming char:
prevMode = mode;
mode = Serial.read();
}
return mode;
}
void PrintMenu(void) {
Serial.println("\n");
Serial.println(" ** Menu ** ");
Serial.println("------------------");
Serial.println("| [W]rite Data |");
Serial.println("| [R]ead Data |");
Serial.println("| [C]lear Data |");
Serial.println("------------------");
Serial.println("Enter [m]enu to return to menu at any point,");
Serial.println("or press any other key to pause during any point of the program.");
return;
}
char PauseMode(char mode){
Serial.println("Program Paused. Choose an option from the following to continue: ");
Serial.println("[m]enu");
Serial.println("[w]rite");
Serial.println("[r]ead");
Serial.println("[c]lear \n");
while (mode == 'p') {
if (Serial.available() > 0) {
mode = Serial.read();
}
}
return mode;
}
char WriteData(int sensorPin, int collectDataBtn) {
// Declarations
int cnt, voltVal;
int addr = 0; // Initializes address at 0 for EEPROM
int beginData = 0;
float voltage, degreesC;
char mode = 'w';
// Collect data when button pushed. This will be replaced by z-axis acceleration threshold
// of rocket.
Serial.println("Waiting for launch to begin...");
while (beginData == 0) {
if (cnt > 20000) {
cnt = 0;
}
else if (cnt < 10000) {
digitalWrite(writePin, LOW);
digitalWrite(readPin, LOW);
digitalWrite(pausePin, LOW);
}
else {
digitalWrite(writePin, HIGH);
digitalWrite(readPin, LOW);
digitalWrite(pausePin, LOW);
}
cnt++;
if (digitalRead(collectDataBtn) == HIGH) {
beginData = 1;
}
prevMode = mode;
mode = CheckModeChange();
if (prevMode != mode) {
beginData = 1;
}
}
digitalWrite(writePin, HIGH);
digitalWrite(readPin, LOW);
digitalWrite(pausePin, LOW);
// Write Data loop
while (mode == 'w') {
// First we'll measure the voltage at the analog pin. Normally
// we'd use analogRead(), which returns a number from 0 to 1023.
// Here we've written a function (further down) called
// getVoltage() that returns the true voltage (0 to 5 Volts)
// present on an analog input pin.
voltage = getVoltage(sensorPin);
// Now we'll convert the voltage to degrees Celsius.
// This formula comes from the temperature sensor datasheet:
// CONVERT IN PYTHON SCRIPT
// degreesC = (voltage - 0.5) * 100.0;
//Serial.print("Address[");
//Serial.print(addr);
//Serial.print("]: \t");
Serial.println(voltage);
/***
Write the value to the appropriate byte of the EEPROM.
these values will remain there when the board is
turned off.
***/
// Convert for storage to EEPROM
voltVal = (voltage * 10000) / 4;
// Write to EEPROM
EEPROM.write(addr, voltVal);
/***
Advance to the next address, when at the end restart at the beginning.
Larger AVR processors have larger EEPROM sizes, E.g:
- Arduno Duemilanove: 512b EEPROM storage.
- Arduino Uno: 1kb EEPROM storage.
- Arduino Mega: 4kb EEPROM storage.
Rather than hard-coding the length, you should use the pre-provided length function.
This will make your code portable to all AVR processors.
***/
addr = addr + 1;
/***
As the EEPROM sizes are powers of two, wrapping (preventing overflow) of an
EEPROM address is also doable by a bitwise and of the length - 1.
++addr &= EEPROM.length() - 1;
***/
// Check for mode change
mode = CheckModeChange();
delay(100);
}
return mode;
}
char ReadData(int Pin, char prevMode) {
// Declarations
byte value;
float voltageVal, voltage, degreesC;
int addr = 0;
char mode = 'r';
// Checks previous mode. If previous mode was write, now that we are reading
// we should reset the EEPROM address to 0 so we can read from the beginning.
if ( (prevMode == 'w') || (prevMode == 'W') ) {
addr = 0;
}
while (mode == 'r') {
value = EEPROM.read(addr);
voltageVal = value;
voltage = voltageVal * 4 / 10000;
//Serial.print("Voltage: ");
//Serial.print("\t");
//Serial.println(voltage);
degreesC = (voltage - 0.5) * 100.0;
//Serial.print("Degrees C: ");
//Serial.print("\t");
//Serial.print("Address[");
//Serial.print(addr);
//Serial.print("]: \t");
Serial.println(degreesC);
/***
Advance to the next address, when at the end restart at the beginning.
Larger AVR processors have larger EEPROM sizes, E.g:
- Arduno Duemilanove: 512b EEPROM storage.
- Arduino Uno: 1kb EEPROM storage.
- Arduino Mega: 4kb EEPROM storage.
Rather than hard-coding the length, you should use the pre-provided length function.
This will make your code portable to all AVR processors.
***/
addr = addr + 1;
if (addr == EEPROM.length()) {
addr = 0;
}
// Check for mode change
mode = CheckModeChange();
delay(100);
}
return mode;
}
float getVoltage(int pin) {
// This function has one input parameter, the analog pin number
// to read. You might notice that this function does not have
// "void" in front of it; this is because it returns a floating-
// point value, which is the true voltage on that pin (0 to 5V).
return (analogRead(pin) * 0.004882814);
// This equation converts the 0 to 1023 value that analogRead()
// returns, into a 0.0 to 5.0 value that is the true voltage
// being read at that pin.
}