We Keep Coding

Python program not writing anything to CSV file

Got thrown on this project with little python experience. Trying to write from an encoder to a csv. Weird thing is that the program wont even write a string into the file. Any advice is appreciated!
while not file_passed:
try:
file_name = input("Enter log file name: ")
# Add extension if not already given
if "." not in file_name:
file_name += ".csv"
log_file = open(file_name, "a")
print("log file open debug passed")
# Add header row to log file
if os.stat(log_file.name).st_size == 0:
log_file.write("time (ms), pressure, angle(rad) \n")
print("Logged header to file")
file_passed = True
except:
print("Invalid file, or could not open the file specified.\n-----")
Here is a snippet of what I have narrowed it down to I believe. You can see my little debug testing statements to see if its breaking into the correct if statements. It does print all of the debug statements so it seems to be breaking into the if with log_file.write("time(ms), pressure, angle(rad) \n")
EDIT: Tried adding a with before log_file.write("") line, program crashes on start when added. Here is the code in its entirety as I should have added to begin with my bad
import serial
from datetime import datetime
import os
pressure_passed = False
arduino_passed = False
file_passed = False
BAUD_RATE = 115200
GARBAGE_CYCLES = 3 # how many cycles to ignore before logging data
garbage_cycle = 0
# Save data to log file
def LogData(startTime, pressureData, arduinoData, file):
global garbage_cycle
if garbage_cycle < GARBAGE_CYCLES:
garbage_cycle += 1
else:
delta = datetime.now() - startTime
ms = delta.total_seconds() * 1000
dataString = "{:0.2f}, {}, {}\n".format(ms, pressureData, arduinoData)
file.write(dataString)
print(dataString, end = "")
# Get the COM port for the Mark-10 Series 5
while not pressure_passed:
try:
pressure_com = input("Enter Mark-10 Series 5 COM Port #: ")
pressure_ser = serial.Serial("COM" + str(pressure_com), BAUD_RATE)
pressure_passed = True
except:
print("Invalid COM Port, please enter a valid port.\n-----")
# Get the COM port for the Arduino
while not arduino_passed:
try:
arduino_com = input("Enter Ardunio COM Port #: ")
arduino_ser = serial.Serial("COM" + str(arduino_com), BAUD_RATE)
arduino_passed = True
except:
print("Invalid COM Port, please enter a valid port.\n-----")
# Get the name for the log file
while not file_passed:
try:
file_name = input("Enter log file name: ")
# Add extension if not already given
if "." not in file_name:
file_name += ".csv"
log_file = open(file_name, "a")
# Add header row to log file
if os.stat(log_file.name).st_size == 0:
log_file.write("time (ms), pressure, angle (deg)\n")
file_passed = True
except:
print("Invalid file, or could not open the file specified.\n-----")
start = datetime.now()
# Variables to read serial input
pressure_data = ""
last_pressure = ""
arduino_data = ""
last_arduino = ""
# Main program loop
# Serial is read from byte by byte to better sync the two devices
while True:
try:
x_changed = False
y_changed = False
# Read from Mark-10 serial if available
# x is a byte read from the serial line, converted to ascii
if pressure_ser.in_waiting > 0:
x = pressure_ser.read().decode('ascii')
x_changed = True
# Read from Arduino serial if available
# y is a byte read from the serial line, converted to ascii
if arduino_ser.in_waiting > 0:
y = arduino_ser.read().decode('ascii')
y_changed = True
# If new data received, check if we should log it
if x_changed:
if x == '\n': # New line detected, log the accumulated data
if last_pressure != pressure_data:
LogData(start, last_pressure, last_arduino, log_file)
last_pressure = pressure_data
pressure_data = ""
elif x != '\r': # Otherwise, add the read character to the string
pressure_data += x
if y_changed:
if y == '\n': # New line detected, log the accumulated data
if last_arduino != arduino_data:
LogData(start, last_pressure, last_arduino, log_file)
last_arduino = arduino_data
arduino_data = ""
elif y != '\r': # Otherwise, add the read character to the string
arduino_data += y
except Exception as e:
print(e)
if arduino_ser.isOpen():
arduino_ser.close()
if pressure_ser.isOpen():
pressure_ser.close()
log_file.close()
break

python socket file transfer verified with sha256 not working, but only sometimes?

Client side:
def send_file_to_hashed(data, tcpsock):
time.sleep(1)
f = data
flag = 0
i=0
tcpsock.send(hashlib.sha256(f.read()).hexdigest())
f.seek(0)
time.sleep(1)
l = f.read(BUFFER_SIZE-64)
while True:
while (l):
tcpsock.send(hashlib.sha256(l).hexdigest() + l)
time.sleep(1)
hashok = tcpsock.recv(6)
if hashok == "HASHOK":
l = f.read(BUFFER_SIZE-64)
flag = 1
if hashok == "BROKEN":
flag = 0
if not l:
time.sleep(1)
tcpsock.send("DONE")
break
return (tcpsock,flag)
def upload(filename):
flag = 0
while(flag == 0):
with open(os.getcwd()+'\\data\\'+ filename +'.csv', 'rU') as UL:
tuplol = send_file_to_hashed(UL ,send_to_sock(filename +".csv",send_to("upload",TCP_IP,TCP_PORT)))
(sock,flagn) = tuplol
flag = flagn
time.sleep(2)
sock.close()
Server Side:
elif(message == "upload"):
message = rec_OK(self.sock)
fis = os.getcwd()+'/data/'+ time.strftime("%H:%M_%d_%m_%Y") + "_" + message
f = open(fis , 'w')
latest = open(os.getcwd()+'/data/' + message , 'w')
time.sleep(1)
filehash = rec_OK(self.sock)
print("filehash:" + filehash)
while True:
time.sleep(1)
rawdata = self.sock.recv(BUFFER_SIZE)
log.write("rawdata :" + rawdata + "\n")
data = rawdata[64:]
dhash = rawdata[:64]
log.write("chash: " + dhash + "\n")
log.write("shash: " + hashlib.sha256(data).hexdigest() + "\n")
if dhash == hashlib.sha256(data).hexdigest():
f.write(data)
latest.write(data)
self.sock.send("HASHOK")
log.write("HASHOK\n" )
print"HASHOK"
else:
self.sock.send("HASHNO")
print "HASHNO"
log.write("HASHNO\n")
if rawdata == "DONE":
f.close()
f = open(fis , 'r')
if (hashlib.sha256(f.read()).hexdigest() == filehash):
print "ULDONE"
log.write("ULDONE")
f.close()
latest.close()
break
else:
self.sock.send("BROKEN")
print hashlib.sha256(f.read()).hexdigest()
log.write("BROKEN")
print filehash
print "BROKEN UL"
f.close()
So the data upload is working fine in all tests that i ran from my computer, even worked fine while uploading data over my mobile connection and still sometimes people say it takes a long time and they kill it after a few minutes. the data is there on their computers but not on the server. I don't know what is happening please help!

First of all: this is unrelated to sha.
Streaming over the network is unpredictable. This line
rawdata = self.sock.recv(BUFFER_SIZE)
doesn't guarantee that you read BUFFER_SIZE bytes. You may have read only 1 byte in the worst case scenario. Therefore your server side is completely broken because of the assumption that rawdata contains whole message. It is even worse. If the client sends command and hash fast you may get e.g. rawdata == 'DONEa2daf78c44(...) which is a mixed output.
The "hanging" part just follows from that. Trace your code and see what happens when the server receives partial/broken messages ( I already did that in my imagination :P ).
Streaming over the network is almost never as easy as calling sock.send on one side and sock.recv on the other side. You need some buffering/framing protocol. For example you can implement this simple protocol: always interpret first two bytes as the size of incoming message, like this:
client (pseudocode)
# convert len of msg into two-byte array
# I am assuming the max size of msg is 65536
buf = bytearray([len(msg) & 255, len(msg) >> 8])
sock.sendall(buf)
sock.sendall(msg)
server (pseudocode)
size = to_int(sock.recv(1))
size += to_int(sock.recv(1)) << 8
# You need two calls to recv since recv(2) can return 1 byte.
# (well, you can try recv(2) with `if` here to avoid additional
# syscall, not sure if worth it)
buffer = bytearray()
while size > 0:
tmp = sock.recv(size)
buffer += tmp
size -= len(tmp)
Now you have properly read data in buffer variable which you can work with.
WARNING: the pseudocode for the server is simplified. For example you need to check for empty recv() result everywhere (including where size is calculated). This is the case when the client disconnects.
So unfortunately there's a lot of work in front of you. You have to rewrite whole sending and receving code.

Python serial port read until it finishes

#!/usr/bin/python
I am struggling with quite the simple problem. I want to read some data from the serial port first than start writing the data. The data reading and writing works well. The problem is I need to read first around 7 lines like
X7_SEM_V3_6
ICAP OK
SA OK
IC OK
RBDK OK
status OK
S OK
Then send 'I' followed by N and C, then 9 hex digits from the text file. The code below read one line and went into the writing section and read the whole text file;
X7_SEM_V3_6
000062240
000062241
ICAP
000062240
000062241
so on
after doing this to all seven read lines than it send I. I want it read all seven lines once than send I and start working. this is in while loop. If I use something else it just read first line and stuck. Please some one help.
import serial, time
import binascii
ser = serial.Serial()
ser.port = "/dev/ttyUSB1"
ser.baudrate = 38400
ser.bytesize = serial.EIGHTBITS
ser.parity = serial.PARITY_NONE
ser.stopbits = serial.STOPBITS_ONE
ser.xonxoff = False
ser.rtscts = False
ser.dsrdtr = False
number_address = 1341602
number_char = 9
timeout=1
f=open('lut.txt','r')
try:
ser.open()
except Exception, e:
print "error open serial port: " + str(e)
exit()
if ser.isOpen():
try:
ser.flushInput()
ser.flushOutput()
# reading
max_packet = 20
lines = 0
while True:
byteData = ser.read_until('\r',max_packet)
newdata=str(byteData)
print newdata.strip()
#ser.write('I')
ser.write('I')
time.sleep(0.01)
for line in f.readlines():
print line
ser.write('N')
time.sleep(0.01)
ser.write(' ')
time.sleep(0.01)
ser.write('C')
time.sleep(0.01)
for i in line:
newdata=i
ser.write(newdata)
time.sleep(0.01)
except Exception, e1:
print "error communicating...: " + str(e1)
else:
print "cannot open serial port "

Timeout after X second in Python

I am writing a program in python, which works like this:
Take input string through Serial port, when press enter (Carriage
Return)
Check if $ sign is present as the first character of input string then continue
Problem
It can make a trouble if I didn't get the Carriage Return CR and got another string at the same time or after specific interval of time.
In order to avoid this problem I want to add timeout session which makes the previous buffer Null after specific interval of time if Carriage Return not received.
kindly review my codes below and guide me on how do I add timeout option in it?
CODE
import serial
x = [0,0,0]
ser = serial.Serial('/dev/ttyAMA0', 9600)
buffer = ''
while True:
buffer += ser.read(ser.inWaiting())
if '\n' in buffer:
if buffer[0] == '$':
x1 = buffer.rstrip()
x2= x1.split(",")
print((x2[0]),(x2[1]),(x2[2]))
buffer = ""

I thought you just need to add this line at the end of program.
This line will add more 60sec every time you input correct string
clear_buffer = time.time() + 60
Check out below coding
import serial
import time
x = [0,0,0]
ser = serial.Serial('/dev/ttyAMA0', 9600)
buffer = ''
clear_buffer = time.time() + 60
while True:
if time.time() >= clear_buffer:
buffer = ''
clear_buffer = time.time() + 60
buffer += ser.read(ser.inWaiting())
if '\n' in buffer:
if buffer[0] == '$':
x1 = buffer.rstrip()
x2= x1.split(",")
print((x2[0]),(x2[1]),(x2[2]))
buffer = ""
clear_buffer = time.time() + 60

I may be reading your question wrong, but it appears as though you just want to clear the buffer string after a set amount of time? This will clear it every 60 seconds:
import serial
import time
x = [0,0,0]
ser = serial.Serial('/dev/ttyAMA0', 9600)
buffer = ''
clear_buffer = time.time() + 60
while True:
if time.time() >= clear_buffer:
buffer = ''
clear_buffer = time.time() + 60
buffer += ser.read(ser.inWaiting())
if '\n' in buffer:
if buffer[0] == '$':
x1 = buffer.rstrip()
x2= x1.split(",")
print((x2[0]),(x2[1]),(x2[2]))
buffer = ""

Finding string Python Arduino

I have a little script in Python which I am brand new to. I want to check if a certain word appears in ser.readline(). The syntax for the If statement is not right and I am not sure how to lay this out so it continuously reads the serial for the word "sound". I've attached an output image below so you can see how it is printing. I'd like to trigger an MP3 as it finds the word "sound" but as of yet I haven't even managed to get it to print a confirmation saying its found the word.
import serial
import time
ser = serial.Serial('COM6', 9600, timeout=0)
while 1:
try:
print (ser.readline())
time.sleep(1)
**if "sound" in ser.readline():
print("foundsound")**
except ser.SerialTimeoutException:
print('Data could not be read')
time.sleep(1)

You may be reading from the port more often than you intend to. I would call ser.readline() just once per iteration of your main loop:
while True:
try:
data = ser.readline().decode("utf-8") # Read the data
if data == '': continue # skip empty data
print(data) # Display what was read
time.sleep(1)
if "sound" in data:
print('Found "sound"!')
except ser.SerialTimeoutException:
print('Data could not be read')
time.sleep(1)

can you try:
import serial
import time
ser = serial.Serial('COM6', 9600, timeout=0)
while 1:
try:
line = ser.readline()
print line
time.sleep(1)
**if "sound" in line:
print("foundsound")**
except ser.SerialTimeoutException:
print('Data could not be read')
time.sleep(1)