We Keep Coding

Writing to serial port but only receiving first character?

I'm reading data in from a machine to windows 7. Using python, I read the serial port, process the data then write the data to a different serial port. Using com0com null modem emulator, the data is sent to another program. Here is the code I'm using:
import serial
import time
ser = serial.Serial(port='COM7', baudrate=9600)
ser2 = serial.Serial(port='COM8', baudrate=9600)
value_one = None
while (True):
# Check if incoming bytes are waiting to be read from the serial input
# buffer.
# NB: for PySerial v3.0 or later, use property `in_waiting` instead of
# function `inWaiting()` below!
if (ser.in_waiting > 16):
# read the bytes and convert from binary array to ASCII
data_str = ser.read(ser.in_waiting).decode('ascii')
if (value_one == None):
time.sleep(1)
print(data_str)
value_one_parse = data_str[7:9]
print(value_one_parse)
value_one = float(value_one_parse)
print(value_one)
else:
time.sleep(1)
print(data_str)
value_two_parse = data_str[7:9]
print(value_two_parse)
value_two = float(value_two_parse)
print(value_two)
avg = ((value_one + value_two)/2)
print(avg)
avgprep = str(avg) + '\r\n'
print(avgprep)
ser2.write(avgprep.encode('utf-8'))
value_one = None
value_two = None
time.sleep(0.01)
So if avgprep = 71.1, why am I only receiving the first digit 7 to the program?

I changed ser.in_waiting > 16 to ser.in_waiting > 0 and put a time.sleep(5) after that.

Python program not writing anything to CSV file

Got thrown on this project with little python experience. Trying to write from an encoder to a csv. Weird thing is that the program wont even write a string into the file. Any advice is appreciated!
while not file_passed:
try:
file_name = input("Enter log file name: ")
# Add extension if not already given
if "." not in file_name:
file_name += ".csv"
log_file = open(file_name, "a")
print("log file open debug passed")
# Add header row to log file
if os.stat(log_file.name).st_size == 0:
log_file.write("time (ms), pressure, angle(rad) \n")
print("Logged header to file")
file_passed = True
except:
print("Invalid file, or could not open the file specified.\n-----")
Here is a snippet of what I have narrowed it down to I believe. You can see my little debug testing statements to see if its breaking into the correct if statements. It does print all of the debug statements so it seems to be breaking into the if with log_file.write("time(ms), pressure, angle(rad) \n")
EDIT: Tried adding a with before log_file.write("") line, program crashes on start when added. Here is the code in its entirety as I should have added to begin with my bad
import serial
from datetime import datetime
import os
pressure_passed = False
arduino_passed = False
file_passed = False
BAUD_RATE = 115200
GARBAGE_CYCLES = 3 # how many cycles to ignore before logging data
garbage_cycle = 0
# Save data to log file
def LogData(startTime, pressureData, arduinoData, file):
global garbage_cycle
if garbage_cycle < GARBAGE_CYCLES:
garbage_cycle += 1
else:
delta = datetime.now() - startTime
ms = delta.total_seconds() * 1000
dataString = "{:0.2f}, {}, {}\n".format(ms, pressureData, arduinoData)
file.write(dataString)
print(dataString, end = "")
# Get the COM port for the Mark-10 Series 5
while not pressure_passed:
try:
pressure_com = input("Enter Mark-10 Series 5 COM Port #: ")
pressure_ser = serial.Serial("COM" + str(pressure_com), BAUD_RATE)
pressure_passed = True
except:
print("Invalid COM Port, please enter a valid port.\n-----")
# Get the COM port for the Arduino
while not arduino_passed:
try:
arduino_com = input("Enter Ardunio COM Port #: ")
arduino_ser = serial.Serial("COM" + str(arduino_com), BAUD_RATE)
arduino_passed = True
except:
print("Invalid COM Port, please enter a valid port.\n-----")
# Get the name for the log file
while not file_passed:
try:
file_name = input("Enter log file name: ")
# Add extension if not already given
if "." not in file_name:
file_name += ".csv"
log_file = open(file_name, "a")
# Add header row to log file
if os.stat(log_file.name).st_size == 0:
log_file.write("time (ms), pressure, angle (deg)\n")
file_passed = True
except:
print("Invalid file, or could not open the file specified.\n-----")
start = datetime.now()
# Variables to read serial input
pressure_data = ""
last_pressure = ""
arduino_data = ""
last_arduino = ""
# Main program loop
# Serial is read from byte by byte to better sync the two devices
while True:
try:
x_changed = False
y_changed = False
# Read from Mark-10 serial if available
# x is a byte read from the serial line, converted to ascii
if pressure_ser.in_waiting > 0:
x = pressure_ser.read().decode('ascii')
x_changed = True
# Read from Arduino serial if available
# y is a byte read from the serial line, converted to ascii
if arduino_ser.in_waiting > 0:
y = arduino_ser.read().decode('ascii')
y_changed = True
# If new data received, check if we should log it
if x_changed:
if x == '\n': # New line detected, log the accumulated data
if last_pressure != pressure_data:
LogData(start, last_pressure, last_arduino, log_file)
last_pressure = pressure_data
pressure_data = ""
elif x != '\r': # Otherwise, add the read character to the string
pressure_data += x
if y_changed:
if y == '\n': # New line detected, log the accumulated data
if last_arduino != arduino_data:
LogData(start, last_pressure, last_arduino, log_file)
last_arduino = arduino_data
arduino_data = ""
elif y != '\r': # Otherwise, add the read character to the string
arduino_data += y
except Exception as e:
print(e)
if arduino_ser.isOpen():
arduino_ser.close()
if pressure_ser.isOpen():
pressure_ser.close()
log_file.close()
break

python socket file transfer verified with sha256 not working, but only sometimes?

Client side:
def send_file_to_hashed(data, tcpsock):
time.sleep(1)
f = data
flag = 0
i=0
tcpsock.send(hashlib.sha256(f.read()).hexdigest())
f.seek(0)
time.sleep(1)
l = f.read(BUFFER_SIZE-64)
while True:
while (l):
tcpsock.send(hashlib.sha256(l).hexdigest() + l)
time.sleep(1)
hashok = tcpsock.recv(6)
if hashok == "HASHOK":
l = f.read(BUFFER_SIZE-64)
flag = 1
if hashok == "BROKEN":
flag = 0
if not l:
time.sleep(1)
tcpsock.send("DONE")
break
return (tcpsock,flag)
def upload(filename):
flag = 0
while(flag == 0):
with open(os.getcwd()+'\\data\\'+ filename +'.csv', 'rU') as UL:
tuplol = send_file_to_hashed(UL ,send_to_sock(filename +".csv",send_to("upload",TCP_IP,TCP_PORT)))
(sock,flagn) = tuplol
flag = flagn
time.sleep(2)
sock.close()
Server Side:
elif(message == "upload"):
message = rec_OK(self.sock)
fis = os.getcwd()+'/data/'+ time.strftime("%H:%M_%d_%m_%Y") + "_" + message
f = open(fis , 'w')
latest = open(os.getcwd()+'/data/' + message , 'w')
time.sleep(1)
filehash = rec_OK(self.sock)
print("filehash:" + filehash)
while True:
time.sleep(1)
rawdata = self.sock.recv(BUFFER_SIZE)
log.write("rawdata :" + rawdata + "\n")
data = rawdata[64:]
dhash = rawdata[:64]
log.write("chash: " + dhash + "\n")
log.write("shash: " + hashlib.sha256(data).hexdigest() + "\n")
if dhash == hashlib.sha256(data).hexdigest():
f.write(data)
latest.write(data)
self.sock.send("HASHOK")
log.write("HASHOK\n" )
print"HASHOK"
else:
self.sock.send("HASHNO")
print "HASHNO"
log.write("HASHNO\n")
if rawdata == "DONE":
f.close()
f = open(fis , 'r')
if (hashlib.sha256(f.read()).hexdigest() == filehash):
print "ULDONE"
log.write("ULDONE")
f.close()
latest.close()
break
else:
self.sock.send("BROKEN")
print hashlib.sha256(f.read()).hexdigest()
log.write("BROKEN")
print filehash
print "BROKEN UL"
f.close()
So the data upload is working fine in all tests that i ran from my computer, even worked fine while uploading data over my mobile connection and still sometimes people say it takes a long time and they kill it after a few minutes. the data is there on their computers but not on the server. I don't know what is happening please help!

First of all: this is unrelated to sha.
Streaming over the network is unpredictable. This line
rawdata = self.sock.recv(BUFFER_SIZE)
doesn't guarantee that you read BUFFER_SIZE bytes. You may have read only 1 byte in the worst case scenario. Therefore your server side is completely broken because of the assumption that rawdata contains whole message. It is even worse. If the client sends command and hash fast you may get e.g. rawdata == 'DONEa2daf78c44(...) which is a mixed output.
The "hanging" part just follows from that. Trace your code and see what happens when the server receives partial/broken messages ( I already did that in my imagination :P ).
Streaming over the network is almost never as easy as calling sock.send on one side and sock.recv on the other side. You need some buffering/framing protocol. For example you can implement this simple protocol: always interpret first two bytes as the size of incoming message, like this:
client (pseudocode)
# convert len of msg into two-byte array
# I am assuming the max size of msg is 65536
buf = bytearray([len(msg) & 255, len(msg) >> 8])
sock.sendall(buf)
sock.sendall(msg)
server (pseudocode)
size = to_int(sock.recv(1))
size += to_int(sock.recv(1)) << 8
# You need two calls to recv since recv(2) can return 1 byte.
# (well, you can try recv(2) with `if` here to avoid additional
# syscall, not sure if worth it)
buffer = bytearray()
while size > 0:
tmp = sock.recv(size)
buffer += tmp
size -= len(tmp)
Now you have properly read data in buffer variable which you can work with.
WARNING: the pseudocode for the server is simplified. For example you need to check for empty recv() result everywhere (including where size is calculated). This is the case when the client disconnects.
So unfortunately there's a lot of work in front of you. You have to rewrite whole sending and receving code.

Why does this python program not work in IDLE, when using a raspberry pi?

The following code communicates with multiple pH modules, via the serial port. The pH modules are selected via a multiplexer via the i2c bus using an 8574N chip. When the raspberry pi has booted up the program works correctly in the terminal however, if the program was to be stop and be restarted it fails to initialize correctly. Also, this program does not work correctly in the python IDLE (selecting random devices on the multiplexer). It seems like the i2c or serial comms on the pi is not initializing correctly. For example, in IDLE when selecting a module, It brings up the incorrect module to the one that you have chosen spits out incorrect data. However, this all works fine in the terminal.
Does any one have any ideas??
Any help would be much appreciated!
from smbus import SMBus
# from itertools import cycle
import time
import serial
usbport = '/dev/ttyAMA0'
ser = serial.Serial(usbport, 9600, timeout = 2)#
line = ""
data = ""
bus = SMBus(1) # Port 1 used on REV2
print "Electronics Workshop PH Reader."
print "=========================="
global count
count = 0 #init count to 0
probevalue = ""
def inputsel(count):
bus.write_byte(0x38, 0x00)
# count = input(" Select ph unit '0 to 23'") # count now incremented during program
count = input(" Select ph unit '0 to 23'")
if (count< 16):
data_sel = count
data_sel = data_sel | 32 # or the 2 numbers together
else:
data_sel = count - 16
data_sel = data_sel | 16 # or the 2 numbers together
bus.write_byte(0x38, data_sel) # send "count"Varable out to I2c Bus
print str(data_sel)
time.sleep (1)
data_sel = 0
print "Reading Channel:" + str(count)
def write_serial(serial_data):
global ser
data = ""
line = ""
status = ""
ser.write(serial_data) # set ph unit to take one reading at a time
ser.write("\r") # set ph unit to take one reading at a time
time.sleep (1)
ser.write(serial_data) # set ph unit to take one reading at a time
ser.write("\r")
time.sleep (1)
while status != "done":
data = ser.read(1) # gets data from PH module '
if(data == "\r"): # check each bite of data to see if its a carriage return expecting "XX.XXr"
#carriage return sent diplay message and data
print "Received from sensor:" + line
status = "done"
probevalue = line
line =" "
ser.flushInput()
ser.flushOutput()
else:
# all 5 bytes of data have not been received
line = line + data # add one to the varable line
#if(data == " "):
# write_serial()
return probevalue
def main():
global serial_data
serial_data = " "
global count
probevalue = " "
count = 0
bus.write_byte(0x38, 0)
while 1:
inputsel(count) #select the input based off count variable
loop = range(0,7)
for loopcount in loop:
if ((loopcount == 0) | (loopcount == 1)): #set command to #?
serial_data = "#?\r" #set command to request id
if (loopcount == 0): #if buffer needs clearing
print "Clearing Buffer....."
else: print "Requesting ID....."
probevalue = write_serial(serial_data) #call write_serial with #? command put value into probevalue
elif (loopcount >= 2): #set r command once buffer clear and ID obtained
serial_data = "r\r" #set command to read value
if (loopcount == 2): print "Reading pH:" #output reaidng pH when loopcounter at 2
probevalue = write_serial(serial_data) #call write_serial with r command put value into probevalue
print "=========================="

Python - Migrating program from Linux to Windows, in Windows it doesn't work

This has been bugging me for a few days now. I'm trying to migrate my utility, LScreamer, to Windows 7, yet it doesn't seem to be cooperating.
This utility transmits a Atmega firmware file to the bootloader running in the atmega. In Linux when I run this utility, it is able to send the entire file with only a handful of retransmits necessary. In Windows, almost every other transmit is a retransmit, and most of the times it results in the processor not getting the complete firmware.
My understanding of python and pyserial is that they should be portable between OS's. But since I am pretty inexperienced in using Python, perhaps I'm doing something strange.
Is there any observable errors in the LScreamer utility attached below? Thanks for your help!
#! /usr/bin/env python
""" This program is a port of Screamer V2.0 produced on Sparkfun.com which is
useful for wirelessly downloading code to an arduino.
The problem with the original code was that it was written for Windows,
whereas there may be many Linux users that would like to use the program.
Currently this port only supports ATmega168 and ATmega1280, but it has
been written to easily add more processors. Questions and contributions
can be submitted back to the developer at dustin.robotics#gmail.com
This has been updated to work on Windows as well, but it will still be
called LScreamer.
Here is the sparkfun page: http://www.sparkfun.com/tutorials/122#Step2
Usage: python LScreamer.py [options] [input file]
Options:
-h, --help show this help
-v, --verbose show additional information
-p Processor,
available arguments 'ATMEGA168', 'ATMEGA328' or 'ATMEGA1280'
default: ATMEGA168
-i, --interface Specify serial interface to broadcast
firmware file over, default /dev/ttyUSB0
Examples:
LScreamer.py -p ATMEGA168 -v -i /dev/ttyUSB0 ~/file.hex
Wirelessly downloads file.hex to an ATMEGA168 processor
LScreamer.py -p ATMEGA328 -v -i COM1 C:/Project/file.hex
Wirelessly downloads file.hex to an ATMEGA328 processor
LScreamer.py
Lists this information as well as the available ports on the system.
This could be ran before inserting serial device, and after, to
determine which port it has been connected to.
"""
import serial
import glob
import sys
import getopt
import platform
from time import sleep
def scan():
"""scan for available ports. return a list of device names."""
if is_windows:
# scan for available ports. return a list of tuples (num, name)
available = []
for i in range(256):
try:
s = serial.Serial(i)
available.append( (i, s.portstr))
s.close()
except serial.SerialException:
pass
return available
else:
return glob.glob('/dev/ttyS*') + glob.glob('/dev/ttyUSB*')
def usage():
print __doc__
class FileProc:
"File processing utilities"
def __init__(self, fileloc):
"""Main processing occurs within the FileProc class"""
#define global variables modified in this def
self.setArc()
pfile = self.parse(fileloc)
if _parsed_file == 0:
print "Problem parsing hex file, please check file again"
sys.exit()
#file parsed successfully, initiate remote reset
print "Waiting for target IC to boot into bootloader"
if _verbose == 1: print "Open port " + _iface + ' with baud rate = 19200, 8 data bits, 1 stop bit, 1 sign bit with a 1 second timeout'
ser = serial.Serial(_iface, 19200, timeout=1)
#toggle RTS line to restart remote device
ser.setRTS(1) #set RTS
ser.setRTS(0) #Clear RTS
ser.setRTS(1) #set RTS
ser.setRTS(0) #Clear RTS
#sleep(1) #wait for 1 seconds with RTS set
print " To quit waiting, do Cnrl + C"
if _verbose == 1:
print " If target IC does not boot into bootloader and start outputting a 5, not '5', then the target should be checked"
while ser.read(1) != '\x05':
pass
#target is in bootloader, send 6, not '6', to start firmware transmission
ser.write('\x06')
print "Target has successfully booted into bootloader, starting transmission of firmware"
current_memory_address = 0
while current_memory_address < _lastmem:
#wait until target IC sends confirmation
while (ser.inWaiting() == 0):
pass
target_status = ser.read()
if target_status == 'T':
#everything working correctly
pass
elif target_status == '\x07':
#need to resend line
print "resending last line"
current_memory_address -= _pagesize
else:
print target_status
print "incorrect response from target IC, will now exit"
sys.exit()
#show progress in terminal
print str(current_memory_address) + " of " + str(_lastmem)
#start processing hex file for output
#convert 16 bit current_memory_address into two 8-bit characters
memory_address_high = current_memory_address / 256
memory_address_low = current_memory_address % 256
#calculate checksum of this line
checksum = 0
checksum = checksum + _pagesize
checksum += memory_address_high
checksum += memory_address_low
for x in range(_pagesize):
checksum += pfile[current_memory_address + x]
#reduce checksum so that it is only 8 bits
while checksum > 256:
checksum -= 256
#take two's compliment of checksum
checksum = 256 - checksum
#start sending current line to target ic
#start character
ser.write(":")
#record length
if _pagesize < 256:
ser.write(chr(_pagesize))
else: #split up into high and low byte
ser.write(chr(_pagesize >> 8))
ser.write(chr(_pagesize % 256))
#send this block's address
ser.write(chr(memory_address_low))
ser.write(chr(memory_address_high))
#send this blocks checksum
ser.write(chr(checksum))
#now send the block
for x in range(_pagesize):
ser.write(chr(pfile[current_memory_address + x]))
#update current memory address
current_memory_address += _pagesize
#we have completed transmitting, tell target ic. Multiple S's for redundancy
ser.write(":")
ser.write("S")
ser.write("S")
ser.write("S")
ftemp.write(":SSS")
ftemp.close()
#tell user that transmission completed successfully
print "LScreamer has successfully sent " + str(_lastmem) + " bytes to the target " + _type
#close serial port
ser.close()
#exit gracefully
def setArc(self):
global _memsize
global _pagesize
global _parsed_file
global _maxsize
global _lastmem
global _type
_parsed_file = 0
if _type == 'ATMEGA168':
_memsize = 16384
_pagesize = 128 #words
_maxsize = _memsize - 1
elif _type == 'ATMEGA1280':
_memsize = 131072
_pagesize = 256 #words
_maxsize = _memsize - 1
elif _type == 'ATMEGA328':
_memsize = 32768
_pagesize = 128 #words
_maxsize = _memsize - 1
def parse(self, fileloc):
"""remove formatting and checksums, sort into rows of 128 bytes"""
#define global variables modified in this def
global _lastmem
global _parsed_file
try:
fhex = open(fileloc,"r")
except IOError:
print "File could not be opened"
sys.exit()
"""file is open, enter loop reading in hex lines"""
li=[] #initialize list, so can start adding elements by extending it
if _verbose == 1: print "reading input file '" + fileloc + "' now."
while 1:
lines = fhex.readlines(100000)
if not lines:
break
for line in lines:
#remove colon and new line
if(line[0]!=':'): #if line has no : then ignore
continue
s = line.split(":")
s = s[1].split("\r\n")
if(s[7:9]=='04'):
continue
if(len(s[0])!=0): #remove empty lines
li.extend([s[0]])
#Hex file is cleaned up now, stored in list li
#prefill hex_array with 0xFF
hex_array=[]
hex_array = [255 for i in range(_memsize)]
if _verbose == 1: print " processing hex file..."
#step through cleaned file and load into hex array
for line in li:
record_length = int(line[0:2], 16) # int('A',16) returns 10
#find memory address to store this line
memory_address_high = int(line[2:4], 16)
memory_address_low = int(line[4:6], 16)
memory_address = memory_address_high * 256 + memory_address_low
#check for end of file tag
if int(line[6:8], 16) == 1:
break
#save last memory location
_lastmem = memory_address + record_length
for x in range(record_length): #Save values to
lower_byte = 8+x*2
upper_byte = 10+x*2
#print "lower %d upper %d x %d" % (lower_byte, upper_byte, x)
hex_array[memory_address + x]= int(line[lower_byte:upper_byte], 16)
#print "int(line[lb:ub},16 = %d" % (int(line[lower_byte:upper_byte], 16))
#print hex_array
#sleep(1)
#file was successfully parsed
fhex.close()
_parsed_file = 1
return hex_array
def main(argv):
#determine OS
global is_windows
is_windows = (platform.system().lower().find("win") > -1)
try:
#To test this in python, do args = '-hvp ATMEGA168 /home/test'.split()
#then do getopt.getopt(argv, "hvp:", ["help", "--verbose"])
opts, args = getopt.getopt(argv, 'hvp:i:', ['help', 'verbose', 'interface'])
#detect if no inputs are given
if len(args) == 0:
usage()
print "\nThe available interfaces are: " + str(scan()) + "\n"
sys.exit(2)
except getopt.GetoptError:
usage()
sys.exit(2)
#setup global variables
global _verbose ; _verbose = 0
global _iface ; _iface = '/dev/ttyUSB0'
global _type ; _type = 'ATMEGA168'
for opt, arg in opts:
if opt in ("-h", "--help"):
usage()
sys.exit()
if opt in ('-v', '--verbose'):
_verbose = 1
if opt in ('-p'):
_type = arg
if opt in ('-i', '--interface'):
_iface = arg
hex = "".join(args)
FileProc(hex)
if __name__=='__main__':
main(sys.argv[1:])
"""
* This program is free software; you can redistribute it and/or modify it *
* under the terms of the GNU General Public License as published by the Free *
* Software Foundation; either version 3 of the License, or (at your option) *
* any later version. *
* *
* This program is distributed in the hope that it will be useful, but WITHOUT *
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or *
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for *
* more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, see <http://www.gnu.org/licenses/>. *"""

I hooked up a MAX3232 and discovered that the XBee is dropping packets, so I am going to investigate into getting a better driver for it. When the MCU is connected to the computer via a Serial cable, the program downloads fine. So it seems to be an xbee, not a python issue. Thanks for your help!

You could try opening the hex file with universal newline support:
fhex = open(fileloc, "rU")
Also, your checksum is wrong. It should be:
while checksum >= 256:
checksum -= 256
#take two's complement of checksum
checksum = 255 - checksum
Additionally, instead of:
hex_array = [255 for i in range(_memsize)]
you could use:
hex_array = [255] * _memsize
and put sleeps in the busy waits:
while ser.read(1) != '\x05':
time.sleep(0.1)