I have a file, which I am writing data to, however I would like the naming convention to have the time, the file was created at and then stream to the same file for one hour before creating a new text file.
My problem is that I am using a while loop to create the text file while reading from a GPIO and so when the code runs, it creates a new text file every time it moves through the loop. How can I create the file, then write to the existing file for a predetermined amount of time?
import spidev
import time
import datetime
import os
spi = spidev.SpiDev()
spi.open(0,0) #open port 0 of the SPI
count = 0
def timeStamp(fname, fmt ='%Y-%m-%d-%H-%M-%S_{fname}'):
#this function adds the time and date
return datetime.datetime.now().strftime(fmt).format(fname = fname)
def alarm_check(int):
#this function checks the value read in by the SPI
#creating an alarm if the value is 0
if int == 0:
return ("System Alarm!")
else:
return ("System OK")
def write_to_file(int):
# this function will write the current value to a file,
# the time it was read in
# and if the system was im alarm
with open (('SPI_Input_Values'), 'a') as output:
output.write("Input = " + str(int) + '\t' + timeStamp('ADC') + '\t\t' + str(alarm_check(int)) + '\n')
def readadc(adcnum):
#this function will open the SPI and read it to see the current value
# this will then be written to a text value
# using the write_to_file function
if adcnum > 7 or adcnum < 0:
return -1
r = spi.xfer2([1,8 + adcnum << 4,0])
adcout = ((r[1] & 3) << 8) + r[2]
return adcout
while True:
Inp1 = int(round(readadc(0)/10.24)) # defines Inp1 as an integer to be read in
write_to_file(Inp1)
count = count +1
time.sleep(0.1) # puts the systemm to sleep for 0.1 seconds
You need to dynamically create your filename. At the moment it was a hardcoded string. Here's an example:
fname = 'SPI_Input_Values'
fmt ='%Y-%m-%d-%H'
date_str = datetime.datetime.now().strftime(fmt)
file_name = date_str + "_" + fname
with open ((file_name, 'a') as output:
output.write("Input = " + str(int) + '\t' + timeStamp('ADC') + '\t\t' + str(alarm_check(int)) + '\n')
This will constantly append to the file that represents the current hour. When the next hour begins, it will automatically create a new file and begin appending data to it instead.
Something like:
def write_to_file(output, int):
output.write("Input = " + str(int) + '\t' + timeStamp('ADC') + '\t\t' + str(alarm_check(int)) + '\n')
with open (('SPI_Input_Values'), 'a') as output:
while True:
Inp1 = int(round(readadc(0)/10.24)) # defines Inp1 as an integer to be read in
write_to_file(output, Inp1)
count = count +1
time.sleep(0.1)
Related
I am using multithreading and its working fine.In the first thread as mentioned in the code, I need my csv file to be saved with filename as Logger1 for the first time and after this logger1 file has crossed 1000 lines it should be converted into zip file, and removed while 'i' gets incremented by 1.Now after this, the further data should be saved in the csv file with filename as logger2.
If variable 'i' is assigned value the way, as shown in the code then unbound local error appears stating variable i is referenced before assignment. If I declare it like global i instead of just i=1 in the beginning, same error appears. If I assigned value inside the u1thread this value never gets actually incremented because every time the thread is executed i is set to be 1.
i=1
class u1Thread(threading.Thread):
def run(self):
while True:
export_csv = x.to_csv (r'/home/pi/Logger' + str(i)
+ '.csv', index = None, mode='a',
header=False)
input_file = open("Logger" + str(i) + ".csv","r+")
reader_file = csv.reader(input_file)
l = len(list(reader_file))
if (l > 1000) :
jungle_zip = zipfile.ZipFile('Logger' + str(i) +
'.zip', 'w')
jungle_zip.write('Logger' + str(i) + '.csv',
compress_type=zipfile.ZIP_DEFLATED)
jungle_zip.close()
os.remove("Logger" + str(i) + ".csv")
i +=1
class vfThread(threading.Thread):
def run(self):
while True:
ret, frame=video.read()
Writing a python script to parse an incoming string from a text file and output word by word to serial (serial parts commented out)
Getting the following error:
Traceback (most recent call last):
File "/Users/di/Desktop/RBDS/rbdsScroll.py", line 23, in <module>
wordLength = len(wordList[stringInc])
IndexError: list index out of range
I know it has to do with the new list (when the contents of the text file have far less words than the previous list) not having a high enough index number to work. I'm not quite sure how to go about remedying this. Any help would be appreciated. Full code below:
import time
#import serial
from config import dataPath
from config import scrollTime
from config import preText
from config import postText
from config import port
stringInc=wordFirst=wordLast=wordList=wordLength=word2Length=0
while True:
f = open(dataPath, 'r')
file_contents = f.read()
f.close()
wordList = file_contents.split()
maxLength = len(wordList)
wordLength = len(wordList[stringInc])
if stringInc < maxLength - 1:
word2Length = len(wordList[stringInc + 1])
wordFirst = 0
wordLast = 8
if wordLength > 8:
longString = (wordList[stringInc])
while wordLength + 1 > wordLast:
# ser = serial.Serial(port)
# ser.write(preText + longString[wordFirst:wordLast] + postText)
# ser.close()
print(preText + longString[wordFirst:wordLast] + postText)
wordFirst = wordFirst + 1
wordLast = wordLast + 1
time.sleep(scrollTime / 1000)
elif (wordLength + word2Length < 8) and (stringInc + 1 < maxLength):
# ser = serial.Serial(port)
# ser.write(preText + wordList[stringInc] + " " + wordList[stringInc + 1] + postText)
# ser.close()
print(preText + wordList[stringInc] + " " + wordList[stringInc + 1] + postText)
stringInc = stringInc + 1
time.sleep(scrollTime / 1000)
else:
# ser = serial.Serial(port)
# ser.write(preText + wordList[stringInc] + postText)
# ser.close()
print(preText + wordList[stringInc] + postText)
time.sleep(scrollTime / 1000)
stringInc = stringInc + 1
if stringInc == maxLength:
stringInc = 0
If the file content is fixed, you should read it only once, and move the while just before setting wordLength. That sends all the read words one after the other.
In case the file content changes, which would explain the error, if the new content is shorter than the current value of stringInc, you lower maxLength (just after reading the file) but never force stringInc to be lower than this new value, hence the error message...
Adding a check like if stringInc >= maxLength: after changing maxLength should help correcting the code (in that case set stringInc to 0, I guess).
But the behavior is still strange, and I would rather send all the words of the file after reading it once, not reading it in the while, and then later would read the file again, to send if only if modified...
Brand new to programming but very enjoyable challenge.
Here's a question which I suspect may be caused by a misunderstanding of python loops.
System info: Using notepad++ and IDLE python 3.4.3 on Win 7 32-bit
My solution is to open 1 database, use it to look for a correct master entry from database 2, pulls a index number (task_no), then write a 3rd file identical to the first database, this time with the correct index number.
My problem is that it performs 1st and 2nd loop correctly, then on the 2nd iteration of loop 1, tries to perform a block in loop 2 while iterating through the rows of loop 1, not the task_rows of loop 2.
footnote: Both files are quite large (several MB) so I'm note sure if storing them in memory is a good idea.
This was a relevant question that I found closest to this problem:
python nested loop using loops and files
What I got out of it was that I tried moving the file opening within the 1st loop, but the problem persists. Something to do with how I'm using CSV reader?
I also have the sinking suspicion that there may be a root cause in problem solving so I am welcome to suggestions for alternative ways to solve the problem.
Thanks in advance!
The gist:
for row in readerCurrentFile: #LOOP 1
# iterates through readerCurrentFile to define search variables
[...]
for task_row in readerTaskHeader: #LOOP 2
# searches each row iteratively through readerTaskHeader
# Match compid
#if no match, continue <<<- This is where it goes back to 1st loop
[...]
# Match task frequency
#if no match, continue
[...]
# once both of the above matches check out, will grab data (task_no from task_row[0]
task_no = ""
task_no = task_row[0]
if task_row:
break
[...]
# writes PM code
print("Successful write of PM schedule row")
print(compid + " " + dict_freq_names[str(pmfreqx) + str(pmfreq)] + ": " + pmid + " " + task_no)
The entire code:
import csv
import re
#Writes schedule
csvNewPMSchedule = open('new_pm_schedule.csv', 'a', newline='')
writerNewPMSchedule = csv.writer(csvNewPMSchedule)
# Dictionaries of PM Frequency
def re_compile_dict(d,f):
for k in d:
d[k] = re.compile(d[k], flags=f)
dict_month = {60:'Quin',36:'Trien',24:'Bi-An',12:'Annual(?<!Bi-)(?<!Semi-)',6:'Semi-An',3:'Quart',2:'Bi-Month',1:'Month(?<!Bi-)'}
dict_week = {2:'Bi-Week',1:'Week(?<!Bi-)'}
dict_freq_names = {'60Months':'Quintennial','36Months':'Triennial','24Months':'Bi-Annual','12Months':'Annual','6Months':'Semi-Annual','3Months':'Quarterly','2Months':'Bi-Monthly','1Months':'Monthly','2Weeks':'Bi-Weekly','1Weeks':'Weekly'}
re_compile_dict(dict_month,re.IGNORECASE)
re_compile_dict(dict_week, re.IGNORECASE)
# Unique Task Counter
task_num = 0
total_lines = 0
#Error catcher
error_in_row = []
#Blank out all rows
pmid = 0
compid = 0
comp_desc = 0
pmfreqx = 0
pmfreq = 0
pmfreqtype = 0
# PM Schedule Draft (as provided by eMaint)
currentFile = open('pm_schedule.csv', encoding='windows-1252')
readerCurrentFile = csv.reader(currentFile)
# Loop 1
for row in readerCurrentFile:
if row[0] == "pmid":
continue
#defines row items
pmid = row[0]
compid = row[1]
comp_desc = row[2]
#quantity of pm frequency
pmfreqx_temp = row[3]
#unit of pm frequency, choices are: Months, Weeks
pmfreq = row[4]
#pmfreqtype is currently only static not sure what other options we have
pmfreqtype = row[5]
#pmnextdate is the next scheduled due date from this one. we probably need logic later that closes out any past due date
pmnextdate = row[6]
# Task Number This is what we want to change
# pass
# We want to change this to task header's task_desc
sched_task_desc = row[8]
#last done date
last_pm_date = row[9]
#
#determines frequency search criteria
#
try:
pmfreqx = int(pmfreqx_temp)
except (TypeError, ValueError):
print("Invalid PM frequency data, Skipping row " + pmid)
error_in_row.append(pmid)
continue
#
#defines frequency search variable
#
freq_search_var = ""
if pmfreq == "Weeks":
freq_search_var = dict_week[pmfreqx]
elif pmfreq == "Months":
freq_search_var = dict_month[pmfreqx]
if not freq_search_var:
print("Error in assigning frequency" + compid + " " + str(pmfreqx) + " " + pmfreq)
error_in_row.append(pmid)
continue
#defines Equipment ID Search Variable
print(compid + " frequency found: " + str(pmfreqx) + " " + str(pmfreq))
compid_search_var = re.compile(compid,re.IGNORECASE)
#
# Matching function - search taskHeader for data
#
#PM Task Header Reference
taskHeader = open('taskheader.csv', encoding='windows-1252')
readerTaskHeader = csv.reader(taskHeader)
for task_row in readerTaskHeader:
# task_row[0]: taskHeader pm number
# task_row[1]: "taskHeader task_desc
# task_row[2]: taskHeader_task_notes
#
# search for compid
compid_match = ""
compid_match = compid_search_var.search(task_row[1])
if not compid_match:
print(task_row[1] + " does not match ID for " + compid + ", trying next row.") #debug 2
continue # <<< STOPS ITERATING RIGHT OVER HERE
print("Found compid " + task_row[1]) # debug line
#
freq_match = ""
freq_match = freq_search_var.search(task_row[1])
if not freq_match:
print(task_row[1] + " does not match freq for " + compid + " " + dict_freq_names[str(pmfreqx) + str(pmfreq)] + ", trying next row.") #debug line
continue
print("Frequency Match: " + compid + " " + dict_freq_names[str(pmfreqx) + str(pmfreq)]) # freq debug line
#
task_no = ""
print("Assigning Task Number to " + task_row[0])
task_no = task_row[0]
if task_row:
break
#
#error check
#
if not task_no:
print("ERROR IN SEARCH " + compid + " " + pmid)
error_in_row.append(pmid)
continue
#
# Writes Rows
#
writerNewPMSchedule.writerow([pmid,compid,comp_desc,pmfreqx,pmfreq,pmfreqtype,pmnextdate,task_no,sched_task_desc,last_pm_date])
print("Successful write of PM schedule row")
print(compid + " " + dict_freq_names[str(pmfreqx) + str(pmfreq)] + ": " + pmid + " " + task_no)
print("==============")
# Error reporting lined out for now
# for row in error_in_row:
# writerNewPMSchedule.writerow(["Error in row:",str(error_in_row[row])])
# print("Error in row: " + str(error_in_row[row]))
print("Finished")
I put trailing print() methods right next to my write() method lines at the end of my code to test why my output files were incomplete. But, the print() output is "all the stuff" I expect; while the write() output is off by a confusing amount (only 150 out of 200 'things'). Reference Image of Output: IDLE versus external output file
FYI: Win 7 64 // Python 3.4.2
My modules take an SRT captions file ('test.srt') and returns a list object I create from it; in particular, one with 220 list entries of the form: [[(index), [time], string]]
times = open('times.txt', 'w')
### A portion of Riobard's SRT Parser: srt.py
import re
def tc2ms(tc):
''' convert timecode to millisecond '''
sign = 1
if tc[0] in "+-":
sign = -1 if tc[0] == "-" else 1
tc = tc[1:]
TIMECODE_RE = re.compile('(?:(?:(?:(\d?\d):)?(\d?\d):)?(\d?\d))?(?:[,.](\d?\d?\d))?')
match = TIMECODE_RE.match(tc)
try:
assert match is not None
except AssertionError:
print(tc)
hh,mm,ss,ms = map(lambda x: 0 if x==None else int(x), match.groups())
return ((hh*3600 + mm*60 + ss) * 1000 + ms) * sign
# my code
with open('test.srt') as f:
file = f.read()
srt = []
for line in file:
splitter = file.split("\n\n")
# SRT splitter
i = 0
j = len(splitter)
for items in splitter:
while i <= j - 2:
split_point_1 = splitter[i].index("\n")
split_point_2 = splitter[i].index("\n", split_point_1 + 1)
index = splitter[i][:split_point_1]
time = [splitter[i][split_point_1:split_point_2]]
time = time[0][1:]
string = splitter[i][split_point_2:]
string = string[1:]
list = [[(index), [time], string]]
srt += list
i += 1
# time info outputter
i = 0
j = 1
for line in srt:
if i != len(srt) - 1:
indexer = srt[i][1][0].index(" --> ")
timein = srt[i][1][0][:indexer]
timeout = srt[i][1][0][-indexer:]
line_time = (tc2ms(timeout) - tc2ms(timein))/1000
space_time = ((tc2ms((srt[j][1][0][:indexer]))) - (tc2ms(srt[i][1][0][-indexer:])))/1000
out1 = "The space between Line " + str(i) + " and Line " + str(j) + " lasts " + str(space_time) + " seconds." + "\n"
out2 = "Line " + str(i) + ": " + str(srt[i][2]) + "\n\n"
times.write(out1)
times.write(out2)
print(out1, end="")
print(out2)
i += 1
j += 1
else:
indexer = srt[i][1][0].index(" --> ")
timein = srt[i][1][0][:indexer]
timeout = srt[i][1][0][-indexer:]
line_time = (tc2ms(timeout) - tc2ms(timein))/1000
outend = "Line " + str(i) + ": " + str(srt[i][2]) + "\n<End of File>"
times.write(outend)
print(outend)
My two write() method output files, respectively, only print out either ~150 or ~200 items of the 220 things it otherwise correctly prints to the screen.
You want to close your times file when done writing; operating systems use write buffers to speed up file I/O, collecting larger blocks of data to be written to disk in one go; closing the file flushes that buffer:
times.close()
Consider opening the file in a with block:
with open('times.txt', 'w') as times:
# all code that needs to write to times
I have the following code tha uses FFmpeg . It has 5 argv and it takes in filename,video,segment size, start time, end time, ratings. thats supposed to let me classify segments many times with my ratings "PG G M18..." but there's this error,
"File "C:\c.py",line 92, in <module> os.rename<filename + str(x), filename + str(x) + classification)
WindowsError: [Error2] The system cannot find the file specified.
Ive tried to edit many times but this error still persists. Anyone have any idea what could this error mean and anyway to solve it?
import sys
import subprocess
import os
#change hh:mm:ss to seconds:
def getSeconds(sec):
l = sec.split(':')
return int(l[0])* 3600 + int(l[1])* 60 + float(l[2])
def get_total_time(filename):
proc = subprocess.Popen(["ffmpeg", "-i", filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
lines = proc.communicate()[1]
target = [line for line in lines.split('\n') if 'Duration:' in line][0]
time = target.split('Duration: ')[-1].split(',', 1)[0]
return time
#check command line arguments
if len(sys.argv) < 5:
print "Error: not enough arguments"
sys.exit()
#save filename to file_name
file_name = sys.argv[1]
if not file_name.endswith('mpg'):
print 'Error! File extension not supported'
sys.exit()
# save a size of chunk in chunk
segsize = int(sys.argv[2])
chunk = (segsize * 1024)
# get time of starting censorship in seconds
start_censorship = getSeconds(sys.argv[3])
# get time of ending censorship in seconds
end_censorship = getSeconds(sys.argv[4])
classification = sys.argv[5]
if classification not in ['P','PG','G','NC16','M18','R21']:
print "Error: invalid classification"
sys.exit()
#initialize variable for extension
file_ext = ''
# if extension exists then save it into file_ext
if '.' in file_name:
# split file_name on two parts from right
file_ext = sys.argv[1].split('.')[-1]
# file_name without extension
filename = '.'.join(file_name.split('.')[:-1])
# total_time of file in seconds
total_time = getSeconds(get_total_time(file_name))
print total_time
#open file
in_file = open(file_name,'rb')
#read first chunks
s = in_file.read(chunk)
in_file.seek(0, 2)
file_size = in_file.tell()
chunks = (file_size / chunk) + 1
chunk_time = total_time/ file_size * chunk
#close input file
in_file.close()
#loop for each chunk
for x in range(0, chunks):
# starting time of current chunk
t1 = chunk_time*x
# ending time of current chunk
t2 = chunk_time*(x+1)
if t2 < start_censorship or t1 > end_censorship:
pass
else:
if os.path.exists(filename + str(x) + 'x'):
os.rename(filename + str(x) + 'x', filename + str(x))
os.rename(filename + str(x), filename + str(x) + classification)
#read next bytes
You're not checking whether filename + str(x) exists before you try to rename it. Either check first, or put it in a try block and catch OSError (which WindowsError subclasses). Either that or you're missing an indent on the second rename.
In that last for loop it looks like you are possibly renaming multiple files -- where do the original files come from?
Or, asked another way, for every chunk that is not censored you are renaming a file -- is that really what you want?