Why is my progress bar not progressing? Is it in a loop? - python

I'm trying to make a progress bar so that as the lines are processed the bar goes up. I can't get it to work.
import io
from datetime import datetime
from alive_progress import alive_bar
result = io.open("Edificado/edificadoResultadoSinPorBlancos.txt","w",encoding='utf-8',errors="surrogateescape")
start_time = datetime.now()
print(f"Empece en: {start_time}")
with io.open("Edificado/edificco.txt","r",encoding='utf-8',errors="surrogateescape" ) as f:
data = len(f.readlines())
with alive_bar(len(data)) as bar:
for line in f:
if '|' in line:
line = line.replace("|","-")
result.write(line)
bar()
result.close()
end_time = datetime.now()
print('Duracion: {}'.format(end_time - start_time))


You can't read the file conten twice, one with f.readlines() and one with for line in f because the file descriptor is at the end
Do
start_time = datetime.now()
print(f"Empece en: {start_time}")
with io.open("Edificado/edificco.txt","r",encoding='utf-8',errors="surrogateescape" ) as f,
io.open("Edificado/edificadoResultadoSinPorBlancos.txt","w",encoding='utf-8',errors="surrogateescape") as result:
lines = f.readlines()
with alive_bar(len(lines)) as bar:
for line in lines:
result.write(line.replace("|","-"))
bar()
Also
you were calling len twice, that couldn't work
don't need to check for a character presence to replace it (about |)

Related

Overwrite content of new generated file

The code generates a file every 30 seconds with the date/time and then types the 'key' in self.log into it. The problem is when the new file gets generated and 'key' gets typed into that it just appends itself to the bottom and doesn't overwrite the old content in the newly generated file.
Would appreciate your help :)
def report(self):
if self.log:
print(self.log)
starttime = time.time()
while True:
timestr = time.strftime("%Y%m%d-%H%M%S")
fileloc = f'C:/logging/pylogger/{timestr}.txt'
with open(fileloc, mode='w') as f:
f.seek(0)
f.truncate()
for key in self.log:
f.write(key)
time.sleep(30.0 - ((time.time() - starttime) % 30.0))

You question is not fully clear to me, but if I correctly understood you need to remove the elements from the list once written:
def report(self):
if self.log:
print(self.log)
starttime = time.time()
while True:
timestr = time.strftime("%Y%m%d-%H%M%S")
fileloc = f'C:/logging/pylogger/{timestr}.txt'
with open(fileloc, mode='w') as f:
f.seek(0)
f.truncate()
for key in self.log:
f.write(key)
self.log.remove(key)
time.sleep(30.0 - ((time.time() - starttime) % 30.0))
Just add self.log.remove(key) in your for loop, so the values are removed from the list once written and when you reach the next cycle after 30 seconds the new file will contains only the new values.

How to split the csv file with line numbers which pass as parameter and save into different files

python script.py 2
Here each file will be having 2 lines. filename is having time, data,hour, second
I have 5 line in a csv file
I will take input from user as parameter
if user is giving 2 then my each file will be having 2 lines each.
first 2 line will be in first file, second two line in second file and last line is in third file
Last file will be having one line
Filename will be having hour minute second
below is the csv file
1,Network activity,ip-dst,80.179.42.44,,1,20160929
2,Payload delivery,md5,4ad2924ced722ab65ff978f83a40448e,,1,20160929
3,Network activity,domain,alkamaihd.net,,1,20160929
4,Payload delivery,md5,197c018922237828683783654d3c632a,,1,20160929
5,Network activity,domain,dnsrecordsolver.tk,,1,20160929
I need to divide in to 3 different files, split number of lines has to taken as argument
Below is the approach
take the division line as parameter and save with data with time stamp
Below is the code for saving the data with date.hour,time. I need some logic to divide the line and continue from there
import csv
import time
import sys
sourceData = "Oil.csv"
def GetLineCount():
with open(sourceData) as f:
for i, l in enumerate(f):
pass
return i
def MakeLog(startLine, numLines):
destData = time.strftime("%Y%m%d-%H%M%S.log")
with open(sourceData, 'r') as csvfile:
with open(destData, 'w') as dstfile:
reader = csv.reader(csvfile)
writer = csv.writer(dstfile)
next (reader) #skip header
python script.py 2
then 3 files will be generate with 2 lines in each file and last file will be having one
Expected out
3 text file will be generate 2 lines each

There is a linux split command
split -l 2 Oil.csv
would split into files of 2 lines each.
For appending hour, minute, and second to filenames,
split -l 2 Oil.csv --additional-suffix=`date +"%H:%M:%S"`

And here's how you would do it in Python3.
import argparse
import time
from itertools import zip_longest
def grouper(n, iterable, fill_value=None):
args = [iter(iterable)] * n
return zip_longest(fillvalue=fill_value, *args)
def splitter(n_lines, file):
with open(file) as f:
for i, payload in enumerate(grouper(n_lines, f, fill_value=''), 1):
f_name = f"{time.strftime('%Y%m%d-%H%M%S')}_{i*n_lines}.log"
with open(f_name, 'w') as out:
out.writelines(payload)
def get_parser():
parser = argparse.ArgumentParser(description="File splitter")
parser.add_argument("file", metavar="FILE", type=str, help="Target file to be chopped up")
parser.add_argument("n_lines", type=int, default=2, help="Number of lines to output per file")
return parser
def command_line_runner():
parser = get_parser()
args = vars(parser.parse_args())
splitter(args['n_lines'], args['file'])
if __name__ == "__main__":
command_line_runner()
Sample run: python3 main.py sample.csv 2 produces 3 files:
20200921-095943_2.log
20200921-095943_4.log
20200921-095943_6.log
The first two have two lines each and the last one, well, one line.
The contents of sample.csv is as in your example:
1,Network activity,ip-dst,80.179.42.44,,1,20160929
2,Payload delivery,md5,4ad2924ced722ab65ff978f83a40448e,,1,20160929
3,Network activity,domain,alkamaihd.net,,1,20160929
4,Payload delivery,md5,197c018922237828683783654d3c632a,,1,20160929
5,Network activity,domain,dnsrecordsolver.tk,,1,20160929

You can find the line numbers to split at n lines with (linenumber+1) % n:
import sys
import datetime
def save_file(newcontent):
savenewfile = str(datetime.datetime.now()) + ".log"
with open(savenewfile, 'w') as dstfile:
dstfile.write(newcontent)
split_at = int(sys.argv[1])
newcontent = ""
with open("Oil.csv", "r") as f:
for i, line in enumerate(f):
newcontent += line
if (i+1) % split_at == 0: # i+1 because i starts at 0
save_file(newcontent)
newcontent = ""
# save remaining lines
if newcontent:
save_file(newcontent)
save_file(newcontent) will be your function to save a string to a new file.
Your timestring %Y%m%d-%H%M%S won't make a unique file name. You can add a counter or use str(datetime.datetime.now()) + ".log" instead.

Is there a method to print the duration of the script and save it inside the csv?

I would like to print the duration of the script and save it inside the csv. What would be the best method?
import requests
with open('infile.txt', 'r') as f:
urls = f.readlines()
datalist=[]
for url in urls:
data = requests.get(url)
datalist.append(data.text)
with open('outfile.txt', 'w') as f:
for item in datalist:
f.write("%s\n" % item)

You can use datetime module.
import requests
from datetime import datetime
def run():
with open('infile.txt', 'r') as f:
urls = f.readlines()
datalist=[]
for url in urls:
data = requests.get(url)
datalist.append(data.text)
with open('outfile.txt', 'w') as f:
for item in datalist:
f.write("%s\n" % item)
startTime = datetime.now()
run()
print(datetime.now() - startTime)

One simple way you could do this is by using the built-in time module. Get the time before the execution, get the time after the execution, subtract them and you get the time taken for your script to run. You could then just export it to your csv just like every other text.
import time
# STEP 1: Get start time
start_time = time.time()
# Step 2: Run your code you want to time
result = func_one(some_param)
# Step 3: Calculate total time elapsed
end_time = time.time() - start_time

How to keep writing value into spreadsheet in a user controlled loop in Python?

I'm trying to write values into a .csv file every second until interrupted by the user by pressing a key. I'm able to achieve it in a finite loop.
I tried using raw_input but the program would only write the last value before interruption. What should I change in my code?
Here's the code that works for a finite loop:
import time
import csv
class Timestamp:
def __init__(self):
my_file = open('test_csv.csv','w+')
with my_file:
new_file = csv.writer(my_file)
for val in range(0,20):
with open('test_csv.csv','a') as f:
date_now = time.strftime('%d/%m/%y')
time_now = time.strftime('%H:%M:%S')
to_write = [date_now, time_now]
csv_file =csv.writer(f)
csv_file.writerow(to_write)
time.sleep(1)
Timestamp()

You can use threads.
import time
import csv
from threading import Thread
def write_loop(self):
my_file = open('test_csv.csv', 'w+')
with my_file:
new_file = csv.writer(my_file)
for val in range(0, 20):
with open('test_csv.csv', 'a') as f:
date_now = time.strftime('%d/%m/%y')
time_now = time.strftime('%H:%M:%S')
to_write = [date_now, time_now]
csv_file = csv.writer(f)
csv_file.writerow(to_write)
time.sleep(1)
t = Thread(target=write_loop)
t.daemon = True
t.start()
input("Press any key to stop")

Optimizing Permutation Search in File

I wrote a program that, using a set of characters, searches in a file for any of the permutations of that set. I would love your suggestions for optimizing that program.
I split the file into multiple, depending on the number of characters of each "word":
#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys
import itertools
import time
start_time = time.time()
chrset = sys.argv[1]
lgr = len(chrset)
f = open('gd'+str(lgr), 'r')
perms = []
s = list(itertools.permutations(chrset))
for perm in s:
perms.append(''.join(map(str,perm)))
for line in f:
line = line.rstrip('\n')
for pp in perms:
if pp == line:
print (line)
print("--- %s seconds ---" % (time.time() - start_time))

Categories