In this piece of my script when max_conc is calculated, the next for loop doesn't work unless I put it right after the reader. But doing so, max_conc doesn't work anymore.
with open ('outfile.csv', 'r') as infile:
reader = csv.reader(infile)
max_conc = max(cols[1] for cols in reader)
print max_conc
for row in reader:
print row
Is reopening the input file the only solution?
Because when you called:
max(cols[1] for cols in reader)
You exhausted the iterator, you can't loop through it again. But you can use seek to get back to the beginning of the file:
infile.seek(0)
Or, you can store a copy of the file in the memory:
lines = infile.readlines()
max_conc = max(cols[1] for cols in lines)
Another way, as #Lyncha suggested, loop through a list-copy of the file:
lines = list(reader)
Here's a little example:
>>> f = open('test.txt')
>>> f.read()
'a\nb\nc\nd\ne\nf\ng'
>>> f.read()
'' #whoops, iterator is exhausted
>>> f.seek(0)
>>> f.read()
'a\nb\nc\nd\ne\nf\ng'
Related
Taking a video game design course and I've never had to use python before so I am very confused... I am tasked with the following :
read in the CSV file into Python and store its contents as a list of lists
(or 2D list/array). To do so, you will make use of the CSV[1] library.
The reading of the CSV file should be done as its own function - please create a function called readCSV(...)
that takes in the file name as the argument and returns the 2D list.
As mentionned I have no previous coding experience with python. I have managed to do this so far and would greatly appreciate some support.
import csv
# reading each row and printing it
def readCSV(fileName):
TwoDimList = []
with open(fileName, 'r') as f:
r = csv.reader(f, delimiter=',')
for row in r:
entities = readCSV('entities.csv')
print(entities)
Just append each row (which is a list of columns values) to your 2d list and return it in the end:
def readCSV(fileName):
two_dim_list = [] # snake case ftw (PEP8)
with open(fileName, 'r') as f:
r = csv.reader(f, delimiter=',')
# next(r) # skip header line if necessary
for row in r:
two_dim_list.append(row)
return two_dim_list
The short version of that is:
def readCSV(fileName):
with open(fileName, 'r') as f:
r = csv.reader(f, delimiter=',')
# next(r) # skip header line
return list(r)
You can just call list on the reader to get the full 2d list:
def read_csv(file_name):
with open(file_name) as f:
return list(csv.reader(f))
This works because csv.reader is an iterable.
define a function to read csv and return list, and use it later in the program
def readCSVinList(fpath,fname):
with open(fpath+fname) as csv_file:
csv_reader=csv.reader(csv_file)
return list(csv_reader)
f= readCSVinList("A:\\Test\\","test.csv")
for row in f:
print(row)
I have a file 'master.sql' that contains:
a.b.c
d.e.f
g.h.i
and I want to write on 'databases.sql' just the first letters, like this:
a
d
g
Here is my code, but returns just the last letter, the 'g'.
with open ('master.sql', 'r') as f:
for line in f:
x=(line.split('.')[0])
with open('databases.sql', 'w') as f:
f.write(str(x))
How can I fix this?
You'll need to write your data as you read it, otherwise you're not going to be able to do what you want. Fortunately, with allows you to open multiple files concurrently. This should work for you.
with open ('master.sql', 'r') as f1, open('databases.sql', 'w') as f2:
for line in f1:
f2.write(line.split('.')[0] + '\n')
Don't forget to write a newline, because file.write doesn't add one automatically.
Using list:
x = []
with open('master.sql', 'r') as f:
for line in f.readlines():
x.append(line.split('.')[0])
with open('databases.sql', 'w') as f:
for word in x:
f.write(str(word)+'\n')
The variable x receives all values, but each loop overwrite the last value. Hence, the result is 'g'.
To save all values you can do like this:
lst = []
with open ('master.sql', 'r') as f:
for line in f:
lst.append(line.split('.')[0])
x = '\n'.join(lst)
with open('databases.sql', 'w') as f:
f.write(x)
This is my code i am able to print each line but when blank line appears it prints ; because of CSV file format, so i want to skip when blank line appears
import csv
import time
ifile = open ("C:\Users\BKA4ABT\Desktop\Test_Specification\RDBI.csv", "rb")
for line in csv.reader(ifile):
if not line:
empty_lines += 1
continue
print line
If you want to skip all whitespace lines, you should use this test: ' '.isspace().
Since you may want to do something more complicated than just printing the non-blank lines to the console(no need to use CSV module for that), here is an example that involves a DictReader:
#!/usr/bin/env python
# Tested with Python 2.7
# I prefer this style of importing - hides the csv module
# in case you do from this_file.py import * inside of __init__.py
import csv as _csv
# Real comments are more complicated ...
def is_comment(line):
return line.startswith('#')
# Kind of sily wrapper
def is_whitespace(line):
return line.isspace()
def iter_filtered(in_file, *filters):
for line in in_file:
if not any(fltr(line) for fltr in filters):
yield line
# A dis-advantage of this approach is that it requires storing rows in RAM
# However, the largest CSV files I worked with were all under 100 Mb
def read_and_filter_csv(csv_path, *filters):
with open(csv_path, 'rb') as fin:
iter_clean_lines = iter_filtered(fin, *filters)
reader = _csv.DictReader(iter_clean_lines, delimiter=';')
return [row for row in reader]
# Stores all processed lines in RAM
def main_v1(csv_path):
for row in read_and_filter_csv(csv_path, is_comment, is_whitespace):
print(row) # Or do something else with it
# Simpler, less refactored version, does not use with
def main_v2(csv_path):
try:
fin = open(csv_path, 'rb')
reader = _csv.DictReader((line for line in fin if not
line.startswith('#') and not line.isspace()),
delimiter=';')
for row in reader:
print(row) # Or do something else with it
finally:
fin.close()
if __name__ == '__main__':
csv_path = "C:\Users\BKA4ABT\Desktop\Test_Specification\RDBI.csv"
main_v1(csv_path)
print('\n'*3)
main_v2(csv_path)
Instead of
if not line:
This should work:
if not ''.join(line).strip():
my suggestion would be to just use the csv reader who can delimite the file into rows. Like this you can just check whether the row is empty and if so just continue.
import csv
with open('some.csv', 'r') as csvfile:
# the delimiter depends on how your CSV seperates values
csvReader = csv.reader(csvfile, delimiter = '\t')
for row in csvReader:
# check if row is empty
if not (row):
continue
You can always check for the number of comma separated values. It seems to be much more productive and efficient.
When reading the lines iteratively, as these are a list of comma separated values you would be getting a list object. So if there is no element (blank link), then we can make it skip.
with open(filename) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=",")
for row in csv_reader:
if len(row) == 0:
continue
You can strip leading and trailing whitespace, and if the length is zero after that the line is empty.
import csv
with open('userlist.csv') as f:
reader = csv.reader(f)
user_header = next(reader) # Add this line if there the header is
user_list = [] # Create a new user list for input
for row in reader:
if any(row): # Pick up the non-blank row of list
print (row) # Just for verification
user_list.append(row) # Compose all the rest data into the list
This example just prints the data in array form while skipping the empty lines:
import csv
file = open("data.csv", "r")
data = csv.reader(file)
for line in data:
if line: print line
file.close()
I find it much clearer than the other provided examples.
import csv
ifile=csv.reader(open('C:\Users\BKA4ABT\Desktop\Test_Specification\RDBI.csv', 'rb'),delimiter=';')
for line in ifile:
if set(line).pop()=='':
pass
else:
for cell_value in line:
print cell_value
I need to remove the second line of my csv file.
I am using the code below but unfortunately it doesn't work.
data = ""
adresse = "SLV.csv"
if os.path.exists(adresse) :
f = open(adresse,"ab")
writer = csv.writer(f,delimiter = ",")
reader = csv.reader(open(adresse,"rb") , delimiter = ",")
for line in reader:
if reader.line_num == 2:
writer.writerow(line)
f.close()
Since all you want to do is remove the second line, using the csv module is overkill. It doesn't matter if the file is comma separated data or Vogon poetry. Write the front parts, skip the middle part and write the end.
import shutil
# generate test file
with open('x.txt', 'w') as f:
for i in range(10):
f.write('line %d\n' % i)
# skip one line
with open('x.txt','rb') as rd, open('x.txt', 'rb+') as wr:
wr.write(rd.readline())
rd.readline()
shutil.copyfileobj(rd, wr)
wr.truncate()
print open('x.txt').read()
write to a temp file and update the original after:
if os.path.exists(adresse) :
with open(adresse,"r") as f,open("temp.csv" "a+") as temp:
writer = csv.writer(temp,delimiter = ",")
reader = csv.reader(f , delimiter = ",")
for ind, line in enumerate(reader):
if ind == 2:
continue
else:
temp.writerow(line)
temp.seek(0)
with open(adresse,"w") as out:
reader = csv.reader(temp , delimiter = ",")
writer = csv.writer(out,delimiter = ",")
for row in reader:
writer.writerow(line)
If the files can be read into memory just call list on reader and remove the second element:
if os.path.exists(adresse) :
with open(adresse,"r") as f:
reader = list(csv.reader(f , delimiter = ","))
reader.pop(1)
with open(adresse,"w") as out:
writer = csv.writer(out,delimiter = ",")
for row in reader:
writer.writerow(row)
if i understand you correctly you are trying to make a new file and you don't want to insert the line number 2.
if this is your scenario there is a trivial bug in your procedure, that is:
if reader.line_num != 2:
writer.writerow(line)
Here my solution, less code for you to write:
>>> import pyexcel as pe # pip install pyexcel
>>> sheet = pe.load("SLV.csv")
>>> del sheet.row[1] # first row starts at index 0
>>> sheet.save_as("SLV.csv")
I do agree with tdelaney, and this is a far more compact solution
lines = open('x.txt', 'r').readlines()
lines.pop(1)
open('x.txt', 'w').writelines(lines)
This is my code i am able to print each line but when blank line appears it prints ; because of CSV file format, so i want to skip when blank line appears
import csv
import time
ifile = open ("C:\Users\BKA4ABT\Desktop\Test_Specification\RDBI.csv", "rb")
for line in csv.reader(ifile):
if not line:
empty_lines += 1
continue
print line
If you want to skip all whitespace lines, you should use this test: ' '.isspace().
Since you may want to do something more complicated than just printing the non-blank lines to the console(no need to use CSV module for that), here is an example that involves a DictReader:
#!/usr/bin/env python
# Tested with Python 2.7
# I prefer this style of importing - hides the csv module
# in case you do from this_file.py import * inside of __init__.py
import csv as _csv
# Real comments are more complicated ...
def is_comment(line):
return line.startswith('#')
# Kind of sily wrapper
def is_whitespace(line):
return line.isspace()
def iter_filtered(in_file, *filters):
for line in in_file:
if not any(fltr(line) for fltr in filters):
yield line
# A dis-advantage of this approach is that it requires storing rows in RAM
# However, the largest CSV files I worked with were all under 100 Mb
def read_and_filter_csv(csv_path, *filters):
with open(csv_path, 'rb') as fin:
iter_clean_lines = iter_filtered(fin, *filters)
reader = _csv.DictReader(iter_clean_lines, delimiter=';')
return [row for row in reader]
# Stores all processed lines in RAM
def main_v1(csv_path):
for row in read_and_filter_csv(csv_path, is_comment, is_whitespace):
print(row) # Or do something else with it
# Simpler, less refactored version, does not use with
def main_v2(csv_path):
try:
fin = open(csv_path, 'rb')
reader = _csv.DictReader((line for line in fin if not
line.startswith('#') and not line.isspace()),
delimiter=';')
for row in reader:
print(row) # Or do something else with it
finally:
fin.close()
if __name__ == '__main__':
csv_path = "C:\Users\BKA4ABT\Desktop\Test_Specification\RDBI.csv"
main_v1(csv_path)
print('\n'*3)
main_v2(csv_path)
Instead of
if not line:
This should work:
if not ''.join(line).strip():
my suggestion would be to just use the csv reader who can delimite the file into rows. Like this you can just check whether the row is empty and if so just continue.
import csv
with open('some.csv', 'r') as csvfile:
# the delimiter depends on how your CSV seperates values
csvReader = csv.reader(csvfile, delimiter = '\t')
for row in csvReader:
# check if row is empty
if not (row):
continue
You can always check for the number of comma separated values. It seems to be much more productive and efficient.
When reading the lines iteratively, as these are a list of comma separated values you would be getting a list object. So if there is no element (blank link), then we can make it skip.
with open(filename) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=",")
for row in csv_reader:
if len(row) == 0:
continue
You can strip leading and trailing whitespace, and if the length is zero after that the line is empty.
import csv
with open('userlist.csv') as f:
reader = csv.reader(f)
user_header = next(reader) # Add this line if there the header is
user_list = [] # Create a new user list for input
for row in reader:
if any(row): # Pick up the non-blank row of list
print (row) # Just for verification
user_list.append(row) # Compose all the rest data into the list
This example just prints the data in array form while skipping the empty lines:
import csv
file = open("data.csv", "r")
data = csv.reader(file)
for line in data:
if line: print line
file.close()
I find it much clearer than the other provided examples.
import csv
ifile=csv.reader(open('C:\Users\BKA4ABT\Desktop\Test_Specification\RDBI.csv', 'rb'),delimiter=';')
for line in ifile:
if set(line).pop()=='':
pass
else:
for cell_value in line:
print cell_value