Related
I have 2 CSVs which are New.csv and Old.csv shown below:
Old.csv
longName,shortName,eventType,number,severity
ACTAGENT201,ACAT201,RES,1,INFO
ACTAGENT202,ACAT202,RES,2,ALERT
ACODE801,AC801,ADMIN,1,MINOR
ACODE802,AC802,ADMIN,2,MINOR
ACODE102,AC102,COMM,2,CRITICAL
ACODE103,AC103,COMM,3,CRITICAL
ACODE104,AC104,COMM,4,CRITICAL
ACODE105,AC105,COMM,5,CRITICAL
ACODE106,AC106,COMM,6,CRITICAL
New.csv
longName,shortName,eventType,number,severity
ACTAGENT201,ACAT201,RES,1,INFO
ACTAGENT202,ACAT202,RES,2,ALERT
ACODE801,AC801,ADMIN,1,MINOR
ACODE802,AC802,ThisHasBeenChanged,2,MINOR
ACODE102,AC102,COMM,2,CRITICAL
ACODE103,AC103,COMM,3,CRITICAL
ACODE104,AC104,COMM,4,THISHASBEENCHANGED
ACODE105,AC105,COMM,5,CRITICAL
ACODE106,AC106,COMM,6,CRITICAL
If there is data in one of the columns in the row that has been modified/changed between the old.csv and the new.csv then that whole row should be appended to the changes.csv like this with each column from old.csv and new.csv beside each other:
I know how to find new and deleted items in the csv, but could not figure out how to get the modified items. Code below:
import csv
def DeletedItems(old_csv, new_csv, changes_csv):
with open(new_csv, newline="", encoding="utf8") as new_fp:
csv_reader = csv.reader(new_fp)
csv_headings = next(csv_reader)
new_long_names = {row[0] for row in csv.reader(new_fp)}
with open(old_csv, newline="", encoding="utf8") as old_fp:
with open(changes_csv, "a", newline="", encoding="utf8") as changes_fp:
writer = csv.writer(changes_fp)
writer.writerow("")
for row in csv.reader(old_fp):
if row[0] not in new_long_names:
writer.writerow(row)
def NewItems(old_csv, new_csv, changes_csv):
with open(old_csv, newline="", encoding="utf8") as old_fp:
csv_reader = csv.reader(old_fp)
csv_headings = next(csv_reader)
old_long_names = {row[0] for row in csv.reader(old_fp)}
with open(new_csv, newline="", encoding="utf8") as new_fp:
with open(changes_csv, "w", newline="", encoding="utf8") as changes_fp:
writer = csv.writer(changes_fp)
for row in csv.reader(new_fp):
if row[0] not in old_long_names:
writer.writerow(row)
NewItems("old.csv", "new.csv", "changes.csv")
DeletedItems("old.csv", "new.csv", "changes.csv")
First, read both CSV files into a dictionary, using the longName values as keys.
import csv
with open(old_csv_file, "r") as fh:
reader = csv.reader(fh)
old_csv = {row[0]: row for row in reader}
with open(new_csv_file, "r") as fh:
reader = csv.reader(fh)
new_csv = {row[0]: row for row in reader}
Then, it's easy to find newly added and deleted keys using set operations.
old_longNames = set(old_csv.keys())
new_longNames = set(new_csv.keys())
# common: set intersection
common_longNames = old_longNames.intersection(new_longNames)
# removed: whatever's in old but not in new
removed_longNames = old_longNames - new_longNames
# added: whatever's in new but not in old
added_longNames = new_longNames - old_longNames
Finally, iterate over the common set to find where there are changes:
changed_longNames = []
for key in common_longNames:
old_row = old_csv[key]
new_row = new_csv[key]
# if any(o != n for o, n in zip(old_row, new_row)):
if old_row != new_row:
# this row has at least one column changed. Do whatever
print(f"LongName {key} has changes")
changed_longNames.append(key)
Or, as a list comprehension:
changed_longNames = [key for key in common_longNames if old_csv[key] != new_csv[key]]
Writing everything to a new csv file is also fairly trivial. Note that the sets don't preserve the order, so you might not get the result in the same order.
with open("deleted.csv", "w") as fh:
writer = csv.writer(fh)
for key in removed_longNames:
writer.writerow(old_csv[key])
with open("inserted.csv", "w") as fh:
writer = csv.writer(fh)
for key in added_longNames:
writer.writerow(new_csv[key])
with open("changed.csv", "w") as fh:
writer = csv.writer(fh)
for key in changed_longNames:
old_row = old_csv[key]
new_row = new_csv[key]
merged_row = []
for oi, ni in zip(old_row, new_row):
merged_row.append(oi)
merged_row.append(ni)
writer.writerow(merged_row)
Hello I'm trying to make a program that updates the values in a csv. The user searches for the ID, and if the ID exists, it gets the new values you want to replace on the row where that ID number is. Here row[0:9] is the length of my ID.
My idea was to scan each row from 0-9 or where my ID number is, and when its found, I will replace the values besides it using the .replace() method. This how i did it:
def update_thing():
replace = stud_ID +','+ stud_name +','+ stud_course +','+ stud_year
empty = []
with open(fileName, 'r+') as upFile:
for row in f:
if row[0:9] == stud_ID:
row=row.replace(row,replace)
msg = Label(upd_win, text="Updated Successful", font="fixedsys 12 bold").place(x=3,y=120)
if not row[0:9] == getID:
empty.append(row)
upFile.close()
upFile = open(fileName, 'w')
upFile.writelines(empty)
upFile.close()
But it's not working, I need ideas on how to get through this.
With the csv module you can iterate over the rows and access each one as a dict. As also noted here, the preferred way to update a file is by using temporary file.
from tempfile import NamedTemporaryFile
import shutil
import csv
filename = 'my.csv'
tempfile = NamedTemporaryFile(mode='w', delete=False)
fields = ['ID', 'Name', 'Course', 'Year']
with open(filename, 'r') as csvfile, tempfile:
reader = csv.DictReader(csvfile, fieldnames=fields)
writer = csv.DictWriter(tempfile, fieldnames=fields)
for row in reader:
if row['ID'] == str(stud_ID):
print('updating row', row['ID'])
row['Name'], row['Course'], row['Year'] = stud_name, stud_course, stud_year
row = {'ID': row['ID'], 'Name': row['Name'], 'Course': row['Course'], 'Year': row['Year']}
writer.writerow(row)
shutil.move(tempfile.name, filename)
If that's still not working you might try one of these encodings:
with open(filename, 'r', encoding='utf8') as csvfile, tempfile:
with open(filename, 'r', encoding='ascii') as csvfile, tempfile:
Edit: added str, print and encodings
Simply write to a new file at same time reading over the lines of original, conditionally changing the row based on Stud_ID value. New file is suffixed _new in name.
line_replace = stud_ID +','+ stud_name +','+ stud_course +','+ stud_year
with open(fileName, 'r') as readFile, open(fileName.replace('.csv', '_new.csv'), 'w') as writeFile:
for row in readFile:
if row[0:9] == stud_ID:
writeFile.write(line_replace)
msg = Label(upd_win, text="Updated Successful", font="fixedsys 12 bold").place(x=3,y=120)
else:
writeFile.write(row)
name = input("Enter a name : ")
fieldnames = ["first_name", "number"]
with open('names.csv') as csvfile, open('output.csv', 'w') as outputfile:
reader = csv.DictReader(csvfile, fieldnames=fieldnames)
writer = csv.DictWriter(outputfile, fieldnames=fieldnames)
for line in reader:
if name not in line:
fieldnames = ["name", "number"]
writer.writeheader()
writer.writerow({'first_name': row['first_name'],
'number': row['number']})
with open('names.csv', 'w') as csvfile, open('output.csv') as outputfile:
reader = csv.DictReader(outputfile, fieldnames=fieldnames)
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
for row in reader:
fieldnames = ['first_name', 'number']
writer.writeheader()
writer.writerow({'first_name': row['first_name'],
'last_name': row['number']})
This is what I've coded so far. I want to write the rows which doesn't include name variable in it to the output file and write it back to the csvfile (names.csv)
James Smith,2025550131
Kevin Harris,2025550105
This is how my csvfile looks like.
first_name,last_name
James Smith,2025550131
first_name,last_name
James Smith,2025550131
This is the names.csv file after I run the code.
row isn't declared in your first write loop, so running this in an IDE could have memorized the output of a semi-successful attempt and you have a weird issue.
You also write the header each time. Just don't, it's done by default.
Also: you're comparing to the keys, not to the values. if name not in row: checks if the name is a key, which cannot happen, keys are "first_name", and "number".
You need to do if name != row['first_name']:
Last but not least, no need to read/write the file again to replace the input, just perform a shutil.move to overwrite with the output.
My fix proposal (note newline='', better when using python 3 to avoid blank lines on some not so up-to-date versions):
import shutil,csv
name = input("Enter a name : ")
fieldnames = ["first_name", "number"]
with open('names.csv') as csvfile, open('output.csv', 'w',newline='') as outputfile:
reader = csv.DictReader(csvfile, fieldnames=fieldnames)
writer = csv.DictWriter(outputfile, fieldnames=fieldnames)
for row in reader: # row, not line !!
if name != row['first_name']:
writer.writerow({'first_name': row['first_name'], 'number': row['number']})
# replace original file
shutil.move('output.csv','names.csv')
You need to read the input name.csv file.
Check in name matches the input name, then ignore that row.
If it does not match then write the content to output file.
In the end just copy the output file over the original name.csv file.
import csv, shutil
name = 'vikash' # This you can take as input from user
fieldnames = ["first_name", "number"]
with open('names.csv', 'r') as csvfile, open('output.csv', 'w') as outputfile:
reader = csv.DictReader(csvfile, fieldnames=fieldnames)
writer = csv.DictWriter(outputfile, fieldnames=fieldnames)
for row in reader:
if not name == row['first_name']:
writer.writerow({'first_name': row['first_name'], 'number': row['number']})
shutil.move('output.csv','names.csv')
names.csv
first_name,number
vikash,1
viki,2
pawan,3
output.csv
first_name,number
viki,2
pawan,3
The reason why your code didn't work:
name = input("Enter a name : ")
fieldnames = ["first_name", "number"]
with open('names.csv') as csvfile, open('output.csv', 'w') as outputfile:
reader = csv.DictReader(csvfile, fieldnames=fieldnames)
writer = csv.DictWriter(outputfile, fieldnames=fieldnames)
for line in reader:
# you are reading line from reader but you are using row later.
if name not in line:
fieldnames = ["name", "number"]
# above line is not needed as its already initialised earlier.
writer.writeheader()
# this is not required as header will be written automatically.
# Plus you also don't want to write header ever time you write a row to the output file.
writer.writerow({'first_name': row['first_name'], 'number': row['number']})
# use line variable here or change line above to row.
I want to add a new column to an existing file. But it gets a little complicated with the additional loops i add.
input file:
testfile.csv
col1,col2,col3
1,2,3
3,4,5
4,6,7
output i want:
USA_testfile.csv
col1,col2,col3,country
1,2,3,USA
3,4,5,USA
4,6,7,USA
UK_testfile.csv
col1,col2,col3,country
1,2,3,UK
3,4,5,UK
4,6,7,UK
This is what i have tried:
import csv
import sys
country_list= ['USA', 'UK']
def add_col(csv_file):
for country in country_list:
with open(csv_file, 'rb') as fin:
with open(country+"_timeline_outfile_"+csv_file, 'wb') as fout:
writer = csv.writer(fout, lineterminator='\n')
reader = csv.reader(fin)
all_rows =[]
row = next(reader)
row.append('country')
all_rows.append(row)
print all_rows
for row in reader:
row.append(country)
all_rows.append(row)
writer.writerows(all_rows)
add_col(sys.argv[1])
And this is the error i got:
File "write_to_csv.py", line 33, in add_col
writer.writerows(all_rows)
ValueError: I/O operation on closed file
I was trying to follow this post here
import csv
countries = ['USA', 'UK']
data = list(csv.reader(open('testfile.csv', 'rb')))
for country in countries:
with open('{0}_testfile.csv'.format(country), 'wb') as f:
writer = csv.writer(f)
for i, row in enumerate(data):
if i == 0:
row = row + ['country']
else:
row = row + [country]
writer.writerow(row)
I couldn't reproduce your error, but i cleaned your code a bit.
There is no reason to reopen the input file for every language.
def add_col(csv_file):
with open(csv_file, 'rb') as fin:
reader = csv.reader(fin)
for country in country_list:
fin.seek(0) # jump to begin of file again
with open(country+"_timeline_outfile_"+csv_file, 'wb') as fout:
writer = csv.writer(fout, lineterminator='\n')
header = next(reader)
header.append('country')
writer.writerow(header)
for row in reader:
row.append(country)
writer.writerow(row)
So I have a program that creates CSV from .Json.
First I load the json file.
f = open('Data.json')
data = json.load(f)
f.close()
Then I go through it, looking for a specific keyword, if I find that keyword. I'll write everything related to that in a .csv file.
for item in data:
if "light" in item:
write_light_csv('light.csv', item)
This is my write_light_csv function :
def write_light_csv(filename,dic):
with open (filename,'a') as csvfile:
headers = ['TimeStamp', 'light','Proximity']
writer = csv.DictWriter(csvfile, delimiter=',', lineterminator='\n',fieldnames=headers)
writer.writeheader()
writer.writerow({'TimeStamp': dic['ts'], 'light' : dic['light'],'Proximity' : dic['prox']})
I initially had wb+ as the mode, but that cleared everything each time the file was opened for writing. I replaced that with a and now every time it writes, it adds a header. How do I make sure that header is only written once?.
You could check if file is already exists and then don't call writeheader() since you're opening the file with an append option.
Something like that:
import os.path
file_exists = os.path.isfile(filename)
with open (filename, 'a') as csvfile:
headers = ['TimeStamp', 'light', 'Proximity']
writer = csv.DictWriter(csvfile, delimiter=',', lineterminator='\n',fieldnames=headers)
if not file_exists:
writer.writeheader() # file doesn't exist yet, write a header
writer.writerow({'TimeStamp': dic['ts'], 'light': dic['light'], 'Proximity': dic['prox']})
Just another way:
with open(file_path, 'a') as file:
w = csv.DictWriter(file, my_dict.keys())
if file.tell() == 0:
w.writeheader()
w.writerow(my_dict)
You can check if the file is empty
import csv
import os
headers = ['head1', 'head2']
for row in interator:
with open('file.csv', 'a') as f:
file_is_empty = os.stat('file.csv').st_size == 0
writer = csv.writer(f, lineterminator='\n')
if file_is_empty:
writer.writerow(headers)
writer.writerow(row)
I would use some flag and run a check before writing headers! e.g.
flag=0
def get_data(lst):
for i in lst:#say list of url
global flag
respons = requests.get(i)
respons= respons.content.encode('utf-8')
respons=respons.replace('\\','')
print respons
data = json.loads(respons)
fl = codecs.open(r"C:\Users\TEST\Desktop\data1.txt",'ab',encoding='utf-8')
writer = csv.DictWriter(fl,data.keys())
if flag==0:
writer.writeheader()
writer.writerow(data)
flag+=1
print "You have written % times"%(str(flag))
fl.close()
get_data(urls)
Can you change the structure of your code and export the whole file at once?
def write_light_csv(filename, data):
with open (filename, 'w') as csvfile:
headers = ['TimeStamp', 'light','Proximity']
writer = csv.DictWriter(csvfile, delimiter=',', lineterminator='\n',fieldnames=headers)
writer.writeheader()
for item in data:
if "light" in item:
writer.writerow({'TimeStamp': item['ts'], 'light' : item['light'],'Proximity' : item['prox']})
write_light_csv('light.csv', data)
You can use the csv.Sniffer Class and
with open('my.csv', newline='') as csvfile:
if csv.Sniffer().has_header(csvfile.read(1024))
# skip writing headers
While using Pandas: (for storing Dataframe data to CSV file)
just add this check before setting header property if you are using an index to iterate over API calls to add data in CSV file.
if i > 0:
dataset.to_csv('file_name.csv',index=False, mode='a', header=False)
else:
dataset.to_csv('file_name.csv',index=False, mode='a', header=True)
Here's another example that only depends on Python's builtin csv package. This method checks that the header is what's expected or it throws an error. It also handles the case where the file doesn't exist or does exist but is empty by writing the header. Hope this helps:
import csv
import os
def append_to_csv(path, fieldnames, rows):
is_write_header = not os.path.exists(path) or _is_empty_file(path)
if not is_write_header:
_assert_field_names_match(path, fieldnames)
_append_to_csv(path, fieldnames, rows, is_write_header)
def _is_empty_file(path):
return os.stat(path).st_size == 0
def _assert_field_names_match(path, fieldnames):
with open(path, 'r') as f:
reader = csv.reader(f)
header = next(reader)
if header != fieldnames:
raise ValueError(f'Incompatible header: expected {fieldnames}, '
f'but existing file has {header}')
def _append_to_csv(path, fieldnames, rows, is_write_header: bool):
with open(path, 'a') as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
if is_write_header:
writer.writeheader()
writer.writerows(rows)
You can test this with the following code:
file_ = 'countries.csv'
fieldnames_ = ['name', 'area', 'country_code2', 'country_code3']
rows_ = [
{'name': 'Albania', 'area': 28748, 'country_code2': 'AL', 'country_code3': 'ALB'},
{'name': 'Algeria', 'area': 2381741, 'country_code2': 'DZ', 'country_code3': 'DZA'},
{'name': 'American Samoa', 'area': 199, 'country_code2': 'AS', 'country_code3': 'ASM'}
]
append_to_csv(file_, fieldnames_, rows_)
If you run this once you get the following in countries.csv:
name,area,country_code2,country_code3
Albania,28748,AL,ALB
Algeria,2381741,DZ,DZA
American Samoa,199,AS,ASM
And if you run it twice you get the following (note, no second header):
name,area,country_code2,country_code3
Albania,28748,AL,ALB
Algeria,2381741,DZ,DZA
American Samoa,199,AS,ASM
Albania,28748,AL,ALB
Algeria,2381741,DZ,DZA
American Samoa,199,AS,ASM
If you then change the header in countries.csv and run the program again, you'll get a value error, like this:
ValueError: Incompatible header: expected ['name', 'area', 'country_code2', 'country_code3'], but existing file has ['not', 'right', 'fieldnames']