So I have a program that creates CSV from .Json.
First I load the json file.
f = open('Data.json')
data = json.load(f)
f.close()
Then I go through it, looking for a specific keyword, if I find that keyword. I'll write everything related to that in a .csv file.
for item in data:
if "light" in item:
write_light_csv('light.csv', item)
This is my write_light_csv function :
def write_light_csv(filename,dic):
with open (filename,'a') as csvfile:
headers = ['TimeStamp', 'light','Proximity']
writer = csv.DictWriter(csvfile, delimiter=',', lineterminator='\n',fieldnames=headers)
writer.writeheader()
writer.writerow({'TimeStamp': dic['ts'], 'light' : dic['light'],'Proximity' : dic['prox']})
I initially had wb+ as the mode, but that cleared everything each time the file was opened for writing. I replaced that with a and now every time it writes, it adds a header. How do I make sure that header is only written once?.
You could check if file is already exists and then don't call writeheader() since you're opening the file with an append option.
Something like that:
import os.path
file_exists = os.path.isfile(filename)
with open (filename, 'a') as csvfile:
headers = ['TimeStamp', 'light', 'Proximity']
writer = csv.DictWriter(csvfile, delimiter=',', lineterminator='\n',fieldnames=headers)
if not file_exists:
writer.writeheader() # file doesn't exist yet, write a header
writer.writerow({'TimeStamp': dic['ts'], 'light': dic['light'], 'Proximity': dic['prox']})
Just another way:
with open(file_path, 'a') as file:
w = csv.DictWriter(file, my_dict.keys())
if file.tell() == 0:
w.writeheader()
w.writerow(my_dict)
You can check if the file is empty
import csv
import os
headers = ['head1', 'head2']
for row in interator:
with open('file.csv', 'a') as f:
file_is_empty = os.stat('file.csv').st_size == 0
writer = csv.writer(f, lineterminator='\n')
if file_is_empty:
writer.writerow(headers)
writer.writerow(row)
I would use some flag and run a check before writing headers! e.g.
flag=0
def get_data(lst):
for i in lst:#say list of url
global flag
respons = requests.get(i)
respons= respons.content.encode('utf-8')
respons=respons.replace('\\','')
print respons
data = json.loads(respons)
fl = codecs.open(r"C:\Users\TEST\Desktop\data1.txt",'ab',encoding='utf-8')
writer = csv.DictWriter(fl,data.keys())
if flag==0:
writer.writeheader()
writer.writerow(data)
flag+=1
print "You have written % times"%(str(flag))
fl.close()
get_data(urls)
Can you change the structure of your code and export the whole file at once?
def write_light_csv(filename, data):
with open (filename, 'w') as csvfile:
headers = ['TimeStamp', 'light','Proximity']
writer = csv.DictWriter(csvfile, delimiter=',', lineterminator='\n',fieldnames=headers)
writer.writeheader()
for item in data:
if "light" in item:
writer.writerow({'TimeStamp': item['ts'], 'light' : item['light'],'Proximity' : item['prox']})
write_light_csv('light.csv', data)
You can use the csv.Sniffer Class and
with open('my.csv', newline='') as csvfile:
if csv.Sniffer().has_header(csvfile.read(1024))
# skip writing headers
While using Pandas: (for storing Dataframe data to CSV file)
just add this check before setting header property if you are using an index to iterate over API calls to add data in CSV file.
if i > 0:
dataset.to_csv('file_name.csv',index=False, mode='a', header=False)
else:
dataset.to_csv('file_name.csv',index=False, mode='a', header=True)
Here's another example that only depends on Python's builtin csv package. This method checks that the header is what's expected or it throws an error. It also handles the case where the file doesn't exist or does exist but is empty by writing the header. Hope this helps:
import csv
import os
def append_to_csv(path, fieldnames, rows):
is_write_header = not os.path.exists(path) or _is_empty_file(path)
if not is_write_header:
_assert_field_names_match(path, fieldnames)
_append_to_csv(path, fieldnames, rows, is_write_header)
def _is_empty_file(path):
return os.stat(path).st_size == 0
def _assert_field_names_match(path, fieldnames):
with open(path, 'r') as f:
reader = csv.reader(f)
header = next(reader)
if header != fieldnames:
raise ValueError(f'Incompatible header: expected {fieldnames}, '
f'but existing file has {header}')
def _append_to_csv(path, fieldnames, rows, is_write_header: bool):
with open(path, 'a') as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
if is_write_header:
writer.writeheader()
writer.writerows(rows)
You can test this with the following code:
file_ = 'countries.csv'
fieldnames_ = ['name', 'area', 'country_code2', 'country_code3']
rows_ = [
{'name': 'Albania', 'area': 28748, 'country_code2': 'AL', 'country_code3': 'ALB'},
{'name': 'Algeria', 'area': 2381741, 'country_code2': 'DZ', 'country_code3': 'DZA'},
{'name': 'American Samoa', 'area': 199, 'country_code2': 'AS', 'country_code3': 'ASM'}
]
append_to_csv(file_, fieldnames_, rows_)
If you run this once you get the following in countries.csv:
name,area,country_code2,country_code3
Albania,28748,AL,ALB
Algeria,2381741,DZ,DZA
American Samoa,199,AS,ASM
And if you run it twice you get the following (note, no second header):
name,area,country_code2,country_code3
Albania,28748,AL,ALB
Algeria,2381741,DZ,DZA
American Samoa,199,AS,ASM
Albania,28748,AL,ALB
Algeria,2381741,DZ,DZA
American Samoa,199,AS,ASM
If you then change the header in countries.csv and run the program again, you'll get a value error, like this:
ValueError: Incompatible header: expected ['name', 'area', 'country_code2', 'country_code3'], but existing file has ['not', 'right', 'fieldnames']
Related
how can I put my first row of data in the csv under the header and not in the same row as header?
This is the results.
And down here is my coding.
import os
# ...
filename = 'C:/Desktop/GPS_Trial/Trial6/' + str(d1) + '_' + str(file_counter) +'.csv'
#check whether the file exist or not
rows_to_be_written = []
if not os.path.exists(filename):
rows_to_be_written.append(header1)
rows_to_be_written.append(header2)
rows_to_be_written.append(header3)
rows_to_be_written.append(gps)
rows_to_be_written.append(gps2)
rows_to_be_written.append(gps3)
#write the data into csv
with open(filename, 'a', newline='', encoding='UTF8') as f:
writer = csv.writer(f, delimiter=',')
writer.writerow(rows_to_be_written)
print(gps, gps2, gps3)
You write header with values in one row if it file not exists.
You should write it separately
rows_to_be_written = []
header = None
if not os.path.exists(filename):
header = [header1, header2, header3]
rows_to_be_written.append(gps)
rows_to_be_written.append(gps2)
rows_to_be_written.append(gps3)
# write the data into csv
with open(filename, 'a', newline='', encoding='UTF8') as f:
writer = csv.writer(f, delimiter=',')
if header:
writer.writerow(header)
writer.writerow(rows_to_be_written)
print(gps, gps2, gps3)
Also may be you tried write rows, but you write only one row with header in it. Then change code like this
rows_to_be_written = []
if not os.path.exists(filename):
rows_to_be_written.append([header1, header2, header3])
rows_to_be_written.append([gps, gps2, gps3])
# write the data into csv
with open(filename, 'a', newline='', encoding='UTF8') as f:
writer = csv.writer(f, delimiter=',')
for row in rows_to_be_written:
writer.writerow(row)
print(gps, gps2, gps3)
You need to add the headings separately, and only if they are not there already:
# check whether the file exist or not
if not os.path.exists(filename):
headings = [header1, header2, header3]
else:
headings = None
rows_to_be_written = [gps, gps2, gps3]
# write the data into csv
with open(filename, 'a', newline='', encoding='UTF8') as f:
writer = csv.writer(f)
# Write headings if exist
if headings != None:
writer.writerow(headings)
# Write rows
writer.writerow(rows_to_be_written)
print(gps, gps2, gps3)
I suggest you consider this approach
# Open file to see if there are headings
with open(filename, "r") as f:
try:
has_headings = csv.Sniffer().has_header(f.read(1024))
except csv.Error:
# The file seems to be empty
has_headings = False
# Open to write. In append mode ("a")
with open(filename, "a") as f:
writer = csv.writer(f)
if has_headings:
# Write the rows at the top
writer.writerow(headings_list)
# Use writerows if youe have a 2D list, else use a for loop of writer.writerow
writer.writerows(lists_of_rows)
I am trying to attempt something that I have not before within python.
The code below collects data from my test database and put it into a text under my headers of 'Test1','Test2','Test3'. This is working fine.
What I am trying to attempt now is to add a header (on top of the current header) and footer to the file.
python code:
file = 'file.txt'
header_names = {'t1':'Test1', 't2': 'Test2','t3':'Test3'}
with open(file, 'w', newline='') as f:
w = csv.DictWriter(f, fieldnames=header_names.keys(), restval='', extrasaction='ignore')
w.writerow(header_names)
for doc in res['test']['test']:
my_dict = doc['test']
w.writerow(my_dict)
current file output using the above code.
file.txt
Test1,Test2,Test3
Bob,john,Male
Cat,Long,female
Dog,Short,Male
Case,Fast,Male
Nice,who,Male
ideal txt output.
{header}
Filename:file.txt
date:
{data}
Test1,Test2,Test3
Bob,john,Male
Cat,Long,female
Dog,Short,Male
Case,Fast,Male
Nice,who,Male
{Footer}
this file was generated by using python.
the {header}, {data} and {footer} is not needed within the file that is just to make clear what is needed. i hope this makes sense.
Something like this
import csv
from datetime import date
# prepare some sample data
data = [['Bob', 'John', 'Male'],
['Cat', 'Long', 'Female']]
fieldnames = ['test1', 'test2', 'test3']
data = [dict(zip(fieldnames, row)) for row in data]
# actual part that writes to a file
with open('spam.txt', 'w', newline='') as f:
f.write('filename:spam.txt\n')
f.write(f'date:{date.today().strftime("%Y%m%d")}\n\n')
wrtr = csv.DictWriter(f, fieldnames = fieldnames)
wrtr.writeheader()
wrtr.writerows(data)
f.write('\nwritten with python\n')
Output in the file:
filename:spam.txt
date:20190321
test1,test2,test3
Bob,John,Male
Cat,Long,Female
written with python
Now, all that said, do you really need to write header and footer. It will just break a nicely formatted csv file and would require extra effort later on when reading it.
Or if you prefer - is the csv format what best suits your needs? Maybe using json would be better...
vardate= datetime.datetime.now().strftime("%x")
file = 'file.txt'
header_names = {'t1':'Test1', 't2': 'Test2','t3':'Test3'}
with open(file, 'w', newline='') as f:
f.seek(0,0) //This will move cursor to start position of file
f.writelines("File Name: ", file)
f.writelines("date: ", vardate)
f.writelines(".Try out next..")
w = csv.DictWriter(f, fieldnames=header_names.keys(), restval='',
extrasaction='ignore')
w.writerow(header_names)
for doc in res['test']['test']:
my_dict = doc['test']
w.writerow(my_dict)
f.seek(0,2)
f.writelines("This is generated using Python")
I've seen an older answer for the post, Inline CSV File Editing with Python, about how to modify a csv file and save it. It uses the tempfile module. I have used the same code in Python 3.6 but I get an error because I read the file in binary mode and I cannot iterate over it.
The code below creates a simple CSV file
import csv
with open('proglanguages.csv', mode='w', newline='') as csv_file:
fieldnames = ['lang', 'value']
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
writer.writerow({'lang': 'Java', 'value': '90'})
writer.writerow({'lang': 'c', 'value': '80'})
writer.writerow({'lang': 'Perl', 'value': '78'})
writer.writerow({'lang': 'C++', 'value': '92'})
writer.writerow({'lang': 'Python', 'value': '0'})
writer.writerow({'lang': 'Fortran', 'value': '70'})
The code below modifies the previous generated CSV file,
from tempfile import NamedTemporaryFile
import shutil
import csv
filename = 'proglanguages.csv'
tempfile = NamedTemporaryFile(delete=False)
with open(filename, mode='rb') as csvFile, tempfile:
reader = csv.reader(csvFile, delimiter=',', quotechar='"')
writer = csv.writer(tempfile, delimiter=',', quotechar='"')
for row in reader:
if row[0] == 'Python':
row[1] = '100'
writer.writerow(row)
else:
writer.writerow(row)
shutil.move(tempfile.name, filename)
In which way, could I use the for-loop for iteration, modify the item and write in the tempfile
You are not lucky here. Answers from Martijn Pieters are always high quality ones. This one was but was targetted at Python 2 while you use Python 3. And the csv semantics have changed... You should no longer use binary mode in Python 3 and do what you did for the initial csv file:
...
tempfile = NamedTemporaryFile(delete=False, newline='', mode='w')
with open(filename, mode='r', newline='') as csvFile, tempfile:
reader = csv.reader(csvFile)
writer = csv.writer(tempfile)
...
name = input("Enter a name : ")
fieldnames = ["first_name", "number"]
with open('names.csv') as csvfile, open('output.csv', 'w') as outputfile:
reader = csv.DictReader(csvfile, fieldnames=fieldnames)
writer = csv.DictWriter(outputfile, fieldnames=fieldnames)
for line in reader:
if name not in line:
fieldnames = ["name", "number"]
writer.writeheader()
writer.writerow({'first_name': row['first_name'],
'number': row['number']})
with open('names.csv', 'w') as csvfile, open('output.csv') as outputfile:
reader = csv.DictReader(outputfile, fieldnames=fieldnames)
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
for row in reader:
fieldnames = ['first_name', 'number']
writer.writeheader()
writer.writerow({'first_name': row['first_name'],
'last_name': row['number']})
This is what I've coded so far. I want to write the rows which doesn't include name variable in it to the output file and write it back to the csvfile (names.csv)
James Smith,2025550131
Kevin Harris,2025550105
This is how my csvfile looks like.
first_name,last_name
James Smith,2025550131
first_name,last_name
James Smith,2025550131
This is the names.csv file after I run the code.
row isn't declared in your first write loop, so running this in an IDE could have memorized the output of a semi-successful attempt and you have a weird issue.
You also write the header each time. Just don't, it's done by default.
Also: you're comparing to the keys, not to the values. if name not in row: checks if the name is a key, which cannot happen, keys are "first_name", and "number".
You need to do if name != row['first_name']:
Last but not least, no need to read/write the file again to replace the input, just perform a shutil.move to overwrite with the output.
My fix proposal (note newline='', better when using python 3 to avoid blank lines on some not so up-to-date versions):
import shutil,csv
name = input("Enter a name : ")
fieldnames = ["first_name", "number"]
with open('names.csv') as csvfile, open('output.csv', 'w',newline='') as outputfile:
reader = csv.DictReader(csvfile, fieldnames=fieldnames)
writer = csv.DictWriter(outputfile, fieldnames=fieldnames)
for row in reader: # row, not line !!
if name != row['first_name']:
writer.writerow({'first_name': row['first_name'], 'number': row['number']})
# replace original file
shutil.move('output.csv','names.csv')
You need to read the input name.csv file.
Check in name matches the input name, then ignore that row.
If it does not match then write the content to output file.
In the end just copy the output file over the original name.csv file.
import csv, shutil
name = 'vikash' # This you can take as input from user
fieldnames = ["first_name", "number"]
with open('names.csv', 'r') as csvfile, open('output.csv', 'w') as outputfile:
reader = csv.DictReader(csvfile, fieldnames=fieldnames)
writer = csv.DictWriter(outputfile, fieldnames=fieldnames)
for row in reader:
if not name == row['first_name']:
writer.writerow({'first_name': row['first_name'], 'number': row['number']})
shutil.move('output.csv','names.csv')
names.csv
first_name,number
vikash,1
viki,2
pawan,3
output.csv
first_name,number
viki,2
pawan,3
The reason why your code didn't work:
name = input("Enter a name : ")
fieldnames = ["first_name", "number"]
with open('names.csv') as csvfile, open('output.csv', 'w') as outputfile:
reader = csv.DictReader(csvfile, fieldnames=fieldnames)
writer = csv.DictWriter(outputfile, fieldnames=fieldnames)
for line in reader:
# you are reading line from reader but you are using row later.
if name not in line:
fieldnames = ["name", "number"]
# above line is not needed as its already initialised earlier.
writer.writeheader()
# this is not required as header will be written automatically.
# Plus you also don't want to write header ever time you write a row to the output file.
writer.writerow({'first_name': row['first_name'], 'number': row['number']})
# use line variable here or change line above to row.
I am a noobie.
I have written a couple of scripts to modify CSV files I work with.
The scripts:
1.) change the headers of a CSV file then save that to a new CSV file,.
2.) Load that CSV File, and change the order of select columns using DictWriter.
from tkinter import *
from tkinter import filedialog
import os
import csv
root = Tk()
fileName = filedialog.askopenfilename(filetypes=(("Nimble CSV files", "*.csv"),("All files", "*.*")))
outputFileName = os.path.splitext(fileName)[0] + "_deleteme.csv" #my temp file
forUpload = os.path.splitext(fileName)[0] + "_forupload.csv"
#Open the file - change the header then save the file
with open(fileName, 'r', newline='') as infile, open(outputFileName, 'w', newline='') as outfile:
reader = csv.reader(infile)
writer = csv.writer(outfile, delimiter=',', lineterminator='\n')
row1 = next(reader)
#new header names
row1[0] = 'firstname'
row1[1] = 'lastname'
row1[4] = 'phone'
row1[5] = 'email'
row1[11] = 'address'
row1[21] = 'website'
#write the temporary CSV file
writer.writerow(row1)
for row in reader:
writer.writerow(row)
#Open the temporary CSV file - rearrange some columns
with open(outputFileName, 'r', newline='') as dInFile, open(forUpload, 'w', newline='') as dOutFile:
fieldnames = ['email', 'title', 'firstname', 'lastname', 'company', 'phone', 'website', 'address', 'twitter']
dWriter = csv.DictWriter(dOutFile, restval='', extrasaction='ignore', fieldnames=fieldnames, lineterminator='\n')
dWriter.writeheader()
for row in csv.DictReader(dInFile):
dWriter.writerow(row)
My question is: Is there a more efficient way to do this?
It seems like I shouldn't have to make a temporary CSV file ("_deleteme.csv") I then delete.
I assume making the temporary CSV file is a rookie move -- is there a way to do this all with one 'With open' statement?
Thanks for any help, it is greatly appreciated.
--Luke
csvfile can be any object with a write() method. You could craft a custom element, or use StringIO. You'd have to verify efficiency yourself.