I have a Python file were I want to add functions that I don't want to have in my main code like open csv files which is this case. The problem is when I want to invoke the function where I have the method "open with" the error:
"ValueError: I/O operation on closed file."
appears and for what I found this happens because indentation.
main.py:
from external_tools import read_csv
import unittest
class ClassTest(unittest.TestCase):
def test_a(self):
csv_file = "file.csv"
csv_reader = read_csv(csv_file)
for line in csv_reader:
print(line[0])
if __name__ == '__main__':
unittest.main()
external_tools.py
import datetime
import csv
def epoch_converter():
epoch_time = 40246871
date_time = datetime.datetime.fromtimestamp(epoch_time)
print("Given epoch time: ", epoch_time)
print("Converted Datetime: ", date_time)
def read_csv(csv_file):
with open(csv_file, 'r') as csv_file:
csv_reader = csv.reader(csv_file)
print(csv_reader)
return csv_reader
Is there any way to do it in this way or do I have to use open with in my main?
Here's how to do what I suggested in my comment, which was to turn read_csv into a context manager itself using contextlib.contextmanager as a function decorator.
main.py
from external_tools import read_csv
import unittest
class ClassTest(unittest.TestCase):
def test_a(self):
csv_file = "testfile.csv"
with read_csv(csv_file) as csv_reader:
for row in csv_reader:
print(row)
if __name__ == '__main__':
unittest.main()
external_tools.py
import datetime
from contextlib import contextmanager
import csv
def epoch_converter():
epoch_time = 40246871
date_time = datetime.datetime.fromtimestamp(epoch_time)
print("Given epoch time: ", epoch_time)
print("Converted Datetime: ", date_time)
#contextmanager
def read_csv(csv_file):
with open(csv_file, 'r', newline='') as csv_file:
csv_reader = csv.reader(csv_file)
yield csv_reader
print('csv file closed')
Related
I have simple code written in python. while writing into an excel file. I found additional rows get added each time. How can I skip the empty row added each time. and print data one after the other in an excel file
import csv
from datetime import datetime
import time
filename = 'testformat.csv'
fields = ['SLNo', 'Date', 'Time', 'RData', 'BData', 'GData', 'IRData']
date_format = datetime.now().strftime('%Y/%m/%d')
current_time = datetime.now().strftime('%I:%M:%S,%f')
def main():
with open(filename, 'w') as csvfile:
csvwriter = csv.writer(csvfile)
csvwriter.writerow(fields)
for i in range(30):
csvwriter.writerow([i, date_format, current_time])
if __name__ == '__main__':
main()
What you need is already here : https://stackoverflow.com/a/3191811/18081892
You have to use :
with open(filename, 'w', newline='', encoding='utf-8') as csvfile:
I am writing a CSV file with the following function:
import csv
import os
import aiofiles
async def write_extract_file(output_filename: str, csv_list: list):
"""
Write the extracted content into the file
"""
try:
async with aiofiles.open(output_filename, "w+") as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=columns.keys())
writer.writeheader()
writer.writerows(csv_list)
except FileNotFoundError:
print("Output file not present", output_filename)
print("Current dir: ", os.getcwd())
raise FileNotFoundError
However, as there is no await allowed over writerows method, there are no rows being written into the CSV file.
How to resolve this issue? Is there any workaround available?
Thank you.
Entire code can be found here.
You can use aiocsv. Here is a quick example of writing a row to a CSV file asynchronously:
import asyncio
import aiofiles
from aiocsv import AsyncWriter
async def main():
async with aiofiles.open('your-path.csv', 'w') as f:
writer = AsyncWriter(f)
await writer.writerow(['name', 'age'])
await writer.writerow(['John', 25])
asyncio.run(main())
For more examples follow: https://pypi.org/project/aiocsv/
In my opinion it’s better not to try to use the aiofiles with the csv module and run the synchronous code using loop.run_in_executor and wait it asynchronously like below:
def write_extract_file(output_filename: str, csv_list: list):
"""
Write the extracted content into the file
"""
try:
with open(output_filename, "w+") as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=columns.keys())
writer.writeheader()
writer.writerows(csv_list)
except FileNotFoundError:
print("Output file not present", output_filename)
print("Current dir: ", os.getcwd())
raise FileNotFoundError
async def main():
loop = asyncio.get_running_loop()
await loop.run_in_executor(None, write_extract_file, 'test.csv', csv_list)
You can use aiofiles, you just gotta convert the dict to a row :)
import aiofiles
async def write_extract_file(
output_filename: str, csv_list: list
):
cols = columns.keys()
async with aiofiles.open(output_filename, mode='w+') as f_out:
await f_out.write(','.join(cols)+'\n')
for data in csv_list:
line = []
for c in cols:
line.append(str(data[c]) if c in data else '')
line = ','.join(line) + '\n'
await f_out.write(line)
New to Django/Python. I need to write an import script for a CSV file to seed some data (not using fixtures, did that already as that is JSON based and not CSV).
This works:
import csv
from datetime import datetime
from django.utils.timezone import make_aware
from django.core.management.base import BaseCommand
from chatterbox.models import Organisation, Course, Student
class Command(BaseCommand):
def handle(self, **options):
CSV_PATH = './students_AEKI.csv'
Student.objects.filter(organisation__name__exact="AEKI").delete()
with open(CSV_PATH) as file:
file.readline() # skip the header
csv_reader = csv.reader(file, delimiter=',')
org = Organisation.objects.filter(name="AEKI")
for row in csv_reader:
_, Student.objects.get_or_create(
first_name=row[0],
last_name=row[1],
email=row[2],
organisation=org[0],
enrolled=row[4],
last_booking=row[5],
credits_total=row[6],
credits_balance=row[7],
)
This does NOT work:
import csv
from datetime import datetime
from django.utils.timezone import make_aware
from django.core.management.base import BaseCommand
from chatterbox.models import Organisation, Course, Student
class Command(BaseCommand):
def handle(self, **options):
CSV_PATH = './students_AEKI.csv'
Student.objects.filter(organisation__name__exact="AEKI").delete()
with open(CSV_PATH) as file:
file.readline() # skip the header
csv_reader = csv.reader(file, delimiter=',')
org = Organisation.objects.filter(name="AEKI")
for row in csv_reader:
enrolled_utc = make_aware(datetime.strptime(row[4], '%Y-%m-%d'))
last_booking_utc = make_aware(datetime.strptime((row[5], '%Y-%m-%d'))
_, Student.objects.get_or_create(
first_name=row[0],
last_name=row[1],
email=row[2],
organisation=org[0],
enrolled=enrolled_utc,
last_booking=last_booking_utc,
credits_total=row[6],
credits_balance=row[7],
)
Syntax error at the "_".
I need to do some manipulation (eg like adding timezone to date fields) on data before creating it in the table. So what is wrong with the 2nd version?
There's a Syntax error at the "_". Remove the trailing characters.
Also this line has an extra bracket:
last_booking_utc = datetime.strptime((row[5], '%Y-%m-%d')
From
for row in csv_reader:
enrolled_utc = make_aware(datetime.strptime(row[4], '%Y-%m-%d'))
last_booking_utc = make_aware(datetime.strptime((row[5], '%Y-%m-%d'))
_, Student.objects.get_or_create(
first_name=row[0],
last_name=row[1],
email=row[2],
organisation=org[0],
enrolled=enrolled_utc,
last_booking=last_booking_utc,
credits_total=row[6],
credits_balance=row[7],
)
To
for row in csv_reader:
enrolled_utc = make_aware(datetime.strptime(row[4], '%Y-%m-%d'))
last_booking_utc = make_aware(datetime.strptime(row[5], '%Y-%m-%d'))
Student.objects.get_or_create(
first_name=row[0],
last_name=row[1],
email=row[2],
organisation=org[0],
enrolled=enrolled_utc,
last_booking=last_booking_utc,
credits_total=row[6],
credits_balance=row[7],
)
When using this py code to split a large csv into smaller csv's (about ) I am receiving the error:
"OSError: [Error 24] Too Many Open Files:"
After running this there should be 29,930 separate files, however its stopping after 2048.
I have done some research and it looks like there is a per process limit of 2048. How can I get around this?
#!/usr/bin/env python3
import binascii
import csv
import os.path
import sys
from tkinter.filedialog import askopenfilename, askdirectory
from tkinter.simpledialog import askinteger
def split_csv_file(f, dst_dir, keyfunc):
csv_reader = csv.reader(f)
header = next(csv_reader)
csv_writers = {}
for row in csv_reader:
k = keyfunc(row)
if k not in csv_writers:
writer = csv.writer(open(os.path.join(dst_dir, k),
mode='w', newline=''))
writer.writerow(header)
csv_writers[k] = writer
csv_writers[k].writerow(row[0:1])
def get_args_from_cli():
input_filename = sys.argv[1]
column = int(sys.argv[2])
dst_dir = sys.argv[3]
return (input_filename, column, dst_dir)
def get_args_from_gui():
input_filename = askopenfilename(
filetypes=(('CSV', '.csv'),),
title='Select CSV Input File')
column = askinteger('Choose Table Column', 'Table column')
dst_dir = askdirectory(title='Select Destination Directory')
return (input_filename, column, dst_dir)
if __name__ == '__main__':
if len(sys.argv) == 1:
input_filename, column, dst_dir = get_args_from_gui()
elif len(sys.argv) == 4:
input_filename, column, dst_dir = get_args_from_cli()
else:
raise Exception("Invalid number of arguments")
with open(input_filename, mode='r', newline='') as f:
split_csv_file(f, dst_dir, lambda r: r[column-1]+'.csv')
# if the column has funky values resulting in invalid filenames
# replace the line from above with:
# split_csv_file(f, dst_dir, lambda r: binascii.b2a_hex(r[column-1].encode('utf-8')).decode('utf-8')+'.csv')
You don't need to keep the csv writers in a dictionary. You can re-open the file to append to it:
Replace:
if k not in csv_writers:
csv_writers[k] = csv.writer(open(os.path.join(dst_dir, k),
mode='w', newline=''))
csv_writers[k].writerow(row)
With:
filename = os.path.join(dst_dir, k)
with open(filename, mode='a', newline='') as output:
csv.writer(output).writerow(row)
Can anyone explain why this won't print anything?
import csv
def main():
with open('MaxWatt1.csv', 'rb') as f:
reader = csv.reader(f)
for row in reader:
print row
You need to call the main function at the end of the program:
import csv
def main():
with open('MaxWatt1.csv', 'rb') as f:
reader = csv.reader(f)
for row in reader:
print row
main() # Call main function.
Python does not have a main function like C/C++ does (one which gets called implicitly when you run the program). Instead, Python treats the function you have defined as it would any other function. The name main is only significant to the humans reading your code (and maybe some code analysis tools).
Actually, it would probably be best to do:
import csv
def main():
with open('MaxWatt1.csv', 'rb') as f:
reader = csv.reader(f)
for row in reader:
print row
if __name__ == '__main__':
main()
This ensures that the main function is only called when you run the program directly. If you import your file however, the call to main will be skipped. For more information, see:
What does if __name__ == "__main__": do?
So to add to what iCodez said:
import csv
def main():
with open('MaxWatt1.csv', 'rb') as f:
reader = csv.reader(f)
for row in reader:
print row
main()
will work for you