I was trying to copy the contents of csvfile into csvfile2. I cant see any help on this subject.and I also want to append a value value to the first row of csvfile2.Eg : value = 1
output should be like
`1_dog 1_cat 1_frog `
when value=2 result should be
2_dog 2_cat 2_frog
The value ranges between 1 and the total no of lines in csv file.(totalrows)
My trial code:
def csvupload(request, template_name='upload_csv.html'):
if request.method == 'POST':
form = CSVUploadForm(request.POST, request.FILES)
if form.is_valid():
error_list = set()
value = 1
csvfile = request.FILES['file']
file_name = csvfile.name
file_name = unicodedata.normalize('NFKD', file_name).encode('ascii','ignore')
csvfile2 = settings.MEDIA_ROOT + '/new/sample.csv'
writer = csv.writer(outfile, dialect='excel')
csvfile.open()
line_entries = csv.DictReader(csvfile, dialect='excel')
try:
csvfile.read()
line_entries = csv.DictReader(csvfile, dialect='excel', delimiter=',')
rows1 = list(csvfile)
totalrows = len(rows1)
success = True
except:
messages.error(request, "Could not parse file, invalid format")
success = False
return redirect('home')
if success:
messages.success(request, "File read..")
else:
for err in error_list:
messages.error(request, err)
else:
messages.error(request, "No file selected")
else:
form = CSVUploadForm()
context = {
'form': form
}
return render_to_response(template_name, context, context_instance=RequestContext(request))
This code is not complete.I am a new to python and confused.Can someone help?
I don't have a python interpreter handy so I cannot test. I can provide more complete answer later.
You need something like this -
To Read -
>>> import csv
>>> with open('names.csv') as csvfile:
... reader = csv.DictReader(csvfile)
... for row in reader:
... print(row['first_name'], row['last_name'])
To Write -
import csv
with open('names.csv', 'w') as csvfile:
fieldnames = ['first_name', 'last_name']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
writer.writerow({'first_name': 'Baked', 'last_name': 'Beans'})
writer.writerow({'first_name': 'Lovely', 'last_name': 'Spam'})
writer.writerow({'first_name': 'Wonderful', 'last_name': 'Spam'})
So, first, save the file in a temporary location in server -
def handle_uploaded_file(f): # method to save to a temporary location
with open('some/file/name.csv', 'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
Then load it and read line by line as above as mentioned in the docs. https://docs.python.org/2/library/csv.html#csv.DictReader
So your code might be something like -
def csvupload(request, template_name='upload_csv.html'):
if request.method == 'POST':
form = CSVUploadForm(request.POST, request.FILES)
if form.is_valid():
error_list = set()
value = 1
handle_uploaded_file(request.FILES['file']) # save the file
file_name = csvfile.name
file_name = unicodedata.normalize('NFKD', file_name).encode('ascii','ignore')
csvfile2 = settings.MEDIA_ROOT + '/new/sample.csv'
writer = csv.writer(outfile)
row_number = 0
try:
with open(csvfile2, 'w') as writtenFile:
fieldnames = ... # use some code to determine the column names dynamically, may be read the first row and then populate the field names
writer = csv.DictWriter(writtenFile, fieldnames=fieldnames)
writer.writerow({...}) # write header row
with open('some/file/name.csv') as csvfile:
line_entries = csv.DictReader(csvfile)
for row in line_entries:
row_number = row_number + 1
valueDict = [...] # populate the rows as a dict
writer.writerow(valueDict);
success = True
except:
messages.error(request, "Could load file, invalid format")
success = False
return redirect('home')
if success:
messages.success(request, "File read..")
else:
for err in error_list:
messages.error(request, err)
else:
messages.error(request, "No file selected")
else:
form = CSVUploadForm()
context = {
'form': form
}
return render_to_response(template_name, context, context_instance=RequestContext(request))
Related
I have my view which is generating csv file, but it doesn't work after I deployed my project to python anywhere
Error:
enter image description here
view:
`def generate_csv(request, pk):
if request.method == 'POST':
columns = models.Column.objects.all().filter(schema__id=pk)
names = [None for _ in columns]
types = [None for _ in columns]
for column in columns:
names[int(column.order) - 1] = column.name
types[int(column.order) - 1] = column.type
form = forms.DataSetForm(request.POST)
fake = Faker()
if form.is_valid():
rows = int(form['rows'].value())
schema = models.Schema.objects.get(id=pk)
file_name = f'{schema.title}{fake.pyint()}.csv'
file = open(f'media/{file_name}', mode='w', newline='')
dataset = models.DataSet(schema=schema, csv_file=f'{file_name}')
dataset.save()
file_writer = csv.writer(file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
file_writer.writerow(names)
for _ in range(rows):
data = []
for i in types:
data.append(fake_data(i))
file_writer.writerow(data)
file.close()
dataset.status = True
dataset.save()
return JsonResponse({}, status=200)
else:
return JsonResponse({'error': form.errors}, status=400)
return JsonResponse({'error': ''}, status=400)`
I think the problem is that I have to create the csv file in a different way, but I don't know how, can you help me?
I am trying to access the csv file which i passed in my form and saved in media directory.
I am able to access the file if i manually enter the path(localhost://8000/media/1.csv)
but it throws an error saying "No such file or directory" when accessing from open function.
def home(request):
print("Rendering Home...")
if request.method == "POST":
uploaded_file = request.FILES['csvFile']
fs = FileSystemStorage()
name = fs.save(uploaded_file.name,uploaded_file)
url = fs.url(name)
csv_fp = open(f'{url}', 'r') //ERROR:"No such file or dir media/1.csv"
reader = csv.DictReader(csv_fp)
headers = [col for col in reader.fieldnames]
out = [row for row in reader]
return render(request, 'home.html', {'data' : out, 'headers' : headers})
return render(request,"home.html")
have you tried .path instead of .name
file = open(filename.path' 'rb').read()
The problem was the path being given to open function
csv_fp = default_storage.open(os.path.join(settings.MEDIA_ROOT, name), 'r')
simply did the trick :)
My code is currently hard coded to only accept csv files in the column format:
first_name,last_name,phone,email,address,company
However I would like users to be able to upload csv files that are in any* format order and naming scheme and correctly populate our forms. For example:
Email,LastName,FirstName,Company,Phone,Address
would be a valid column format. How would I go about doing that? Relevant code as follows:
dr = csv.reader(open(tmp_file,'r'))
data_dict = {}
headers = next(dr)
print (headers)
#skips over first line in csv
iterlines = iter(dr)
next(iterlines)
for row in iterlines:
data_dict["first_name"] = row[0]
#print(data_dict["first_name"])
data_dict["last_name"] = row[1]
#print(data_dict["last_name"])
data_dict["phone"] = row[2]
#print(data_dict["phone"])
data_dict["email"] = row[3]
#print(data_dict["email"])
data_dict["address"] = row[4]
#print(data_dict["address"])
data_dict["company"] = row[5]
#print(data_dict["company"])
#adds to form
try:
form = AddContactForm(data_dict)
if form.is_valid():
obj = form.save(commit=False)
obj.owner = request.user.username
first_name = form.cleaned_data.get(data_dict["first_name"])
last_name = form.cleaned_data.get(data_dict["last_name"])
phone = form.cleaned_data.get(data_dict["phone"])
email = form.cleaned_data.get(data_dict["email"])
address = form.cleaned_data.get(data_dict["address"])
company = form.cleaned_data.get(data_dict["company"])
obj.save()
else:
logging.getLogger("error_logger").error(form.errors.as_json())
except Exception as e:
logging.getLogger("error_logger").error(repr(e))
pass
headers = "first_name,last_name,email"
headers_array = headers.split(',')
headers_map = {}
for i, column_name in enumerate(headers_array):
headers_map[column_name] = i
#creates {'first_name': 0, 'last_name': 1, 'email': 2}
Now you can now use headers_map to get the row element
row[headers_map['first_name']]
Edit: For those loving one liners
headers_map = {column_name: i for i, column_name in enumerate(headers.split(','))}
There are a number of approaches to handling inconsistent header names in the file. The best approach is to prevent it by rejecting such files at upload time, obliging the uploader to correct them. Assuming this isn't possible, you could try to transform the provided headers into what you want
import csv
import io
import re
with open(tmp_file, 'r') as f:
reader = csv.reader
headers = next(reader)
# Make a new header list with placeholders
fixed_headers = [None * len(headers)]
for i, value in enumerate(headers)
fixed = re.sub(r'(\w+)(?<=[a-z])([A-Z]\w+)', r'\1_\2', v).lower()
new_headers[i] = fixed
The regex finds capital letters in the middle of strings and inserts an underscore; then str.lower is called on the result (so values like 'Email' will be converted to 'email'.
Now rewrite the csv with the fixed headers:
with open(tmp_file, 'r') as f:
reader = csv.reader(f)
next(reader)
new_file = io.StringIO()
writer = csv.writer(new_file)
writer.writerow(fixed_headers)
for row in reader:
writer.writerow(row)
# Rewind the file pointer
new_file.seek(0)
Use csv.DictReader to get rows as dictionaries of values mapped to headers.
dr = csv.DictReader(new_file)
for data_dict in dr:
#adds to form
try:
form = AddContactForm(data_dict)
if form.is_valid():
obj = form.save(commit=False)
obj.owner = request.user.username
first_name = form.cleaned_data.get(data_dict["first_name"])
last_name = form.cleaned_data.get(data_dict["last_name"])
phone = form.cleaned_data.get(data_dict["phone"])
email = form.cleaned_data.get(data_dict["email"])
address = form.cleaned_data.get(data_dict["address"])
company = form.cleaned_data.get(data_dict["company"])
obj.save()
else:
logging.getLogger("error_logger").error(form.errors.as_json())
except Exception as e:
logging.getLogger("error_logger").error(repr(e))
pass
this is my function to make a csv file of a queryset
def esport_to_csv(self, tweets):
with open('tweets.csv', 'w') as new_file:
fieldnames = ["tweet_id", "text" , "user_screen_name", "user_name", "user_verified", "created_at", "user_time_zone", "user_location", "favorite_count", "retweet_count", "user_followers_count", "user_friends_count"]
csv_writer = csv.DictWriter(new_file, fieldnames=fieldnames, delimiter='\t')
csv_writer.writeheader()
for tweet in tweets:
line = []
line.append(tweet["tweet_id"])
line.append(tweet["text"])
line.append(tweet["user_screen_name"])
line.append(tweet["user_name"])
line.append(tweet["user_verified"])
line.append(tweet["created_at"])
line.append(tweet["user_time_zone"])
line.append(tweet["user_location"])
line.append(tweet["favorite_count"])
line.append(tweet["retweet_count"])
line.append(tweet["user_followers_count"])
line.append(tweet["user_friends_count"])
csv_writer.writerow(line)
this is my server response
ValueError: dict contains fields not in fieldnames: 967563194582515712, 'RT #KEEMSTAR: When you have your fathers car & you tell everyone on the internet that your 15 year old ass bought it. t.co/bUhhrPw0…', 'TKBrotherTK', 'Team Kalvin🌏', False, '2018-02-25T06:23:36+05:30', 'Melbourne', 'Australia', 0, 0, None, None
this is from where function called
#list_route(methods=["post"])
def get_created_in_range(self, request):
response = {}
data = request.POST
start = dateutil.parser.parse(data['start'])
end = dateutil.parser.parse(data['end'])
page_no = data['page_no']
export_to_csv = data.get('export_to_csv', "false")
tweets = Tweet.get_created_in_range(start, end, int(page_no))
serializer = TweetSerializer(tweets, many= True)
if export_to_csv == "true":
self.esport_to_csv(serializer.data)
print(type(serializer.data[0]))
response["data"] = serializer.data
return Response(response, status= status.HTTP_200_OK)
if u can see find the function call above here it is
if export_to_csv == "true":
self.esport_to_csv(serializer.data)
i have seen a solution online but i dont know how to apply it over here
You need to pass a dict to DictWriter, and tell it to ignore extra fields if they are present. Try this:
def esport_to_csv(self, tweets):
with open('tweets.csv', 'w') as new_file:
fieldnames = ["tweet_id", "text" , "user_screen_name", "user_name", "user_verified", "created_at", "user_time_zone", "user_location", "favorite_count", "retweet_count", "user_followers_count", "user_friends_count"]
csv_writer = csv.DictWriter(new_file, fieldnames=fieldnames, delimiter='\t', extrasaction='ignore')
csv_writer.writeheader()
for tweet in tweets:
csv_writer.writerow(tweet)
typeError with csv file in python3I tried to write a python3 programme that writes and edit data in csv file,but it is showing TypeError: a bytes-like object is required, not'str'
screenshot of the output in the terminal is attached.I don't know what this means and what is wrong.Someone help me fix it.the code is below:
import csv
import datetime
import shutil
from tempfile import NamedTemporaryFile
def get_length(file_path):
with open("data.csv", "r") as csvfile:
#some csv file data.csv is created in the same directory
reader = csv.reader(csvfile)
reader_list = list(reader)
return len(reader_list)
def append_data(file_path, name, email, amount):
fieldnames = ['id', 'name', 'email', 'amount', 'sent', 'date']
#the number of rows?
next_id = get_length(file_path)
with open(file_path, "a") as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writerow({
"id": next_id,
"name": name,
"email": email,
"sent": "",
"amount": amount,
"date": datetime.datetime.now()
})
#append_data("data.csv", "some name", "address#mail.com", 123.456)
def edit_data(edit_id=None, email=None, amount=None, sent=None):
filename = "data.csv"
temp_file = NamedTemporaryFile(delete=False)
with open(filename, "rb") as csvfile, temp_file:
reader = csv.DictReader(csvfile)
fieldnames = ['id', 'name', 'email', 'amount', 'sent', 'date']
writer = csv.DictWriter(temp_file, fieldnames=fieldnames)
writer.writeheader()
for row in reader:
#print(row['id'] == 4)
if edit_id is not None:
if int(row['id']) == int(edit_id):
row['amount'] = amount
row['sent'] = sent
elif email is not None and edit_id is None:
if str(row['email']) == str(email):
row['amount'] = amount
row['sent'] = sent
else:
pass
writer.writerow(row)
shutil.move(temp_file.name, filename)
return True
return False
edit_data(8, 9992.32, "")
edit_data(email='address#mail.com', amount=19.90, sent='10')
On Python 3 when using csv readers and writers you need to open your file open("data.csv", "r", newline=""). Using rb is compatible with Python 2.x only. You also need to setup similar settings for your NamedTemporaryFile.
The following should help:
import csv
import datetime
import shutil
from tempfile import NamedTemporaryFile
fieldnames = ['id', 'name', 'email', 'amount', 'sent', 'date']
def edit_data(edit_id=None, email=None, amount=None, sent=None):
filename = "data.csv"
with open(filename, "r", newline="") as csvfile, \
NamedTemporaryFile(delete=False, mode="w", newline="") as temp_file:
reader = csv.DictReader(csvfile, fieldnames=fieldnames)
writer = csv.DictWriter(temp_file, fieldnames=fieldnames)
writer.writeheader()
for row in reader:
#print(row['id'] == 4)
if edit_id is not None:
if int(row['id']) == int(edit_id):
row['amount'] = amount
row['sent'] = sent
elif email is not None and edit_id is None:
if str(row['email']) == str(email):
row['amount'] = amount
row['sent'] = sent
else:
pass
writer.writerow(row)
shutil.move(temp_file.name, filename)
edit_data(email='address#mail.com', amount=19.90, sent='10')