We Keep Coding

Getting duplicate data from csv file

def general_number(request):
csvfilename = 'algo/generalnumber.csv'
csvfile = open(csvfilename, 'r')
reader = csv.DictReader(csvfile)
fieldnames = ("Ticker","Company","Industry","PreviousClose","Open","Bid","Ask","DaysRange","ftWeekRange","Volume","AvgVolume","MarketCap","Beta","PERatioTTM","EPSTTM","EarningsDate","ForwardDividendYield","ExDividendDate","OneyTargetEst","ticker_company")
output = []
for each in reader:
row = {}
for field in fieldnames:
row[field] = each[field]
output.append(row)
return JsonResponse(output[20:30],safe=False)
Here I am getting duplicate data in json from csv.
Each datum is showing 10 times. What is wrong in this?

It's just an indentation problem. Fixed it, try now!
def general_number(request):
csvfilename = 'algo/generalnumber.csv'
csvfile = open(csvfilename, 'r')
reader = csv.DictReader(csvfile)
fieldnames = ("Ticker","Company","Industry","PreviousClose","Open","Bid","Ask","DaysRange","ftWeekRange","Volume","AvgVolume","MarketCap","Beta","PERatioTTM","EPSTTM","EarningsDate","ForwardDividendYield","ExDividendDate","OneyTargetEst","ticker_company")
output = []
for each in reader:
row = {}
for field in fieldnames:
row[field] = each[field]
output.append(row)
return JsonResponse(output[20:30],safe=False)

Python-CSV : How do you append data into a csv file?

I have read other question and tried but unsuccessful. There is already 1 row in data.csv (called productId) and I would like to append data into new row under already created headers (productUrl, discount, evaluateScore, volume, packageType, lotNum, validTime, storeName, storeUrl, allImageUrls).
import csv
def get_details():
pid = get_id()
print(pid)
data = aliexpress.get_product_details(['productId', 'productUrl', 'discount', 'evaluateScore',
'volume', 'packageType', 'lotNum', 'validTime', 'storeName', 'storeUrl',
'allImageUrls'], pid)
with open('data.csv', 'a', newline='') as csvfile:
fieldnames = ['productUrl', 'discount', 'evaluateScore',
'volume', 'packageType', 'lotNum', 'validTime', 'storeName', 'storeUrl',
'allImageUrls']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
productUrl = data['productUrl']
discount = data['discount']
evaluateScore = data['evaluateScore']
volume = data['volume']
packageType = data['packageType']
lotNum = data['lotNum']
validTime = data['validTime']
storeName = data['storeName']
storeUrl = data['storeUrl']
allImageUrls = data['allImageUrls']
allImageUrlstuple = allImageUrls.split(',')
print(allImageUrls)
writer.writerow({'productUrl': productUrl, 'discount': discount, 'evaluateScore': evaluateScore,
'volume': volume, 'packageType': packageType, 'lotNum': lotNum, 'validTime': validTime, 'storeName': storeName, 'storeUrl': storeUrl,
'allImageUrls': allImageUrlstuple})

You do not need to re-write the header if the header is already in the CSV. Remove writer.writeheader()
Ex:
import csv
def get_details():
pid = get_id()
print(pid)
data = aliexpress.get_product_details(['productId', 'productUrl', 'discount', 'evaluateScore',
'volume', 'packageType', 'lotNum', 'validTime', 'storeName', 'storeUrl',
'allImageUrls'], pid)
with open('data.csv', 'a', newline='') as csvfile:
fieldnames = ['productUrl', 'discount', 'evaluateScore',
'volume', 'packageType', 'lotNum', 'validTime', 'storeName', 'storeUrl',
'allImageUrls']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
productUrl = data['productUrl']
discount = data['discount']
evaluateScore = data['evaluateScore']
volume = data['volume']
packageType = data['packageType']
lotNum = data['lotNum']
validTime = data['validTime']
storeName = data['storeName']
storeUrl = data['storeUrl']
allImageUrls = data['allImageUrls']
allImageUrlstuple = allImageUrls.split(',')
print(allImageUrls)
writer.writerow({'productUrl': productUrl, 'discount': discount, 'evaluateScore': evaluateScore,
'volume': volume, 'packageType': packageType, 'lotNum': lotNum, 'validTime': validTime, 'storeName': storeName, 'storeUrl': storeUrl,
'allImageUrls': allImageUrlstuple})

It may be easier to rewrite the file:
with open('data.csv',newline='') as csvfile:
r = csv.reader(csvfile)
data = [line for line in r]
with open('data.csv','w',newline='') as csvfile:
w = csv.writer(csvfile)
w.writerow(['col1','col2'])
w.writerows(data)

Working with csv file in python3 programme shows "TypeError: a bytes-like object is required, not 'str'"

typeError with csv file in python3I tried to write a python3 programme that writes and edit data in csv file,but it is showing TypeError: a bytes-like object is required, not'str'
screenshot of the output in the terminal is attached.I don't know what this means and what is wrong.Someone help me fix it.the code is below:
import csv
import datetime
import shutil
from tempfile import NamedTemporaryFile
def get_length(file_path):
with open("data.csv", "r") as csvfile:
#some csv file data.csv is created in the same directory
reader = csv.reader(csvfile)
reader_list = list(reader)
return len(reader_list)
def append_data(file_path, name, email, amount):
fieldnames = ['id', 'name', 'email', 'amount', 'sent', 'date']
#the number of rows?
next_id = get_length(file_path)
with open(file_path, "a") as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writerow({
"id": next_id,
"name": name,
"email": email,
"sent": "",
"amount": amount,
"date": datetime.datetime.now()
})
#append_data("data.csv", "some name", "address#mail.com", 123.456)
def edit_data(edit_id=None, email=None, amount=None, sent=None):
filename = "data.csv"
temp_file = NamedTemporaryFile(delete=False)
with open(filename, "rb") as csvfile, temp_file:
reader = csv.DictReader(csvfile)
fieldnames = ['id', 'name', 'email', 'amount', 'sent', 'date']
writer = csv.DictWriter(temp_file, fieldnames=fieldnames)
writer.writeheader()
for row in reader:
#print(row['id'] == 4)
if edit_id is not None:
if int(row['id']) == int(edit_id):
row['amount'] = amount
row['sent'] = sent
elif email is not None and edit_id is None:
if str(row['email']) == str(email):
row['amount'] = amount
row['sent'] = sent
else:
pass
writer.writerow(row)
shutil.move(temp_file.name, filename)
return True
return False
edit_data(8, 9992.32, "")
edit_data(email='address#mail.com', amount=19.90, sent='10')

On Python 3 when using csv readers and writers you need to open your file open("data.csv", "r", newline=""). Using rb is compatible with Python 2.x only. You also need to setup similar settings for your NamedTemporaryFile.
The following should help:
import csv
import datetime
import shutil
from tempfile import NamedTemporaryFile
fieldnames = ['id', 'name', 'email', 'amount', 'sent', 'date']
def edit_data(edit_id=None, email=None, amount=None, sent=None):
filename = "data.csv"
with open(filename, "r", newline="") as csvfile, \
NamedTemporaryFile(delete=False, mode="w", newline="") as temp_file:
reader = csv.DictReader(csvfile, fieldnames=fieldnames)
writer = csv.DictWriter(temp_file, fieldnames=fieldnames)
writer.writeheader()
for row in reader:
#print(row['id'] == 4)
if edit_id is not None:
if int(row['id']) == int(edit_id):
row['amount'] = amount
row['sent'] = sent
elif email is not None and edit_id is None:
if str(row['email']) == str(email):
row['amount'] = amount
row['sent'] = sent
else:
pass
writer.writerow(row)
shutil.move(temp_file.name, filename)
edit_data(email='address#mail.com', amount=19.90, sent='10')

Write to specific columns in a csv file

I've a script that gets data in a json format through an API. I need to write headers to the file, then append with some of the json data.
import requests, csv, json
def test(limit="", api_token=""):
url_rack = "url"
api_token = "Token " + api_token
headers = {'Authorization': api_token}
params = {'limit': limit}
session = requests.Session()
json_data = session.get(url_rack, headers=headers, params=params)
with open('my_file.csv', 'wb') as csvfile:
filewriter = csv.writer(csvfile, delimiter=',',
quotechar='|', quoting=csv.QUOTE_MINIMAL)
filewriter.writerow(['Header1', 'Header2'])
json_data = json_data.json()['results']
for i in json_data:
bunit = i['tenant']
open(csvfile)
if not bunit:
print 'Placeholder'
else:
bunit = i['tenant']['name']
print bunit
for i in json_data:
owner = i['tenant']
if not owner:
print 'Placeholder Owner'
else:
owner = i['tenant']['name']
print owner
def main():
api_token = "mytoken"
test(api_token=api_token)
if __name__ == "__main__":
main()
I tried to write the data taken from for i in tenants: in the column under Header1, and the data from for i in owners under Header2. I have two trivial problems. When I tried to open the csv file inside the for loop I got the error TypeError: coercing to Unicode: need string or buffer, file found. I got that the file is open so I can't reopen it, but don't want to create it in the loop as that would rewrite the file each iteration?
Secondly filewriter.writerow obviously writes in rows, but I need to write the data in each column under the header. I printed it out to test that I can actually call the api and get the data but can't work out how to append it under Header1 and Header2.
EDIT
My data that prints:
Placeholder tenant
Placeholder tenant
Tenant1
Placeholder tenant
Placeholder tenant
Placeholder tenant
Placeholder tenant
Tenant2
Placeholder tenant
Tenant3
Tenant4
and
Placeholder Owner
Placeholder Owner
Owner1
Placeholder Owner
Placeholder Owner
Placeholder Owner
Placeholder Owner
Owner2
Placeholder Owner
Owner3
Owner4

Why not to rewrite your code like this:
...
with open('my_file.csv', 'wb') as csvfile:
filewriter = csv.writer(csvfile, delimiter=',',
quotechar='|', quoting=csv.QUOTE_MINIMAL)
filewriter.writerow(['Header1', 'Header2'])
json_data = json_data.json()['results']
for i in json_data:
bunit = i['tenant']
if not bunit:
print 'Placeholder'
else:
bunit = i['tenant']['name']
owner = i['tenant']
if not owner:
print 'Placeholder Owner'
else:
owner = i['tenant']['name']
filewriter.writerow([bunit, owner])
...

To open() function, you need to provide a filename as parameter.
Here you are providing an already open file as a parameter. Hence, first error.
The csvfile is already open and you do not need to open it again to write to it.
To write to the file, you can store the data in a list and then write to file at end of function.
Suppose I want to write r1c1,r1c2,r2c1,r2c2,r3c1,r3c2 in cells-A1,B1,A2,B2,A3,B3 respectively, then I would do something like the following:-
listtowrite = [['r1c1','r1c2'],['r2c1','r2c2'],['r3c1','r3c2']]
csvfile = open('my_file.csv', 'w')
for arow in listtowrite:
for acolumn in arow:
csvfile.write(acolumn+',')
csvfile.write('\n')
csvfile.close()

Figured it out. Just needed to write each to a list
json_data = prefixes_response.json()['results']
tenant = []
for i in json_data:
bunit = i['tenant']
if not bunit:
tenant.append('Placeholder tenant')
else:
bunit = i['tenant']['name']
tenant.append(bunit)
and zip both lists and write
rows = zip(Header1, Header2)
for row in rows:
filewriter.writerow(row)

copying contents between two csv files django

I was trying to copy the contents of csvfile into csvfile2. I cant see any help on this subject.and I also want to append a value value to the first row of csvfile2.Eg : value = 1
output should be like
`1_dog 1_cat 1_frog `
when value=2 result should be
2_dog 2_cat 2_frog
The value ranges between 1 and the total no of lines in csv file.(totalrows)
My trial code:
def csvupload(request, template_name='upload_csv.html'):
if request.method == 'POST':
form = CSVUploadForm(request.POST, request.FILES)
if form.is_valid():
error_list = set()
value = 1
csvfile = request.FILES['file']
file_name = csvfile.name
file_name = unicodedata.normalize('NFKD', file_name).encode('ascii','ignore')
csvfile2 = settings.MEDIA_ROOT + '/new/sample.csv'
writer = csv.writer(outfile, dialect='excel')
csvfile.open()
line_entries = csv.DictReader(csvfile, dialect='excel')
try:
csvfile.read()
line_entries = csv.DictReader(csvfile, dialect='excel', delimiter=',')
rows1 = list(csvfile)
totalrows = len(rows1)
success = True
except:
messages.error(request, "Could not parse file, invalid format")
success = False
return redirect('home')
if success:
messages.success(request, "File read..")
else:
for err in error_list:
messages.error(request, err)
else:
messages.error(request, "No file selected")
else:
form = CSVUploadForm()
context = {
'form': form
}
return render_to_response(template_name, context, context_instance=RequestContext(request))
This code is not complete.I am a new to python and confused.Can someone help?

I don't have a python interpreter handy so I cannot test. I can provide more complete answer later.
You need something like this -
To Read -
>>> import csv
>>> with open('names.csv') as csvfile:
... reader = csv.DictReader(csvfile)
... for row in reader:
... print(row['first_name'], row['last_name'])
To Write -
import csv
with open('names.csv', 'w') as csvfile:
fieldnames = ['first_name', 'last_name']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
writer.writerow({'first_name': 'Baked', 'last_name': 'Beans'})
writer.writerow({'first_name': 'Lovely', 'last_name': 'Spam'})
writer.writerow({'first_name': 'Wonderful', 'last_name': 'Spam'})
So, first, save the file in a temporary location in server -
def handle_uploaded_file(f): # method to save to a temporary location
with open('some/file/name.csv', 'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
Then load it and read line by line as above as mentioned in the docs. https://docs.python.org/2/library/csv.html#csv.DictReader
So your code might be something like -
def csvupload(request, template_name='upload_csv.html'):
if request.method == 'POST':
form = CSVUploadForm(request.POST, request.FILES)
if form.is_valid():
error_list = set()
value = 1
handle_uploaded_file(request.FILES['file']) # save the file
file_name = csvfile.name
file_name = unicodedata.normalize('NFKD', file_name).encode('ascii','ignore')
csvfile2 = settings.MEDIA_ROOT + '/new/sample.csv'
writer = csv.writer(outfile)
row_number = 0
try:
with open(csvfile2, 'w') as writtenFile:
fieldnames = ... # use some code to determine the column names dynamically, may be read the first row and then populate the field names
writer = csv.DictWriter(writtenFile, fieldnames=fieldnames)
writer.writerow({...}) # write header row
with open('some/file/name.csv') as csvfile:
line_entries = csv.DictReader(csvfile)
for row in line_entries:
row_number = row_number + 1
valueDict = [...] # populate the rows as a dict
writer.writerow(valueDict);
success = True
except:
messages.error(request, "Could load file, invalid format")
success = False
return redirect('home')
if success:
messages.success(request, "File read..")
else:
for err in error_list:
messages.error(request, err)
else:
messages.error(request, "No file selected")
else:
form = CSVUploadForm()
context = {
'form': form
}
return render_to_response(template_name, context, context_instance=RequestContext(request))