We Keep Coding

File handling in django

I am trying to access the csv file which i passed in my form and saved in media directory.
I am able to access the file if i manually enter the path(localhost://8000/media/1.csv)
but it throws an error saying "No such file or directory" when accessing from open function.
def home(request):
print("Rendering Home...")
if request.method == "POST":
uploaded_file = request.FILES['csvFile']
fs = FileSystemStorage()
name = fs.save(uploaded_file.name,uploaded_file)
url = fs.url(name)
csv_fp = open(f'{url}', 'r') //ERROR:"No such file or dir media/1.csv"
reader = csv.DictReader(csv_fp)
headers = [col for col in reader.fieldnames]
out = [row for row in reader]
return render(request, 'home.html', {'data' : out, 'headers' : headers})
return render(request,"home.html")

have you tried .path instead of .name
file = open(filename.path' 'rb').read()

The problem was the path being given to open function
csv_fp = default_storage.open(os.path.join(settings.MEDIA_ROOT, name), 'r')
simply did the trick :)

Using string matching from csv to find patterns in Python using results from OCR

I am new to python and I want to use the results from the OCR (string) to be able to match the first column of my csv file and then only if the condition is true (the string from ocr matches to the one in the csv then it should use the pic. I get an error as soon as I try to integrate the code together.
For the OCR, I am using pytesseract and I am using Flask to render the web app.
The error I get is : AttributeError: '_io.TextIOWrapper' object has no attribute 'filename'
New error: The view function for 'upload_image' did not return a valid response. The function either returned None or ended without a return statement.
This error only persists when I try to add this code:
match = extracted_text
matched_row = None
with open("/Users/ri/Desktop/DPL/DPL.csv", "r") as file:
# Read file as a CSV delimited by tabs.
reader = csv.reader(file, delimiter='\t')
for row in reader:
if row[0] == match:
matched_row = row
print(matched_row)
app.py
#app.route('/', methods=['POST'])
def upload_image():
if request.method == 'POST':
# checks whether or not the post request has the file part
if 'file' not in request.files:
flash('No file part')
return redirect(request.url)
file = request.files['file']
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '':
flash('No file selected for uploading')
return redirect(request.url)
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
file.save(os.path.join(os.getcwd() +
UPLOAD_INPUT_IMAGES_FOLDER, file.filename))
flash('File successfully uploaded')
# calls the ocr_processing function to perform text extraction
extracted_text = ocr_processing(file)
print(extracted_text)
match = extracted_text
matched_row = None
with open("/Users/ri/Desktop/DPL/DPL.csv", "r") as f:
# Read file as a CSV delimited by tabs.
reader = csv.reader(f, delimiter='\t')
for row in reader:
if row[0] == match:
matched_row = row
print(matched_row)
loaded_vec = CountVectorizer(
vocabulary=pickle.load(open("./tfidf_vector.pkl", "rb")))
loaded_tfidf = pickle.load(open("./tfidf_transformer.pkl", "rb"))
model_pattern_type = pickle.load(
open("./clf_svm_Pattern_Category.pkl", "rb"))
model_pattern_category = pickle.load(
open("./clf_svm_Pattern_Type.pkl", "rb"))
match = [match]
X_new_counts = loaded_vec.transform(
match)
# .values.astype('U')
X_new_tfidf = loaded_tfidf.transform(X_new_counts)
predicted_pattern_type = model_pattern_type.predict(X_new_tfidf)
your_predicted_pattern_type = predicted_pattern_type[0]
predicted_pattern_category = model_pattern_category.predict(
X_new_tfidf)
your_predicted_pattern_category = predicted_pattern_category[0]
return render_template('uploads/results.html',
msg='Processed successfully!',
match=match,
your_predicted_pattern_category=your_predicted_pattern_category,
your_predicted_pattern_type=your_predicted_pattern_type,
img_src=UPLOAD_INPUT_IMAGES_FOLDER + file.filename)
# break
else:
print("no mattern found")
else:
flash('Allowed file types are txt, pdf, png, jpg, jpeg, gif')
return redirect(request.url)

Taking *ANY csv column format into Django form

My code is currently hard coded to only accept csv files in the column format:
first_name,last_name,phone,email,address,company
However I would like users to be able to upload csv files that are in any* format order and naming scheme and correctly populate our forms. For example:
Email,LastName,FirstName,Company,Phone,Address
would be a valid column format. How would I go about doing that? Relevant code as follows:
dr = csv.reader(open(tmp_file,'r'))
data_dict = {}
headers = next(dr)
print (headers)
#skips over first line in csv
iterlines = iter(dr)
next(iterlines)
for row in iterlines:
data_dict["first_name"] = row[0]
#print(data_dict["first_name"])
data_dict["last_name"] = row[1]
#print(data_dict["last_name"])
data_dict["phone"] = row[2]
#print(data_dict["phone"])
data_dict["email"] = row[3]
#print(data_dict["email"])
data_dict["address"] = row[4]
#print(data_dict["address"])
data_dict["company"] = row[5]
#print(data_dict["company"])
#adds to form
try:
form = AddContactForm(data_dict)
if form.is_valid():
obj = form.save(commit=False)
obj.owner = request.user.username
first_name = form.cleaned_data.get(data_dict["first_name"])
last_name = form.cleaned_data.get(data_dict["last_name"])
phone = form.cleaned_data.get(data_dict["phone"])
email = form.cleaned_data.get(data_dict["email"])
address = form.cleaned_data.get(data_dict["address"])
company = form.cleaned_data.get(data_dict["company"])
obj.save()
else:
logging.getLogger("error_logger").error(form.errors.as_json())
except Exception as e:
logging.getLogger("error_logger").error(repr(e))
pass

headers = "first_name,last_name,email"
headers_array = headers.split(',')
headers_map = {}
for i, column_name in enumerate(headers_array):
headers_map[column_name] = i
#creates {'first_name': 0, 'last_name': 1, 'email': 2}
Now you can now use headers_map to get the row element
row[headers_map['first_name']]
Edit: For those loving one liners
headers_map = {column_name: i for i, column_name in enumerate(headers.split(','))}

There are a number of approaches to handling inconsistent header names in the file. The best approach is to prevent it by rejecting such files at upload time, obliging the uploader to correct them. Assuming this isn't possible, you could try to transform the provided headers into what you want
import csv
import io
import re
with open(tmp_file, 'r') as f:
reader = csv.reader
headers = next(reader)
# Make a new header list with placeholders
fixed_headers = [None * len(headers)]
for i, value in enumerate(headers)
fixed = re.sub(r'(\w+)(?<=[a-z])([A-Z]\w+)', r'\1_\2', v).lower()
new_headers[i] = fixed
The regex finds capital letters in the middle of strings and inserts an underscore; then str.lower is called on the result (so values like 'Email' will be converted to 'email'.
Now rewrite the csv with the fixed headers:
with open(tmp_file, 'r') as f:
reader = csv.reader(f)
next(reader)
new_file = io.StringIO()
writer = csv.writer(new_file)
writer.writerow(fixed_headers)
for row in reader:
writer.writerow(row)
# Rewind the file pointer
new_file.seek(0)
Use csv.DictReader to get rows as dictionaries of values mapped to headers.
dr = csv.DictReader(new_file)
for data_dict in dr:
#adds to form
try:
form = AddContactForm(data_dict)
if form.is_valid():
obj = form.save(commit=False)
obj.owner = request.user.username
first_name = form.cleaned_data.get(data_dict["first_name"])
last_name = form.cleaned_data.get(data_dict["last_name"])
phone = form.cleaned_data.get(data_dict["phone"])
email = form.cleaned_data.get(data_dict["email"])
address = form.cleaned_data.get(data_dict["address"])
company = form.cleaned_data.get(data_dict["company"])
obj.save()
else:
logging.getLogger("error_logger").error(form.errors.as_json())
except Exception as e:
logging.getLogger("error_logger").error(repr(e))
pass

Django save uploaded file

I want to upload a file, what I am able to do with the code stated below, but I also need to save all the uploads in different folder with different names. If 2 users upload the same file from browser, then in the folder it should be saved with different name or Unique Identification number.
Following is my code:
views.py
from django.shortcuts import render
import openpyxl
def index(request):
if "GET" == request.method:
return render(request, 'myapp/index.html', {})
else:
excel_file = request.FILES["excel_file"]
# you may put validations here to check extension or file size
wb = openpyxl.load_workbook(excel_file)
# getting all sheets
sheets = wb.sheetnames
print(sheets)
# getting a particular sheet
worksheet = wb["Sheet1"]
print(worksheet)
# getting active sheet
active_sheet = wb.active
print(active_sheet)
# reading a cell
print(worksheet["A1"].value)
excel_data = list()
# iterating over the rows and
# getting value from each cell in row
for row in worksheet.iter_rows():
row_data = list()
for cell in row:
row_data.append(str(cell.value))
print(cell.value)
excel_data.append(row_data)
return render(request, 'myapp/index.html', {"excel_data":excel_data})

Give you a Django FileField way Implemention:
def user_directory_path(instance, filename):
# file will be uploaded to MEDIA_ROOT/user_<id>/<filename>
suffix = filename[filename.rindex(".")+1:]
return 'upfiles/{0}/{1}.{2}'.format(instance.user.username, get_randomfilename(),suffix)
class Picture(models.Model):
user = models.ForeignKey(User,on_delete=models.CASCADE)
file = models.ImageField(upload_to=user_directory_path)
date_added = models.DateTimeField(auto_now_add=True)
#parser_classes((MultiPartParser,))
#permission_classes((IsAuthenticated, ))
def upload_picture(request):
'''
:input :{"file":f}
:return:{"id":pictureId}
'''
if "file" in request.FILES:
f = request.FILES["file"]
picture = Picture()
picture.user= request.user
picture.file = f
picture.save()
return Response(data={"id":picture.id})
else:
return Response({},status=status.HTTP_400_BAD_REQUEST)

copying contents between two csv files django

I was trying to copy the contents of csvfile into csvfile2. I cant see any help on this subject.and I also want to append a value value to the first row of csvfile2.Eg : value = 1
output should be like
`1_dog 1_cat 1_frog `
when value=2 result should be
2_dog 2_cat 2_frog
The value ranges between 1 and the total no of lines in csv file.(totalrows)
My trial code:
def csvupload(request, template_name='upload_csv.html'):
if request.method == 'POST':
form = CSVUploadForm(request.POST, request.FILES)
if form.is_valid():
error_list = set()
value = 1
csvfile = request.FILES['file']
file_name = csvfile.name
file_name = unicodedata.normalize('NFKD', file_name).encode('ascii','ignore')
csvfile2 = settings.MEDIA_ROOT + '/new/sample.csv'
writer = csv.writer(outfile, dialect='excel')
csvfile.open()
line_entries = csv.DictReader(csvfile, dialect='excel')
try:
csvfile.read()
line_entries = csv.DictReader(csvfile, dialect='excel', delimiter=',')
rows1 = list(csvfile)
totalrows = len(rows1)
success = True
except:
messages.error(request, "Could not parse file, invalid format")
success = False
return redirect('home')
if success:
messages.success(request, "File read..")
else:
for err in error_list:
messages.error(request, err)
else:
messages.error(request, "No file selected")
else:
form = CSVUploadForm()
context = {
'form': form
}
return render_to_response(template_name, context, context_instance=RequestContext(request))
This code is not complete.I am a new to python and confused.Can someone help?

I don't have a python interpreter handy so I cannot test. I can provide more complete answer later.
You need something like this -
To Read -
>>> import csv
>>> with open('names.csv') as csvfile:
... reader = csv.DictReader(csvfile)
... for row in reader:
... print(row['first_name'], row['last_name'])
To Write -
import csv
with open('names.csv', 'w') as csvfile:
fieldnames = ['first_name', 'last_name']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
writer.writerow({'first_name': 'Baked', 'last_name': 'Beans'})
writer.writerow({'first_name': 'Lovely', 'last_name': 'Spam'})
writer.writerow({'first_name': 'Wonderful', 'last_name': 'Spam'})
So, first, save the file in a temporary location in server -
def handle_uploaded_file(f): # method to save to a temporary location
with open('some/file/name.csv', 'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
Then load it and read line by line as above as mentioned in the docs. https://docs.python.org/2/library/csv.html#csv.DictReader
So your code might be something like -
def csvupload(request, template_name='upload_csv.html'):
if request.method == 'POST':
form = CSVUploadForm(request.POST, request.FILES)
if form.is_valid():
error_list = set()
value = 1
handle_uploaded_file(request.FILES['file']) # save the file
file_name = csvfile.name
file_name = unicodedata.normalize('NFKD', file_name).encode('ascii','ignore')
csvfile2 = settings.MEDIA_ROOT + '/new/sample.csv'
writer = csv.writer(outfile)
row_number = 0
try:
with open(csvfile2, 'w') as writtenFile:
fieldnames = ... # use some code to determine the column names dynamically, may be read the first row and then populate the field names
writer = csv.DictWriter(writtenFile, fieldnames=fieldnames)
writer.writerow({...}) # write header row
with open('some/file/name.csv') as csvfile:
line_entries = csv.DictReader(csvfile)
for row in line_entries:
row_number = row_number + 1
valueDict = [...] # populate the rows as a dict
writer.writerow(valueDict);
success = True
except:
messages.error(request, "Could load file, invalid format")
success = False
return redirect('home')
if success:
messages.success(request, "File read..")
else:
for err in error_list:
messages.error(request, err)
else:
messages.error(request, "No file selected")
else:
form = CSVUploadForm()
context = {
'form': form
}
return render_to_response(template_name, context, context_instance=RequestContext(request))