I'm attempting to process an uploaded CSV file using Django. The main logic of how I go about doing this is expressed in both the models.py and views.py scripts. Once I've uploaded the file, I'm unable to process any of the content (in my views.py). Here are the two scripts, but if there's any more information I can provide, I'd be happy to.
In my models.py file, I have two classes, one for the document itself, and the other class for the fields in the file.
models.py:
from django.db import models
import os
class Document(models.Model):
docfile = models.FileField(upload_to='documents')
class DocumentEntry(models.Model):
document = models.ForeignKey(Document, on_delete=models.CASCADE)
field = models.CharField(max_length=250, default="TEST")
Next, in my views.py I get the file that was uploaded via the request.FILES['docfile'] and pass it to the handle_files() function. However, when I try to loop through the reader, I'm unable to access any of the elements in the file that was uploaded.
views.py:
from django.shortcuts import render
from django.conf import settings
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse
import csv
from .models import Document, DocumentEntry
from .forms import UploadFileForm
def process_file(request):
# Handle file upload
if request.method == 'POST':
form = UploadFileForm(request.POST, request.FILES)
if form.is_valid():
handle_files(request.FILES['docfile'])
# Redirect to the document list after POST
return HttpResponseRedirect(reverse('process_files'))
else:
form = UploadFileForm() # A empty, unbound form
# Load documents for the list page
documents = Document.objects.all()
# Render list page with the documents and the form
return render(
request,
'upload/process_files.html',
{'documents': documents, 'form': form}
)
def handle_files(csv_file):
newdoc = Document(docfile=csv_file)
newdoc.save()
reader = csv.DictReader(open(csv_file))
for row in reader:
field = row['field']
entry = DocumentEntry(document=newdoc, field=field)
entry.save()
Updated
Here is full example of handler function:
def handle_files(csv_file):
newdoc = Document(docfile=csv_file)
newdoc.save()
with open(newdoc.docfile.name) as f:
reader = csv.DictReader(f)
for row in reader:
field = row['field']
entry = DocumentEntry(document=newdoc, field=field)
entry.save()
open() expects the path to the file, not the actual file data, which is contained in request.FILES['docfile'].
Replace:
reader = csv.DictReader(open(csv_file))
with:
import io
io_file = io.TextIOWrapper(csv_file.file)
reader = csv.DictReader(io_file)
Related
I wanted some help with a project I'm working on:
So I have to take a CSV as input from the user and convert the CSV save it as a Table in the Database.
So I was able to take the input from the user and save it in the MEDIA_ROOT.
But now I'm unsure how to create a model without knowing the columns and so on. (I know I can get the columns from pandas(in views.py) but how to send that column details to models.py)
I'm new to Django and very much confused with so many files. Please help.
Note: I don't want to save the CSV file I want to convert it into a table in the database.
views.py
import pandas as pd
from django.shortcuts import render
from django.http import HttpResponse
from django.views.generic import TemplateView, ListView, CreateView
from django.core.files.storage import FileSystemStorage
def home(request):
return render(request, 'index.html')
def upload(request):
if request.method == 'POST':
upload_file = request.FILES['csvfile']
fs = FileSystemStorage()
fs.save(upload_file.name, upload_file)
dataset = pd.read_csv('media/'+upload_file.name)
colums = list(dataset.columns)
return render(request, 'upload.html')
models.py
from assignment.settings import MEDIA_ROOT
from django.db import models
from django.db.models.fields import CharField
class CSV(models.Model):
csvfile = models.FileField(upload_to='CSV')
The best scenario here is to use a single model called Files and store CSV in a JSON field. You can easily read CSV as a Dict using csv.DictReader and save it in database.
Im working through Alex Pale's minimal django file upload example for django 1.8 -
https://github.com/axelpale/minimal-django-file-upload-example/tree/master/src/for_django_1-8/myproject/myproject/myapp
I know how to get the file extension in the form, but how can I get this in the view. I'm aware I can access the file thru -
docfile=request.FILES['docfile']
View -
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse
from myproject.myapp.models import Document
from myproject.myapp.forms import DocumentForm
def list(request):
# Handle file upload
if request.method == 'POST':
form = DocumentForm(request.POST, request.FILES)
if form.is_valid():
docfile=request.FILES['docfile']
# Redirect to the document list after POST
return HttpResponseRedirect(reverse('myproject.myapp.views.list'))
else:
form = DocumentForm() # A empty, unbound form
# Load documents for the list page
documents = Document.objects.all()
# Render list page with the documents and the form
return render_to_response(
'list.html',
{'documents': documents, 'form': form},
context_instance=RequestContext(request)
)
Form -
from django import forms
class DocumentForm(forms.Form):
docfile = forms.FileField(
label='Select a file'
)
If you are asking for the extension try this:
>>> f = "this.that.ext"
>>> e = f.split(".")[-1]
>>> f
'this.that.ext'
>>> e
'ext'
>>>
You know how to get the complete filename. What you want is the last string after the "."
I want to make a application when the user upload the file, so get it to system, and execute the linux command like "xxd".
cSite---cSite---settings.py
| |__etc...
|
--myapp---
Here is my file upload codes.
views.py :
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse
from myapp.models import Document
from myapp.forms import DocumentForm
def list(request):
# Handle file upload
if request.method == 'POST':
form = DocumentForm(request.POST, request.FILES)
if form.is_valid():
newdoc = Document(docfile = request.FILES['docfile'])
newdoc.save()
# Redirect to the document list after POST
return HttpResponseRedirect(reverse('myapp.views.list'))
else:
form = DocumentForm() # A empty, unbound form
# Load documents for the list page
documents = Document.objects.all()
# Render list page with the documents and the form
return render_to_response(
'list.html',
{'documents': documents, 'form': form},
context_instance=RequestContext(request)
)
models.py :
from django.db import models
class Document(models.Model):
docfile = models.FileField(upload_to='documents/%Y/%m/%d')
As you know, this code is widespreaded simple file upload form.
Now, here is my trouble.
Want to use the salt when I save the file
(Can I use the "Keeping Original Filename for FileField in Django"? and if I can, how to apply this?
I don't know what is representing the file path variable in FileField on ./views.py/ It always contain the error. (The command must be executed at the right after same time of file uploaded in user view.)
i am doing a copy cat with Need a minimal Django file upload example
i changed the view.py with my code to dump the csv file into sqlite database.
i have already created the table in default sqlite database.
import sqlite3
import csv
import sys
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse
from myproject.myapp.models import Document
from myproject.myapp.forms import DocumentForm
def list(request):
# Handle file upload
if request.method == 'POST':
form = DocumentForm(request.POST, request.FILES)
if form.is_valid():
newdoc = Document(docfile = request.FILES['docfile'])
newdoc.save()
gfile= csv.reader(open(newdoc))
gon = sqlite3.connect("database.sqlite")
gon.text_factory = str
gon.execute("DELETE FROM abc where rowID > 0 ")
gon.executemany("insert into abc values (?, ?, ?, ?, ?)", gfile)
gon.commit()
gon.close()*
return HttpResponseRedirect(reverse('myproject.myapp.views.list'))
else:
form = DocumentForm() # A empty, unbound form
# Load documents for the list page
documents = Document.objects.all()
# Render list page with the documents and the form
return render_to_response(
'myapp/list.html',
{'documents': documents, 'form': form},
context_instance=RequestContext(request)
)
my code is starting from first introduction of gfile
Error # line 20 : Coercing to unicode : need string or buffer, Document Found
please help
You are passing Document instance to open. Instead you should pass the file, that was uploaded directly to csv.reader:
gfile = csv.reader(request.FILES['docfile'])
On the 'panel' page, I have a choice field with a list of uploaded documents or 'bots' as I usually refer to them. This list only displays 'bots' that have been uploaded by the current user.
panel\forms.py
from django import forms
import os
from upload.models import Document
#### RETRIEVE LIST OF BOTS UPLOADED BY CURRENT USER ####
def get_files(user):
bots = Document.objects.filter(user=user.id)
file_list = []
for b in bots:
file_list.append((b.id,b.docfile))
return file_list
class botForm(forms.Form):
def __init__(self, user, *args, **kwargs):
super(botForm, self).__init__(*args, **kwargs)
self.fields['bot'] = forms.ChoiceField(choices=get_files(user))
This works fine and displays a list of all the users bots. The problem arises when I try to pass these values over to the 'game' page and access them here.
game\views.py
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse
from game.models import game
from game.forms import GameForm
from upload.models import Document
from panel.forms import botForm
import league
def RPS(request):
if request.method == 'POST': # If the request is a POST method...
if 'PanelPlay' in request.POST:
panel = botForm(request.POST)
if panel.is_valid():
print panel.cleaned_data['bot']
elif 'GamePlay' in request.POST:
form = GameForm(request.POST) # A form bound to the POST data
if form.is_valid(): # All validation rules pass
leagueOuput = []
leagueOutput = league.run(form.cleaned_data['bot1'],form.cleaned_data['bot2'])
newGame = game()
newGame.bot1 = leagueOutput[0]
newGame.bot2 = leagueOutput[1]
newGame.bot1wins = leagueOutput[2]
newGame.bot2wins = leagueOutput[3]
newGame.save()
return HttpResponseRedirect(reverse('game.views.RPS')) # Redirect after POST
form = GameForm() # An unbound form
results = game.objects.all() # Load messages for the list page
return render_to_response('game.html', {'results': results, 'form': form}, context_instance=RequestContext(request))
When attempting to access and validate the panel data, I get the following error.
'QueryDict' object has no attribute 'id'
Referring to this specific line.
bots = Document.objects.filter(user=user.id)
I have found and read about a number of similar issues but I can't seem to carry over their solutions to my own project.
Thanks in advance for any and all help.
When you are constructing the botForm, you're passing request.POST (a QueryDict) as the user parameter. Did you mean
panel = botForm(request.user, data=request.POST)
?
(assuming you're using django authentification).