Im trying to import some data from a csv file to a django database using django-import-export, with a foreign key (location). What I want to achieve is, that the location_id is passed by the request url.
value,datetime,location
4.46,2020-01-01,1
4.46,2020-01-02,1
My urls look like this, so I want "location_id" to be passed into the uploaded csv file:
urlpatterns = [
...
...
path('..../<int:location_id>/upload', views.simple_upload, name='upload'),
]
My view looks like this:
def simple_upload(request, location_id):
if request.method == 'POST':
rainfall_resource = RainfallResource()
dataset = Dataset()
new_rainfall = request.FILES['myfile']
imported_data = dataset.load(new_rainfall.read().decode("utf-8"), format="csv")
try:
result = rainfall_resource.import_data(dataset, dry_run=True) # Test the data import
except Exception as e:
return HttpResponse(e, status=status.HTTP_400_BAD_REQUEST)
if not result.has_errors():
rainfall_resource.import_data(dataset, dry_run=False) # Actually import now
return render(request, '/import.html')
My ModelResource looks like this:
class RainfallResource(resources.ModelResource):
location_id = fields.Field(
column_name='location_id',
attribute='location_id',
widget=ForeignKeyWidget(Location, 'Location'))
class Meta:
model = Rainfall
def before_import_row(self, row, **kwargs):
row['location'] = location_id
The manipulation works when I hardcode "location_id" like:
def before_import_row(self, row, **kwargs):
row['location'] = 123
However, I do not understand how to pass the location_id argument from the "url" to the "before_import_row" function. Help would be highly appreciated:-)
I think you will have to modify your imported_data in memory, before importing.
You can use the tablib API to update the dataset:
# import the data as per your existing code
imported_data = dataset.load(new_rainfall.read().decode("utf-8"), format="csv")
# create an array containing the location_id
location_arr = [location_id] * len(imported_data)
# use the tablib API to add a new column, and insert the location array values
imported_data.append_col(location_arr, header="location")
By using this approach, you won't need to override before_import_row()
Related
I have an excel file with data of employees... The unit name field in the excel sheet is linked to the unit table which is another model.
Have included the ForeignKeyWidget in resources.py to look for the unit while importing and it works with django's inbuilt ImportExportModelAdmin.
While importing through the user made view it throws following error.
['“A Unit” value has an invalid date format. It must be in YYYY-MM-DD format.']
resources.py
from import_export import resources, fields
from import_export.widgets import ForeignKeyWidget
from sqlalchemy import column
from . models import *
class EmployeeResource(resources.ModelResource):
id = fields.Field(attribute='id', column_name='id')
EmpID = fields.Field(attribute='EmpID', column_name='EmpID')
Name = fields.Field(attribute='Name', column_name='Name')
DOB = fields.Field(attribute='DOB', column_name='DOB')
PresentUnit = fields.Field(attribute='PresentUnit', column_name='PresentUnit', widget=ForeignKeyWidget(Unit, 'UnitName'))
PostInDate = fields.Field(attribute='PostInDate', column_name='PostInDate')
class Meta:
model = Employee
views.py
#login_required(login_url='accounts:login')
def upload_employee_view(request):
if request.method == "POST":
file = request.FILES['employeefile']
dataset = Dataset()
if not file.name.endswith('xlsx'):
messages.info(request, "Incorrect File Format, Upload aborted")
return render(request, 'employee/uploademployee.html')
employee_data = dataset.load(file.read(), format='xlsx')
for employee in employee_data:
value = Employee(
employee[0],
employee[1],
employee[2],
employee[3],
employee[4],
employee[5],
)
value.save()
return render(request, 'employee/uploademployee.html')
In addition is there a way such that if an employee already exists with a given EmpID the record of that particular employee will get updated rather than a new record being created.
And also any way to avoid the compulsion of an id column to be included in the excel sheet which has to be uploaded
Not sure if this way is smart, but it simplest workaround:
Check format of date string inside Excel file:
print(employee[5]) for example
Let say it will be 2021/11/02 16:36:23
Convert it to datetime
Convert it to string you need
from datetime import datetime
string_datetime = '2021/11/02 16:36:23'
input_format = '%Y/%m/%d %H:%M:%S'
output_format = '%Y-%m-%d'
formatted_string_date = datetime.strptime(string_datetime, input_format).strftime(output_format)
print(formatted_string_date)
2021-11-02
I'm using the Django Python framework to retrieve data from a CSV.
The code works, but I am trying to make the code reusable and I haven't been able to accomplish that because I can't find a way to pass the url to the csv from an instance.
The code of the view is as follows:
class ThreeLinerCSV(APIView):
authentication_classes = []
permission_classes = []
def get(self, request, format=None):
with open('csvpathhere.csv', newline='') as csvfile:
reader = csv.DictReader(csvfile)
csv1 = list(reader)
header = list(csv1[0].keys())
#Headers
headerData0 = header[0]
headerDataTxt0 = str(headerData0)
headerData1 = header[1]
headerDataTxt1 = str(headerData1)
headerData2 = header[2]
headerDataTxt2 = str(headerData2)
headerData3 = header[3]
headerDataTxt3 = str(headerData3)
#Data arrays
Date = reversed([i[headerDataTxt0] for i in csv1])
DataValue1 = reversed([i[headerDataTxt1] for i in csv1])
DataValue2 = reversed([i[headerDataTxt2] for i in csv1])
DataValue3 = reversed([i[headerDataTxt3] for i in csv1])
#Data to send to views
data = {
"labels": Date,
"dataAxis1": DataValue1,
"dataAxis2": DataValue2,
"dataAxis3": DataValue3,
"headerData1": headerData1,
"headerData2": headerData2,
"headerData3": headerData3,
}
return Response(data)
#I call the class here
OISLIBOR = ThreeLinerCSV()
Then, the urls.py has this:
url(r'^api/OISLIBOR/data/$', OISLIBOR.as_view()),
I need to find a way to get the "csvpathhere.csv" out of the class and be able to input it from the instance. Any idea of how to do that?
If I understand your questions correctly, you could pass the target CSV name as a GET query string parameter.
You would have to change one line in your view:
with open(request.GET.get('target_name'), newline='') as csvfile:
And the call your endpoint like this
http: ... api/OISLIBOR/data/?target_name=here.goes.the.target.url
I'm want to populate a WTForms SelectField with the rows returned by this query:
cur.execute("SELECT length FROM skipakke_alpin_ski WHERE stock > 0")
The query returns rows with the ski length of different types of skis. cur.fetchall() returns the following tuple:
[(70,), (75,), (82,), (88,), (105,), (115,), (125,), (132,), (140,), (150,), (160,), (170,)]
How would I go about to do add these numbers to a SelectField, so that each ski length would be its own selectable choice? If I had done this manually, I would have done the following:
ski_size = SelectField('Ski size', choices=['70', '70', '75', '75'])
... And so on for all of the different lengths.
In one of the projects I have used like below:
models
class PropertyType(db.Model):
id = db.Column(db.Integer, primary_key=True)
title = db.Column(db.String(255), nullable=False)
def __repr__(self):
return str(self.id)
and in forms
from wtforms.ext.sqlalchemy.fields import QuerySelectField
class PropertyEditor(Form):
property_type = QuerySelectField(
'Property Type',
query_factory=lambda: models.PropertyType.query,
allow_blank=False
)
//Other remaining fields
Hope this helps.
The solution might look like the code below.
Let's assume you have two files: routes.py and views.py
In routes.py file you put this
from flask_wtf import FlaskForm
from wtforms import SelectField
# Here we have class to render ski
class SkiForm(FlaskForm):
ski = SelectField('Ski size')
In views.py file you put this
# Import your SkiForm class from `routes.py` file
from routes import SkiForm
# Here you define `cur`
cur = ...
# Now let's define a method to return rendered page with SkiForm
def show_ski_to_user():
# List of skies
cur.execute("SELECT length FROM skipakke_alpin_ski WHERE stock > 0")
skies = cur.fetchall()
# create form instance
form = SkiForm()
# Now add ski length to the options of select field
# it must be list with options with (key, value) data
form.ski.choices = [(ski, ski) for ski in skies]
# If you want to use id or other data as `key` you can change it in list generator
if form.validate():
# your code goes here
return render_template('any_file.html', form=form)
Remember that by default key value is unicode. If you want to use int or other data type use coerce argument in SkiForm class, like this
class SkiForm(FlaskForm):
ski = SelectField('Ski size', coerce=int)
Solution:
I had quite simmilar problem, and here is my workaround
class SkiForm(FlaskForm):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
cur.execute("SELECT length FROM skipakke_alpin_ski WHERE stock > 0")
skies = cur.fetchall()
self.ski_size.choices = [
(ski , ski) for ski in skies
]
ski_size = SelectField("Ski size")
Explanation:
We modified original FlaskForm, so that it executs database query each time when it is being created.
So SkiForm field data choices always stays up to date.
I managed to solve this by doing the following:
def fetch_available_items(table_name, column):
with sqlite3.connect('database.db') as con:
cur = con.cursor()
cur.execute("SELECT length FROM skipakke_alpin_ski WHERE stock > 0")
return cur.fetchall()
class SkipakkeForm(Form):
alpin_ski = SelectField('Select your ski size', choices=[])
#app.route('/skipakke')
def skipakke():
form = SkipakkeForm
# Clear the SelectField on page load
form.alpin_ski.choices = []
for row in fetch_available_items('skipakke_alpin_ski', 'length'):
stock = str(row[0])
form.alpin_ski.choices += [(stock, stock + ' cm')]
The following is a working code in google app engine. This is used to display some records from a SentMail Model. The SentMail have a large no of fields, here i only showed the fields we now need. Since for displaying data i do not require to take the complete record. Hence i used projection. Note: This code is working
class SentMail(ndb.Model):
to_email = ndb.StringProperty()
bounced = ndb.BooleanProperty(default=False)
bounce_type = ndb.StringProperty()
bounceSubType = ndb.StringProperty()
#staticmethod
def get_bounced_emails():
db_query = SentMail.query(SentMail.bounced==True, projection=['to_email', 'bounce_type'], distinct=True).order(SentMail.bounce_type).order(SentMail.to_email).fetch()
return db_query if len(db_query) > 0 else None
class BounceCompaintRender(session_module.BaseSessionHandler):
"""docstring for BounceCompaintRender"""
def get(self):
bounced_emails = self.get_data('complete_bounced_rec')
template_values = {
'bounced_emails': bounced_emails
}
path = os.path.join(os.path.dirname(__file__), 'templates/bounce_complaint_emails.html')
self.response.out.write(template.render(path, template_values))
else:
self.redirect('/login')
def get_data(self, key):
data = memcache.get(key)
if data is not None:
for each in data:
logging.info(each)
return data
else:
data = SentMail.get_bounced_emails()
memcache.add(key="complete_bounced_rec", value=data, time=3600)
return data
Here the only change I made is in SentMail.get_bounced_emails()
#staticmethod
def get_bounced_emails():
db_query = SentMail.query(SentMail.bounced==True, projection=['to_email', 'bounceSubType'], distinct=True).order(SentMail.bounceSubType).order(SentMail.to_email).fetch()
return db_query if len(db_query) > 0 else None
I now get an error UnprojectedPropertyError: Property bounceSubType is not in the projection. I checked the logs and i found in the projection field one parameter is missing and eventhough it has a value None(None is not the only value). I tried clearing memcache but the problem still arises. The following is the log
SentMail(key=Key('SentMail', 655553213235462), bounceSubType=None, to_email=u'test#example.com', _projection=('to_email',))
This error is due to the difference in the model SentMail wrote in models.py and in datastore (i.e., properties are different in both). For this you need to update all records in datastore related to SentMail then datastore and model will have the same fields.
I have an Openstack-powered, Django-modified application that shows the disk images and snapshots available for a user to launch. The user currently sees both snapshots they created and ones they did not. I would like to separate the current table into two based on whether they are owned by the user or not.
My two table definitions are as follows (note I altered row_actions accordingly):
class UserSnapshotsTable(OldSnapshotsTable):
cloud = tables.Column(get_cloud, verbose_name=_("Cloud"))
class Meta:
name = "usersnapshots"
verbose_name = _("User Snapshots")
table_actions = (DeleteSnapshot,)
row_actions = (LaunchSnapshot, LaunchCluster, EditImage, DeleteSnapshot)
pagination_param = "snapshot_marker"
row_class = UpdateRow
status_columns = ["status"]
class OtherSnapshotsTable(OldSnapshotsTable):
cloud = tables.Column(get_cloud, verbose_name=_("Cloud"))
class Meta:
name = "othersnapshots"
verbose_name = _("Other Snapshots")
table_actions = (DeleteSnapshot,)
row_actions = (LaunchSnapshot, LaunchCluster)
pagination_param = "snapshot_marker"
row_class = UpdateRow
status_columns = ["status"]
I have altered the HTML template to pull the "UserSnapshotsTable" and "OtherSnapshotsTable" tables (I copied the original table and renamed both), but both full tables still generate under the respective headings. There are two functions generating the data:
def get_usersnapshots_data(self):
req = self.request
marker = req.GET.get(UserSnapshotsTable._meta.pagination_param, None)
try:
usersnaps, self._more_snapshots = api.snapshot_list_detailed(req,
marker=marker)
except:
usersnaps = []
exceptions.handle(req, _("Unable to retrieve user-owned snapshots."))
return usersnaps
def get_othersnapshots_data(self):
req = self.request
marker = req.GET.get(OtherSnapshotsTable._meta.pagination_param, None)
try:
othersnaps, self._more_snapshots = api.snapshot_list_detailed(req,
marker=marker)
except:
othersnaps = []
exceptions.handle(req, _("Unable to retrieve non-user-owned snapshots."))
return othersnaps
There are also Edit/Delete options defined for images, and imported for snapshots, that seem to have a key comparison. Here's the "Delete" one (line 7):
class DeleteImage(tables.DeleteAction):
data_type_singular = _("Image")
data_type_plural = _("Images")
def allowed(self, request, image=None):
if image:
return image.owner == request.user.tenant_id
# Return True to allow table-level bulk delete action to appear.
return True
def delete(self, request, obj_id):
api.image_delete(request, obj_id)
How can I separate those tables out? This is my first time asking a question here, so please let me know if I can provide further information. Apologies for the length of it.
As far as I see you are using glanceclient. If that so you can use extra_filters parameter of snapshot_list_detailed() to filter only user images like this:
usersnaps, self._more_snapshots = api.snapshot_list_detailed(
req,
marker = marker,
extra_filters = {"owner": "user_name"}
)
Under cover snapshot_list_detailed uses GET images of Openstack Image Service API.