I'm using django import export (DIE) to import some data. I have a problem importing the data.
my admin.py:
class TestResource(resources.ModelResource):
class Meta:
model = Test
exclude = ('id',)
import_id_fields = ['VS',]
skip_unchanged = True
class TestAdmin(ImportExportMixin,admin.ModelAdmin):
fieldsets = [
('VS', {'fields':['VS']}),
('pool', {'fields':['pool']}),
('pool_port', {'fields':['pool_port']}),
('idc', {'fields':['idc']})
]
list_display = ('VS','pool','pool_port','idc')
list_filter = ['pool_port']
search_fields = ['VS','pool','pool_port','idc']
resource_class = TestResource
admin.site.register(Test,TestAdmin)
I want to import an excel file like:
But:
I want to import all the rows . Please tell me how to ignore the duplicates. Thanks in advance!
change line
skip_unchanged = True
to
skip_unchanged = False
OK I find one way to import. Import the file with save(). If you guys have a better way please tell me~
The code as follow:
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE","mysite.settings")
import django
django.setup()
import xlrd
from Test.models import Test
def input(VS,pool_text,port,idc_text):
i = Test(
VS = vs,
pool = pool_text,
pool_port = port,
idc = idc_text
)
i.save()
files = xlrd.open_workbook('test.xls')
sh = files.sheet_by_index(0)
n = 0
for i in range(1,sh.nrows):
vs = sh.cell(i,0).value
pool_text = sh.cell(i,1).value
port = sh.cell(i,2).value
idc = sh.cell(i,3).value
input(vs,pool_text,port,idc)
n += 1
print n
Related
I've created a function in my serializers.py that call an external API and give me a dict back.
How can I use the output from return downloads in the get_all_files as a field in class Meta?
After the first answer, I've got the following error message:
Exception Value: 'NoneType' object is not subscriptable
Exception Location: /app/api/serializers.py, line 68, in get_all_files
Line 68 is the following: return get_all_files(instance.bands)
serializers.py
from rest_framework import serializers
from . import views
from api.models import Application, Indice, Satellite, Band
from satsearch import Search
class IndiceSerializer(serializers.ModelSerializer):
class Meta:
model = Indice
fields = ['name', 'accr', 'description', 'is_NormalizedDifference', 'calc', ]
class SatelliteSerializer(serializers.ModelSerializer):
class Meta:
model = Satellite
fields = ['name', 'accr', 'operator', ]
class BandSerializer(serializers.ModelSerializer):
class Meta:
model = Band
fields = ['band', 'description', 'wavelength', 'resolution', ]
class OsdSerializer(serializers.ModelSerializer):
bands = BandSerializer(source='indice_to_use.needed_bands', many=True)
satellite = SatelliteSerializer(source='indice_to_use.satellite_to_use')
indice = IndiceSerializer(source='indice_to_use')
files = serializers.SerializerMethodField()
def get_files(self, instance):
def get_all_files(bands):
# configuration
url = 'https://earth-search.aws.element84.com/v0' # URL to Sentinel 2 AWS catalog
collection = 'sentinel-s2-l2a-cogs'
# search parameter
startDate = '2021-04-10'
endDate = '2021-04-12'
location = [ 13.6677,
43.7232,
16.2605,
45.4522
]
bbox_search = Search(
bbox=location,
datetime=startDate+"/"+endDate,
query={'eo:cloud_cover': {'lt': 50}},
collections=[collection],
url=url,
sort={'field': 'eo:cloud_cover', 'direction': 'desc'},
)
items = bbox_search.items()
downloads = {}
for i, item in enumerate(items):
data = {}
data['Product ID']= item.properties["sentinel:product_id"]
data['Preview']= item.asset("thumbnail")["href"]
data['Date']= item.properties["datetime"]
data['Cloud cover']= item.properties["eo:cloud_cover"]
for band in bands.split(','):
data[band] = item.asset(band)["href"]
downloads[i] = data
return downloads
return get_all_files(instance.bands)
class Meta:
model = Application
fields = ['machine_name', 'name', 'description', 'indice', 'satellite', 'bands', 'files', ]
In order to do this, you would need to do the following:
Alongside the other serializers (bands, satellite and indices)
You should put:
files = SerializerMethodField()
This SerializerMethodField gets the value by calling the method get_{field_name}, in this case would be get_files. More on this
So you could move all the logic you have up there in this method:
def get_files(self, instance):
return get_all_fields(instance.bands)
And now you would only have to put files in the fields list like this:
fields = ['machine_name', 'name', 'description', 'indice', 'satellite', 'bands', 'files']
Your nested function is throwing off the interpreter, which results in an incomplete stack trace. I'd start by converting your get_files method to a more typical function:
class OsdSerializer(serializers.ModelSerializer):
bands = BandSerializer(source='indice_to_use.needed_bands', many=True)
satellite = SatelliteSerializer(source='indice_to_use.satellite_to_use')
indice = IndiceSerializer(source='indice_to_use')
files = serializers.SerializerMethodField()
def get_files(self, instance):
bands = instance.bands
# configuration
url = 'https://earth-search.aws.element84.com/v0' # URL to Sentinel 2 AWS catalog
collection = 'sentinel-s2-l2a-cogs'
# search parameter
startDate = '2021-04-10'
endDate = '2021-04-12'
location = [ 13.6677,
43.7232,
16.2605,
45.4522
]
bbox_search = Search(
bbox=location,
datetime=startDate+"/"+endDate,
query={'eo:cloud_cover': {'lt': 50}},
collections=[collection],
url=url,
sort={'field': 'eo:cloud_cover', 'direction': 'desc'},
)
items = bbox_search.items()
downloads = {}
for i, item in enumerate(items):
data = {}
data['Product ID']= item.properties["sentinel:product_id"]
data['Preview']= item.asset("thumbnail")["href"]
data['Date']= item.properties["datetime"]
data['Cloud cover']= item.properties["eo:cloud_cover"]
for band in bands.split(','):
data[band] = item.asset(band)["href"]
downloads[i] = data
return downloads
Now when you run it, you will still get the TypeError, but it will point to the line where the error is actually occuring, which I assume is in the downloads loop.
If my assumption is correct, then you will need to make sure you are parsing the data correctly, because you are trying to access a key on an object that doesn't exist. For example, in the line:
data['Preview']= item.asset("thumbnail")["href"]
if item.asset('thumbnail') returns None, then you would get this error.
If you still can't figure it out, please provide a link to the satsearch package, and/or show example data that is returned from Search(...)
The code works fine. The error was caused by a mistake in my backend. The key of the sat-search result was in 3 digest, e.g. B03. The content in my field was in two digst, e.g. B3. That's why the matching doesn't worked.
Hello I want to share a mongodb between two django apps (monitor , manager) using one application called common.
I can create database instances in the manager application easily but when accessing the book authors i cannot.
it return this error
mongoengine.errors.FieldDoesNotExist: The fields "{'_id'}" do not
exist on the document "author"
models.py
from mongoengine import *
class author(Document):
name = StringField(required = True)
meta = {'abstract': True , 'allow_inheritance':True}
class book(Document):
name = StringField(required = True)
authors = ListField(ReferenceField(author))
meta = {'abstract': True , 'allow_inheritance':True}
manager.py
from mongoengine import *
from models import *
class author(author):
def rand(self):
print("i am useless")
class book(book):
def rand2(self):
print("i am also useless")
if __name__ == "__main__":
connect('test', host = '0.0.0.0',port = 27017)
a1 = author(name = "Charef")
a1.save()
a2 = author(name = "hamid")
a2.save()
a3 = author(name = "djoudi")
a3.save()
a4 = author(name = "cheb khaled")
a4.save()
book1_authors = [a1,a2,a4]
book2_authors = [a1,a3]
book1 = book(name = "Hello Django", authors = book1_authors)
book1.save()
book2 = book(name = "Hello python", authors = book2_authors)
book2.save()
monitor
from mongoengine import *
from models import *
class author(author):
def say_hi(self):
print("Hi, my name is {} and this is my book".format(self.name))
class book(book):
def book_info(self):
for author in self.authors:
print(author.say_hi())
print("done !! ")
if __name__ == "__main__":
connect("test",host = "0.0.0.0", port = 27017)
s_book = book.objects()[0]
print(s_book.name)
print(len(s_book.authors))
Use unique names for the different classes (e.g. BaseBook for the abstract class and Book for the concrete class). Some internals of mongoengine relies on the uniqueness of the class names so it's not a good idea to go against that.
Using the following works:
class BaseAuthor(Document):
name = StringField(required=True)
meta = {'abstract': True , 'allow_inheritance':True}
class BaseBook(Document):
name = StringField(required=True)
authors = ListField(ReferenceField(BaseAuthor))
meta = {'abstract': True , 'allow_inheritance':True}
class Author(BaseAuthor):
def rand(self):
print("i am useless")
class Book(BaseBook):
def rand2(self):
print("i am also useless")
Also, if possible use the same Book/Author classes in both monitor and manager
I have a function which search for json files in a directory, parse the file and write data in the database. My problem is writing in database, because it take around 30 minutes. Any idea how can I speed up writting in a database? I have few quite big files to parse, but parsing the file is not a problem it take around 3 minutes. Currently I am using sqlite but in the future I will change it to PostgreSQL.
Here is my function:
def create_database():
with transaction.atomic():
directory = os.fsencode('data/web_files/unzip')
for file in os.listdir(directory):
filename = os.fsdecode(file)
with open('data/web_files/unzip/{}'.format(filename.strip()), encoding="utf8") as f:
data = json.load(f)
cve_items = data['CVE_Items']
for i in range(len(cve_items)):
database_object = DataNist()
try:
impact = cve_items[i]['impact']['baseMetricV2']
database_object.severity = impact['severity']
database_object.exp_score = impact['exploitabilityScore']
database_object.impact_score = impact['impactScore']
database_object.cvss_score = impact['cvssV2']['baseScore']
except KeyError:
database_object.severity = ''
database_object.exp_score = ''
database_object.impact_score = ''
database_object.cvss_score = ''
for vendor_data in cve_items[i]['cve']['affects']['vendor']['vendor_data']:
database_object.vendor_name = vendor_data['vendor_name']
for description_data in cve_items[i]['cve']['description']['description_data']:
database_object.description = description_data['value']
for product_data in vendor_data['product']['product_data']:
database_object.product_name = product_data['product_name']
database_object.save()
for version_data in product_data['version']['version_data']:
if version_data['version_value'] != '-':
database_object.versions_set.create(version=version_data['version_value'])
My models.py:
class DataNist(models.Model):
vendor_name = models.CharField(max_length=100)
product_name = models.CharField(max_length=100)
description = models.TextField()
date = models.DateTimeField(default=timezone.now)
severity = models.CharField(max_length=10)
exp_score = models.IntegerField()
impact_score = models.IntegerField()
cvss_score = models.IntegerField()
def __str__(self):
return self.vendor_name + "-" + self.product_name
class Versions(models.Model):
data = models.ForeignKey(DataNist, on_delete=models.CASCADE)
version = models.CharField(max_length=50)
def __str__(self):
return self.version
I will appreciate if you can give me any advice how can I improve my code.
Okay, given the structure of the data, something like this might work for you.
This is standalone code aside from that .objects.bulk_create() call; as commented in the code, the two classes defined would actually be models within your Django app.
(By the way, you probably want to save the CVE ID as an unique field too.)
Your original code had the misassumption that every "leaf entry" in the affected version data would have the same vendor, which may not be true. That's why the model structure here has a separate product-version model that has vendor, product and version fields. (If you wanted to optimize things a little, you might deduplicate the AffectedProductVersions even across DataNists (which, as an aside, is not a perfect name for a model)).
And of course, as you had already done in your original code, the importing should be run within a transaction (transaction.atomic()).
Hope this helps.
import json
import os
import types
class DataNist(types.SimpleNamespace): # this would actually be a model
severity = ""
exp_score = ""
impact_score = ""
cvss_score = ""
def save(self):
pass
class AffectedProductVersion(types.SimpleNamespace): # this too
# (foreign key to DataNist here)
vendor_name = ""
product_name = ""
version_value = ""
def import_item(item):
database_object = DataNist()
try:
impact = item["impact"]["baseMetricV2"]
except KeyError: # no impact object available
pass
else:
database_object.severity = impact.get("severity", "")
database_object.exp_score = impact.get("exploitabilityScore", "")
database_object.impact_score = impact.get("impactScore", "")
if "cvssV2" in impact:
database_object.cvss_score = impact["cvssV2"]["baseScore"]
for description_data in item["cve"]["description"]["description_data"]:
database_object.description = description_data["value"]
break # only grab the first description
database_object.save() # save the base object
affected_versions = []
for vendor_data in item["cve"]["affects"]["vendor"]["vendor_data"]:
for product_data in vendor_data["product"]["product_data"]:
for version_data in product_data["version"]["version_data"]:
affected_versions.append(
AffectedProductVersion(
data_nist=database_object,
vendor_name=vendor_data["vendor_name"],
product_name=product_data["product_name"],
version_name=version_data["version_value"],
)
)
AffectedProductVersion.objects.bulk_create(
affected_versions
) # save all the version information
return database_object # in case the caller needs it
with open("nvdcve-1.0-2019.json") as infp:
data = json.load(infp)
for item in data["CVE_Items"]:
import_item(item)
In my project im trying to use an external .py file for manage data using my django app style like this:
In my django project i create a model:
class temp_test_keywords(models.Model):
main_id = models.ForeignKey(temp_main)
test_id = models.ForeignKey(temp_case)
key_id = models.ForeignKey(temp_keywords)
variable_id = models.ForeignKey(temp_variables, null=True, blank=True)
def __str__(self):
return '%s -> %s' % (str(self.main_id), str(self.test_id))
Well, now in my external rst.py file i start django env like this:
import sys
import os
import django
sys.path.append('core')
os.environ['DJANGO_SETTINGS_MODULE'] = 'core.settings'
django.setup()
ok, at this point i import table and create class for do some thinks with it:
from django.db import models
from django.contrib.contenttypes.fields import
GenericForeignKey,GenericRelation
from django.contrib.contenttypes.models import ContentType
from django.db.models import Count
from frontend.models import temp_test_keywords
class PrepareRst:
def __init__(self,test_id,t_type,log=False):
self.rst = self.mainprep(test_id,t_type)
def mainprep(self,test_id,t_type):
return self.tc_prep(test_id)
#TestCase rst prep method
def tc_prep(self,test_id):
maxpar = temp_test_keywords.objects.filter(main_id = test_id).values('key_id').annotate(total=Count('variable_id')).order_by('-total').first()
totpar = maxpar['total']
#Part1 list creation
count = 0
ltouple = ()
l1 = ["Test Case"]
while (count < totpar):
l1.append("")
count += 1
ltouple += (l1,)
#Query for extract keywords, values
kv = temp_test_keywords.select_related()
but when i run an AttributeError: type object 'temp_test_keywords' has no attribute 'select_related' error raise
if i start python manage.py shell from terminal the "kv = temp_test_keywords.select_related()" command works fine, why in my .py code doesn't?
Thanks in advance
Try,
kv = temp_test_keywords.objects.all().select_related()
I am trying to build a very simple, wiki-type site using django-cms.
I have 1 app, with 2 models defined:
class Subject(models.Model):
label=models.CharField
class Topic(models.Model):
...
cat = models.ForeignKey('topics.Category',
blank=True,
default=None,
help_text=u'Please choose a category for this topic',
null=True
)
I am trying to have the default menu show the Subject classes as the top-level options, with the Topic classes as sub-levels for each Subject. There will be 4 Subjects altogether. E.g.:
Subject 1
-topic1
-topic2
Subject 2
-topic3
-topic4
etc..
I have read all the django-cms docs, and I'm still confused. In my menu.py, should I be extending Menu or CMSAttachMenu? Do I need 4 different generators? How do I reference the ForeignKey field when using the generator?
I am a beginner, any help is greatly appreciated
You could do something like that:
# menu.py
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _
from cms.menu_bases import CMSAttachMenu
from menus.base import NavigationNode
from menus.menu_pool import menu_pool
from .models import Subject
class SubjectsMenu(CMSAttachMenu):
name = _("Subjects Menu")
def get_nodes(self, request):
nodes = []
cnt = 0
for subject in Subjects.objects.all():
subject_node_id = cnt
node = NavigationNode(
subject.label,
reverse('subject_view_detail', args=(subject.pk,)),
subject_node_id
)
nodes.append(node)
for topic in subject.topics.all():
cnt += 1
node = NavigationNode(
topic.name,
reverse('topic_view_detail', args=(topic.pk,)),
cnt,
subject_node_id # parent
)
nodes.append(node)
cnt += 1
return nodes
menu_pool.register_menu(SubjectsMenu)
Then you can add this menu to your AppHook or attach it from the admin.