We Keep Coding

How to access data from custom_export in Otree?

I am trying to use ExtraModel and custom_export to export data from live_pages. However, when I go on devserver and check the data tab, it is nowhere to be found. If I download the excel (bottom right of the page) the new variables are not included in the data.
Where can I find the data from the custom export? Or am I defining the function wrong? Any help greatly appreciated.
See MWE below
from otree.api import *
import random
doc = """
Your app description
"""
class C(BaseConstants):
NAME_IN_URL = 'mwe_export'
PLAYERS_PER_GROUP = None
NUM_ROUNDS = 1
NUM_EMPLOYERS = 3
class Subsession(BaseSubsession):
pass
class Group(BaseGroup):
pass
class Player(BasePlayer):
pass
class Offer(ExtraModel):
group = models.Link(Group)
sender = models.Link(Player)
wage = models.IntegerField()
effort = models.IntegerField()
job_id = models.IntegerField()
information_type = models.StringField()
# FUNCTIONS
def to_dict(offer: Offer):
return dict(sender=offer.sender.id_in_group,
wage=offer.wage,
effort=offer.effort,
job_id=offer.job_id,
information_type=offer.information_type)
# PAGES
class MyPage(Page):
#staticmethod
def js_vars(player: Player):
return dict(my_id=player.id_in_group)
#staticmethod
def live_method(player: Player, data):
print(data)
group = player.group
job_id = random.randint(1, 1000)
wage = data['wage']
effort = data['effort']
information_type = data['information_type']
if data['information_type'] == 'offer':
offer = Offer.create(group=group,
sender=player,
job_id=job_id,
wage=wage,
effort=effort,
information_type=information_type)
print(offer)
print(to_dict(offer))
return {0: to_dict(offer)}
page_sequence = [MyPage]
def custom_export(players):
yield ['session.code', 'participant_code', 'id_in_session']
offers = Offer.filter()
for offer in offers:
player = offer.sender
participant = player.participant
yield [participant.code, participant.id_in_session, offer.job_id, offer.wage, offer.effort]

In the menu at the top of the admin page there is also a "Data" item. The custom export for your app should be available there under the heading "Per-app".

peewee check automatically created id not in result of subquery

I have next data structure:
from enum import IntEnum, unique
from pathlib import Path
from datetime import datetime
from peewee import *
#unique
class Status(IntEnum):
CREATED = 0
FAIL = -1
SUCCESS = 1
db_path = Path(__file__).parent / "test.sqlite"
database = SqliteDatabase(db_path)
class BaseModel(Model):
class Meta:
database = database
class Unit(BaseModel):
name = TextField(unique=True)
some_field = TextField(null=True)
created_at = DateTimeField(default=datetime.now)
class Campaign(BaseModel):
id_ = AutoField()
created_at = DateTimeField(default=datetime.now)
class Task(BaseModel):
id_ = AutoField()
status = IntegerField(default=Status.CREATED)
unit = ForeignKeyField(Unit, backref="tasks")
campaign = ForeignKeyField(Campaign, backref="tasks")
Next code create units, campaign and tasks:
def fill_units(count):
units = []
with database.atomic():
for i in range(count):
units.append(Unit.create(name=f"unit{i}"))
return units
def init_campaign(count):
units = Unit.select().limit(count)
with database.atomic():
campaign = Campaign.create()
for unit in units:
Task.create(unit=unit, campaign=campaign)
return campaign
The problem appears when I'm trying to add more units into existing campaign. I need to select units which haven't been used in this campaign. In SQL I can do this using next query:
SELECT * FROM unit WHERE id NOT IN (SELECT unit_id FROM task WHERE campaign_id = 1) LIMIT 10
But how to do this using peewee?
The only way I've found yet is:
def get_new_units_for_campaign(campaign, count):
unit_names = [task.unit.name for task in campaign.tasks]
units = Unit.select().where(Unit.name.not_in(unit_names)).limit(count)
return units
It's somehow works but I'm 100% sure that it's the dumbest way to implement this. Could you show me the proper way to implement this?

Finally I found this:
Unit.select().where(Unit.id.not_in(campaign.tasks.select(Task.unit))).limit(10)
Which produces
SELECT "t1"."id", "t1"."name", "t1"."some_field", "t1"."created_at" FROM "unit" AS "t1" WHERE ("t1"."id" NOT IN (SELECT "t2"."unit_id" FROM "task" AS "t2" WHERE ("t2"."campaign_id" = 1))) LIMIT 10
Which matches with SQL query I've provided in my question.
P.S. I've done some research and it seems to be a proper implementation, but I'd appreciate if somebody correct me and show the better way (if exist).

How to speed up writing in a database?

I have a function which search for json files in a directory, parse the file and write data in the database. My problem is writing in database, because it take around 30 minutes. Any idea how can I speed up writting in a database? I have few quite big files to parse, but parsing the file is not a problem it take around 3 minutes. Currently I am using sqlite but in the future I will change it to PostgreSQL.
Here is my function:
def create_database():
with transaction.atomic():
directory = os.fsencode('data/web_files/unzip')
for file in os.listdir(directory):
filename = os.fsdecode(file)
with open('data/web_files/unzip/{}'.format(filename.strip()), encoding="utf8") as f:
data = json.load(f)
cve_items = data['CVE_Items']
for i in range(len(cve_items)):
database_object = DataNist()
try:
impact = cve_items[i]['impact']['baseMetricV2']
database_object.severity = impact['severity']
database_object.exp_score = impact['exploitabilityScore']
database_object.impact_score = impact['impactScore']
database_object.cvss_score = impact['cvssV2']['baseScore']
except KeyError:
database_object.severity = ''
database_object.exp_score = ''
database_object.impact_score = ''
database_object.cvss_score = ''
for vendor_data in cve_items[i]['cve']['affects']['vendor']['vendor_data']:
database_object.vendor_name = vendor_data['vendor_name']
for description_data in cve_items[i]['cve']['description']['description_data']:
database_object.description = description_data['value']
for product_data in vendor_data['product']['product_data']:
database_object.product_name = product_data['product_name']
database_object.save()
for version_data in product_data['version']['version_data']:
if version_data['version_value'] != '-':
database_object.versions_set.create(version=version_data['version_value'])
My models.py:
class DataNist(models.Model):
vendor_name = models.CharField(max_length=100)
product_name = models.CharField(max_length=100)
description = models.TextField()
date = models.DateTimeField(default=timezone.now)
severity = models.CharField(max_length=10)
exp_score = models.IntegerField()
impact_score = models.IntegerField()
cvss_score = models.IntegerField()
def __str__(self):
return self.vendor_name + "-" + self.product_name
class Versions(models.Model):
data = models.ForeignKey(DataNist, on_delete=models.CASCADE)
version = models.CharField(max_length=50)
def __str__(self):
return self.version
I will appreciate if you can give me any advice how can I improve my code.

Okay, given the structure of the data, something like this might work for you.
This is standalone code aside from that .objects.bulk_create() call; as commented in the code, the two classes defined would actually be models within your Django app.
(By the way, you probably want to save the CVE ID as an unique field too.)
Your original code had the misassumption that every "leaf entry" in the affected version data would have the same vendor, which may not be true. That's why the model structure here has a separate product-version model that has vendor, product and version fields. (If you wanted to optimize things a little, you might deduplicate the AffectedProductVersions even across DataNists (which, as an aside, is not a perfect name for a model)).
And of course, as you had already done in your original code, the importing should be run within a transaction (transaction.atomic()).
Hope this helps.
import json
import os
import types
class DataNist(types.SimpleNamespace): # this would actually be a model
severity = ""
exp_score = ""
impact_score = ""
cvss_score = ""
def save(self):
pass
class AffectedProductVersion(types.SimpleNamespace): # this too
# (foreign key to DataNist here)
vendor_name = ""
product_name = ""
version_value = ""
def import_item(item):
database_object = DataNist()
try:
impact = item["impact"]["baseMetricV2"]
except KeyError: # no impact object available
pass
else:
database_object.severity = impact.get("severity", "")
database_object.exp_score = impact.get("exploitabilityScore", "")
database_object.impact_score = impact.get("impactScore", "")
if "cvssV2" in impact:
database_object.cvss_score = impact["cvssV2"]["baseScore"]
for description_data in item["cve"]["description"]["description_data"]:
database_object.description = description_data["value"]
break # only grab the first description
database_object.save() # save the base object
affected_versions = []
for vendor_data in item["cve"]["affects"]["vendor"]["vendor_data"]:
for product_data in vendor_data["product"]["product_data"]:
for version_data in product_data["version"]["version_data"]:
affected_versions.append(
AffectedProductVersion(
data_nist=database_object,
vendor_name=vendor_data["vendor_name"],
product_name=product_data["product_name"],
version_name=version_data["version_value"],
)
)
AffectedProductVersion.objects.bulk_create(
affected_versions
) # save all the version information
return database_object # in case the caller needs it
with open("nvdcve-1.0-2019.json") as infp:
data = json.load(infp)
for item in data["CVE_Items"]:
import_item(item)

Querying joined tables in SQLAlchemy and displaying in an ObjectListView

I have an ObjectListView that displays information retrieved from an SQLite DB with SQLAlchemy.
def setupOLV(self):
self.loanResultsOlv.SetEmptyListMsg("No Loan Records Found")
self.loanResultsOlv.SetColumns([
ColumnDefn("Date Issued", "left", 100, "date_issued",
stringConverter="%d-%m-%y"),
ColumnDefn("Card Number", "left", 100, "card_id"),
ColumnDefn("Student Number", "left", 100, "person_id"),
ColumnDefn("Forename", "left", 150, "person_fname"),
ColumnDefn("Surname", "left", 150, "person_sname"),
ColumnDefn("Reason", "left", 150, "issue_reason"),
ColumnDefn("Date Due", "left", 100, "date_due",
stringConverter="%d-%m-%y"),
ColumnDefn("Date Returned", "left", 100, "date_returned",
stringConverter="%d-%m-%y")
])
I also have three models, Loan:
class Loan(DeclarativeBase):
"""
Loan model
"""
__tablename__ = "loans"
id = Column(Integer, primary_key=True)
card_id = Column(Unicode, ForeignKey("cards.id"))
person_id = Column(Unicode, ForeignKey("people.id"))
date_issued = Column(Date)
date_due = Column(Date)
date_returned = Column(Date)
issue_reason = Column(Unicode(50))
person = relation("Person", backref="loans", cascade_backrefs=False)
card = relation("Card", backref="loans", cascade_backrefs=False)
Person:
class Person(DeclarativeBase):
"""
Person model
"""
__tablename__ = "people"
id = Column(Unicode(50), primary_key=True)
fname = Column(Unicode(50))
sname = Column(Unicode(50))
and Card:
class Card(DeclarativeBase):
"""
Card model
"""
__tablename__ = "cards"
id = Column(Unicode(50), primary_key=True)
active = Column(Boolean)
I am trying to join the tables (loans and people) in order to retrieve and display the information in my ObjectListView. Here is my query method:
def getQueriedRecords(session, filterChoice, keyword):
"""
Searches the database based on the filter chosen and the keyword
given by the user
"""
qry = session.query(Loan)
if filterChoice == "person":
result = qry.join(Person).filter(Loan.person_id=='%s' % keyword).all()
elif filterChoice == "card":
result = qry.join(Person).filter(Loan.card_id=='%s' % keyword).all()
return result
I can retrieve and display every field stored in the loans table but forename and surname (should be drawn from people table and joined on person.id) are blank in my ObjectListView. I have SQL output on so I can see the query and it is not selecting at all from the people table.
How can I modify my query/ObjectListView to retrieve and display this information. ?
UPDATE: I have created an example script that is runnable here.

You're only querying for a Loan (qry = session.query(Loan)). Why do you expect something else to be in the results besides what's in the SELECT statement?

I admit that I am pretty new to SQLAlchemy myself, but I thought I would share what I use to display results from my queries. I have a program that uses a SQLite DB with 4+ tables and I pull data from 2-3 of them in a single query and display this information in an ObjectListView. I owe Mike Driscoll for his in depth tutorials, particularly wxPython and SqlAlchemy: An Intro to MVC and CRUD.
Here is what I would possibly add/change in your code.
In your model section add a "display" class such as:
def OlvDisplay(object):
def __init__(self, date_issued, card_id, person_id, fname, sname,
issue_reason, date_due, date_returned):
self.date_issued = date_issued
self.card_id = card_id
self.person_id = person_id
self.person_fname = fname
self.person_sname = sname
self.issue_reason = issue_reason
self.date_due = date_due
self.date_returned = date_returned
This display class is used in the convertResults definition below and assists with making sure the data is formatted properly for the ObjectListView.
The adjustment to your existing query function:
def getQueriedRecords(session, filterChoice, keyword):
"""
Searches the database based on the filter chosen and the keyword
given by the user
"""
qry = session.query(Loan)
if filterChoice == "person":
result = qry.join(Person).filter(Loan.person_id=='%s' % keyword).all()
elif filterChoice == "card":
result = qry.join(Person).filter(Loan.card_id=='%s' % keyword).all()
convertedResults = convertResults(result)
return convertedResults
What we're doing here is creating a local variable that is essentially running the conversion definition and storing the results for the next line, which returns those results.
And the "Convertor" function:
def convertResults(results):
finalResults = []
for record in results:
result = OlvDisplay(
record.date_issued,
record.card_id,
record.person_id,
record.person.fname,
record.person.sname,
record.issue_reason,
record.date_due,
record.date_returned
)
finalResults.append(result)
return finalResults
The important part here are the 2 lines:
record.person.fname
record.person.sname
Since we are wanting to pull information from another table using the established relationship it is important to refer to that relationship to actually see the data.
And to populate the ObjectListView Widget:
theOutput = getQueriedRecords(session, filterChoice, keyword)
self.setupOLV.SetObjects(theOutput)
Hope this helps you out.
-MikeS

Retrieving a set of model objects that are indirectly related

I have 4 models:
class TransitLine(models.Model):
name = models.CharField(max_length=32)
class Stop(models.Model):
line = models.ForeignKey(TransitLine, related_name='stops')
class ExitType(models.Model):
name = models.CharField(max_length=32)
people_limit = models.PositiveSmallIntegerField()
class Exits(models.Model):
TOKEN_BOOTH = 0
GATE = 1
REVOLVING_DOOR = 2
EXIT_TYPES = (
(TOKEN_BOOTH, 'Token booth'),
(GATE, 'Gate'),
(REVOLVING_DOOR, 'Revolving door'),
)
name = models.CharField(max_length=32)
stop = models.ForeignKey(Stop, related_name='exits')
type = models.ForeignKey(ExitType, related_name='exits')
I have one TransitLine object. I want to retrieve all the unique ExitType objects that are related to the Stop objects of the TransitLine (that was a mouth full).
Some semi-pseudo code of what I want to do:
tl = TransitLine.objects.get(id=1)
exit_types = []
for s in tl.stops:
exit_types.append([e.type for e in s.exits])
unique_exit_types = list(set(exit_types))
Obviously, prefer to do this in one QuerySet call. Any suggestions?

I would try something like this:
ExitType.objects.filter(exits__stop__line=line).distinct()