I am trying to use ExtraModel and custom_export to export data from live_pages. However, when I go on devserver and check the data tab, it is nowhere to be found. If I download the excel (bottom right of the page) the new variables are not included in the data.
Where can I find the data from the custom export? Or am I defining the function wrong? Any help greatly appreciated.
See MWE below
from otree.api import *
import random
doc = """
Your app description
"""
class C(BaseConstants):
NAME_IN_URL = 'mwe_export'
PLAYERS_PER_GROUP = None
NUM_ROUNDS = 1
NUM_EMPLOYERS = 3
class Subsession(BaseSubsession):
pass
class Group(BaseGroup):
pass
class Player(BasePlayer):
pass
class Offer(ExtraModel):
group = models.Link(Group)
sender = models.Link(Player)
wage = models.IntegerField()
effort = models.IntegerField()
job_id = models.IntegerField()
information_type = models.StringField()
# FUNCTIONS
def to_dict(offer: Offer):
return dict(sender=offer.sender.id_in_group,
wage=offer.wage,
effort=offer.effort,
job_id=offer.job_id,
information_type=offer.information_type)
# PAGES
class MyPage(Page):
#staticmethod
def js_vars(player: Player):
return dict(my_id=player.id_in_group)
#staticmethod
def live_method(player: Player, data):
print(data)
group = player.group
job_id = random.randint(1, 1000)
wage = data['wage']
effort = data['effort']
information_type = data['information_type']
if data['information_type'] == 'offer':
offer = Offer.create(group=group,
sender=player,
job_id=job_id,
wage=wage,
effort=effort,
information_type=information_type)
print(offer)
print(to_dict(offer))
return {0: to_dict(offer)}
page_sequence = [MyPage]
def custom_export(players):
yield ['session.code', 'participant_code', 'id_in_session']
offers = Offer.filter()
for offer in offers:
player = offer.sender
participant = player.participant
yield [participant.code, participant.id_in_session, offer.job_id, offer.wage, offer.effort]
In the menu at the top of the admin page there is also a "Data" item. The custom export for your app should be available there under the heading "Per-app".
Related
I try to figure out how to structure my program which is a simple trading bot.
class "Exchange"
should store instances of the class "TradingPair"
class "Symbol"
stores all symbol related stuff
class "Websocket"
gets the ws stream and stores the ticks it in a dataframe in the TradingPair instance
located in Exchange.symbols[symbol_name].history
class "Indicators"
calculates for example a moving average and stores the values in
Exchange.symbols[symbol_name].history
Here are my questions:
To access Exchange.symbols I need a class variable so I can read/edit it from within other class instances. In Websocket / handle_symbol_ticker I have to write Exchange.symbols[self.symbol_name].history. Could this be done in a shorter manner. I did try history_pat = Exchange.symbols[self.symbol_name].history, but this generates a new object...
In Indicators / calc_ma I could not use loc[-1,colum_name] but had to use .index[-1]. What would be the best way do the index?
Here is the code:
import pandas as pd
class Exchange:
symbols = {}
class Symbol:
def __init__(self, base_asset, quote_asset):
self.base_asset = base_asset
self.quote_asset = quote_asset
self.symbol_name = self.base_asset + self.quote_asset
self.history = pd.DataFrame()
class Websocket(Exchange):
def __init__(self, symbol_name):
self.symbol_name = symbol_name
history_path = Exchange.symbols[self.symbol_name].history # doesn't work
def handle_symbol_ticker(self, msg: dict):
Exchange.symbols[self.symbol_name].history = pd.concat([
Exchange.symbols[self.symbol_name].history,
pd.DataFrame([msg])
]).set_index("event_time")
# def handle_symbol_ticker(self, msg: dict):
# history_path = pd.concat([ # <- doesn't work
# history_path,
# pd.DataFrame([msg])
# ]).set_index("event_time")
class Indicators(Exchange):
def __init__(self, symbol_name):
self.symbol_name = symbol_name
def calc_ma(self, timespan):
timespan_name = "ma_" + str(timespan)
Exchange.symbols[self.symbol_name].history.loc[
Exchange.symbols[self.symbol_name].history.index[-1],
timespan_name] \
= Exchange.symbols[self.symbol_name].history["close"].tail(timespan).mean()
if __name__ == "__main__":
bnc_exchange = Exchange()
bnc_exchange.symbols["axsbusd"] = Symbol("axs", "busd")
bnc_websocket = Websocket( "axsbusd")
bnc_indicators = Indicators("axsbusd")
bnc_exchange.symbols["axsbusd"].history = pd.DataFrame({
"event_time": [101,102,103,104,105],
"close": [50,51,56,54,53],
})
bnc_websocket.handle_symbol_ticker({
"event_time": 106,
"close": 54
})
bnc_indicators.calc_ma(3)
print(bnc_exchange.symbols["axsbusd"].history)
I need to create a simple presentation filter for a Django webpage. Since it's discarded without need for storage, I do not need to use Model overhead.
Here is my relevant views.py code:
#login_required()
def cards(request):
# form = CountryForm()
f_name = STAT_FILES / 'csv/segment_summary_quart.csv'
# pathname = os.path.abspath(os.path.dirname(__file__))
df = pd.read_csv(f_name, index_col=None)
pl_name = STAT_FILES / 'pickles/lbounds'
pu_name = STAT_FILES / 'pickles/ubounds'
lbounds = pickle.load(open(pl_name, "rb"))
ubounds = pickle.load(open(pu_name, "rb"))
filter_name = []
i = 0
max_i = len(ubounds)
while i < max_i:
filter_name.append(f'Personal best trophy range: {str(int(lbounds[i])).rjust(4," ")}-{str(int(ubounds[i])).rjust(4," ")}')
i += 1
sort_name = []
sort_name.append("Alphabetic Order")
sort_name.append("Most Popular")
sort_name.append("Least Popular")
sort_name.append("Highest Win Rate")
sort_name.append("Lowest Win Rate")
form = cardStatsForm(filter_name, sort_name)
Given that my values may change dynamically, I am trying to establish the cardStatsForm() object without initially knowing the values that will occur when publishing the page to the end user. The following code resides within forms.py:
from django import forms
class cardStatsForm(forms.Form):
def __init__(self, filter_opts, sortOrder, *args, **kwargs):
super(cardStatsForm, self).__init__(*args, **kwargs)
self.fields['filts'].choices = filter_opts
self.fields['sorts'].choices = sortOrder
filts = forms.MultipleChoiceField(widget=forms.CheckboxSelectMultiple, choices=(), required=True)
sorts = forms.Select(choices=())
Unfortunately I haven't had to worry about the html side yet since I keep getting the following error with respect to my form class:
KeyError at /clashstats/cards/
'sorts'
Request Method: GET
Request URL: http://127.0.0.1:8000/clashstats/cards/
Django Version: 4.0
Exception Type: KeyError
Exception Value:
'sorts'
Exception Location: /Users/cooneycw/PycharmProjects/The6ix/clashstats/forms.py, line 8, in __init__
Python Executable: /Users/cooneycw/miniforge3/envs/The6ix/bin/python
I am more familiar with the flask framework and see that Django relies more heavily on objects which I am less familiar with. What am i doing incorrectly here?
The solution recommend by AMG above enabled me to move past this error. The next error related to the expectation for tuples for the choice listings. The following code modified to send zipped tuples for the list items now completes successfully:
forms.py
from django import forms
class cardStatsForm(forms.Form):
def __init__(self, filterList, sortList, *args, **kwargs):
super(cardStatsForm, self).__init__(*args, **kwargs)
self.filts = []
self.sorts = []
self.fields['filts'].choices = filterList
self.fields['sorts'].choices = sortList
filts = forms.MultipleChoiceField(widget=forms.CheckboxSelectMultiple, choices=(), required=True)
sorts = forms.ChoiceField(choices=())
views.py
#login_required()
def cards(request):
# form = CountryForm()
f_name = STAT_FILES / 'csv/segment_summary_quart.csv'
# pathname = os.path.abspath(os.path.dirname(__file__))
df = pd.read_csv(f_name, index_col=None)
pl_name = STAT_FILES / 'pickles/lbounds'
pu_name = STAT_FILES / 'pickles/ubounds'
lbounds = pickle.load(open(pl_name, "rb"))
ubounds = pickle.load(open(pu_name, "rb"))
filter_name = []
i = 0
max_i = len(ubounds)
while i < max_i:
filter_name.append(f'Personal best trophy range: {str(int(lbounds[i])).rjust(4," ")}-{str(int(ubounds[i])).rjust(4," ")}')
i += 1
filter_id = range(len(filter_name))
filter_list = list(zip(filter_id, filter_name))
sort_name = []
sort_name.append("Alphabetic Order")
sort_name.append("Most Popular")
sort_name.append("Least Popular")
sort_name.append("Highest Win Rate")
sort_name.append("Lowest Win Rate")
sort_id = range(len(sort_name))
sort_list = list(zip(sort_id, sort_name))
form = cardStatsForm(filter_list, sort_list)
My thanks to https://stackoverflow.com/users/4872140/amg for his help to resolve this issue.
I have a function which search for json files in a directory, parse the file and write data in the database. My problem is writing in database, because it take around 30 minutes. Any idea how can I speed up writting in a database? I have few quite big files to parse, but parsing the file is not a problem it take around 3 minutes. Currently I am using sqlite but in the future I will change it to PostgreSQL.
Here is my function:
def create_database():
with transaction.atomic():
directory = os.fsencode('data/web_files/unzip')
for file in os.listdir(directory):
filename = os.fsdecode(file)
with open('data/web_files/unzip/{}'.format(filename.strip()), encoding="utf8") as f:
data = json.load(f)
cve_items = data['CVE_Items']
for i in range(len(cve_items)):
database_object = DataNist()
try:
impact = cve_items[i]['impact']['baseMetricV2']
database_object.severity = impact['severity']
database_object.exp_score = impact['exploitabilityScore']
database_object.impact_score = impact['impactScore']
database_object.cvss_score = impact['cvssV2']['baseScore']
except KeyError:
database_object.severity = ''
database_object.exp_score = ''
database_object.impact_score = ''
database_object.cvss_score = ''
for vendor_data in cve_items[i]['cve']['affects']['vendor']['vendor_data']:
database_object.vendor_name = vendor_data['vendor_name']
for description_data in cve_items[i]['cve']['description']['description_data']:
database_object.description = description_data['value']
for product_data in vendor_data['product']['product_data']:
database_object.product_name = product_data['product_name']
database_object.save()
for version_data in product_data['version']['version_data']:
if version_data['version_value'] != '-':
database_object.versions_set.create(version=version_data['version_value'])
My models.py:
class DataNist(models.Model):
vendor_name = models.CharField(max_length=100)
product_name = models.CharField(max_length=100)
description = models.TextField()
date = models.DateTimeField(default=timezone.now)
severity = models.CharField(max_length=10)
exp_score = models.IntegerField()
impact_score = models.IntegerField()
cvss_score = models.IntegerField()
def __str__(self):
return self.vendor_name + "-" + self.product_name
class Versions(models.Model):
data = models.ForeignKey(DataNist, on_delete=models.CASCADE)
version = models.CharField(max_length=50)
def __str__(self):
return self.version
I will appreciate if you can give me any advice how can I improve my code.
Okay, given the structure of the data, something like this might work for you.
This is standalone code aside from that .objects.bulk_create() call; as commented in the code, the two classes defined would actually be models within your Django app.
(By the way, you probably want to save the CVE ID as an unique field too.)
Your original code had the misassumption that every "leaf entry" in the affected version data would have the same vendor, which may not be true. That's why the model structure here has a separate product-version model that has vendor, product and version fields. (If you wanted to optimize things a little, you might deduplicate the AffectedProductVersions even across DataNists (which, as an aside, is not a perfect name for a model)).
And of course, as you had already done in your original code, the importing should be run within a transaction (transaction.atomic()).
Hope this helps.
import json
import os
import types
class DataNist(types.SimpleNamespace): # this would actually be a model
severity = ""
exp_score = ""
impact_score = ""
cvss_score = ""
def save(self):
pass
class AffectedProductVersion(types.SimpleNamespace): # this too
# (foreign key to DataNist here)
vendor_name = ""
product_name = ""
version_value = ""
def import_item(item):
database_object = DataNist()
try:
impact = item["impact"]["baseMetricV2"]
except KeyError: # no impact object available
pass
else:
database_object.severity = impact.get("severity", "")
database_object.exp_score = impact.get("exploitabilityScore", "")
database_object.impact_score = impact.get("impactScore", "")
if "cvssV2" in impact:
database_object.cvss_score = impact["cvssV2"]["baseScore"]
for description_data in item["cve"]["description"]["description_data"]:
database_object.description = description_data["value"]
break # only grab the first description
database_object.save() # save the base object
affected_versions = []
for vendor_data in item["cve"]["affects"]["vendor"]["vendor_data"]:
for product_data in vendor_data["product"]["product_data"]:
for version_data in product_data["version"]["version_data"]:
affected_versions.append(
AffectedProductVersion(
data_nist=database_object,
vendor_name=vendor_data["vendor_name"],
product_name=product_data["product_name"],
version_name=version_data["version_value"],
)
)
AffectedProductVersion.objects.bulk_create(
affected_versions
) # save all the version information
return database_object # in case the caller needs it
with open("nvdcve-1.0-2019.json") as infp:
data = json.load(infp)
for item in data["CVE_Items"]:
import_item(item)
I am developing a small Python script in order to get the weather data from forecast.io Once I get the JSON document, I call a Class in order to create a new record to be saved in the database. The problem is that some fields (which are also attributes in my Class) are not always informed in the API so I must include some kind of defensive code or the script will break when field is not found.
I've found this answer of #Alex Martelli which seams pretty good: Reading from Python dict if key might not be present
If you want to do something different than using a default value (say,
skip the printing completely when the key is absent), then you need a
bit more structure, i.e., either:
for r in results:
if 'key_name' in r:
print r['key_name']
or
for r in results:
try: print r['key_name']
except KeyError: pass
But I am wondering if I must include an "if" or a "try" on every field I want to save or is there a prettier way to do this? (I want to save 27 fields and 27 "if" seems ugly)
This is the code I have so far:
from datetime import datetime
import tornado.web
import tornado.httpclient
from tornado import gen
from src.db.city import list_cities
from src.db.weather import Weather
from motorengine import *
#gen.coroutine
def forecastio_api():
http_client = tornado.httpclient.AsyncHTTPClient()
base_url = "https://api.forecast.io/forecast/APIKEY"
city yield list_cities()
for city in city:
url = base_url + "/%s,%s" %(str(city.loc[0]), str(city.loc[1]))
response = yield http_client.fetch(url)
json = tornado.escape.json_decode(response.body)
for day in json['daily']['data']:
weather = Weather(city=city,
time = datetime.fromtimestamp(day['time']),
summary = day.get('summary'),
icon = day.get('icon'),
sunrise_time = datetime.fromtimestamp(day.get('sunriseTime')),
sunset_time = datetime.fromtimestamp(day.get('sunsetTime')),
moon_phase = day.get('moonPhase'),
precip_intensity = day.get('precipIntensity'),
precip_intensity_max = day.get('precipIntensityMax'),
precip_intensity_max_time = datetime.fromtimestamp(day.get('precipIntensityMaxTime')),
precip_probability = day.get('precipProbability'),
precip_type = day.get('precipType'),
temperature_min = day.get('temperatureMin'),
temperature_min_time = datetime.fromtimestamp(day.get('temperatureMinTime')),
temperature_max = day.get('temperatureMax'),
temperature_max_time = datetime.fromtimestamp(day.get('temperatureMaxTime')),
apparent_temperature_min = day.get('apparentTemperatureMin'),
apparent_temperature_min_time = datetime.fromtimestamp(day.get('apparentTemperatureMinTime')),
apparent_temperature_max = day.get('apparentTemperatureMax'),
apparent_temperature_max_time = datetime.fromtimestamp(day.get('apparentTemperatureMaxTime')),
dew_point = day.get('dewPoint'),
humidity = day.get('humidity'),
wind_speed = day.get('windSpeed'),
wind_bearing = day.get('windBearing'),
visibility = day.get('visibility'),
cloud_cover = day.get('cloudCover'),
pressure = day.get('pressure'),
ozone = day.get('ozone')
)
weather.create()
if __name__ == '__main__':
io_loop = tornado.ioloop.IOLoop.instance()
connect("DATABASE", host="localhost", port=27017, io_loop=io_loop)
forecastio_api()
io_loop.start()
and this is the Weather Class using Motornegine:
from tornado import gen
from motorengine import Document
from motorengine.fields import DateTimeField, DecimalField, ReferenceField, StringField
from src.db.city import City
class Weather(Document):
__collection__ = 'weather'
__lazy__ = False
city = ReferenceField(reference_document_type=City)
time = DateTimeField(required=True)
summary = StringField()
icon = StringField()
sunrise_time = DateTimeField()
sunset_time = DateTimeField()
moon_phase = DecimalField(precision=2)
precip_intensity = DecimalField(precision=4)
precip_intensity_max = DecimalField(precision=4)
precip_intensity_max_time = DateTimeField()
precip_probability = DecimalField(precision=2)
precip_type = StringField()
temperature_min = DecimalField(precision=2)
temperature_min_time = DateTimeField()
temperature_max = DecimalField(precision=2)
temperature_max_time = DateTimeField()
apparent_temperature_min = DecimalField(precision=2)
apparent_temperature_min_time = DateTimeField()
apparent_temperature_max = DecimalField(precision=2)
apparent_temperature_max_time = DateTimeField()
dew_point = DecimalField(precision=2)
humidity = DecimalField(precision=2)
wind_speed = DecimalField(precision=2)
wind_bearing = DecimalField(precision=2)
visibility = DecimalField(precision=2)
cloud_cover = DecimalField(precision=2)
pressure = DecimalField(precision=2)
ozone = DecimalField(precision=2)
create_time = DateTimeField(auto_now_on_insert=True)
#gen.coroutine
def create(self):
yield self.save()
You can check Schematics. This library helps you define objects that can be easily populated from dicts(you can easily turn json to python dict). It allows you to define validation rules on each property. The object will throw ModelValidationError error when some properties are missing or in the wrong format. Schematics allows you to add default values and a lot more nice stuff when you define your models.
I have an Openstack-powered, Django-modified application that shows the disk images and snapshots available for a user to launch. The user currently sees both snapshots they created and ones they did not. I would like to separate the current table into two based on whether they are owned by the user or not.
My two table definitions are as follows (note I altered row_actions accordingly):
class UserSnapshotsTable(OldSnapshotsTable):
cloud = tables.Column(get_cloud, verbose_name=_("Cloud"))
class Meta:
name = "usersnapshots"
verbose_name = _("User Snapshots")
table_actions = (DeleteSnapshot,)
row_actions = (LaunchSnapshot, LaunchCluster, EditImage, DeleteSnapshot)
pagination_param = "snapshot_marker"
row_class = UpdateRow
status_columns = ["status"]
class OtherSnapshotsTable(OldSnapshotsTable):
cloud = tables.Column(get_cloud, verbose_name=_("Cloud"))
class Meta:
name = "othersnapshots"
verbose_name = _("Other Snapshots")
table_actions = (DeleteSnapshot,)
row_actions = (LaunchSnapshot, LaunchCluster)
pagination_param = "snapshot_marker"
row_class = UpdateRow
status_columns = ["status"]
I have altered the HTML template to pull the "UserSnapshotsTable" and "OtherSnapshotsTable" tables (I copied the original table and renamed both), but both full tables still generate under the respective headings. There are two functions generating the data:
def get_usersnapshots_data(self):
req = self.request
marker = req.GET.get(UserSnapshotsTable._meta.pagination_param, None)
try:
usersnaps, self._more_snapshots = api.snapshot_list_detailed(req,
marker=marker)
except:
usersnaps = []
exceptions.handle(req, _("Unable to retrieve user-owned snapshots."))
return usersnaps
def get_othersnapshots_data(self):
req = self.request
marker = req.GET.get(OtherSnapshotsTable._meta.pagination_param, None)
try:
othersnaps, self._more_snapshots = api.snapshot_list_detailed(req,
marker=marker)
except:
othersnaps = []
exceptions.handle(req, _("Unable to retrieve non-user-owned snapshots."))
return othersnaps
There are also Edit/Delete options defined for images, and imported for snapshots, that seem to have a key comparison. Here's the "Delete" one (line 7):
class DeleteImage(tables.DeleteAction):
data_type_singular = _("Image")
data_type_plural = _("Images")
def allowed(self, request, image=None):
if image:
return image.owner == request.user.tenant_id
# Return True to allow table-level bulk delete action to appear.
return True
def delete(self, request, obj_id):
api.image_delete(request, obj_id)
How can I separate those tables out? This is my first time asking a question here, so please let me know if I can provide further information. Apologies for the length of it.
As far as I see you are using glanceclient. If that so you can use extra_filters parameter of snapshot_list_detailed() to filter only user images like this:
usersnaps, self._more_snapshots = api.snapshot_list_detailed(
req,
marker = marker,
extra_filters = {"owner": "user_name"}
)
Under cover snapshot_list_detailed uses GET images of Openstack Image Service API.