We Keep Coding

How to access data from custom_export in Otree?

I am trying to use ExtraModel and custom_export to export data from live_pages. However, when I go on devserver and check the data tab, it is nowhere to be found. If I download the excel (bottom right of the page) the new variables are not included in the data.
Where can I find the data from the custom export? Or am I defining the function wrong? Any help greatly appreciated.
See MWE below
from otree.api import *
import random
doc = """
Your app description
"""
class C(BaseConstants):
NAME_IN_URL = 'mwe_export'
PLAYERS_PER_GROUP = None
NUM_ROUNDS = 1
NUM_EMPLOYERS = 3
class Subsession(BaseSubsession):
pass
class Group(BaseGroup):
pass
class Player(BasePlayer):
pass
class Offer(ExtraModel):
group = models.Link(Group)
sender = models.Link(Player)
wage = models.IntegerField()
effort = models.IntegerField()
job_id = models.IntegerField()
information_type = models.StringField()
# FUNCTIONS
def to_dict(offer: Offer):
return dict(sender=offer.sender.id_in_group,
wage=offer.wage,
effort=offer.effort,
job_id=offer.job_id,
information_type=offer.information_type)
# PAGES
class MyPage(Page):
#staticmethod
def js_vars(player: Player):
return dict(my_id=player.id_in_group)
#staticmethod
def live_method(player: Player, data):
print(data)
group = player.group
job_id = random.randint(1, 1000)
wage = data['wage']
effort = data['effort']
information_type = data['information_type']
if data['information_type'] == 'offer':
offer = Offer.create(group=group,
sender=player,
job_id=job_id,
wage=wage,
effort=effort,
information_type=information_type)
print(offer)
print(to_dict(offer))
return {0: to_dict(offer)}
page_sequence = [MyPage]
def custom_export(players):
yield ['session.code', 'participant_code', 'id_in_session']
offers = Offer.filter()
for offer in offers:
player = offer.sender
participant = player.participant
yield [participant.code, participant.id_in_session, offer.job_id, offer.wage, offer.effort]

In the menu at the top of the admin page there is also a "Data" item. The custom export for your app should be available there under the heading "Per-app".

Python Protocol for Building a `pandas.DataFrame`

Hello SO and community!
Guess, my question somewhat resonates with this one.
However, trust the below task is a little bit different from that referenced above, namely to extract, transform, load data utilizing pandas.DataFrame, and I am stuck implementing Protocol for the purpose.
The code is below:
import io
import pandas as pd
import re
import requests
from functools import cache
from typing import Protocol
from zipfile import ZipFile
from pandas import DataFrame
#cache
def extract_can_from_url(url: str, **kwargs) -> DataFrame:
'''
Returns DataFrame from downloaded zip file from url
Parameters
----------
url : str
url to download from.
**kwargs : TYPE
additional arguments to pass to pd.read_csv().
Returns
-------
DataFrame
'''
name = url.split('/')[-1]
if os.path.exists(name):
with ZipFile(name, 'r').open(name.replace('-eng.zip', '.csv')) as f:
return pd.read_csv(f, **kwargs)
else:
r = requests.get(url)
with ZipFile(io.BytesIO(r.content)).open(name.replace('-eng.zip', '.csv')) as f:
return pd.read_csv(f, **kwargs)
class ETL(Protocol):
# =============================================================================
# Maybe Using these items for dataclass:
# url: str
# meta: kwargs(default_factory=dict)
# =============================================================================
def __init__(self, url: str, **kwargs) -> None:
return None
def download(self) -> DataFrame:
return DataFrame
def retrieve_series_ids(self) -> list[str]:
return list[str]
def transform(self) -> DataFrame:
return DataFrame
def sum_up_series_ids(self) -> DataFrame:
return DataFrame
class ETLCanadaFixedAssets(ETL):
def __init__(self, url: str, **kwargs) -> None:
self.url = url
self.kwargs = kwargs
#cache
def download(self) -> DataFrame:
self.df = extract_can_from_url(URL, index_col=0, usecols=range(14))
return self.df
def retrieve_series_ids(self) -> list[str]:
# =========================================================================
# Columns Specific to URL below, might be altered
# =========================================================================
self._columns = {
"Prices": 0,
"Industry": 1,
"Flows and stocks": 2,
"VECTOR": 3,
}
self.df_cut = self.df.loc[:, tuple(self._columns)]
_q = (self.df_cut.iloc[:, 0].str.contains('2012 constant prices')) & \
(self.df_cut.iloc[:, 1].str.contains('manufacturing', flags=re.IGNORECASE)) & \
(self.df_cut.iloc[:, 2] == 'Linear end-year net stock')
self.df_cut = self.df_cut[_q]
self.series_ids = sorted(set(self.df_cut.iloc[:, -1]))
return self.series_ids
def transform(self) -> DataFrame:
# =========================================================================
# Columns Specific to URL below, might be altered
# =========================================================================
self._columns = {
"VECTOR": 0,
"VALUE": 1,
}
self.df = self.df.loc[:, tuple(self._columns)]
self.df = self.df[self.df.iloc[:, 0].isin(self.series_ids)]
return self.df
def sum_up_series_ids(self) -> DataFrame:
self.df = pd.concat(
[
self.df[self.df.iloc[:, 0] == series_id].iloc[:, [1]]
for series_id in self.series_ids
],
axis=1
)
self.df.columns = self.series_ids
self.df['sum'] = self.df.sum(axis=1)
return self.df.iloc[:, [-1]]
UPD
Instantiating the class ETLCanadaFixedAssets
df = ETLCanadaFixedAssets(URL, index_col=0, usecols=range(14)).download().retrieve_series_ids().transform().sum_up_series_ids()
returns an error, however, expected:
AttributeError: 'DataFrame' object has no attribute 'retrieve_series_ids'
Please can anyone provide a guidance for how to put these things together (namely how to retrieve the DataFrame which might have been retrieved otherwise using the procedural approach by calling the functions within the last class as they appear within the latter) and point at those mistakes which were made above?
Probably, there is another way to do this elegantly using injection.
Thank you very much in advance!

All the functions of ETLCanadaFixedAssets and ETL classes should return self. This will allow you to call the functions of the class on the return value of the functions, so you can chain them together. You could add one more function that retrieves the encapsulated dataframe but that will always be called last, as the moment you call this function you cannot chain other functions any more. What you are trying to build is called fluent API you may read more about it here
For example:
class ETL(Protocol):
def download(self) -> ETL:
...
def retrieve_series_ids(self) -> ETL:
...
def transform(self) -> ETL:
...
def sum_up_series_ids(self) -> ETL:
...
#property
def dataframe(self) -> DataFrame:
...
Note you will need the following import line to be able to use the class annotation inside the class definition
from __future__ import annotations

SqlAlchemy db.Enum: How to detect it and retrieve the values

I have a function to add a column to a model:
def col(_type, label = "", **kwargs):
c = db.Column(_type, **kwargs)
c.info = {"label":label, "type":""}
return c
I use it like:
# Job Type
class JTEnum(enum.Enum):
full = "ﺖﻣﺎﻣ ﻮﻘﺗ"
part_time = "ﻦﯿﻤﻫ ﻮﻘﺗ"
project = "ﭖﺭﻭﮋﻫ ﺎﯾ"
jobtype = col(db.Enum(JTEnum), "ﻥﻮﻋ ﺶﻐﻟ", unique=False,
nullable=False,
default=JTEnum.full.name)
jobtype.info["choices"] = [(i.name, i.value) for i in JTEnum]
I call the col function for various columns. I would like to put the last line inside this function and it can be called on Enum type and fill the choices something like:
def col(_type, label = "", **kwargs):
c = db.Column(_type, **kwargs)
c.info = {"label":label, "type":""}
if isinstance(_type, db.Enum):
c.info["choices"] = [(i.name, i.value) for i in _type]
return c
However, it gives the error:
TypeError: 'Enum' object is not iterable

The first issue I'm seeing in your code is that you default to JTEnum.full.name, which should be the Enum element not its name, then you need to iterate the underlying Enum class of the SQLAlchemy Enum.
import enum
import sqlalchemy as db
class JTEnum(enum.Enum):
full = "ﺖﻣﺎﻣ ﻮﻘﺗ"
part_time = "ﻦﯿﻤﻫ ﻮﻘﺗ"
project = "ﭖﺭﻭﮋﻫ ﺎﯾ"
def col(_type, label="", **kwargs):
c = db.Column(_type, **kwargs)
c.info = {"label": label, "type": ""}
if isinstance(_type, db.Enum):
c.info["choices"] = [(i.name, i.value) for i in _type.enum_class]
return c
c = col(
db.Enum(JTEnum),
"ﻥﻮﻋ ﺶﻐﻟ",
unique=False,
nullable=False,
default=JTEnum.full,
)
print(c.info)
# {'label': 'ﻥﻮﻋ ﺶﻐﻟ',
# 'type': '',
# 'choices': [('full', 'ﺖﻣﺎﻣ ﻮﻘﺗ'),
# ('part_time', 'ﻦﯿﻤﻫ ﻮﻘﺗ'),
# ('project', 'ﭖﺭﻭﮋﻫ ﺎﯾ')]}

_type which is db.Enum is not iterable. You may refer to _type.enum_class like that:
def col(_type, label = "", **kwargs):
c = db.Column(_type, **kwargs)
c.info = {"label":label, "type":""}
if isinstance(_type, db.Enum):
c.info["choices"] = [(i.name, i.value) for i in _type.enum_class]
return c

How to output a dictionary I made in a class in Python pandas

I'm getting back into Python and am currently working on code that takes the name and associated numbers for logins from various points from an excel sheet, sums up the numbers, and then adds the name to a dictionary with an associated value of "low", "medium", or "high". I've been able to get the code to work in function form, and am currently putting it into a class:
import pandas as pd
import xlrd
import sys
class aggregateActivity:
def __init__(self):
self.l = "path to excel file"
self.s = "name of excel worksheet"
self.activityAssociation = {}
dfs = pd.read_excel(self.l, self.s, usecols=[1,2,3])
numberOfLogins = dfs.sum(axis=1)
activity = {}
for index, row in dfs.iterrows():
activity[row[0]]=numberOfLogins[index]
for key in activity:
if activity.get(key) < 100:
activityAssociation[key] = "Low"
if activity.get(key) >= 100 and activity.get(key) < 200:
activityAssociation[key] = "Medium"
if activity.get(key) >= 200:
activityAssociation[key] = "High"
x = aggregateActivity()
x.activityAssociation
The issue here is that when I run the code, jupyter outputs an empty dictionary:
{}
I've checked to see if the dictionary actually has contents added to it be appending print(activityAssociation) to the end of the class, and can confirm that it does.

you forgot self. which is causing python to create a new activityAssociation local variable
import pandas as pd
import xlrd
import sys
class aggregateActivity:
def __init__(self):
self.l = "path to excel file"
self.s = "name of excel worksheet"
self.activityAssociation = {}
dfs = pd.read_excel(self.l, self.s, usecols=[1,2,3])
numberOfLogins = dfs.sum(axis=1)
activity = {}
for index, row in dfs.iterrows():
activity[row[0]]=numberOfLogins[index]
for key in activity:
if activity.get(key) < 100:
self.activityAssociation[key] = "Low"
if activity.get(key) >= 100 and activity.get(key) < 200:
self.activityAssociation[key] = "Medium"
if activity.get(key) >= 200:
self.activityAssociation[key] = "High"
x = aggregateActivity()
x.activityAssociation

Return DataFrame using ipywidgets Button

I'm currently creating a Class that inherits a DataFrame from pandas. I'm interested in developing a method called 'new_filter' that is a fancier execution of a DataFrame command:
import pandas as pd
from ipywidgets import widgets
from IPython.display import display
import numpy as np
class Result(pd.DataFrame):
#property
def _constructor(self):
return Result
def _filter_done(self, c):
self._column_name = self._filter_dd.value
self._expression = self._filter_txt.value
return self[eval('self.'+ self._column_name +' '+self._expression)]
def new_filter(self):
self._filter_dd = widgets.Dropdown(options=list(self.columns),
description='Column:')
self._filter_txt = widgets.Text(description='Expr:')
self._filter_button = widgets.Button(description = 'Done')
self._filter_box = widgets.VBox([self._filter_dd, self._filter_txt, self._filter_button])
display(self._filter_box)
self._filter_button.on_click(self._filter_done)
After creating an object like:
test = Result(np.random.randn(3,4), columns=['A','B','C','D']) #just an example
test_2 = test.new_filter()
Then, for example:
Widget Output
What I want is that 'test_2' be an object from 'Result' class. Is there any solution to this?

First, you will have to return something in the function new_filter. Second, if you want the same object to be modified, it is a bit hard I think. One thing you can do is to have an object which has a trait which can be updated in _filter_done.
Here is a small example of how you can do it:
import pandas as pd
from ipywidgets import widgets
from IPython.display import display
import numpy as np
class Result(pd.DataFrame):
#property
def _constructor(self):
return Result
def _filter_done(self, obj, c):
## obj is the obejct to be modified.
## Updating its data attribute to have the filtered data.
self._column_name = self._filter_dd.value
self._expression = self._filter_txt.value
obj.data = self[eval('self.'+ self._column_name +' '+self._expression)]
def new_filter(self):
self._filter_dd = widgets.Dropdown(options=list(self.columns),
description='Column:')
self._filter_txt = widgets.Text(description='Expr:')
self._filter_button = widgets.Button(description = 'Done')
self._filter_box = widgets.VBox([self._filter_dd, self._filter_txt, self._filter_button])
display(self._filter_box)
result_obj = FilterResult()
self._filter_button.on_click(lambda arg: self._filter_done(result_obj, arg))
return result_obj
from traitlets import HasTraits
from traittypes import DataFrame
class FilterResult(HasTraits):
data = DataFrame()
With the same example code as in your question, i.e.,
test = Result(np.random.randn(3,4), columns=['A', 'B', 'C','D']) #just an example
test_2 = test.new_filter()
You can see that whenever you click on done, the updated dataframe is in test_2.data.