We Keep Coding

Creating new df from series of widget boxes

I have created an "input form" with several ipywidget boxes. I want to be able to reference all the values to create a new dataframe.
I'm currently doing this in a horrible way.
portfolio_df = pd.DataFrame([[VBox1.children[0].value, VBox2.children[0].value, VBox3.children[0].value, VBox4.children[0].value]],
columns=['Product Name','Units','Price', 'Invested Amount'])
row_2 = [VBox1.children[1].value, VBox2.children[1].value, VBox3.children[1].value, VBox4.children[21].value]
portfolio_df.loc[len(portfolio_df)] = row_2
row_3 = [VBox1.children[2].value, VBox2.children[2].value, VBox3.children[2].value, VBox4.children[2].value]
portfolio_df.loc[len(portfolio_df)] = row_3
row_4 = [VBox1.children[3].value, VBox2.children[3].value, VBox3.children[3].value, VBox4.children[3].value]
portfolio_df.loc[len(portfolio_df)] = row_4
and so on up till row 23 in this instance !! (but the length will vary up to the number of children within a VBox)
I suspect I can do this more pythonically using a for loop but cant figure it out.
Full code as per requests (I've edited columns so my live data is different but this is exact replica of the set up)
import pandas as pd
import numpy as np
import datetime as dt
import ipywidgets as ipw
from ipywidgets import *
barrier_list = pd.DataFrame(np.random.randn(24, 4), columns=('Product
Name','ISIN','A','B'))
barrier_list= barrier_list.astype(str)
dd_list = []
for i in range(len(barrier_list['Product Name'])):
dropdown = ipw.FloatText(description=barrier_list['ISIN'][i],
value=barrier_list['Product Name'][i],
disabled=False,
layout = {'width':'350px'})
dropdown.style.description_width = 'initial'
dd_list.append(dropdown)
dd_list1 = []
for i in range(len(barrier_list['Product Name'])):
dropdown1 = ipw.FloatText(description='Units',
value=0,
layout = {'width':'200px'})
dd_list1.append(dropdown1)
dd_list2 = []
for i in range(len(barrier_list['Product Name'])):
dropdown2 = ipw.FloatText(description='Price',
value=0,
layout = {'width':'200px'})
dd_list2.append(dropdown2)
dd_list3 = []
for i in range(len(barrier_list['Product Name'])):
dropdown3 = ipw.FloatText(description='Value',
value=0,
layout = {'width':'200px'})
dd_list3.append(dropdown3)
VBox1 = ipw.VBox(dd_list)
VBox2 = ipw.VBox(dd_list1)
VBox3 = ipw.VBox(dd_list2)
VBox4 = ipw.VBox(dd_list3)
HBox = widgets.HBox([VBox1, VBox2, VBox3, VBox4])

solved this one by looping through the VBoxes one by one and then concatenating the dataframes into one main one.
product_df = pd.DataFrame()
for i in range(len(dd_list)):
product_name_df = pd.DataFrame([[VBox1.children[i].value]],columns=
['Product Name'])
product_df = product_df.append(product_name_df)
unit_df = pd.DataFrame()
for i in range(len(dd_list)):
unit_amount_df = pd.DataFrame([[VBox2.children[i].value]],columns=
['Units'])
unit_df = unit_df.append(unit_amount_df)
price_df = pd.DataFrame()
for i in range(len(dd_list)):
price_amount_df = pd.DataFrame([[VBox3.children[i].value]],columns=
['Price'])
price_df = price_df.append(price_amount_df)
value_df = pd.DataFrame()
for i in range(len(dd_list)):
value_amount_df = pd.DataFrame([[VBox4.children[i].value]],columns=
['Value'])
value_df = value_df.append(value_amount_df)
df_list = [product_df.reset_index(drop=True),unit_df.reset_index(drop=True),
price_df.reset_ind ex(drop=True),value_df.reset_index(drop=True)]
portfolio_df = pd.concat((df_list), axis=1)
portfolio_df

Bokeh soo slow when updating plot in the browser

I am trying to create a dashboard in bokeh. I am fairly new to bokeh. The plots work alright but when I try to update it using bokeh server, it gets very slow, it takes more than a minute to update the plots.
I don't know if I'm doing anything wrong. Below is the code I'm using:
import pandas as pd
from bokeh.io import curdoc
from bokeh.plotting import figure
from bokeh.models import (
Div, SingleIntervalTicker, DatetimeTickFormatter, NumeralTickFormatter, DateRangeSlider, ColumnDataSource
)
from bokeh.layouts import layout
def _get_data(path, name):
df = pd.read_csv(path)
df.drop(columns='Province/State', inplace=True)
df.rename(columns={'Country/Region': 'country', 'Lat': 'lat', 'Long': 'long'}, inplace=True)
df = df.melt(var_name='date', value_name=name, id_vars=['country', 'lat', 'long'])
df = df.groupby(by=['country', 'date'], as_index=False, sort=False, dropna=False).sum()
df['id'] = df.country + df.date
df['date'] = pd.to_datetime(df['date'], format='%m/%d/%y', infer_datetime_format=True)
return df
def _merged_data():
confirmed = _get_data('data/time_series_covid19_confirmed_global.csv', 'confirmed')
deaths = _get_data('data/time_series_covid19_deaths_global.csv', 'deaths')
recovered = _get_data('data/time_series_covid19_recovered_global.csv', 'recovered')
merged = pd.merge(confirmed, deaths[['id', 'deaths']], on='id', validate='1:1')
merged = merged.merge(recovered[['id', 'recovered']], on='id', validate='1:1')
merged.drop(columns='id', inplace=True)
return merged
def line_fig(label, interval, **kwargs):
fig = figure(
plot_width=400,
plot_height=250,
background_fill_color='#222222',
y_axis_label=label,
border_fill_color='#222222',
outline_line_color='#222222',
**kwargs
)
# Fig
fig.toolbar_location = None
fig.tools = []
# Axis
fig.axis.major_label_text_color = '#bdbdbd'
fig.axis.major_tick_line_color = '#5c5c5c'
fig.axis.major_tick_in = 0
fig.axis.minor_tick_line_color = None
fig.axis.axis_label_text_color = '#bdbdbd'
fig.axis.axis_line_color = "#5c5c5c"
# X-Axis
fig.xgrid.grid_line_color = "#5c5c5c"
fig.xgrid.grid_line_width = 1
fig.xgrid.grid_line_alpha = 0.4
fig.xgrid.grid_line_dash = [3, 9]
fig.xaxis.formatter = DatetimeTickFormatter(months="%b %Y")
# Y-Axis
fig.ygrid.grid_line_color = "#5c5c5c"
fig.ygrid.grid_line_width = 1
fig.ygrid.grid_line_alpha = 0.4
fig.ygrid.grid_line_dash = [3, 9]
fig.yaxis.ticker = SingleIntervalTicker(interval=interval)
fig.yaxis.formatter = NumeralTickFormatter(format='0a')
return fig
source = _merged_data()
datacache = ColumnDataSource(source)
date = datacache.data['date']
start_date = date.min()
end_date = date.max()
date_slider = DateRangeSlider(start=start_date,
end=end_date,
value=(start_date, end_date),
step=1,
show_value=False,
default_size=400)
del date
def date_slider_callback(attr, old, new):
old = pd.to_datetime(date_slider.value_as_date[0], infer_datetime_format=True)
new = pd.to_datetime(date_slider.value_as_date[1], infer_datetime_format=True)
temp_data = source[(source['date'] >= old) & (source['date'] <= new)]
datacache.data = ColumnDataSource.from_df(temp_data)
date_slider.on_change('value', date_slider_callback)
confirmed_chart = make_chart('Confirmed Cases', 'date', 'confirmed', interval=10000000.00, color='#D83020')
death_chart = make_chart('Confirmed Deaths', 'date', 'deaths', interval=300000.00, color='#eaeaea')
recovery_chart = make_chart('Confirmed Recovery', 'date', 'recovered', interval=10000000.00, color='#35ac46')
lay_out = layout(
children=[
[date_slider],
[confirmed_chart],
[death_chart],
[recovery_chart]
]
)
document = curdoc()
document.add_root(lay_out)
Right now, I don't know what I'm doing wrong, maybe there's some kind of best practice I should follow, I don't really know what's making the plot so slow.

Plotting data from two different dataframes to a single Dash graph

I'm trying to generate a Dash app which displays historical and forecasted housing prices. I've got the forecasted data stored in a different dataframe from the historical prices, and I'd like to plot them both on the same graph in Dash, and have the graph get updated via callback when the user picks a different city from a dropdown menu. I would like both traces of the graph to update when a value is selected in the dropdown. I've tried various things but can only get one trace from one dataframe plotted for the graph in my callback:
# --- import libraries ---
import dash
import dash_table
import dash_core_components as dcc
import dash_html_components as html
import pandas as pd
import numpy as np
import plotly.express as px
import plotly.graph_objects as go
from dash.dependencies import Output, Input
# --- load data ---
df_h = pd.read_csv('df_h.csv')
df_arima = pd.read_csv('df_arima.csv')
options = [] #each column in the df_h dataframe is an option for the dropdown menu
for column in df_h.columns:
options.append({'label': '{}'.format(column, column), 'value': column})
external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']
# --- initialize the app ---
app = dash.Dash(__name__, external_stylesheets=external_stylesheets)
# --- layout the dashboard ---
app.layout = html.Div(
children = [
html.Div([
html.Label('Select a feature from drop-down to plot'),
dcc.Dropdown(
id = 'city-dropdown',
options = options,
value = 'Denver, CO',
multi = False,
clearable = True,
searchable = True,
placeholder = 'Choose a City...'),
html.Div(id = 'forecast-container',
style = {'padding': '50px'}),
]),
],
)
# --- dropdown callback ---
#app.callback(
Output('forecast-container', 'children'),
Input('city-dropdown', 'value'))
def forecast_graph(value):
dff = df_h[['Date', value]] #'value' is identical between the two dataframes. references
dfa = df_arima[df_arima['City'] == value] # a col in dff and row values in dfa
return [
dcc.Graph(
id = 'forecast-graph',
figure = px.line(
data_frame = dff,
x = 'Date',
y = value).update_layout(
showlegend = False,
template = 'xgridoff',
yaxis = {'title': 'Median Home Price ($USD)'},
xaxis = {'title': 'Year'},
title = {'text': 'Median Home Price vs. Year for {}'.format(value),
'font': {'size': 24}, 'x': 0.5, 'xanchor': 'center'}
),
)
]
I was able to accomplish this in Plotly but can't figure out how to do it in Dash. This is what I want in Dash:
Plotly graph I am trying to reproduce in callback, that is linked to a dropdown menu
This is all I can get to work in Dash:
Only one dataframe plots in Dash
This is the code that works in plotly graph objects:
from statsmodels.tsa.arima_model import ARIMA
df_ml = pd.read_csv('df_ml.csv')
# --- data cleaning ---
df_pred = df_ml[df_ml['RegionName'] == city]
df_pred = df_pred.transpose().reset_index().drop([0])
df_pred.columns = ['Date', 'MedianHomePrice_USD']
df_pred['MedianHomePrice_USD'] = df_pred['MedianHomePrice_USD'].astype('int')
df_pred['Date'] = pd.to_datetime(df_pred['Date'])
df_pred['Date'] = df_pred['Date'].dt.strftime('%Y-%m')
df_model = df_pred.set_index('Date')
model_data = df_model['MedianHomePrice_USD']
def house_price_forecast(model_data, forecast_steps, city):
#--- ARIMA Model (autoregressive integrated moving average) ---
model = ARIMA(model_data, order = (2,1,2), freq = 'MS')
res = model.fit()
forecast = res.forecast(forecast_steps)
forecast_mean = forecast[0]
forecast_se = forecast[1]
forecast_ci = forecast[2]
df_forecast = pd.DataFrame()
df_forecast['Mean'] = forecast_mean.T
df_forecast['Lower_ci'], df_forecast['Upper_ci'] = forecast_ci.T
df_forecast['Date'] = pd.date_range(start = '2021-02', periods = forecast_steps, freq = 'MS')
df_forecast['Date'] = df_forecast['Date'].dt.strftime('%Y-%m')
df_forecast.index = df_forecast['Date']
fig = go.Figure()
fig.add_trace(go.Scatter(x = df_pred['Date'], y = df_pred['MedianHomePrice_USD'],
mode = 'lines', name = 'Median Home Price ($USD)',
line_color = 'rgba(49, 131, 189, 0.75)', line_width = 2))
fig.add_trace(go.Scatter(x = df_forecast.index, y = df_forecast['Mean'],
mode = 'lines', line_color = '#e6550d',
name = 'Forecast mean'))
fig.add_trace(go.Scatter(x = df_forecast.index, y = df_forecast['Upper_ci'],
mode = 'lines', line_color = '#e0e0e0', fill = 'tonexty',
fillcolor = 'rgba(225,225,225, 0.3)',
name = 'Upper 95% confidence interval'))
fig.add_trace(go.Scatter(x = df_forecast.index, y = df_forecast['Lower_ci'],
mode = 'lines', line_color = '#e0e0e0', fill = 'tonexty',
fillcolor = 'rgba(225,225,225, 0.3)',
name = 'Lower 95% confidence interval'))
fig.update_layout(title = 'Median Home Price in {}, {} - {} (Predicted)'.format(
city, model_data.idxmin()[:-3], df_forecast_mean.idxmax()[:-3]),
xaxis_title = 'Year', yaxis_title = 'Median Home Price ($USD)',
template = 'xgridoff')
fig.show()
house_price_forecast(model_data, 24, 'Denver, CO') #24 month prediction
Perhaps a more succinct way of asking this question: How do I add a trace to an existing Dash graph, with data from a different dataframe, and both traces get updated when the user selects a value from a single dropdown?

Figured it out...
Don't use the syntax I used above in your callback. Put the px.line call inside a variable(fig, in this case), and then use fig.add_scatter to add data from a different dataframe to the graph. Both parts of the graph will update from the callback.
Also, fig.add_scatter doesn't have a dataframe argument, so use df.column or df[column] (ex. 'dfa.Date' below)
# --- import libraries ---
import dash
import dash_core_components as dcc
import dash_html_components as html
import pandas as pd
import plotly.express as px
from dash.dependencies import Output, Input
# --- load data ---
df_h = pd.read_csv('df_h.csv')
df_h['Date'] = pd.to_datetime(df_h['Date'])
df_arima = pd.read_csv('df_arima.csv')
df_arima['Date'] = pd.to_datetime(df_arima['Date'])
df_arima['Date'] = df_arima['Date'].dt.strftime('%Y-%m')
external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']
# --- initialize the app ---
app = dash.Dash(__name__, external_stylesheets=external_stylesheets)
app.layout = html.Div([
dcc.Graph(id = 'forecast-container')
]
)
# --- dropdown callback ---
#app.callback(
Output('forecast-container', 'figure'),
Input('city-dropdown', 'value'))
def update_figure(selected_city):
dff = df_h[['Date', selected_city]]
# dff[selected_city] = dff[selected_city].round(0)
dfa = df_arima[df_arima['City'] == selected_city]
fig = px.line(dff, x = 'Date', y = selected_city,
hover_data = {selected_city: ':$,f'})
fig.add_scatter(x = dfa.Date, y = dfa.Mean,
line_color = 'orange', name = 'Forecast Mean')
fig.add_scatter(x = dfa.Date, y = dfa.Lower_ci,
fill = 'tonexty', fillcolor = 'rgba(225,225,225, 0.3)',
marker = {'color': 'rgba(225,225,225, 0.9)'},
name = 'Lower 95% Confidence Interval')
fig.add_scatter(x = dfa.Date, y = dfa.Upper_ci,
fill = 'tonexty', fillcolor = 'rgba(225,225,225, 0.3)',
marker = {'color': 'rgba(225,225,225, 0.9)'},
name = 'Upper 95% Confidence Interval')
fig.update_layout(template = 'xgridoff',
yaxis = {'title': 'Median Home Price ($USD)'},
xaxis = {'title': 'Year'},
title = {'text': 'Median Home Price vs. Year for {}'.format(selected_city),
'font': {'size': 24}, 'x': 0.5, 'xanchor': 'center'})
return fig
if __name__ == '__main__':
app.run_server(debug = True)

bokeh hover multiline with datetime axis

I want to create a multiline Bokeh plot with datetime axis and a hover tool that shows the datetime of the data point. This should be supported and I have tried to obtain the intended behaviour in two ways:
Use hover.formatters to format the x-value. This has no effect on the plot.
Add a description variable with the correctly formatted date/time values. This results in a hover tool where all date/time values are displayed in a list for each point.
I have included a smaller example of my code that illustrates my approach and the result. It is used in conjunction with a checkboxgroup that updates the data. This is why a new ColumnDataSource is made from the dataframe.
import pandas as pd
import numpy as np
from bokeh.io import output_file, show
from bokeh.plotting import figure
from bokeh.models import HoverTool, ColumnDataSource
from bokeh.palettes import Spectral4
from bokeh.layouts import column
#output_file("demo.html")
available_quant = ["LACTIC_ACID", "GLUCOSE", "XYLOSE", "FORMIC_ACID"]
quant_legend = ["Lactic acid", "Glucose", "Xylose", "Formic acid"]
Create a dataframe with 4 quantities and the time
datelist = pd.date_range(end = pd.datetime.today(), periods=100).tolist()
desc = datelist
for i, date in enumerate(datelist):
desc[i] = str(date)
RT_x = np.linspace(-5, 5, num=100)
lactic = RT_x**2
data = {'time': datelist, 'desc': desc, 'LACTIC_ACID': RT_x**2 + 2, 'GLUCOSE': RT_x**2, 'XYLOSE': RT_x**2 - 2, 'FORMIC_ACID': RT_x**2 - 4}
df = pd.DataFrame.from_dict(data)
df['time'] = pd.to_datetime(df['time'], format = "%Y-%m-%d %H:%M:%S")
Copy the relevant data to a columndatasource
substance_colors = Spectral4
quant_to_plot = available_quant
xs = []
ys = []
xsprint = []
colors = []
labels = []
for i, substance in enumerate(quant_to_plot):
xs.append(list(df['time']))
ys.append(list(df[substance]))
xsprint.append(list(df['desc']))
index = available_quant.index(substance)
colors.append(substance_colors[index])
labels.append(quant_legend[index])
new_src = ColumnDataSource(data={'x': xs, 'y': ys, 'desc': xsprint, 'color': colors, 'label': labels})
Make the first plot using hover.formatters
p = figure(plot_width=800, plot_height=400, x_axis_type="datetime", title = 'Demo', x_axis_label = 'Time', y_axis_label = 'c [g/mL]')
p.multi_line('x','y', color = 'color', legend = 'label', source = new_src)
hover = HoverTool(tooltips=[('Type','#label'),
('Time','$x'),
('Conc','$y')],
formatters={'Time': 'datetime'},
mode = 'mouse',
line_policy='next')
p.add_tools(hover)
p.legend.location = "top_left"
p.legend.orientation = "horizontal"
Make second plot using description variable
p2 = figure(plot_width=800, plot_height=400, x_axis_type="datetime", title = 'Demo', x_axis_label = 'Time', y_axis_label = 'c [g/mL]')
p2.multi_line('x','y', color = 'color', legend = 'label', source = new_src)
hover = HoverTool(tooltips=[('Type','#label'),
('Time','#desc'),
('Conc','$y')],
mode = 'mouse',
line_policy='nearest')
p2.add_tools(hover)
mylayout = column(p, p2)
show(mylayout)
Am I missing something trivial? I am running Bokeh 0.13.0 and python 3.6.4.

The first approach works with the following modification of the hovertool:
hover = HoverTool(tooltips=[('Type','#label'),
('Time','$x{%F}'),
('Conc','$y')],
formatters={'$x': 'datetime'},
mode = 'mouse',
line_policy='nearest')

How to update Pretext in Bokeh with a Select tool

I have a bokeh plot that updates my plot through a select tool. The select tool contains subjects that update the plot where the values are x='Polarity'and y='Subjectivity'.
Here is a dummy data for what I want:
import pandas as pd
import random
list_type = ['All', 'Compliment', 'Sport', 'Remaining', 'Finance', 'Infrastructure', 'Complaint', 'Authority',
'Danger', 'Health', 'English']
df = pd.concat([pd.DataFrame({'Subject' : [list_type[i] for t in range(110)],
'Polarity' : [random.random() for t in range(110)],
'Subjectivity' : [random.random() for t in range(110)]}) for i in range(len(list_type))], axis=0)
My code for updating the plot looks like this:
options = []
options.append('All')
options.extend(df['Subject'].unique().tolist())
source = ColumnDataSource(df)
p = figure()
r = p.circle(x='Polarity', y='Subjectivity', source = source)
select = Select(title="Subject", options=options, value="All")
output_notebook()
def update_plot(attr, old, new):
if select.value=="All":
df_filter = df.copy()
else:
df_filter = df[df['Subject']==select.value]
source1 = ColumnDataSource(df_filter)
r.data_source.data = source1.data
select.on_change('value', update_plot)
layout = column(row(select, width=400), p)
#show(layout)
curdoc().add_root(layout)
I want to add a 'Pretext' that has a df.describe(), that can update with the plot through the select tool. I tried this by adding these codes but it displays nothing:
stats = PreText(text='', width=500)
t1 = select.value
def update_stats(df, t1):
stats.text = str(df[[t1, select.value+'_returns']].describe())
select.on_change('value', update_plot, update_stats)
layout = column(row(select, width=400), p, stats)
curdoc().add_root(layout)
show(layout)
Anyone know a solution? Thanks!

You don't need two separate function for that, you can just change your original function update_plot to add statement to change the text for PreText as stats.text = str(df_filter.describe()). The function will look as below -
def update_plot(attr, old, new):
if select.value=="All":
df_filter = df.copy()
else:
df_filter = df[df['Subject']==select.value]
source1 = ColumnDataSource(df_filter)
r.data_source.data = source1.data
stats.text = str(df_filter.describe())
Entire code
from bokeh.models.widgets import Select, PreText
from bokeh.layouts import column, row
from bokeh.models import ColumnDataSource
from bokeh.plotting import figure, curdoc
from bokeh.plotting import figure, show
import pandas as pd
import random
list_type = ['All', 'Compliment', 'Sport', 'Remaining', 'Finance', 'Infrastructure', 'Complaint', 'Authority',
'Danger', 'Health', 'English']
df = pd.concat([pd.DataFrame({'Subject' : [list_type[i] for t in range(110)],
'Polarity' : [random.random() for t in range(110)],
'Subjectivity' : [random.random() for t in range(110)]}) for i in range(len(list_type))], axis=0)
options = []
options.append('All')
options.extend(df['Subject'].unique().tolist())
source = ColumnDataSource(df)
p = figure()
r = p.circle(x='Polarity', y='Subjectivity', source = source)
select = Select(title="Subject", options=options, value="All")
#output_notebook()
stats = PreText(text=str(df.describe()), width=500)
def update_plot(attr, old, new):
if select.value=="All":
df_filter = df.copy()
else:
df_filter = df[df['Subject']==select.value]
source1 = ColumnDataSource(df_filter)
r.data_source.data = source1.data
stats.text = str(df_filter.describe())
select.on_change('value', update_plot)
layout = column(row(select, width=400), p, stats)
#show(layout)
curdoc().add_root(layout)