Paramter Estimation with parmest for differential equations in Pyomo - python
I have a model and I want to estimate the values of the variables using parmest in pyomo.
Using parmest, I got a runtime error and a TypeError, and I can't figure out why.
Below is my code:
data = pd.read_csv('Test.csv')
#data
def modelfunc(data):
model = m = ConcreteModel()
m.t = ContinuousSet(bounds=(0,1))
# Define parameters
m.Ssu_in = Param(m.t, mutable=True)
m.Saa_in = Param(m.t, mutable=True)
m.Sfa_in = Param(m.t, mutable=True)
m.Q = Param(m.t, mutable=True)
m.V_liq = Param(initialize=3400, within=PositiveReals)
# Variables
m.S_su = Var(m.t, initialize=0.012394, domain=PositiveReals, bounds=(0.001,1))
m.S_aa = Var(m.t, initialize=0.0055432, domain=PositiveReals, bounds=(0,0.1))
m.S_fa = Var(m.t, initialize=0.10741, domain=PositiveReals, bounds=(0.001,2))
# Derivatives
m.dS_su_dt = DerivativeVar(m.S_su, wrt=m.t)
m.dS_aa_dt = DerivativeVar(m.S_aa, wrt=m.t)
m.dS_fa_dt = DerivativeVar(m.S_fa, wrt=m.t)
#initial values
m.S_su[0].fix(0.012394)
m.S_aa[0].fix(0.0055432)
m.S_fa[0].fix(0.10741)
#Discretize model using Finite Difference Method
discretizer = TransformationFactory('dae.finite_difference')
discretizer.apply_to(m,nfe=50,wrt=m.t,scheme='BACKWARD')
# Load data into the following variables
timepoints = list(m.t)
data_timepoints = data['time'].tolist()
data_profiles1 = data['S_su'].tolist()
data_profiles2 = data['S_aa'].tolist()
data_profiles3 = data['S_fa'].tolist()
data_profiles27 = data['Q'].tolist()
# Interpolate the data
interp_Ssu_values = np.interp(timepoints, data_timepoints, data_profiles1)
interp_Saa_values = np.interp(timepoints, data_timepoints, data_profiles2)
interp_Sfa_values = np.interp(timepoints, data_timepoints, data_profiles3)
interp_Q_values = np.interp(timepoints, data_timepoints, data_profiles27)
for i,t in enumerate(timepoints):
m.Ssu_in[t] = interp_Ssu_values[i]
m.Saa_in[t] = interp_Saa_values[i]
m.Sfa_in[t] = interp_Sfa_values[i]
m.Q[t] = interp_Q_values[i]
#Constraints
def S_su_out_bal(m,t):
return m.dS_su_dt[t] == (m.Q[t]/m.V_liq) * (m.Ssu_in[t] - m.S_su[t]) + 0.000662979
m.Ssu_outcon = Constraint(m.t, rule=S_su_out_bal)
def S_aa_out_bal(m,t):
return m.dS_aa_dt[t] == (m.Q[t]/m.V_liq) * (m.Saa_in[t] - m.S_aa[t]) - 0.00202160
m.Saa_outcon = Constraint(m.t, rule=S_aa_out_bal)
def S_fa_out_bal(m,t):
return m.dS_fa_dt[t] == (m.Q[t]/m.V_liq) * (m.Sfa_in[t] - m.S_fa[t]) + 0.005667982
m.Sfa_outcon = Constraint(m.t, rule=S_fa_out_bal)
return model
#Vars to estimate
theta_names = ['m.S_su', 'm.S_aa', 'm.S_fa']
#Sum of squred error
def SSE(m, data):
expr = (float(data['S_su']) - m.S_su)**2 + \
(float(data['S_aa']) - m.S_aa)**2 + \
(float(data['S_fa']) - m.S_fa)**2
return expr
# Create an instance of the Parameter Estimation
pest = parmest.Estimator(modelfunc, data, theta_names, SSE, tee=True)
# Parameter Estimation
obj, theta = pest.theta_est()
I get the following error:
ERROR: Rule failed for Expression 'SecondStageCost' with index None:
TypeError: unsupported operand type(s) for -: 'float' and 'IndexedVar'
ERROR: Constructing component 'SecondStageCost' from data=None failed:
TypeError: unsupported operand type(s) for -: 'float' and 'IndexedVar'
--- Logging error ---
Traceback (most recent call last):
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1085, in emit
msg = self.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 929, in format
return fmt.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 238, in format
return self.standard_formatter.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 107, in format
msg = record.getMessage()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 373, in getMessage
msg = msg % self.args
TypeError: not enough arguments for format string
Call stack:
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\runpy.py", line 87, in _run_code
exec(code, run_globals)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel_launcher.py", line 17, in <module>
app.launch_new_instance()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\traitlets\config\application.py", line 976, in launch_instance
app.start()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelapp.py", line 712, in start
self.io_loop.start()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\tornado\platform\asyncio.py", line 199, in start
self.asyncio_loop.run_forever()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\base_events.py", line 570, in run_forever
self._run_once()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\base_events.py", line 1859, in _run_once
handle._run()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\events.py", line 81, in _run
self._context.run(self._callback, *self._args)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 510, in dispatch_queue
await self.process_one()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 499, in process_one
await dispatch(*args)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 406, in dispatch_shell
await result
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 730, in execute_request
reply_content = await reply_content
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\ipkernel.py", line 383, in do_execute
res = shell.run_cell(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\zmqshell.py", line 528, in run_cell
return super().run_cell(*args, **kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 2975, in run_cell
result = self._run_cell(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3030, in _run_cell
return runner(coro)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\async_helpers.py", line 78, in _pseudo_sync_runner
coro.send(None)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3257, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3473, in run_ast_nodes
if (await self.run_code(code, result, async_=asy)):
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3553, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "C:\Users\mfo21001\AppData\Local\Temp\ipykernel_4860\3065394695.py", line 84, in <cell line: 84>
obj, theta = pest.theta_est()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 687, in theta_est
return self._Q_opt(solver=solver, return_values=return_values,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 432, in _Q_opt
ef = local_ef.create_EF(scen_names,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\create_ef.py", line 88, in create_EF
scen_dict = {
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\create_ef.py", line 89, in <dictcomp>
name: scenario_creator(name, **scenario_creator_kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 143, in _experiment_instance_creation_callback
instance = callback(experiment_number = exp_num, cb_data = cb_data)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 391, in _instance_creation_callback
model = self._create_parmest_model(exp_data)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 344, in _create_parmest_model
logger.warning(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1458, in warning
self._log(WARNING, msg, args, **kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1589, in _log
self.handle(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1599, in handle
self.callHandlers(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1661, in callHandlers
hdlr.handle(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 954, in handle
self.emit(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 250, in emit
super(StdoutHandler, self).emit(record)
Message: 'theta_name[%s] (%s) was not found on the model'
Arguments: ((0, 'm.S_su'),)
--- Logging error ---
Traceback (most recent call last):
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1085, in emit
msg = self.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 929, in format
return fmt.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 238, in format
return self.standard_formatter.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 107, in format
msg = record.getMessage()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 373, in getMessage
msg = msg % self.args
TypeError: not enough arguments for format string
Call stack:
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\runpy.py", line 87, in _run_code
exec(code, run_globals)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel_launcher.py", line 17, in <module>
app.launch_new_instance()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\traitlets\config\application.py", line 976, in launch_instance
app.start()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelapp.py", line 712, in start
self.io_loop.start()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\tornado\platform\asyncio.py", line 199, in start
self.asyncio_loop.run_forever()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\base_events.py", line 570, in run_forever
self._run_once()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\base_events.py", line 1859, in _run_once
handle._run()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\events.py", line 81, in _run
self._context.run(self._callback, *self._args)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 510, in dispatch_queue
await self.process_one()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 499, in process_one
await dispatch(*args)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 406, in dispatch_shell
await result
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 730, in execute_request
reply_content = await reply_content
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\ipkernel.py", line 383, in do_execute
res = shell.run_cell(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\zmqshell.py", line 528, in run_cell
return super().run_cell(*args, **kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 2975, in run_cell
result = self._run_cell(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3030, in _run_cell
return runner(coro)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\async_helpers.py", line 78, in _pseudo_sync_runner
coro.send(None)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3257, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3473, in run_ast_nodes
if (await self.run_code(code, result, async_=asy)):
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3553, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "C:\Users\mfo21001\AppData\Local\Temp\ipykernel_4860\3065394695.py", line 84, in <cell line: 84>
obj, theta = pest.theta_est()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 687, in theta_est
return self._Q_opt(solver=solver, return_values=return_values,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 432, in _Q_opt
ef = local_ef.create_EF(scen_names,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\create_ef.py", line 88, in create_EF
scen_dict = {
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\create_ef.py", line 89, in <dictcomp>
name: scenario_creator(name, **scenario_creator_kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 143, in _experiment_instance_creation_callback
instance = callback(experiment_number = exp_num, cb_data = cb_data)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 391, in _instance_creation_callback
model = self._create_parmest_model(exp_data)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 344, in _create_parmest_model
logger.warning(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1458, in warning
self._log(WARNING, msg, args, **kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1589, in _log
self.handle(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1599, in handle
self.callHandlers(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1661, in callHandlers
hdlr.handle(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 954, in handle
self.emit(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 250, in emit
super(StdoutHandler, self).emit(record)
Message: 'theta_name[%s] (%s) was not found on the model'
Arguments: ((1, 'm.S_aa'),)
--- Logging error ---
Traceback (most recent call last):
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1085, in emit
msg = self.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 929, in format
return fmt.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 238, in format
return self.standard_formatter.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 107, in format
msg = record.getMessage()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 373, in getMessage
msg = msg % self.args
TypeError: not enough arguments for format string
Call stack:
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\runpy.py", line 87, in _run_code
exec(code, run_globals)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel_launcher.py", line 17, in <module>
app.launch_new_instance()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\traitlets\config\application.py", line 976, in launch_instance
app.start()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelapp.py", line 712, in start
self.io_loop.start()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\tornado\platform\asyncio.py", line 199, in start
self.asyncio_loop.run_forever()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\base_events.py", line 570, in run_forever
self._run_once()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\base_events.py", line 1859, in _run_once
handle._run()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\events.py", line 81, in _run
self._context.run(self._callback, *self._args)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 510, in dispatch_queue
await self.process_one()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 499, in process_one
await dispatch(*args)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 406, in dispatch_shell
await result
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 730, in execute_request
reply_content = await reply_content
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\ipkernel.py", line 383, in do_execute
res = shell.run_cell(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\zmqshell.py", line 528, in run_cell
return super().run_cell(*args, **kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 2975, in run_cell
result = self._run_cell(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3030, in _run_cell
return runner(coro)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\async_helpers.py", line 78, in _pseudo_sync_runner
coro.send(None)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3257, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3473, in run_ast_nodes
if (await self.run_code(code, result, async_=asy)):
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3553, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "C:\Users\mfo21001\AppData\Local\Temp\ipykernel_4860\3065394695.py", line 84, in <cell line: 84>
obj, theta = pest.theta_est()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 687, in theta_est
return self._Q_opt(solver=solver, return_values=return_values,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 432, in _Q_opt
ef = local_ef.create_EF(scen_names,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\create_ef.py", line 88, in create_EF
scen_dict = {
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\create_ef.py", line 89, in <dictcomp>
name: scenario_creator(name, **scenario_creator_kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 143, in _experiment_instance_creation_callback
instance = callback(experiment_number = exp_num, cb_data = cb_data)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 391, in _instance_creation_callback
model = self._create_parmest_model(exp_data)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 344, in _create_parmest_model
logger.warning(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1458, in warning
self._log(WARNING, msg, args, **kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1589, in _log
self.handle(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1599, in handle
self.callHandlers(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1661, in callHandlers
hdlr.handle(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 954, in handle
self.emit(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 250, in emit
super(StdoutHandler, self).emit(record)
Message: 'theta_name[%s] (%s) was not found on the model'
Arguments: ((2, 'm.S_fa'),)
TypeError Traceback (most recent call last)
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py in _experiment_instance_creation_callback(scenario_name, node_names, cb_data)
142 try:
--> 143 instance = callback(experiment_number = exp_num, cb_data = cb_data)
144 except TypeError:
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py in _instance_creation_callback(self, experiment_number, cb_data)
390 raise RuntimeError(f'Unexpected data format for cb_data={cb_data}')
--> 391 model = self._create_parmest_model(exp_data)
392
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py in _create_parmest_model(self, data)
365 model.FirstStageCost = pyo.Expression(rule=FirstStageCost_rule)
--> 366 model.SecondStageCost = pyo.Expression(rule=_SecondStageCostExpr(self.obj_function, data))
367
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\core\base\block.py in __setattr__(self, name, val)
543 #
--> 544 self.add_component(name, val)
545 else:
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\core\base\block.py in add_component(self, name, val)
1088 try:
-> 1089 val.construct(data)
1090 except:
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\core\base\expression.py in construct(self, data)
368 assert data is None
--> 369 self._construct_from_rule_using_setitem()
370 finally:
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\core\base\indexed_component.py in _construct_from_rule_using_setitem(self)
707 # constant, then only call the rule once.
--> 708 val = rule(block, None)
709 for index in self.index_set():
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\core\base\initializer.py in __call__(self, parent, idx)
372 def __call__(self, parent, idx):
--> 373 return self._fcn(parent)
374
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py in __call__(self, model)
269 def __call__(self, model):
--> 270 return self._ssc_function(model, self._data)
271
~\AppData\Local\Temp\ipykernel_4860\3065394695.py in SSE(m, data)
74 def SSE(m, data):
---> 75 expr = (float(data['S_su']) - m.S_su)**2 + \
76 (float(data['S_aa']) - m.S_aa)**2 + \
TypeError: unsupported operand type(s) for -: 'float' and 'IndexedVar'
During handling of the above exception, another exception occurred:
RuntimeError Traceback (most recent call last)
~\AppData\Local\Temp\ipykernel_4860\3065394695.py in <cell line: 84>()
82
83 # Parameter Estimation
---> 84 obj, theta = pest.theta_est()
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py in theta_est(self, solver, return_values, calc_cov, cov_n)
685 assert cov_n > len(self.theta_names), "The number of datapoints must be greater than the number of parameters to estimate"
686
--> 687 return self._Q_opt(solver=solver, return_values=return_values,
688 bootlist=None, calc_cov=calc_cov, cov_n=cov_n)
689
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py in _Q_opt(self, ThetaVals, solver, return_values, bootlist, calc_cov, cov_n)
430 scenario_creator_kwargs=scenario_creator_options)
431 else:
--> 432 ef = local_ef.create_EF(scen_names,
433 _experiment_instance_creation_callback,
434 EF_name = "_Q_opt",
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\create_ef.py in create_EF(scenario_names, scenario_creator, scenario_creator_kwargs, EF_name, suppress_warnings, nonant_for_fixed_vars)
86 if scenario_creator_kwargs is None:
87 scenario_creator_kwargs = dict()
---> 88 scen_dict = {
89 name: scenario_creator(name, **scenario_creator_kwargs)
90 for name in scenario_names
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\create_ef.py in <dictcomp>(.0)
87 scenario_creator_kwargs = dict()
88 scen_dict = {
---> 89 name: scenario_creator(name, **scenario_creator_kwargs)
90 for name in scenario_names
91 }
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py in _experiment_instance_creation_callback(scenario_name, node_names, cb_data)
143 instance = callback(experiment_number = exp_num, cb_data = cb_data)
144 except TypeError:
--> 145 raise RuntimeError("Only one callback signature is supported: "
146 "callback(experiment_number, cb_data) ")
147 """
RuntimeError: Only one callback signature is supported: callback(experiment_number, cb_data)
Related
InvalidArgumentError: Graph execution error:
I keep getting a "graph execution error" The complete code is in the link if you'd like to see it. I don't really understand the error at all. I was trying to get the epochs running but instead, I get "1/30" and then it just stops. I checked my folders and it appears that I have all jpeg files. I'm in a corner I don't know what to do.
history = model.fit_generator(train_generator,
epochs=30,
verbose=1,
validation_data=validation_generator,
callbacks = [best_model]
)
https://colab.research.google.com/drive/1hvHkDusyqEsdZg5ZRVhhriZrDagpFdU6?usp=sharing
Epoch 1/30
---------------------------------------------------------------------------
InvalidArgumentError Traceback (most recent call last)
<ipython-input-42-5368c251678d> in <module>
----> 1 history = model.fit_generator(train_generator,
2 epochs=30,
3 verbose=1,
4 validation_data=validation_generator,
5 callbacks = [best_model]
2 frames
/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
52 try:
53 ctx.ensure_initialized()
---> 54 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
55 inputs, attrs, num_outputs)
56 except core._NotOkStatusException as e:
InvalidArgumentError: Graph execution error:
Detected at node 'categorical_crossentropy/softmax_cross_entropy_with_logits' defined at (most recent call last):
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.8/dist-packages/ipykernel_launcher.py", line 16, in <module>
app.launch_new_instance()
File "/usr/local/lib/python3.8/dist-packages/traitlets/config/application.py", line 992, in launch_instance
app.start()
File "/usr/local/lib/python3.8/dist-packages/ipykernel/kernelapp.py", line 612, in start
self.io_loop.start()
File "/usr/local/lib/python3.8/dist-packages/tornado/platform/asyncio.py", line 149, in start
self.asyncio_loop.run_forever()
File "/usr/lib/python3.8/asyncio/base_events.py", line 570, in run_forever
self._run_once()
File "/usr/lib/python3.8/asyncio/base_events.py", line 1859, in _run_once
handle._run()
File "/usr/lib/python3.8/asyncio/events.py", line 81, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.8/dist-packages/tornado/ioloop.py", line 690, in <lambda>
lambda f: self._run_callback(functools.partial(callback, future))
File "/usr/local/lib/python3.8/dist-packages/tornado/ioloop.py", line 743, in _run_callback
ret = callback()
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 787, in inner
self.run()
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 748, in run
yielded = self.gen.send(value)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py", line 365, in process_one
yield gen.maybe_future(dispatch(*args))
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py", line 268, in dispatch_shell
yield gen.maybe_future(handler(stream, idents, msg))
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/kernelbase.py", line 543, in execute_request
self.do_execute(
File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/ipkernel.py", line 306, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/usr/local/lib/python3.8/dist-packages/ipykernel/zmqshell.py", line 536, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 2854, in run_cell
result = self._run_cell(
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 2881, in _run_cell
return runner(coro)
File "/usr/local/lib/python3.8/dist-packages/IPython/core/async_helpers.py", line 68, in _pseudo_sync_runner
coro.send(None)
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 3057, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 3249, in run_ast_nodes
if (await self.run_code(code, result, async_=asy)):
File "/usr/local/lib/python3.8/dist-packages/IPython/core/interactiveshell.py", line 3326, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-25-f51df55a1054>", line 1, in <module>
history = model.fit_generator(train_datagen.flow_from_directory(TRAINING_DIR,
File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 2260, in fit_generator
return self.fit(
File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 1409, in fit
tmp_logs = self.train_function(iterator)
File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 1051, in train_function
return step_function(self, iterator)
File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 1040, in step_function
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 1030, in run_step
outputs = model.train_step(data)
File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 890, in train_step
loss = self.compute_loss(x, y, y_pred, sample_weight)
File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 948, in compute_loss
return self.compiled_loss(
File "/usr/local/lib/python3.8/dist-packages/keras/engine/compile_utils.py", line 201, in __call__
loss_value = loss_obj(y_t, y_p, sample_weight=sw)
File "/usr/local/lib/python3.8/dist-packages/keras/losses.py", line 139, in __call__
losses = call_fn(y_true, y_pred)
File "/usr/local/lib/python3.8/dist-packages/keras/losses.py", line 243, in call
return ag_fn(y_true, y_pred, **self._fn_kwargs)
File "/usr/local/lib/python3.8/dist-packages/keras/losses.py", line 1787, in categorical_crossentropy
return backend.categorical_crossentropy(
File "/usr/local/lib/python3.8/dist-packages/keras/backend.py", line 5134, in categorical_crossentropy
return tf.nn.softmax_cross_entropy_with_logits(
Node: 'categorical_crossentropy/softmax_cross_entropy_with_logits'
logits and labels must be broadcastable: logits_size=[16,5] labels_size=[16,11]
[[{{node categorical_crossentropy/softmax_cross_entropy_with_logits}}]] [Op:__inference_train_function_1983]
spacy.load error: RuntimeError: dictionary changed size during iteration
I am loading a spaCy model as part of a step in my Dataflow streaming pipeline. To load the pre-downloaded spaCy model for a specific language I am using nlp_model = spacy.load(SPACY_KEYS[lang]) where SPACY_KEYS is a dictionary containing the names of the models for each language (e.g. 'en': 'en_core_web_sm'). This works without any issues for the majority of the jobs run by the pipeline, but for a few iterations I am getting the following error: Error message from worker: generic::unknown: Traceback (most recent call last): File "apache_beam/runners/common.py", line 1232, in apache_beam.runners.common.DoFnRunner.process File "apache_beam/runners/common.py", line 752, in apache_beam.runners.common.PerWindowInvoker.invoke_process File "apache_beam/runners/common.py", line 870, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window File "apache_beam/runners/common.py", line 1368, in apache_beam.runners.common._OutputProcessor.process_outputs File "/usr/local/lib/python3.7/site-packages/submodules/entities_and_pii_removal.py", line 259, in entities_and_PII nlp_model = spacy.load(SPACY_KEYS[lang]) # load spacy model File "/usr/local/lib/python3.7/site-packages/spacy/__init__.py", line 52, in load name, vocab=vocab, disable=disable, exclude=exclude, config=config File "/usr/local/lib/python3.7/site-packages/spacy/util.py", line 420, in load_model return load_model_from_package(name, **kwargs) # type: ignore[arg-type] File "/usr/local/lib/python3.7/site-packages/spacy/util.py", line 453, in load_model_from_package return cls.load(vocab=vocab, disable=disable, exclude=exclude, config=config) # type: ignore[attr-defined] File "/usr/local/lib/python3.7/site-packages/de_core_news_sm/__init__.py", line 10, in load return load_model_from_init_py(__file__, **overrides) File "/usr/local/lib/python3.7/site-packages/spacy/util.py", line 621, in load_model_from_init_py config=config, File "/usr/local/lib/python3.7/site-packages/spacy/util.py", line 489, in load_model_from_path return nlp.from_disk(model_path, exclude=exclude, overrides=overrides) File "/usr/local/lib/python3.7/site-packages/spacy/language.py", line 2042, in from_disk util.from_disk(path, deserializers, exclude) # type: ignore[arg-type] File "/usr/local/lib/python3.7/site-packages/spacy/util.py", line 1299, in from_disk reader(path / key) File "/usr/local/lib/python3.7/site-packages/spacy/language.py", line 2037, in <lambda> p, exclude=["vocab"] File "spacy/pipeline/trainable_pipe.pyx", line 343, in spacy.pipeline.trainable_pipe.TrainablePipe.from_disk File "/usr/local/lib/python3.7/site-packages/spacy/util.py", line 1299, in from_disk reader(path / key) File "spacy/pipeline/trainable_pipe.pyx", line 333, in spacy.pipeline.trainable_pipe.TrainablePipe.from_disk.load_model File "spacy/pipeline/trainable_pipe.pyx", line 334, in spacy.pipeline.trainable_pipe.TrainablePipe.from_disk.load_model File "/usr/local/lib/python3.7/site-packages/thinc/model.py", line 593, in from_bytes return self.from_dict(msg) File "/usr/local/lib/python3.7/site-packages/thinc/model.py", line 624, in from_dict loaded_value = deserialize_attr(default_value, value, attr, node) File "/usr/local/lib/python3.7/functools.py", line 840, in wrapper return dispatch(args[0].__class__)(*args, **kw) File "/usr/local/lib/python3.7/site-packages/thinc/model.py", line 804, in deserialize_attr return srsly.msgpack_loads(value) File "/usr/local/lib/python3.7/site-packages/srsly/_msgpack_api.py", line 27, in msgpack_loads msg = msgpack.loads(data, raw=False, use_list=use_list) File "/usr/local/lib/python3.7/site-packages/srsly/msgpack/__init__.py", line 76, in unpackb for decoder in msgpack_decoders.get_all().values(): File "/usr/local/lib/python3.7/site-packages/catalogue/__init__.py", line 110, in get_all for keys, value in REGISTRY.items(): RuntimeError: dictionary changed size during iteration I have not been able to identify the cause of this problem. Is there a way of getting around it?
InvalidArgumentError: Graph execution error in B3
activity = model.fit(train_gen, epochs=10, # Increase number of epochs if you have sufficient hardware
validation_data=val_gen,
verbose = 1
)
Epoch 1/10
Traceback (most recent call last):
File "C:\Users\BLRCSE~1\AppData\Local\Temp/ipykernel_15312/3305335964.py", line 1, in
activity = model.fit(train_gen, epochs=10, # Increase number of epochs if you have sufficient hardware
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 67, in error_handler
raise e.with_traceback(filtered_tb) from None
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\tensorflow\python\eager\execute.py", line 54, in quick_execute
tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
InvalidArgumentError: Graph execution error:
Detected at node 'gradient_tape/sequential_1/dense_5/MatMul/MatMul' defined at (most recent call last):
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\runpy.py", line 87, in run_code
exec(code, run_globals)
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\spyder_kernels\console_main.py", line 23, in
start.main()
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\spyder_kernels\console\start.py", line 328, in main
kernel.start()
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\ipykernel\kernelapp.py", line 677, in start
self.io_loop.start()
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\tornado\platform\asyncio.py", line 199, in start
self.asyncio_loop.run_forever()
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\asyncio\base_events.py", line 596, in run_forever
self._run_once()
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\asyncio\base_events.py", line 1890, in _run_once
handle._run()
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\asyncio\events.py", line 80, in _run
self._context.run(self._callback, *self._args)
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 457, in dispatch_queue
await self.process_one()
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 446, in process_one
await dispatch(*args)
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 353, in dispatch_shell
await result
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 648, in execute_request
reply_content = await reply_content
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\ipykernel\ipkernel.py", line 353, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\ipykernel\zmqshell.py", line 533, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2901, in run_cell
result = self._run_cell(
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2947, in _run_cell
return runner(coro)
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\IPython\core\async_helpers.py", line 68, in pseudo_sync_runner
coro.send(None)
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3172, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3364, in run_ast_nodes
if (await self.run_code(code, result, async=asy)):
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3444, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "C:\Users\BLRCSE~1\AppData\Local\Temp/ipykernel_15312/1931121224.py", line 1, in
activity = model.fit(train_gen,
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\engine\training.py", line 1384, in fit
tmp_logs = self.train_function(iterator)
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\engine\training.py", line 1021, in train_function
return step_function(self, iterator)
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\engine\training.py", line 1010, in step_function
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\engine\training.py", line 1000, in run_step
outputs = model.train_step(data)
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\engine\training.py", line 863, in train_step
self.optimizer.minimize(loss, self.trainable_variables, tape=tape)
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\optimizer_v2\optimizer_v2.py", line 530, in minimize
grads_and_vars = self._compute_gradients(
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\optimizer_v2\optimizer_v2.py", line 583, in _compute_gradients
grads_and_vars = self._get_gradients(tape, loss, var_list, grad_loss)
File "C:\Users\BLRCSE513-WS01\anaconda3\lib\site-packages\keras\optimizer_v2\optimizer_v2.py", line 464, in _get_gradients
grads = tape.gradient(loss, var_list, grad_loss)
Node: 'gradient_tape/sequential_1/dense_5/MatMul/MatMul'
Matrix size-incompatible: In[0]: [32,2], In[1]: [120,1]
[[{{node gradient_tape/sequential_1/dense_5/MatMul/MatMul}}]] [Op:__inference_train_function_47374]
If memory growth is enabled for a PhysicalDevice, the runtime initialization will not allocate all memory on the device. Memory growth cannot be configured on a PhysicalDevice with virtual devices configured.
For example:
import tensorflow as tf
physical_devices = tf.config.list_physical_devices('GPU')
try:
tf.config.experimental.set_memory_growth(physical_devices[0], True)
except:
# Invalid device or cannot modify virtual devices once initialized.
pass
Pytorch does not backpropagate through a iterative tensor construction
I am currently trying to build a tensor iteratively in Pytorch. Sadly the backprop does not work with the inplace operation in the loop. I already tried equivalent programs with stack for example. Does somebody know how I could build the tensor with a working backprop? This is a minimal example which produces the error: import torch k=2 a =torch.Tensor([10,20]) a.requires_grad_(True) b = torch.Tensor([10,20]) b.requires_grad_(True) batch_size = a.size()[0] uniform_samples = Uniform(torch.tensor([0.0]), torch.tensor([1.0])).rsample(torch.tensor([batch_size,k])).view(-1,k) exp_a = 1/a exp_b = 1/b km = (1- uniform_samples.pow(exp_b)).pow(exp_a) sticks = torch.zeros(batch_size,k) remaining_sticks = torch.ones_like(km[:,0]) for i in range(0,k-1): sticks[:,i] = remaining_sticks * km[:,i] remaining_sticks *= (1-km[:,i]) sticks[:,k-1] = remaining_sticks latent_variables = sticks latent_variables.sum().backward() The stack trace: /opt/conda/conda-bld/pytorch_1570910687230/work/torch/csrc/autograd/python_anomaly_mode.cpp:57: UserWarning: Traceback of forward call that caused the error: File "/opt/conda/lib/python3.6/runpy.py", line 193, in _run_module_as_main "__main__", mod_spec) File "/opt/conda/lib/python3.6/runpy.py", line 85, in _run_code exec(code, run_globals) File "/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py", line 16, in <module> app.launch_new_instance() File "/opt/conda/lib/python3.6/site-packages/traitlets/config/application.py", line 664, in launch_instance app.start() File "/opt/conda/lib/python3.6/site-packages/ipykernel/kernelapp.py", line 563, in start self.io_loop.start() File "/opt/conda/lib/python3.6/site-packages/tornado/platform/asyncio.py", line 148, in start self.asyncio_loop.run_forever() File "/opt/conda/lib/python3.6/asyncio/base_events.py", line 438, in run_forever self._run_once() File "/opt/conda/lib/python3.6/asyncio/base_events.py", line 1451, in _run_once handle._run() File "/opt/conda/lib/python3.6/asyncio/events.py", line 145, in _run self._callback(*self._args) File "/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py", line 690, in <lambda> lambda f: self._run_callback(functools.partial(callback, future)) File "/opt/conda/lib/python3.6/site-packages/tornado/ioloop.py", line 743, in _run_callback ret = callback() File "/opt/conda/lib/python3.6/site-packages/tornado/gen.py", line 787, in inner self.run() File "/opt/conda/lib/python3.6/site-packages/tornado/gen.py", line 748, in run yielded = self.gen.send(value) File "/opt/conda/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 361, in process_one yield gen.maybe_future(dispatch(*args)) File "/opt/conda/lib/python3.6/site-packages/tornado/gen.py", line 209, in wrapper yielded = next(result) File "/opt/conda/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 268, in dispatch_shell yield gen.maybe_future(handler(stream, idents, msg)) File "/opt/conda/lib/python3.6/site-packages/tornado/gen.py", line 209, in wrapper yielded = next(result) File "/opt/conda/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 541, in execute_request user_expressions, allow_stdin, File "/opt/conda/lib/python3.6/site-packages/tornado/gen.py", line 209, in wrapper yielded = next(result) File "/opt/conda/lib/python3.6/site-packages/ipykernel/ipkernel.py", line 300, in do_execute res = shell.run_cell(code, store_history=store_history, silent=silent) File "/opt/conda/lib/python3.6/site-packages/ipykernel/zmqshell.py", line 536, in run_cell return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs) File "/opt/conda/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2855, in run_cell raw_cell, store_history, silent, shell_futures) File "/opt/conda/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2881, in _run_cell return runner(coro) File "/opt/conda/lib/python3.6/site-packages/IPython/core/async_helpers.py", line 68, in _pseudo_sync_runner coro.send(None) File "/opt/conda/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3058, in run_cell_async interactivity=interactivity, compiler=compiler, result=result) File "/opt/conda/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3249, in run_ast_nodes if (await self.run_code(code, result, async_=asy)): File "/opt/conda/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3326, in run_code exec(code_obj, self.user_global_ns, self.user_ns) File "<ipython-input-124-2bbdbc3af797>", line 16, in <module> sticks[:,i] = remaining_sticks * km[:,i] --------------------------------------------------------------------------- RuntimeError Traceback (most recent call last) <ipython-input-124-2bbdbc3af797> in <module> 19 latent_variables = sticks 20 ---> 21 latent_variables.sum().backward() /opt/conda/lib/python3.6/site-packages/torch/tensor.py in backward(self, gradient, retain_graph, create_graph) 148 products. Defaults to ``False``. 149 """ --> 150 torch.autograd.backward(self, gradient, retain_graph, create_graph) 151 152 def register_hook(self, hook): /opt/conda/lib/python3.6/site-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables) 97 Variable._execution_engine.run_backward( 98 tensors, grad_tensors, retain_graph, create_graph, ---> 99 allow_unreachable=True) # allow_unreachable flag 100 101 RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [2]] is at version 1; expected version 0 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!
You can not do any inplace operations. So you may not use *= in your algorithm. k = 2 a = torch.tensor(np.array([10.,20]), requires_grad=True).float() b = torch.tensor(np.array([10.,20]), requires_grad=True).float() batch_size = a.size()[0] uniform_samples = Uniform(torch.tensor([0.]), torch.tensor([1.])).rsample(torch.tensor([batch_size,k])).view(-1,k) exp_a = 1/a exp_b = 1/b km = (1 - uniform_samples**exp_b)**exp_a sticks = torch.zeros(batch_size,k) remaining_sticks = torch.ones_like(km[:,0]) for i in range(0,k-1): sticks[:,i] = remaining_sticks * km[:,i] remaining_sticks = remaining_sticks * (1-km[:,i]) sticks[:,k-1] = remaining_sticks latent_variables = sticks latent_variables = torch.sum(latent_variables) latent_variables.backward()
Odd Dimension() object in shape of input?
I am experiencing issues with Tensorflow. I've narrowed the issues down to the format of my input shape of the build() method of a custom Attention layer in tf-keras. I am getting a list of Dimension() objects instead of the actual input-shape. Please help.
Attention Layer
x_attn = []
for i in range(self.attn_range): #4
x_attn.append(Attention()(x))
print("DEBUG:00001")
x = L.Concatenate(-1)(x_attn)
Custom Attention Layer build() method:
def build(self,
input_shape):
print("DEBUG:00005")
params_shape = list(input_shape[1:])
print("DEBUG:00007", params_shape)
self.query_weights = self.add_weight(
name='q_weights',
shape=params_shape,
initializer=self.q_weights_init
)
print("DEBUG:00006")
self.key_weights = self.add_weight(
name='key_weights',
shape=params_shape,
initializer=self.key_weights_init
)
self.val_weights = self.add_weight(
name='val_weights',
shape=params_shape,
initializer=self.value_weights_init
)
Output of param shape debug print function is:
DEBUG:00007 [Dimension(10), Dimension(5), Dimension(128)]
Edit:
Full Error Message as Requested:
Traceback (most recent call last):
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/tensor_util.py", line 558, in make_tensor_proto
str_values = [compat.as_bytes(x) for x in proto_values]
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/tensor_util.py", line 558, in <listcomp>
str_values = [compat.as_bytes(x) for x in proto_values]
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/util/compat.py", line 65, in as_bytes
(bytes_or_text,))
TypeError: Expected binary or unicode string, got Dimension(10)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "Empowerment\.py", line 327, in <module>
env_actor = ComputerEnv()
File "Empowerment\.py", line 225, in __init__
self.dqn = ICDQNAgent(self.state_size + (3,), self.state_size[0], 4)
File "Empowerment\.py", line 78, in __init__
self.model, self.autoencoder, self.critic = self.build_model()
File "Empowerment\.py", line 99, in build_model
x_attn.append(Attention()(x))
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py", line 591, in __call__
self._maybe_build(inputs)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py", line 1881, in _maybe_build
self.build(input_shapes)
File "/home/ai/Desktop/ai_proj/layers.py", line 69, in build
initializer=self.q_weights_init
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py", line 384, in add_weight
aggregation=aggregation)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/training/tracking/base.py", line 663, in _add_variable_with_custom_getter
**kwargs_for_getter)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer_utils.py", line 155, in make_variable
shape=variable_shape if variable_shape.rank else None)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/variables.py", line 259, in __call__
return cls._variable_v1_call(*args, **kwargs)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/variables.py", line 220, in _variable_v1_call
shape=shape)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/variables.py", line 198, in <lambda>
previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/variable_scope.py", line 2495, in default_variable_creator
shape=shape)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/variables.py", line 263, in __call__
return super(VariableMetaclass, cls).__call__(*args, **kwargs)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py", line 460, in __init__
shape=shape)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py", line 604, in _init_from_args
initial_value() if init_from_fn else initial_value,
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer_utils.py", line 135, in <lambda>
init_val = lambda: initializer(shape, dtype=dtype)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/init_ops.py", line 533, in __call__
shape, -limit, limit, dtype, seed=self.seed)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/random_ops.py", line 239, in random_uniform
shape = _ShapeTensor(shape)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/random_ops.py", line 44, in _ShapeTensor
return ops.convert_to_tensor(shape, dtype=dtype, name="shape")
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 1087, in convert_to_tensor
return convert_to_tensor_v2(value, dtype, preferred_dtype, name)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 1145, in convert_to_tensor_v2
as_ref=False)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 1224, in internal_convert_to_tensor
ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/constant_op.py", line 305, in _constant_tensor_conversion_function
return constant(v, dtype=dtype, name=name)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/constant_op.py", line 246, in constant
allow_broadcast=True)
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/constant_op.py", line 284, in _constant_impl
allow_broadcast=allow_broadcast))
File "/home/ai/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/tensor_util.py", line 562, in make_tensor_proto
"supported type." % (type(values), values))
TypeError: Failed to convert object of type <class 'list'> to Tensor. Contents: [Dimension(10), Dimension(5), Dimension(128)]. Consider casting elements to a supported type.