We Keep Coding

Reinforcement Learning - only size-1 arrays can be converted to Python scalars - is it data problem?

I'm new to pytorch and even though I was searching for this error I can't seem to understand where axactly I'm doing something wrong.
I'm trying to run a codewith a model that trades 3 different stocks. My data is a csv file with three columns with closing prices of stocks.
I'm trying to run this part of code
env.reset()
# In case you're running this a second time with the same model, delete the gradients
del model.rewards[:]
del model.saved_actions[:]
gamma = 0.9
log_interval = 60
def finish_episode():
R = 0
saved_actions = model.saved_actions
policy_losses = []
value_losses = []
rewards = []
for r in model.rewards[::-1]:
R = r + (gamma * R)
rewards.insert(0, R)
rewards = torch.tensor(rewards)
epsilon = (torch.rand(1) / 1e4) - 5e-5
# With different architectures, I found the following standardization step sometimes
# helpful, sometimes unhelpful.
# rewards = (rewards - rewards.mean()) / (rewards.std(unbiased=False) + epsilon)
# Alternatively, comment it out and use the following line instead:
rewards += epsilon
for (log_prob, value), r in zip(saved_actions, rewards):
reward = torch.tensor(r - value.item()).cuda()
policy_losses.append(-log_prob * reward)
value_losses.append(F.smooth_l1_loss(value, torch.tensor([r]).cuda()))
optimizer.zero_grad()
loss = torch.stack(policy_losses).sum() + torch.stack(value_losses).sum()
loss = torch.clamp(loss, -1e-5, 1e5)
loss.backward()
optimizer.step()
del model.rewards[:]
del model.saved_actions[:]
running_reward = 0
for episode in range(0, 4000):
state = env.reset()
reward = 0
done = False
msg = None
while not done:
action = model.act(state)
state, reward, done, msg = env.step(action)
model.rewards.append(reward)
if done:
break
running_reward = running_reward * (1 - 1/log_interval) + reward * (1/log_interval)
finish_episode()
# Resetting the hidden state seems unnecessary - it's effectively random from the previous
# episode anyway, more random than a bunch of zeros.
# model.reset_hidden()
if msg["msg"] == "done" and env.portfolio_value() > env.starting_portfolio_value * 1.1 and running_reward > 500:
print("Early Stopping: " + str(int(reward)))
break
if episode % log_interval == 0:
print("""Episode {}: started at {:.1f}, finished at {:.1f} because {} # t={}, \
last reward {:.1f}, running reward {:.1f}""".format(episode, env.starting_portfolio_value, \
env.portfolio_value(), msg["msg"], env.cur_timestep, reward, running_reward))
But I'm getting such an error:
TypeError Traceback (most recent call last)
<ipython-input-91-ce955397be85> in <module>()
45 msg = None
46 while not done:
---> 47 action = model.act(state)
48 state, reward, done, msg = env.step(action)
49 model.rewards.append(reward)
1 frames
<ipython-input-89-f463539c7fe3> in forward(self, x)
16
17 def forward(self, x):
---> 18 x = torch.tensor(x).cuda()
19 x = torch.sigmoid(self.input_layer(x))
20 x = torch.tanh(self.hidden_1(x))
TypeError: only size-1 arrays can be converted to Python scalars
This is the part of code with forward function defined
class Policy(nn.Module):
def __init__(self):
super(Policy, self).__init__()
self.input_layer = nn.Linear(11, 128)
self.hidden_1 = nn.Linear(128, 128)
self.hidden_2 = nn.Linear(32,31)
self.hidden_state = torch.tensor(torch.zeros(2,1,32)).cuda()
self.rnn = nn.GRU(128, 32, 2)
self.action_head = nn.Linear(31, 5)
self.value_head = nn.Linear(31, 1)
self.saved_actions = []
self.rewards = []
def reset_hidden(self):
self.hidden_state = torch.tensor(torch.zeros(2,1,32)).cuda()
def forward(self, x):
x = torch.tensor(x).cuda()
x = torch.sigmoid(self.input_layer(x))
x = torch.tanh(self.hidden_1(x))
x, self.hidden_state = self.rnn(x.view(1,-1,128), self.hidden_state.data)
x = F.relu(self.hidden_2(x.squeeze()))
action_scores = self.action_head(x)
state_values = self.value_head(x)
return F.softmax(action_scores, dim=-1), state_values
def act(self, state):
probs, state_value = self.forward(state)
m = Categorical(probs)
action = m.sample()
if action == 1 and env.state[0] < 1: action = torch.LongTensor([2]).squeeze().cuda()
if action == 4 and env.state[1] < 1: action = torch.LongTensor([2]).squeeze().cuda()
if action == 6 and env.state[2] < 1: action = torch.LongTensor([2]).squeeze().cuda()
self.saved_actions.append((m.log_prob(action), state_value))
return action.item()
Can you please direct me where I should make changes? Is it the data I'm feeding my model with, or something different?
Thank you so much for help

You are passing state = env.reset() to:
action = model.act(state)
probs, state_value = self.forward(state)
x = torch.tensor(x).cuda()
And hence torch is throwing an error. It expects a numeric or array type input.

How to fix ''int' object has no attribute 'triu_indices'

I'm using a double for loop, to calculate a value. This for loop gets the i element of a vector and the i+1 element of the same vector, then it does some calculation. But when the second iteration of the second for loop starts, I get the error 'int' object has no attribute 'triu_indices'
I have three matrixes, and some functions. Also I use a double for (I don't think this is the pythonic way to do that, however I'm learning)
I have this:
import numpy as np
#The three matrixes
flowMatrixSymNoCeros = np.array([[0,4,6,2,4,4],
[4,0,4,2,2,8],
[6,4,0,2,2,6],
[2,2,2,0,6,2],
[4,2,2,6,0,10],
[4,8,6,2,10,0]])
flowMatrixSymCeros = np.array([[0,0,6,2,4,0],
[0,0,4,2,2,8],
[6,4,0,2,2,6],
[2,2,2,0,6,2],
[4,2,2,6,0,0],
[0,8,6,2,0,0]])
closenessRatingSymNoceros = np.array([[0,5,3,2,6,4],
[5,0,5,2,6,2],
[3,5,0,1,2,1],
[2,2,1,0,2,2],
[6,6,2,2,0,6],
[4,2,1,2,6,0]])
matrices = np.array([flowMatrixSymNoCeros,
flowMatrixSymCeros,
closenessRatingSymNoceros])
def normalMatrixesAsym(matrices):
matrixes = np.copy(matrices)
matrixes = np.absolute(matrixes)
normalMatrixes = []
for matriz in matrixes:
s = np.sum(matriz)
normalMatrixes.append(matriz / s)
return np.asarray(normalMatrixes)
def sdwm(symetria, normalMatrix):
SD= 0
normalizedMatrix = np.copy(normalMatrix)
m = normalizedMatrix.shape[0]
sd = lambda num,den : (num/den)**(1/2)
# maskUpper = np.mask_indices(m, np.triu, 1)
# maskLower = np.mask_indices(m, np.tril, -1)
upper = normalizedMatrix[np.triu_indices(m,1)]
lower = normalizedMatrix[np.tril_indices(m,-1)]
upperAbs = np.abs(upper)
if(symetria):
media = np.absolute(np.mean(upper))
num = np.sum((upperAbs - np.mean(media))**2)
den = ((m * (m-1))/2)-1
SD = sd(num,den) #calculo del SD
else:
# lower = np.tril(normalizedMatrix,-1)
matrixNoDiag = np.append(upper,lower)
matrixNoDiagAbs = np.abs(matrixNoDiag)
mean = np.absolute(np.mean(matrixNoDiag))
num = np.sum((matrixNoDiagAbs - mean)**2)
den = (m*(m-1))-1 #calcula el denominador
SD = sd(num,den) #calcula el SD
return SD
def calcularCriticalValues(funcion, symetria, normalMatrixes):
normalMatrices = np.copy(normalMatrixes)
criticalValules = []
for normalMatriz in normalMatrices:
criticalValules.append(funcion(symetria,normalMatriz))
return np.asarray(criticalValules)
normalizedMatrices = normalMatrixesAsym(matrices)
SD = calcularCriticalValues(nm.sdwm,False,normalizedMatrices)
m=len(matrices)
R= np.zeros((m,m))
n = len(normalizedMatrices[0])
for i,matrix in enumerate(normalizedMatrices):
upperI = matrix[np.triu_indices(n,1)]
lowerI = matrix[np.tril_indices(n,-1)]
matrixNoDiagI = np.append(upperI,lowerI)
meanI = np.absolute(np.mean(matrixNoDiagI))
matrixNoDiagIAbs = np.abs(matrixNoDiagI)
for j in range(i+1,m):
matrixJ =normalizedMatrices[j]
upperJ = matrixJ[np.triu_indices(n,1)] #the problem is here
lowerJ = matrixJ[np.tril_indices(n,-1)]
matrixNoDiagJ = np.append(upperJ,lowerJ)
meanJ = np.absolute(np.mean(matrixNoDiagJ))
matrixNoDiagJAbs = np.abs(matrixNoDiagJ)
num = np.sum((matrixNoDiagIAbs - meanI)*(matrixNoDiagJAbs -
meanJ))
np = (n*(n-1))-1 #n''
den = np*SD[i]*SD[j]
r = num/den
R[i][j] = r
print(R)
What I expect is a matrix named R, with the calculations that the algorithm do.
This is
>>>R
>>>[[0. 0.456510 0.987845]
[0. 0.156457 0.987845]
[0. 0. 0. ]]
The error I get is:
AttributeError Traceback (most recent call last)
in ()
204 # print(i,j)
205 matrixJ =normalizedMatrices[j]
--> 206 upperJ = matrixJ[np.triu_indices(n,1)] # Obtiene elementos diagonal superior
207 lowerJ = matrixJ[np.tril_indices(n,-1)] # Obtiene elementos diagonal superior
208 matrixNoDiagJ = np.append(upperJ,lowerJ)
AttributeError: 'int' object has no attribute 'triu_indices'

The problem is that you use np as a variable in that loop: np = (n*(n-1))-1 #n''. You are assigning it to an int value, which is shadowing the imported np. You need to rename that variable.

Pyomo: KeyError: "Index '(0, 1, 1)' is not valid for indexed component 'x_ijl'"

With reference to the images in the attached files, i want to model using pyomo.
What I have done so far.
from pyomo.environ import *
from pyomo.opt import SolverFactory
import pyomo.environ
n=13
distanceMatrix=[[0,8,4,10,12,9,15,8,11,5,9,4,10],
[8,0,7,6,8,6,7,10,12,9,8,7,5],
[4,7,0,7,9,5,8,5,4,8,6 ,10,8],
[10,6 ,7,0,6,11,5 ,9,8,12,11,6,9],
[12,8 ,9,6, 0,7,9,6,9,8,4,11,10],
[9,6,5,11,7,0,10,4,3,10,6,5,7],
[15,7 ,8,5,9,10,0,10,9,8,5,9,10],
[8,10 ,5,9,6,4,10,0,11,5,9,6,7],
[11,12,4,8, 9,3,9,11,0, 9,11,11,6],
[5,9,8,12,8,10,8,5,9,0,6,7,5],
[9,8,6,11,4,6,5,9,11,6,0,10,7],
[4,7,10,6,11,5,9,6,11,7,10,0,9],
[10,5,8,9,10,7,10,7,6,5,7,9,0]]
travel_time=[[0,8,4,10,12,9,15,8,11,5,9,4,10],
[8,0,7,6,8,6,7,10,12,9,8,7,5],
[4,7,0,7,9,5,8,5,4,8,6 ,10,8],
[10,6 ,7,0,6,11,5 ,9,8,12,11,6,9],
[12,8 ,9,6, 0,7,9,6,9,8,4,11,10],
[9,6,5,11,7,0,10,4,3,10,6,5,7],
[15,7 ,8,5,9,10,0,10,9,8,5,9,10],
[8,10 ,5,9,6,4,10,0,11,5,9,6,7],
[11,12,4,8, 9,3,9,11,0, 9,11,11,6],
[5,9,8,12,8,10,8,5,9,0,6,7,5],
[9,8,6,11,4,6,5,9,11,6,0,10,7],
[4,7,10,6,11,5,9,6,11,7,10,0,9],
[10,5,8,9,10,7,10,7,6,5,7,9,0]]
Time_windows = [(1400,1500), (0000,2400), (0000,2400),(0700,2400),(0000,2400),(0000,0700),(0700,2400),(0700,2400),(0000,0700),(0000,2400),\
(0000,2400),(0000,2400),(0700,2400)]
Service_time = [0000, 1600,1600,180,30,120,120,60,30,30,90,120,330]
demand = [9999.00, 9999.00,9999.00,12.00, 4.00, 6.00, 8.00,16.00,6.00,16.00,12.00,24.00,8.00]
K = 4 # no. of vehicles
C = 280; # capacity
speed = 40; # default speed
M = 200;
startCity = 0
model = ConcreteModel()
# sets
#model.M = Set(initialize=range(1, n+1))
model.N = Set(initialize=range(1, n+1))
model.K = Set(initialize=range(1, K+1))
model.Nc = Set(initialize=range(3, n+1)) # set of customers
# Param
model.cost = Param(model.N, model.N, initialize=lambda model, i, j: distanceMatrix[i-1][j-1])
model.travel_time = Param(model.N, model.N,initialize=lambda model, i,j: travel_time[i-1][j-1])
model.Time_windows = Param(model.N, initialize=lambda model, i: travel_time[i-1]) # time_windows
model.Service_time = Param(model.N, initialize=lambda model, i: Service_time[i-1]) # Service time
model.demand = Param(model.N, initialize=lambda model, i: demand[i-1])
model.M = Param(initialize=M)
model.C = Param(initialize=C)
# variables
model.x_ijl = Var(model.N, model.N, model.K, within=Binary) # decision variable = 1 iff vehicle l in K uses arc (i,j) in A
model.d_il = Var(model.N, model.K, bounds=(0,None)) # the accumulative demand at node i in V for vehicle l in K
model.w_il = Var(model.N, model.K, bounds=(0,None)) # start time of service at node i in V for vehicle l in K
"""
Constriants
"""
# All l vehicles must leave the depot
def leave_depot(model,l):
return sum(model.x_ijl[0,j,l] for j in model.N) == 1
model.leave_depot = Constraint(model.K, rule=leave_depot)
# All l vehicles must return to the depot
def return_depot(model,l):
return sum(model.x_ijl[i,0,l] for i in model.N) == 1
model.return_depot = Constraint(model.K, rule=return_depot)
# ensures that all customers are serviced exactly once.
def customer_service(model, j):
return sum(sum(model.x_ijl[i,j,l] for l in model.K) for i in model.N) ==1
model.customer_service1 = Constraint(model.Nc, rule=customer_service)
# Inflow and outflow must be equal except for the depot nodes
def flow(model,j,l):
return sum(model.x_ijl[i,j,l] for i in model.N if i < j) == sum(model.x_ijl[j,i,l] for i in model.N if j < i)
model.flow1 = Constraint(model.N,model.K, rule=flow)
# Time windows
def time_windows1(model,i,l):
return model.Time_windows[i][0] <=model.w_il[i,l] <= model.Time_windows[i][1]
model.time_windows = Constraint(model.N,model.K, rule=time_windows1)
# service time
def service_time(model,i,j,l):
return model.w_il[i,l] + model.Service_time[i] + model.travel_time[i,j] <= model.w_il[j,l] + (1 - model.x_ijl[i,j,l])*200
model.service_time = Constraint(model.N, model.N, model.K, rule=service_time)
# vehicle must be empty at start and end of routes
def empty(model, l):
return model.d_il[0,l] + model.d_il[-1,l] == 0
model.empty = Constraint(model.K, rule=empty)
# accumulative demand for all nodes except disposal sites
def demands_forall_nodes(model,i,j,l):
return model.d_il[i,l] + model.demand[i] <= model.d_il[j,l]+(1 - model.x_ijl[i,j,l]*200)
model.demands_forall_nodes = Constraint(model.Nc, model.N,model.K,rule=demands_forall_nodes)
# Capacity contraints
def vehicle_capacity(model, i,l):
return model.d_il[i,l] <= model.C
model.vehicle_capacity = Constraint(model.N, model.K, rule=vehicle_capacity)
# Objective Function
def objective(model):
return sum(model.cost[i,j]*model.x_ijl[i,j,l] for i in model.N for j in model.N for l in model.K)
model.obj = Objective(rule=objective)
opt = SolverFactory("glpk")
results = opt.solve(model, tee=True)
results.write()
However, I got an error with constriant 2 (from image 2) which I know similar will apply to constraint 3 and constriant 9. The error is:
ERROR: Rule failed when generating expression for constraint leave_depot with
index 1: KeyError: "Index '(0, 1, 1)' is not valid for indexed component
'x_ijl'"
ERROR: Constructing component 'leave_depot' from data=None failed: KeyError:
"Index '(0, 1, 1)' is not valid for indexed component 'x_ijl'"
Traceback (most recent call last):
File "vrptwModel.py", line 81, in <module>
model.leave_depot = Constraint(model.K, rule=leave_depot)
File "/usr/local/lib/python2.7/dist-packages/pyomo/core/base/block.py", line 540, in __setattr__
self.add_component(name, val)
File "/usr/local/lib/python2.7/dist-packages/pyomo/core/base/block.py", line 980, in add_component
val.construct(data)
File "/usr/local/lib/python2.7/dist-packages/pyomo/core/base/constraint.py", line 793, in construct
ndx)
File "/usr/local/lib/python2.7/dist-packages/pyomo/core/base/misc.py", line 61, in apply_indexed_rule
return rule(model, index)
File "vrptwModel.py", line 80, in leave_depot
return sum(model.x_ijl[0,j,l] for j in model.N) == 1
File "vrptwModel.py", line 80, in <genexpr>
return sum(model.x_ijl[0,j,l] for j in model.N) == 1
File "/usr/local/lib/python2.7/dist-packages/pyomo/core/base/indexed_component.py", line 543, in __getitem__
index = self._validate_index(index)
File "/usr/local/lib/python2.7/dist-packages/pyomo/core/base/indexed_component.py", line 695, in _validate_index
% ( idx, self.name, ))
KeyError: "Index '(0, 1, 1)' is not valid for indexed component 'x_ijl'"
My problem is modeling constraint 2 and 3.
Please could someone help me to write these constraints correctly

The problem is that you started your indexing sets at 1 not 0. Change m.x_ijl[0,j,l] to m.x_ijl[1,j,l].

pandas: AttributeError: 'dict' object has no attribute 'transform'

I'm quite new with python and pandas, and I'm stuck with an error. I'm working with audiofiles, and I need to extract some features from audiostreams.
def load_features(self, audio_file):
def stereo_to_mono(x):
# stereo to mono
if len(x.shape) > 1 and x.shape[1] > 1:
print('Converting stereo to mono')
x = x.mean(axis=1)
return x
def cut_or_pad_to_length(x, duration, fs):
desired_length = int(round(duration * fs))
length = len(x)
diff = length - desired_length
abs_diff = abs(diff)
if diff < 0:
print('Padding')
# put the short signal in the middle
pad_before = abs_diff // 2
pad_after = abs_diff - pad_before
x = np.lib.pad(x, (pad_before, pad_after), 'constant')
elif diff > 1:
print('Cutting')
# cut the beginning
x = x[0:desired_length]
return x
def adjust_input(x, fs):
x = stereo_to_mono(x)
x = cut_or_pad_to_length(x, 2.0, fs)
return x
#x is data, fs is samplerate
x, fs = sf.read(audio_file)
x0 = adjust_input(x, fs)
# pitchgram
x_features = self.ch.transform(x0)
if self.scaler is not None:
x_features = self.scaler.transform(x_features.reshape(1, -1)) \
# 1 data point with 2D features
x_features = x_features.reshape(1, *x_features.shape)
return x_features
def predict_class_label(self, audio_file):
x_features = self.load_features(audio_file)
instrument_class = np_utils.probas_to_classes(self.model.predict(x_features, verbose=0))[0]
label = self.instr_family_le.inverse_transform(instrument_class)
return label
This gives me following error:
File "C:/dipl0m/ml-master/instrument-classification/predict.py", line 104, in predict_probabilities
x_features = self.load_features(audio_file)
File "C:/dipl0m/ml-master/instrument-classification/predict.py", line 87, in load_features
x_features = self.ch.transform(x0)
AttributeError: 'dict' object has no attribute 'transform'
But x and x0 don't seems like dictionary, because I operate with them like lists, but transform gives me that error ... Or am I wrong somewhere? Can't figure it out for a long time.

Porting pymc2 code to pymc3: custom likelihood function

I am trying to implement the censored data example in Lee&Wagenmakers' book (Chapter 5.5, page 70). In pymc2, I have the following model:
nattempts = 950
nfails = 949
n = 50 # Number of questions
y = np.zeros(nattempts)
y[nattempts-1] = 1
z = 30
unobsmin = 15
unobsmax = 25
unobsrange = np.arange(unobsmin,unobsmax+1)
theta = pymc.Uniform("theta",lower = .25, upper = 1)
#pymc.observed
def Ylike(value=z, theta = theta, n=n, censorn=nfails, unobs=unobsrange):
ylikeobs = pymc.binomial_like(x=value, n=n, p=theta)
ylikeunobs = np.array([])
for i in unobs:
ylikeunobs = np.append(pymc.binomial_like(x=i, n=n, p=theta),ylikeunobs)
return ylikeobs+sum(ylikeunobs)*censorn
testmodel = pymc.Model([theta,Ylike])
mcmc = pymc.MCMC(testmodel)
mcmc.sample(iter = 20000, burn = 50, thin = 2)
which involved the decorater #pymc.observed.
I think I need to express the likelihood using the pm.DensityDist, however, I could not figure it out how to.

OK, I found out how to do it:
with pm.Model():
theta = pm.Uniform("theta",lower = .25, upper = 1)
def logp(value,n,p):
return pm.dist_math.bound(
pm.dist_math.binomln(n, value)
+ pm.dist_math.logpow(p, value)
+ pm.dist_math.logpow(1 - p, n - value),
0 <= value, value <= n,
0 <= p, p <= 1)
def Censorlike(value=z, n=n, censorn=nfails, unobs=unobsrange):
ylikeobs = logp(value=value, n=n, p=theta)
ylikeunobs = 0
for i in unobs:
ylikeunobs += logp(value=i, n=n, p=theta)
return ylikeobs+ylikeunobs*censorn
ylike = pm.DensityDist('ylike', Censorlike, observed={'value':z,'n':n,'censorn':nfails,'unobs':unobsrange})
trace = pm.sample(3e3)