We Keep Coding

How can I gridsearch Light GBM on time series data?

I want to do a grid search on time series data. Is there any function to do that to search through what I have listed on "lgb_params" for example?
lgb_params = {
"learning_rate": [0.001, 0.01, 0.1, 0.2],
"max_depth": [3, 5, 7, 9],
"num_leaves": [5, 10, 15],
"num_boost_round": 10000,
"early_stopping_rounds": 300,
"feature_fraction": [0.2, 0.3, 0.5, 0.7, 0.8],
"verbose": 0
}
lgbtrain = lgb.Dataset(data=X_train, label=y_train, feature_name=cols)
lgbval = lgb.Dataset(data=X_val, label=y_val, reference=lgbtrain, feature_name=cols)
model = lgb.train(lgb_params, lgbtrain,
valid_sets=[lgbtrain, lgbval],
num_boost_round=lgb_params['num_boost_round'],
early_stopping_rounds=lgb_params['early_stopping_rounds'],
feval=lgbm_smape,
verbose_eval=100)
The code above of course does not work in the end since lgb params contains keys with more than 1 values (e.g., learning_rate, max_depth etc.). Well, those are the ones I actually want to search for and that's where the problem is...

I think I came up with a solution, it is currently running and haven't finished since it searches through a lot of values but here's the function that I wrote just in case anyone needs it:
def param_search(lgb_param_dict):
min_error = float("inf")
best_params = dict()
best_iter = float("inf")
for i in range(len(lgb_param_dict["learning_rate"])):
lgb_params = dict()
lgb_params["learning_rate"] = lgb_param_dict["learning_rate"][i]
for j in range(len(lgb_param_dict["max_depth"])):
lgb_params["max_depth"] = lgb_param_dict["max_depth"][j]
for k in range(len(lgb_param_dict["num_leaves"])):
lgb_params["num_leaves"] = lgb_param_dict["num_leaves"][k]
for s in range(len(lgb_param_dict["feature_fraction"])):
lgb_params["feature_fraction"] = lgb_param_dict["feature_fraction"][s]
print(" ")
print("##########")
print("Learning_rate = " + str(lgb_params["learning_rate"]))
print("max_depth = " + str(lgb_params["max_depth"]))
print("num_leaves = " + str(lgb_params["num_leaves"]))
print("feature_fraction = " + str(lgb_params["feature_fraction"]))
model = lgb.train(lgb_params, lgbtrain,
valid_sets=[lgbtrain, lgbval],
num_boost_round=lgb_full_params["num_boost_round"],
early_stopping_rounds=lgb_full_params["early_stopping_rounds"],
feval=lgbm_smape,
verbose_eval=500)
print("Learning_rate = " + str(lgb_params["learning_rate"]))
print("max_depth = " + str(lgb_params["max_depth"]))
print("num_leaves = " + str(lgb_params["num_leaves"]))
print("feature_fraction = " + str(lgb_params["feature_fraction"]))
if min_error > dict(model.best_score["valid_1"])["SMAPE"]:
min_error = dict(model.best_score["valid_1"])["SMAPE"]
best_params = model.params
best_iter = model.best_iteration
else:
continue
return min_error, best_params, best_iter
Print statement are for readability. There is probably better way to write this function but I'll approve it as an answer if it finishes without any problems.
Edit: It worked!

How to implement a lane change manoeuver on Carla

I am trying to implement a simple lane change manoeuver for a self driving car on Carla simulator. Specifically a left lane change.
However when retrieving waypoints from the left lane ( using carla.Waypoint.get_left_lane() function ), the waypoints I get are oscillating between left and right lanes. Below is the script I used :
import sys
import glob
import os
try:
sys.path.append(glob.glob('../../carla/dist/carla-*%d.%d-%s.egg' % (
sys.version_info.major,
sys.version_info.minor,
'win-amd64' if os.name == 'nt' else 'linux-x86_64'))[0])
except IndexError:
pass
import carla
import time
import math
import numpy as np
import random
from agents.navigation.controller import VehiclePIDController
dt = 1.0 / 20.0
args_lateral_dict = {'K_P': 1.95, 'K_I': 0.05, 'K_D': 0.2, 'dt': dt}
args_longitudinal_dict = {'K_P': 1.0, 'K_I': 0.05, 'K_D': 0, 'dt': dt}
max_throt = 0.75
max_brake = 0.3
max_steer = 0.8
offset = 0
VEHICLE_VEL = 5
actorList = []
try:
client = carla.Client("localhost",2000)
client.set_timeout(10.0)
world = client.load_world("Town02")
spectator = world.get_spectator()
actorList.append(spectator)
settings = world.get_settings()
settings.synchronous_mode = True
settings.fixed_delta_seconds = 1/20
world.apply_settings(settings)
blueprint_library = world.get_blueprint_library()
vehicle_bp = blueprint_library.filter("cybertruck")[0]
vehicle = None
while(vehicle is None):
spawn_points = world.get_map().get_spawn_points()
spawn_point = random.choice(spawn_points) if spawn_points else carla.Transform()
vehicle = world.try_spawn_actor(vehicle_bp, spawn_point)
actorList.append(vehicle)
vehicle_controller = VehiclePIDController(vehicle,
args_lateral=args_lateral_dict,
args_longitudinal=args_longitudinal_dict,
offset=offset,
max_throttle=max_throt,
max_brake=max_brake,
max_steering=max_steer)
old_yaw = math.radians(vehicle.get_transform().rotation.yaw)
old_x = vehicle.get_transform().location.x
old_y = vehicle.get_transform().location.y
```
spectator_transform = vehicle.get_transform()
spectator_transform.location += carla.Location(x = 10, y= 0, z = 5.0)
control = carla.VehicleControl()
control.throttle = 1.0
vehicle.apply_control(control)
while True:
####### Im suspecting the bug is within this portion of code ########################
current_waypoint = world.get_map().get_waypoint(vehicle.get_location())
# if not turned_left :
left_waypoint = current_waypoint.get_left_lane()
if(left_waypoint is not None and left_waypoint.lane_type == carla.LaneType.Driving) :
target_waypoint = left_waypoint.previous(0.3)[0]
turned_left = True
else :
if(turned_left) :
target_waypoint = waypoint.previous(0.3)[0]
else : # I tryed commenting this else section but the bug is still present so I dont suspect the bug relates to this else part
target_waypoint = waypoint.next(0.3)[0]
################### End of suspected bug ############################################
world.debug.draw_string(target_waypoint.transform.location, 'O', draw_shadow=False,
color=carla.Color(r=255, g=0, b=0), life_time=120.0,
persistent_lines=True)
control_signal = vehicle_controller.run_step(VEHICLE_VEL,target_waypoint)
vehicle.apply_control(control_signal)
new_yaw = math.radians(vehicle.get_transform().rotation.yaw)
spectator_transform = vehicle.get_transform()
spectator_transform.location += carla.Location(x = -10*math.cos(new_yaw), y= -10*math.sin(new_yaw), z = 5.0)
spectator.set_transform(spectator_transform)
world.tick()
finally:
print("Destroying actors")
client.apply_batch([carla.command.DestroyActor(x) for x in actorList])

I just figured out the cause of the problem. The reason is that I was manipulating Carla waypoints as undirected points. However, in Carla, each waypoint is directed by the road direction.
In the scene, I was testing my code in, all the roads have two lanes, each one in an opposite direction. Hence, the left lane of each lane is the remaining lane.
The issue in the previous code was that I was not changing my view to match the direction of the lane. I was assuming a global reference frame, but in Carla, the waypoints are relative to the frames attached to their respective lanes. And since only one of the two coordinate frames (for each lane) was matching my imagined global reference frame, I was getting an oscillatory behavior.
Another issue was that I was updating the target waypoints to track too early. This caused the vehicle to move a very short distance without going through all the target waypoints. I changed this to keep tracking the same target waypoint until the distance separating it to my vehicle becomes too close before moving to the next waypoint. This helped to perform the lane change behavior.
Below is the script I used :
import sys
import glob
import os
#The added path depends on where the carla binaries are stored
try:
sys.path.append(glob.glob('../../carla/dist/carla-*%d.%d-%s.egg' % (
sys.version_info.major,
sys.version_info.minor,
'win-amd64' if os.name == 'nt' else 'linux-x86_64'))[0])
except IndexError:
pass
import carla
import time
import math
import numpy as np
import random
from agents.navigation.controller import VehiclePIDController
VEHICLE_VEL = 5
class Player():
def __init__(self, world, bp, vel_ref = VEHICLE_VEL, max_throt = 0.75, max_brake = 0.3, max_steer = 0.8):
self.world = world
self.max_throt = max_throt
self.max_brake = max_brake
self.max_steer = max_steer
self.vehicle = None
self.bp = bp
while(self.vehicle is None):
spawn_points = world.get_map().get_spawn_points()
spawn_point = random.choice(spawn_points) if spawn_points else carla.Transform()
self.vehicle = world.try_spawn_actor(vehicle_bp, spawn_point)
self.spectator = world.get_spectator()
dt = 1.0 / 20.0
args_lateral_dict = {'K_P': 1.95, 'K_I': 0.05, 'K_D': 0.2, 'dt': dt}
args_longitudinal_dict = {'K_P': 1.0, 'K_I': 0.05, 'K_D': 0, 'dt': dt}
offset = 0
self.controller = VehiclePIDController(self.vehicle,
args_lateral=args_lateral_dict,
args_longitudinal=args_longitudinal_dict,
offset=offset,
max_throttle=max_throt,
max_brake=max_brake,
max_steering=max_steer)
self.vel_ref = vel_ref
self.waypointsList = []
self.current_pos = self.vehicle.get_transform().location
self.past_pos = self.vehicle.get_transform().location
def dist2Waypoint(self, waypoint):
vehicle_transform = self.vehicle.get_transform()
vehicle_x = vehicle_transform.location.x
vehicle_y = vehicle_transform.location.y
waypoint_x = waypoint.transform.location.x
waypoint_y = waypoint.transform.location.y
return math.sqrt((vehicle_x - waypoint_x)**2 + (vehicle_y - waypoint_y)**2)
def go2Waypoint(self, waypoint, draw_waypoint = True, threshold = 0.3):
if draw_waypoint :
# print(" I draw")
self.world.debug.draw_string(waypoint.transform.location, 'O', draw_shadow=False,
color=carla.Color(r=255, g=0, b=0), life_time=10.0,
persistent_lines=True)
current_pos_np = np.array([self.current_pos.x,self.current_pos.y])
past_pos_np = np.array([self.past_pos.x,self.past_pos.y])
waypoint_np = np.array([waypoint.transform.location.x,waypoint.transform.location.y])
vec2wp = waypoint_np - current_pos_np
motion_vec = current_pos_np - past_pos_np
dot = np.dot(vec2wp,motion_vec)
if (dot >=0):
while(self.dist2Waypoint(waypoint) > threshold) :
control_signal = self.controller.run_step(self.vel_ref,waypoint)
self.vehicle.apply_control(control_signal)
self.update_spectator()
def getLeftLaneWaypoints(self, offset = 2*VEHICLE_VEL, separation = 0.3):
current_waypoint = self.world.get_map().get_waypoint(self.vehicle.get_location())
left_lane = current_waypoint.get_left_lane()
self.waypointsList = left_lane.previous(offset)[0].previous_until_lane_start(separation)
def getRightLaneWaypoints(self, offset = 2*VEHICLE_VEL, separation = 0.3):
current_waypoint = self.world.get_map().get_waypoint(self.vehicle.get_location())
right_lane = current_waypoint.get_left_lane()
self.waypointsList = right_lane.next(offset)[0].next_until_lane_end(separation)
def do_left_lane_change(self):
self.getLeftLaneWaypoints()
for i in range(len(self.waypointsList)-1):
self.current_pos = self.vehicle.get_location()
self.go2Waypoint(self.waypointsList[i])
self.past_pos = self.current_pos
self.update_spectator()
def do_right_lane_change(self):
self.getRightLaneWaypoints()
for i in range(len(self.waypointsList)-1)
self.current_pos = self.vehicle.get_location()
self.go2Waypoint(self.waypointsList[i])
self.past_pos = self.current_pos
self.update_spectator()
def update_spectator(self):
new_yaw = math.radians(self.vehicle.get_transform().rotation.yaw)
spectator_transform = self.vehicle.get_transform()
spectator_transform.location += carla.Location(x = -10*math.cos(new_yaw), y= -10*math.sin(new_yaw), z = 5.0)
self.spectator.set_transform(spectator_transform)
self.world.tick()
def is_waypoint_in_direction_of_motion(self,waypoint):
current_pos = self.vehicle.get_location()
def draw_waypoints(self):
for waypoint in self.waypointsList:
self.world.debug.draw_string(waypoint.transform.location, 'O', draw_shadow=False,
color=carla.Color(r=255, g=0, b=0), life_time=10.0,
persistent_lines=True)
dt = 1.0 / 20.0
args_lateral_dict = {'K_P': 1.95, 'K_I': 0.05, 'K_D': 0.2, 'dt': dt}
args_longitudinal_dict = {'K_P': 1.0, 'K_I': 0.05, 'K_D': 0, 'dt': dt}
offset = 0
VEHICLE_VEL = 10
actorList = []
try:
client = carla.Client("localhost",2000)
client.set_timeout(10.0)
world = client.load_world("Town02")
spectator = world.get_spectator()
actorList.append(spectator)
settings = world.get_settings()
settings.synchronous_mode = True
settings.fixed_delta_seconds = 1/20
world.apply_settings(settings)
blueprint_library = world.get_blueprint_library()
vehicle_bp = blueprint_library.filter("cybertruck")[0]
player = Player(world, vehicle_bp)
actorList.append(player.vehicle)
actorList.append(player.spectator)
while(True):
player.update_spectator()
manoeuver = input("Enter manoeuver: ")
if ( manoeuver == "l"): #Perform left lane change
player.do_left_lane_change()
elif (manoeuver == "r"): #Perform right lane change
player.do_right_lane_change()
elif (manoeuver == "d"): #Just draws the waypoints for debugging purpose
player.getLeftLaneWaypoints()
player.draw_waypoints()
finally:
print("Destroying actors")
client.apply_batch([carla.command.DestroyActor(x) for x in actorList])

Problems at getting texture's color at a given point

I'm trying to get the color of a texture at an u v coordinate in relation with the vertex of a poly. The issue that I'm having is that every coordinate that I get result in the same color even if it is not painted that way.
I've been trying to place a texture and paint it later, then get the color of each vertex in a simple plane. What I could see around is more or less what I already have, polyListComponentConversion and colorAtPoint with the uv coordinates got.
The points that I get are [0.0, 0.0], [1.0, 0.0], [0.0, 1.0], [1.0, 1.0], and the color is always the same which is painted in one of the corners.
If instead using these I try hardcoding another value (like [.5, .5]) then I get another color which seems accurate.
What I do is:
#Create and place the texture files into the poly
def createShadingEngine(vName):
vNewShadingGroup = mc.sets(name = '{0}_SG'.format(vName), empty = True, renderable = True, noSurfaceShader = True)
vNewMaterial = mc.shadingNode('lambert', name = '{0}_mat'.format(vName), asShader = True)
mc.connectAttr('{0}.outColor'.format(vNewMaterial), '{0}.surfaceShader'.format(vNewShadingGroup))
vNewTexture = mc.shadingNode('file', name = '{0}_file'.format(vName), asTexture = True)
vNewUtility = mc.shadingNode('place2dTexture', name = '{0}_tex'.format(vName), asUtility = True)
mc.connectAttr('{0}.outColor'.format(vNewTexture), '{0}.color'.format(vNewMaterial))
vConnectAttr = ['coverage', 'translateFrame', 'rotateFrame', 'mirrorU', 'mirrorV', 'stagger', 'wrapU', 'wrapV',
'repeatUV', 'offset', 'rotateUV', 'noiseUV', 'vertexUvOne', 'vertexUvTwo', 'vertexUvThree', 'vertexCameraOne']
for vAttr in vConnectAttr:
mc.connectAttr('{0}.{1}'.format(vNewUtility, vAttr), '{0}.{1}'.format(vNewTexture, vAttr))
mc.connectAttr('{0}.outUV'.format(vNewUtility), '{0}.uv'.format(vNewTexture))
mc.connectAttr('{0}.outUvFilterSize'.format(vNewUtility), '{0}.uvFilterSize'.format(vNewTexture))
return vNewShadingGroup
#Get the map points for the poly
def getPolyMap(vPoly):
vPoints = []
if mc.objExists(vPoly):
vShape = mc.listRelatives('pPlane1', children = True, fullPath = True, shapes = True)[0]
vEngine = mc.listConnections(vShape, type = 'shadingEngine')
vMaterials = mc.ls(mc.listConnections(vEngine), materials = True)
vTextureFiles = mc.listConnections(vMaterials, type = 'file')
if vTextureFiles :
vMapPoints = mc.polyListComponentConversion('{0}.vtx[*]'.format(vPoly), fv = True, tuv = True)
vMapPoints = mc.ls(vMapPoints, flatten = True)
for vPoint in vMapPoints:
vCoord = mc.polyEditUV(vPoint, q = True)
print vPoint, vCoord
vPoints.append(vCoord)
return vPoints
#Get the color at those points
def getColorAtPointUV(vPoly, vUVPoint = [0, 0]):
vColor = ''
vShape = mc.listRelatives('pPlane1', children = True, fullPath = True, shapes = True)[0]
vEngine = mc.listConnections(vShape, type = 'shadingEngine')
vMaterials = mc.ls(mc.listConnections(vEngine), materials = True)
vTextureFiles = mc.listConnections(vMaterials, type = 'file')
if vTextureFiles and len(vUVPoint) == 2:
vColor = mc.colorAtPoint('{0}.outColor'.format(vTextureFiles ), o = 'RGB', u = vUVPoint[0], v = vUVPoint[1])
return vColor
#With those functions I execute:
vShape = mc.listRelatives('pPlane1', children = True, fullPath = True, shapes = True)[0]
vEngine = mc.listConnections(vShape, type = 'shadingEngine')
vMaterials = mc.ls(mc.listConnections(vEngine), materials = True)
vTextureFiles = mc.listConnections(vMaterials, type = 'file')
cv = getPolyMap('pPlane1')
for cc in cv:
print cc, mc.colorAtPoint(vTextureFiles, o = 'RGB', u = cc[0], v = cc[1])
The result I get from that is:
[0.0, 0.0] [0.00784313678741455, 0.9921568036079407, 0.00784313678741455]
[1.0, 0.0] [0.00784313678741455, 0.9921568036079407, 0.00784313678741455]
[0.0, 1.0] [0.00784313678741455, 0.9921568036079407, 0.00784313678741455]
[1.0, 1.0] [0.00784313678741455, 0.9921568036079407, 0.00784313678741455]
Every corner of the uv prints green, even if only a corner of it is of that color.
On the other hand, if I force it with u = .5, v = .5 the result is the correct one (default grey [0.5019607543945312, 0.5019607543945312, 0.5019607543945312])
Can't post images, but is a grey square with a green dot covering the upper right corner).

subplot with plotly with multiple traces

i am trying to create subplots for the following plots:
the code i used for the the plots are :
radius_mean1 = df[df['diagnosis']==1]['radius_mean']
radius_mean0 = df[df['diagnosis']==0]['radius_mean']
trace_rm1 = go.Histogram(x = radius_mean1, opacity = 0.75, name = 'malignant')
trace_rm2 = go.Histogram(x = radius_mean0, opacity = 0.75, name = 'benign')
data2 = [trace_rm1,trace_rm2]
layout2 = go.Layout(barmode = 'overlay', title = 'radius mean')
fig2 = go.Figure(data=data2, layout=layout2)
py.iplot(fig2)
and similar for the other plot
now i use the following code i found to create subplots:
fig.append_trace(fig1['data'][1], 1, 1)
fig.append_trace(fig2['data'][0], 2, 1)
py.iplot(fig)
and i get this :
how do i add both benign and malignant results to the subplots ?
i cant seem to edit the [0] or [ 1] to show both ie have the first 2 plots exactly into my subplot showing both malignant and benign and not just one or the other.

The data here is not complete, so I using some demo data to present.
import numpy as np
import plotly.graph_objs as go
import plotly
a = np.random.normal(0,1,100)
b = np.random.normal(-2,5,100)
c = np.random.normal(0,1,100)
d = np.random.normal(-2,5,100)
fig = plotly.tools.make_subplots(rows=2,cols=1)
trace_rm1 = go.Histogram(x = a, opacity = 0.75, name = 'malignant')
trace_rm2 = go.Histogram(x = b, opacity = 0.75, name = 'benign')
fig.append_trace(go.Histogram(x = a, opacity = 0.75, name = 'benign'),1,1)
fig.append_trace(go.Histogram(x = b, opacity = 0.75, name = 'malignant'),1,1)
fig.append_trace(go.Histogram(x = c, opacity = 0.75, name = 'benign'),2,1)
fig.append_trace(go.Histogram(x = d, opacity = 0.75, name = 'malignant'),2,1)
fig.layout.update(go.Layout(barmode = 'overlay',))
plotly.offline.plot(fig)

python uppercase and ascii in persian text

I change this code to work in persian. My code returns:
Key: Text is too short to analyze Text:
and doesn't return key and decrypt text. Is it anything wrong with uppercase or ascii in persian text? What can I do?
# -*- coding: utf-8 -*-
from string import uppercase
from operator import itemgetter
def vigenere_decrypt(target_freqs, input):
nchars = len(uppercase)
ordA = ord(u"ا")
sorted_targets = sorted(target_freqs)
def frequency(input):
result = [[c, 0.0] for c in uppercase]
for c in input:
result[c - ordA][1] += 1
return result
def correlation(input):
result = 0.0
freq = frequency(input)
freq.sort(key=itemgetter(1))
for i, f in enumerate(freq):
result += f[1] * sorted_targets[i]
return result
cleaned = [ord(c) for c in input.upper() if c.isupper()]
best_len = 0
best_corr = -100.0
# Assume that if there are less than 20 characters
# per column, the key's too long to guess
for i in xrange(2, len(cleaned) // 20):
pieces = [[] for _ in xrange(i)]
for j, c in enumerate(cleaned):
pieces[j % i].append(c)
# The correlation seems to increase for smaller
# pieces/longer keys, so weigh against them a little
corr = -0.5 * i + sum(correlation(p) for p in pieces)
if corr > best_corr:
best_len = i
best_corr = corr
if best_len == 0:
return ("Text is too short to analyze", "")
pieces = [[] for _ in xrange(best_len)]
for i, c in enumerate(cleaned):
pieces[i % best_len].append(c)
freqs = [frequency(p) for p in pieces]
key = ""
for fr in freqs:
fr.sort(key=itemgetter(1), reverse=True)
m = 0
max_corr = 0.0
for j in xrange(nchars):
corr = 0.0
c = ordA + j
for frc in fr:
d = (ord(frc[0]) - c + nchars) % nchars
corr += frc[1] * target_freqs[d]
if corr > max_corr:
m = j
max_corr = corr
key += chr(m + ordA)
r = (chr((c - ord(key[i % best_len]) + nchars) % nchars + ordA)
for i, c in enumerate(cleaned))
return (key, "".join(r))
def main():
encoded = " پهيتش غعهدد ذصلدي هزفضر کنهرظ ضذکاح يصتمد "
english_frequences = [
14, 4.2, 0.7, 5.2, 0.1, 1.2, 0.4,
1, 1.4, 7.5, 0.1, 8.5, 2.1, 0.1,
3.3, 2.6, 0.7, 0.3, 0.6, 0.2, 1.5,
0.2, 1.6, 1.2, 3, 1.7, 2.7, 5.7, 7.1, 6, 5.7, 9.1]
(key, decoded) = vigenere_decrypt(english_frequences, encoded)
print "Key:", key
print "\nText:", decoded
main()