I'm working on a scientific project where I have a method that takes much time to terminates and which is call more than 20 times. That method could be easily parallelized too. The problem is that the parallelized code is taking much more time than the not parallelized one (commented in the code).
Here is a piece of my code just to show how I am doing such thing:
import copy_reg
import types
from itertools import product
import multiprocessing as mp
def _pickle_method(method):
"""
Author: Steven Bethard (author of argparse)
http://bytes.com/topic/python/answers/552476-why-cant-you-pickle-instancemethods
"""
func_name = method.im_func.__name__
obj = method.im_self
cls = method.im_class
cls_name = ''
if func_name.startswith('__') and not func_name.endswith('__'):
cls_name = cls.__name__.lstrip('_')
if cls_name:
func_name = '_' + cls_name + func_name
return _unpickle_method, (func_name, obj, cls)
def _unpickle_method(func_name, obj, cls):
"""
Author: Steven Bethard
http://bytes.com/topic/python/answers/552476-why-cant-you-pickle-instancemethods
"""
for cls in cls.mro():
try:
func = cls.__dict__[func_name]
except KeyError:
pass
else:
break
return func.__get__(obj, cls)
copy_reg.pickle(types.MethodType, _pickle_method, _unpickle_method)
class ImageData(object):
def __init__(self, width=60, height=60):
self.width = width
self.height = height
self.data = []
for i in range(width):
self.data.append([0] * height)
def parallel_orientation_uncertainty_calculus(x, y, mean_gradient, mean_gradient_direction, gradient_covariance,
gradient_correlation, bins):
v = mean_gradient_direction.data[x][y]
theta_sigma = Uts.Utils.translate_to_polar_coordinates(v[0].item(0), v[1].item(0))
sigma_theta = 0.0
for i in range(bins):
n1 = mt.pow(-mt.pi / 2 + mt.pi * i / bins, 2)
n2 = VariabilityOfGradients.calculate_gradient_orientation_probability_density_function(
mean_gradient, gradient_covariance, gradient_correlation, x, y,
(theta_sigma - mt.pi / 2 + mt.pi * i / bins))
sigma_theta += n1 * n2
return [x, y, sigma_theta]
class VariabilityOfGradients(object):
parallel_orientation_uncertainty_calculus = staticmethod(parallel_orientation_uncertainty_calculus)
#staticmethod
def calculate_orientation_uncertainty(mean_gradient, mean_gradient_direction, gradient_covariance, gradient_correlation, bins):
output = ImD.ImageData(range_min=0, range_max=1)
results = []
pool = Pool()
for x, y in product(range(1, output.width - 1), range(1, output.height - 1)):
print "Iteration ", x, y
result = pool.apply_async(VariabilityOfGradients.parallel_orientation_uncertainty_calculus,
args=[x, y, mean_gradient, mean_gradient_direction, gradient_covariance,
gradient_correlation, bins])
results.append(result.get())
pool.close()
pool.join()
for i, result in enumerate(results):
result = results[i]
print result
output.data[result[0], result[1]] = result[2]
# for x, y in product(range(1, output.width - 1), range(1, output.height - 1)):
# print "Iteration ", x, y
# v = mean_gradient_direction.data[x][y]
# theta_sigma = Uts.Utils.translate_to_polar_coordinates(v[0].item(0), v[1].item(0))
# sigma_theta = 0.0
# for i in range(bins):
# n1 = mt.pow(-mt.pi / 2 + mt.pi * i / bins, 2)
# n2 = VariabilityOfGradients.calculate_gradient_orientation_probability_density_function(
mean_gradient, gradient_covariance, gradient_correlation, x, y,
(theta_sigma - mt.pi / 2 + mt.pi * i / bins))
# sigma_theta += n1 * n2
# output.data[x][y] = sigma_theta
return output
if __name__ == '__main__':
VariabilityOfGradients.calculate_orientation_uncertainty()
I'm wondering what I'm doing wrong. Am I using multiprocessing wrong?
Thank you in advance.
Related
I have defined the following classes in Python:
class DynamicalModel(abc.ABC):
"""Simulation of a dynamical model to be applied in the iLQR solution."""
_id = 0
def __init__(self, n_x, n_u, dt, id=None):
if not id:
id = DynamicalModel._id
DynamicalModel._id += 1
self.n_x = n_x
self.n_u = n_u
self.dt = dt
self.id = id
self.NX_EYE = np.eye(self.n_x, dtype=np.float32)
def __call__(self, x, u):
"""Zero-order hold to integrate continuous dynamics f"""
return x + self.f(x, u) * self.dt
# Single RK4 integration of continuous dynamics.
# k1 = self.dt * self.f(x, u)
# k2 = self.dt * self.f(x + 0.5 * k1, u)
# k3 = self.dt * self.f(x + 0.5 * k2, u)
# k4 = self.dt * self.f(x + k3, u)
# x += (k1 + 2.0 * k2 + 2.0 * k3 + k4) / 6.0
# return x
#staticmethod
#abc.abstractmethod
def f():
"""Continuous derivative of dynamics with respect to time"""
pass
def linearize(self, x: torch.tensor, u: torch.tensor, discrete=False):
"""Compute the Jacobian linearization of the dynamics for a particular state
and controls for all players.
"""
A, B = torch.autograd.functional.jacobian(self.f, (x, u))
if discrete:
return A, B
# Compute the discretized jacobians with euler integration.
A = self.dt * A.reshape(self.n_x, self.n_x) + self.NX_EYE
B = self.dt * B.reshape(self.n_x, self.n_u)
return A, B
#classmethod
def _reset_ids(cls):
cls._id = 0
def __repr__(self):
return f"{type(self).__name__}(n_x: {self.n_x}, n_u: {self.n_u}, id: {self.id})"
class AnalyticalModel(DynamicalModel):
"""Mix-in for analytical linearization"""
def __init__(self, A_num, B_num):
self.A_num = A_num
self.B_num = B_num
def f(self, x, u):
return self._f(x, u)
def linearize(self, x, u):
return self.A_num(x, u), self.B_num(x, u)
class UnicycleDynamics4dSymbolic(AnalyticalModel):
def __init__(self, dt, *args, **kwargs):
super().__init__(4, 2, dt, *args, **kwargs)
p_x, p_y, v, theta, omega, a = sym.symbols('p_x p_y v theta omega a')
x = sym.Matrix([p_x, p_y, v, theta])
u = sym.Matrix([a, omega])
x_dot = sym.Matrix([
x[2]*sym.cos(x[3]),
x[2]*sym.sin(x[3]),
u[0],
u[1],
])
A = x_dot.jacobian(x)
B = x_dot.jacobian(u)
self._f = sym.lambdify((x, u), sym.Array(x_dot)[:,0])
self.A_num = sym.lambdify((x, u), A)
self.B_num = sym.lambdify((x, u), B)
when I tried to call the UnicycleDynamics class :
dec.UnicycleDynamics4dSymbolic(dt)
The following error popped up:
175 class UnicycleDynamics4dSymbolic(AnalyticalModel):
176 def __init__(self, dt, *args, **kwargs):
--> 177 super().__init__(4, 2, dt, *args, **kwargs)
178
179 p_x, p_y, v, theta, omega, a = sym.symbols('p_x p_y v theta omega a')
TypeError: __init__() takes 3 positional arguments but 4 were given
I don't understand how exactly I have passed 4 arguments? I intend to create a child class from AnalyticalModel, but AnalyticalModel is a child class of DynamicalModel. I am a little confused about how to fix this error.
I have this code consisting of a class and a subclass. The class is Euler forward, while the second one is Eulers midpoint method. These are for solving an ODE (x'=x(1/2-x)). Now it doesn't seem to work because when I am to call the function, by typing:
Euler=H.solve(6)
where the 6 is the amount of steps, I get attributeerror.
AttributeError: 'int' object has no attribute 'size'
Could anyone help me make my code more robust and working so I could plot the values later on, really don't see whats wrong. My code below:
import numpy as np
class H:
def __init__(self, f):
self._f = f
def initial(self, u0):
self._u0 = u0
def solve(self, time_points):
n = time_points.size
self._t = time_points
self._u = np.zeros(n)
self._u[0] = self._u0
for k in range(n-1):
self._k = k
self._u[k+1] = self.advance()
return self._u, self._t
class F(H):
def ad(self):
u = self._u; t = self._t; f = self._f; k = self._k
dt = t[k+1] - t[k]
u_k12 = u[k] + dt/2 * f(u[k], t[k])
return u[k] + dt * f(u_k12, (t[k] + dt/2) )
I think what's wrong is the way you use the class. Initial value is set with initial method (u0), then you give solve method the list of points. You can use np.linscape to generate midpoint.
np.linspace(0, 3, 31) # 30 points evenly spaced between 0 and 3
So it's like this:
def func(x, y):
return x * y
midpoint = np.linspace(0, 3, 31)
F_ = F(func)
F_.initial(6)
F_.solve(midpoint)
Code:
class H:
def __init__(self, f):
self._f = f
def initial(self, u0):
self._u0 = u0
def solve(self, time_points):
n = time_points.size
self._t = time_points
self._u = np.zeros(n)
self._u[0] = self._u0
for k in range(n-1):
self._u[k+1] = self.advance(k)
return self._u, self._t
def advance(self, k):
....
class F(H):
def advance(self, k):
dt = self._t[k+1] + self._t[k]
u_k12 = self._u[k] + dt/2 * self._f(self._u[k], self._t[k])
return self._u[k] + dt * self._f(u_k12, (self._t[k] + dt/2))
I need to change the code in line # 39
I have approximately seen scripts that use with open ("file.txt", "r") as f: take data from a text document.
I have a list of "Point.txt"
g = Point(250,127)
g = Point(330,224)
g = Point(557,186)
g = Point(370,197)
g = Point(222,107)
Need to add a function so that the script takes data from the list of the document "Point.txt"
and the whole result was saved in one document "Save.txt"
class Point(object):
def __init__(self, _x, _y, _order = None): self.x, self.y, self.order = _x, _y, _order
def calc(self, top, bottom, other_x):
l = (top * inverse_mod(bottom)) % p
x3 = (l * l - self.x - other_x) % p
return Point(x3, (l * (self.x - x3) - self.y) % p)
def double(self):
if self == INFINITY: return INFINITY
return self.calc(3 * self.x * self.x, 2 * self.y, self.x)
def __add__(self, other):
if other == INFINITY: return self
if self == INFINITY: return other
if self.x == other.x:
if (self.y + other.y) % p == 0: return INFINITY
return self.double()
return self.calc(other.y - self.y, other.x - self.x, other.x)
def __mul__(self, e):
if self.order: e %= self.order
if e == 0 or self == INFINITY: return INFINITY
result, q = INFINITY, self
while e:
if e&1: result += q
e, q = e >> 1, q.double()
return result
def __str__(self):
if self == INFINITY: return "infinity"
return " %x %x" % (self.x, self.y)
def inverse_mod(a):
if a < 0 or a >= p: a = a % p
c, d, uc, vc, ud, vd = a, p, 1, 0, 0, 1
while c:
q, c, d = divmod(d, c) + (c,)
uc, vc, ud, vd = ud - q*uc, vd - q*vc, uc, vc
if ud > 0: return ud
return ud + p
p, INFINITY = 1693, Point(None, None)
g = Point(250,127)
wave = 78
result = ' ID: %x\n getID: %s' % (wave, g*wave)
f = open('Save.txt', 'a')
f.write(result)
f.close()
I have used regex to extract the parameters that you have to pass for Point creation :
import re
f = open('Save.txt', 'a')
with open('Point.txt', 'rb') as points_txt_file:
for line in points_txt_file:
found_points = re.search(r'Point\((\s*\d+\s*),(\s*\d+\s*)\)', f'{line}')
print(found_points.groups())
param1 = int(found_points.group(1))
param2 = int(found_points.group(2))
g = Point(param1, param2)
result = ' ID: %x\n getID: %s' % (wave, g*wave)
f.write(result)
f.close()
Remove your code from line #38 and use this code .
This is my Linear System code. The times_scalar function is defined in the Vector object. When executing the multiply_coefficient_and_row function, I get AttributeError: 'tuple' object has no attribute 'times_scalar'. I tried different things including converting the tuple into a list and back to a tuple but had no success.
from decimal import Decimal, getcontext
from copy import deepcopy
from Vector import Vector
from Plane import Plane
getcontext().prec = 30
class LinearSystem(object):
ALL_PLANES_MUST_BE_IN_SAME_DIM_MSG = 'All planes in the system should live in the same dimension'
NO_SOLUTIONS_MSG = 'No solutions'
INF_SOLUTIONS_MSG = 'Infinitely many solutions'
def __init__(self, planes):
try:
d = planes[0].dimension
for p in planes:
assert p.dimension == d
self.planes = planes
self.dimension = d
except AssertionError:
raise Exception(self.ALL_PLANES_MUST_BE_IN_SAME_DIM_MSG)
def swap_rows(self, row1, row2):
self[row1], self[row2] = self[row2], self[row1]
def multiply_coefficient_and_row(self, coefficient, row):
n = self[row].normal_vector.coordinates
k = self[row].constant_term
new_normal_vector = n.times_scalar(coefficient)
new_constant_term = k * coefficient
self[row] = Plane(normal_vector = new_normal_vector, constant_term = new_constant_term)
def add_multiple_times_row_to_row(self, coefficient, row_to_add, row_to_be_added_to):
n1 = self[row_to_add].normal_vector.coordinates
n2 = self[row_to_be_added_to].normal_vector.coordinates
k1 = self[row_to_add].constant_term
k2 = self[row_to_be_added_to].constant_term
new_normal_vector = n1.times_scalar(coefficient).plus(n2)
new_constant_term = (k1 * coefficient) + k2
self[row_to_be_added_to] = Plane(normal_vector = new_normal_vector, constant_term = new_constant_term)
def indices_of_first_nonzero_terms_in_each_row(self):
num_equations = len(self)
num_variables = self.dimension
indices = [-1] * num_equations
for i,p in enumerate(self.planes):
try:
indices[i] = p.first_nonzero_index(p.normal_vector.coordinates)
except Exception as e:
if str(e) == Plane.NO_NONZERO_ELTS_FOUND_MSG:
continue
else:
raise e
return indices
def __len__(self):
return len(self.planes)
def __getitem__(self, i):
return self.planes[i]
def __setitem__(self, i, x):
try:
assert x.dimension == self.dimension
self.planes[i] = x
except AssertionError:
raise Exception(self.ALL_PLANES_MUST_BE_IN_SAME_DIM_MSG)
def __str__(self):
ret = 'Linear System:\n'
temp = ['Equation {}: {}'.format(i+1,p) for i,p in enumerate(self.planes)]
ret += '\n'.join(temp)
return ret
class MyDecimal(Decimal):
def is_near_zero(self, eps=1e-10):
return abs(self) < eps
and from the Vector class this times_scalar function:
def times_scalar(self, c):
new_coordinates = [Decimal(c) * x for x in self.coordinates]
return Vector(new_coordinates)
I have two classes, namely PositionsD and makemock which are defined as following:
import numpy as np
cdef class PositionsD(object):
property x:
def __get__(self):
return np.array(self._x)
def __set__(self, x):
self._x = x
property y:
def __get__(self):
return np.array(self._y)
def __set__(self, y):
self._y = y
def __init__(self, positions):
self._x = positions[:,0]
self._y = positions[:,1]
class makemock(object):
def __init__(self):
self.scale = 0.238
self.arcsec2rad = np.pi/180./60./60.
self.g1 = None
self.g2 = None
self.source_pos = None
self.z = None
self.h_pos = None
self.h_z = None
def get_pos(self):
return PositionsD(self.source_pos)
pos = property(get_shear_pos)
def get_center(self):
return PositionsD(self.h_pos)
center = property(get_center)
def get_dist(self):
dx_mpc = (self.pos.x-self.center.x)*self.arcsec2rad*self.scale
dy_mpc = (self.pos.y-self.center.y)*self.arcsec2rad*self.scale
return np.vectorize(complex)(dx_mpc, dy_mpc)
dist = property(get_dist)
def get_r(self):
return abs(self.dist)
r = property(get_r)
def get_norm(self):
return -self.dist/np.conjugate(self.dist)
norm = property(get_norm)
def get_gabs(self):
return np.sqrt(self.g1**2 + self.g2**2 )
gabs = property(get_gabs)
def get_g(self):
phiell=np.arctan2(self.g2, self.g1) /2.
phipos=np.arctan2( (self.pos.y-self.center.y), (self.pos.x-self.center.x) )
et = -self.gabs * np.cos( 2*(phiell-phipos) )
ec = -self.gabs * np.sin( 2*(phiell-phipos) )
return np.vectorize(complex)(et, ec)
obs_g = property(get_g)
def data2model(self,params):
rs = params
x = self.r/rs
P = len(self.r)
gamma = np.zeros((P,), dtype=np.float64, order='C')
kappa = np.zeros((P,), dtype=np.float64, order='C')
farcth = np.zeros((P,), dtype=np.float64, order='C')
m1 = np.where(x < 1.0)[0]
kappa[m1] = 2/(x[m1]**2 - 1) * \
(1 - np.log((1 + ((1 - x[m1])/(x[m1] + 1))**0.5)/(1 - ((1 - x[m1])/(x[m1] + 1))**0.5))/(1 - x[m1]**2)**0.5)
farcth[m1]=0.5*np.log((1.+((1.-x[m1])/(x[m1]+1.))**0.5)/(1.-((1.-x[m1])/(x[m1]+1.))**0.5))/(1-x[m1]**2)**0.5
gamma[m1] = 4*(np.log(x[m1]/2) + 2*farcth[m1]) * x[m1]**(-2) - kappa[m1]
model_g = self.norm* gamma /(1. - kappa )
e = (self.obs_g+model_g)/(1+np.conjugate(model_g)*self.obs_g)
mask=(abs(model_g)>1.)
if (np.sum(mask)>0):
e[mask]=1./np.conjugate(e[mask])
return e
My Question is:
I need to run the data2model method from makemock in a loop for different values of params. I figured the process is quite slow while it seems each time in the loop properties of a class such as r, dist, obs_g get computed in each iteration. Is there any way to set them once, in order to increase the speed of loop?
After fiddling around to find a way to initialize some instances at the begining and avoid calculating them in each iteration of each function, I found out it is the best to use a dict and leave the value of each instance in this dictionary and update them at the end of initializing the instances. That is my solution:
class makemock(object):
def __init__(self, g1, g2, source_pos, z, h_pos, h_z, **kw):
self.scale = 0.238
self.arcsec2rad = np.pi/180./60./60.
self.g1 = g1
self.g2 = g2
self.source_pos = source_pos
self.z = z
self.h_pos = h_pos
self.h_z = h_z
pos= PositionsD(self.source_pos)
center=PositionsD(self.h_pos)
dx_mpc = (pos.x-center.x)*self.arcsec2rad*self.scale
dy_mpc = (pos.y-center.y)*self.arcsec2rad*self.scale
dist= np.vectorize(complex)(dx_mpc, dy_mpc)
r= abs(dist)
norm= -dist/np.conjugate(dist)
gabs= np.sqrt(self.g1**2 + self.g2**2 )
phiell=np.arctan2(self.g2, self.g1) /2.
phipos=np.arctan2( (pos.y-center.y), (pos.x-center.x) )
et = -gabs * np.cos( 2*(phiell-phipos) )
ec = -gabs * np.sin( 2*(phiell-phipos) )
obs_g=np.vectorize(complex)(et, ec)
self.__dict__.update(kw)
del kw
self.__dict__.update(locals())
del self.self
def dump(self):
print repr(self.__dict__)
def data2model(self,params):
rs = params
x = self.r/rs
P = len(self.r)
gamma = np.zeros((P,), dtype=np.float64, order='C')
kappa = np.zeros((P,), dtype=np.float64, order='C')
farcth = np.zeros((P,), dtype=np.float64, order='C')
m1 = np.where(x < 1.0)[0]
kappa[m1] = 2/(x[m1]**2 - 1) * \
(1 - np.log((1 + ((1 - x[m1])/(x[m1] + 1))**0.5)/(1 - ((1 - x[m1])/(x[m1] + 1))**0.5))/(1 - x[m1]**2)**0.5)
farcth[m1]=0.5*np.log((1.+((1.-x[m1])/(x[m1]+1.))**0.5)/(1.-((1.-x[m1])/(x[m1]+1.))**0.5))/(1-x[m1]**2)**0.5
gamma[m1] = 4*(np.log(x[m1]/2) + 2*farcth[m1]) * x[m1]**(-2) - kappa[m1]
model_g = self.norm* gamma /(1. - kappa )
e = (self.obs_g+model_g)/(1+np.conjugate(model_g)*self.obs_g)
mask=(abs(model_g)>1.)
if (np.sum(mask)>0):
e[mask]=1./np.conjugate(e[mask])
return e