I'm trying to do something pretty simple with Parallel Python. I would like to be able to create an object from a class I've created inside another method from a class use to do a job in parallel.
Here is a basic example of what I would like to make it work :
import pp
class TestClass(object):
def __init__(self):
pass
def doSomething (self, number) :
print number**2
class PPTask (object) :
def __init__ (self) :
pass
def ppTask(self, number = 1) :
sum = 0
sum += number
tc = TestClass()
tc.doSomething(sum)
return sum
if __name__ == '__main__' :
job_server = pp.Server()
job_list = []
results = []
for i in xrange(10) :
pt = PPTask()
job_list.append(job_server.submit(pt.ppTask, (1,), globals = globals()))
for job in job_list :
results.append(job())
for result in results :
print result
This raise NameError: global name 'TestClass' is not defined and I didn't find any solution to pass it or reuse it in the ppTask method.
Any help would be greatly appriciated.
Thank you in advance
One solution would be to tell the job server to import the source module itself for each job you submit. For example, if your script above were called pptest.py, you could create the jobs like so:
job_list.append(job_server.submit(pt.ppTask, (1,), modules=('pptest',)))
And within ppTask, you could instantiate TestClass like so:
tc = pptest.TestClass()
So overall, the code would look like this:
import pp
class TestClass(object):
def __init__(self):
pass
def doSomething (self, number) :
print number**2
class PPTask (object) :
def __init__ (self) :
pass
def ppTask(self, number = 1) :
sum = 0
sum += number
tc = pptest.TestClass()
tc.doSomething(sum)
return sum
if __name__ == '__main__' :
job_server = pp.Server()
job_list = []
results = []
for i in xrange(10) :
pt = PPTask()
job_list.append(job_server.submit(pt.ppTask, (1,), modules=('pptest',)))
for job in job_list :
results.append(job())
for result in results :
print result
Related
I wanted to use decorator inside a class but I am getting positional error.
1.I dont want to use functools and import wraps.
2.Can we declare it within the class rather than defining decorator outside class?
Is there any other method to declare a decorator within a class?
from pytube import YouTube
if __name__ == "__main__":
class term :
# def __init__(self,u,v):
# self.url = u
# self.path = v
def check(func):
def parameters(self,u,v):
print(f"title:{YouTube(u).title}, views:{YouTube(u).views}, Length:{YouTube(u).length}")
func(u,v)
print("Yes done")
return parameters
# return check
#check
def video1(self,u,v):
# y = YouTube(u)
# print("Y is",y)
video1 = YouTube(u).streams.first().download(v)
print("Success")
return video1
u = input("Enter the video URL: ")
v = input("Enter the path: ")
t1 = term()
t1.video1(u,v)
print(t1)
If I initialise u,v inside class and call it via t1 and use it in methods like check it gives an error saying term has positional argument error.
If I initialise u,v inside instance t1 and call it via t1.video1 and use it in methods like check it gives an error saying video has no positional argument "v".
How to use decorators?Can you pls help me.
Here is just an example for you, (Modify as per your need) -
class term:
def check(func):
def parameters(self, u, v):
print(f"title:")
print("Yes done")
return func(self, u, v)
return parameters
#check
def video1(self, u, v):
print("Success")
return 'Hello'
Also, You are printing an object like
t1 = term()
t1.video1(u,v)
print(t1)
But, you should be printing like -
t1 = term()
result = t1.video1(u,v)
print(result)
I'm trying to use ray module to on an existing code based on if an env variable is true or not.
This is what I've done so far. this code structure is similar to mine but not exactly due to it's size.
import os
if os.getenv("PARALLEL"):
import ray
ray.init()
class A(object):
def __init__(self, attr):
self.attr = attr
def may_be_remote(func):
return ray.remote(func) if os.getenv("PARALLEL") else func
#may_be_remote
def do_work(self):
#work code
def execute(self, n):
for _ in range(n):
do_work.remote()
Then, I call the execute function of class A :
a = A()
a.execute(7)
I get AttributeError : 'function' has no attribute 'remote' on that line.
Where did I go wrong with this code please?
You are accessing remote() on the function do_work, which is not defined.
Did you mean to just call do_work()?
Unfortunately ray makes it hard to get transparent code to switch easily as you intend.
Following https://docs.ray.io/en/latest/ray-overview/index.html#parallelizing-python-classes-with-ray-actors the quite strange insert-.remote syntax is like...
import os
use_ray = os.getenv("PARALLEL") is not None
if use_ray:
import ray
ray.init()
def maybe_remote(cls):
return ray.remote(cls) if use_ray else cls
#maybe_remote
class A:
def __init__(self, attr):
self.attr = attr
def do_work(self, foo): # do something
self.attr += foo
def get_attr(self): # return value maybe from remote worker
return self.attr
if __name__ == '__main__':
n = 7
if use_ray:
a = A.remote(0)
for i in range(1, n + 1):
a.do_work.remote(i)
result = ray.get(a.get_attr.remote())
else:
a = A(0)
for i in range(1, n + 1):
a.do_work(i)
result = a.get_attr()
expect = int((n / 2) * (n + 1))
assert expect == result
Not sure there is also an easy (decorator) solution for the differences in the method calls.
I am getting an error about TypeError: 'staticmethod' object is not callable. Basically, you the input is a map of and given that you provide a pair of floats (pt,eta), the code should return the Y value of the bin that the particular values fall in.
I ve tried related thread (as possible duplicates) but does not seem to be getting the answer I am looking for.
Of course, if one has any recommendations how to even improve the code, that would be welcomed of course.
import ROOT as root
import sys,math
class SFs():
global etaBinsH
global get_EfficiencyData
global get_EfficiencyMC
global eff_dataH
global eff_mcH
global get_ScaleFactor
#staticmethod
def ScaleFactor(inputRootFile) :
#inputRootFile="Muon_IsoMu27.root"
eff_dataH = root.std.map("string", root.TGraphAsymmErrors)()
eff_mcH = root.std.map("string", root.TGraphAsymmErrors)()
#std::map<std::string, root.TGraphAsymmErrors *> eff_data
#std::map<std::string, root.TGraphAsymmErrors *> eff_mc
EtaBins=["Lt0p9", "0p9to1p2","1p2to2p1","Gt2p1"]
print inputRootFile
fileIn = root.TFile(inputRootFile,"read")
fileIn.ls()
HistoBaseName = "ZMassEta"
etaBinsH = fileIn.Get("etaBinsH")
#etaLabel, GraphName
nEtaBins = int(etaBinsH.GetNbinsX())
eff_data= []
eff_mc= []
#eff_mcH =root.TGraphAsymmErrors()
print "EtaBins...........",nEtaBins, len(EtaBins)
for iBin in range (0, nEtaBins) :
etaLabel = EtaBins[iBin]
GraphName = HistoBaseName+etaLabel+"_Data"
print GraphName,etaLabel
eff_data.append(fileIn.Get(str(GraphName)))
eff_dataH[etaLabel]=fileIn.Get(str(GraphName))
GraphName = HistoBaseName+etaLabel+"_MC"
eff_mc.append(fileIn.Get(str(GraphName)))
eff_mcH[etaLabel]=fileIn.Get(str(GraphName))
print eff_mcH[etaLabel].GetXaxis().GetNbins()
print eff_mcH[etaLabel].GetX()[5]
sff = get_ScaleFactor(46.8,2.0)
print "SFFFFFFFFFFFFFf",sff
#staticmethod
def get_ScaleFactor(pt, eta) :
efficiency_data = get_EfficiencyData(pt, eta)
efficiency_mc = get_EfficiencyMC(pt, eta)
if efficiency_mc != 0. :
SF = float(efficiency_data)/float(efficiency_mc)
else :
SF=1.
print "ScaleFactor::get_ScaleFactor(double pt, double eta) Scale Factor set to",SF,efficiency_data,efficiency_mc
return SF
#staticmethod
def get_EfficiencyMC(pt, eta) :
label = FindEtaLabel(eta,"mc")
#label= "Lt0p9"
binNumber = etaBinsH.GetXaxis().FindFixBin(eta)
label = etaBinsH.GetXaxis().GetBinLabel(binNumber)
ptbin = FindPtBin(eff_mcH, label, pt)
Eta = math.fabs(eta)
print "eff_mcH ==================",eff_mcH,binNumber,label,ptbin
#ptbin=10
if ptbin == -99 : eff =1
else : eff= eff_mcH[label].GetY()[ptbin-1]
if eff > 1. : eff = -1
if eff < 0 : eff = 0.
print "inside eff_mc",eff
return eff
#staticmethod
def get_EfficiencyData(pt, eta) :
label = FindEtaLabel(eta,"data")
#label= "Lt0p9"
binNumber = etaBinsH.GetXaxis().FindFixBin(eta)
label = etaBinsH.GetXaxis().GetBinLabel(binNumber)
print eff_dataH
ptbin = FindPtBin(eff_dataH, label, pt)
Eta = math.fabs(eta)
fileOut=root.TFile("out.root","recreate")
fileOut.cd()
eff_dataH[label].Write(label)
#ptbin=10
if ptbin == -99 : eff =1
else : eff= eff_dataH[label].GetY()[ptbin-1]
print "inside eff_data",eff
if eff > 1. : eff = -1
if eff < 0 : eff = 0.
print "inside eff_data",eff,pt,eta,label
return eff
#staticmethod
def FindPtBin( eff_map, EtaLabel, Pt) :
Npoints = eff_map[EtaLabel].GetN()
print Npoints, "for ===============>",eff_map[EtaLabel],eff_map[EtaLabel].GetN(),EtaLabel
#ptMAX=100
#ptMIN=90
ptMAX = (eff_map[EtaLabel].GetX()[Npoints-1])+(eff_map[EtaLabel].GetErrorXhigh(Npoints-1))
ptMIN = (eff_map[EtaLabel].GetX()[0])-(eff_map[EtaLabel].GetErrorXlow(0))
if Pt >= ptMAX : return Npoints
elif Pt < ptMIN :
return -99
else : return eff_map[EtaLabel].GetXaxis().FindFixBin(Pt)
#staticmethod
def FindEtaLabel(Eta, Which) :
Eta = math.fabs(Eta)
binNumber = etaBinsH.GetXaxis().FindFixBin(Eta)
EtaLabel = etaBinsH.GetXaxis().GetBinLabel(binNumber)
it=-1
if str(Which) == "data" :
it = eff_dataH.find(EtaLabel)
if str(Which) == "mc" :
it = eff_mcH.find(EtaLabel)
return EtaLabel
sf = SFs()
sff = sf.ScaleFactor("Muon_IsoMu27.root")
To piggyback a bit on #Felipe's answer, by not making all of your methods static, you can eliminate the need for the global declarations to share variables around, since that's what you are doing anyways:
class SFs():
def __init__(self):
# initialize your global vars instead as
# instance variables
self.etaBinsH = None
self.get_EfficiencyData = None
self.get_EfficiencyMC = None
self.eff_dataH = None
self.get_ScaleFactor = None
# don't make this static, then you have access to the self attributes and it makes
# your code a bit more explicit
def scale_factor(self, input_file):
self.eff_dataH = root.std.map("string", root.TGraphAsymmErrors)()
self.eff_mcH = root.std.map("string", root.TGraphAsymmErrors)()
EtaBins = ["Lt0p9", "0p9to1p2","1p2to2p1","Gt2p1"]
print(input_file) # print with parentheses makes this more portable between versions
fileIn = root.TFile(input_file, "read")
# Now you can use this through self, which is more pythonic
self.etaBinsH = fileIn.Get("etaBinsH")
nEtaBins = int(self.etaBinsH.GetNbinsX())
eff_data, eff_mc = [], []
# rest of code
Your variables can then be shared via self, and the functions can also be accessed via self, otherwise staticmethod keeps access of self out of the function, which is why you can't call any of the other functions.
Classes are namespaces, and self allows you to tie variables to the instance-level namespace. By using global, you are trying to push those variables back to the global namespace to share them around, when really, you already have access to a namespace to share those variables in!
As a simple example:
class A:
# here is the namespace for the *class* A
x = 0 # x is an attribute on the class A, it is accessible on the class and instance level
def __init__(self):
self.y = 4 # y is now explicitly tied to an instance of A, and can be shared between *instance* methods of A
def use_y(self):
# because this is non-static, I have access to instance level
# variables, this is how you share them!
print(self.y)
# I also have access to class-level attributes
print(self.x)
#staticmethod
def use_x():
# I don't have access to self.y, because staticmethod takes that away
try:
print(self.y)
except NameError:
print("Couldn't make this work")
print(A.x) # have to print this as a *class-level* attribute, because self isn't defined here
a = A()
a.use_y()
# 4
# 0
a.use_x()
# Couldn't make this work
# 0
Some examples that might be helpful to see what is going on.
Example 1
class RandomClass():
global global_function
#staticmethod
def random_function(input):
print(global_function("test"))
return "random_function({})".format(input)
#staticmethod
def global_function(input):
return "global_function({})".format(input)
rc = RandomClass()
print(rc.random_function("Input!"))
Outputs
Traceback (most recent call last):
File "test.py", line 14, in <module>
print(rc.random_function("Input!"))
File "test.py", line 6, in random_function
print(global_function("test"))
TypeError: 'staticmethod' object is not callable
Example 2
class RandomClass():
#staticmethod
def random_function(input):
print(global_function("test"))
return "random_function({})".format(input)
#staticmethod
def global_function(input):
return "global_function({})".format(input)
rc = RandomClass()
print(rc.random_function("Input!"))
Output
Traceback (most recent call last):
File "test.py", line 12, in <module>
print(rc.random_function("Input!"))
File "test.py", line 4, in random_function
print(global_function("test"))
NameError: global name 'global_function' is not defined
Example 3
class RandomClass():
#staticmethod
def random_function(input):
print(RandomClass.global_function("test")) # Notice change here.
return "random_function({})".format(input)
#staticmethod
def global_function(input):
return "global_function({})".format(input)
rc = RandomClass()
print(rc.random_function("Input!"))
Output
global_function(test)
random_function(Input!)
Explanation
In short, a #staticmethod cannot access functions within its this class (whether defined with this or global), and instead must initialize a new and independent class to call a function within the class it resides in (example 3). As #C.Nivs mentioned, you should perhaps look into simply not using a class.
I have a number of classes where there are functions inside are almost the same.
Say function x:
class A():
def x_A (self):
...
...do the same thing
...
run a function that is unique in class A itself, say u_A
...
...do the same thing
...
class B():
def x_B (self):
...
...do the same thing
..
run a function that is unique in class B itself, say u_B
...
...do the same thing
...
So I came up with an idea to re-write function x in a new class(say x_C in class C) to replace x_A and x_B. And I just have to import that new class when I need it. something like:
import C
class A():
def x_A (self):
C.x_C(u_A)
class B():
def x_B (self):
C.x_C(u_A)
but I am confused of how to pass in the unique function (u_A and u_B) as a variable and make python to run it properly.
class C():
def x_C (self,unique_function):
...
...do the same thing
..
run unique_function here
...
...do the same thing
...
Thx in advance
blow is newly edited:
hi trying to specify my question:
I have a number of crawlers, at the end of each I got "run_before_insert" to check if they can run properly.
Currently I just copy and paste this function at end of every finished crawler with some edits.
But now I would like to simplify my code by importing "run_before_insert" from other files, and then comes my questions.
def run_before_insert(self):
try:
#store_list = []
comp_name = 'HangTen'
start = time.time()
print('{} runBeforeInsert START'.format(comp_name), '\n')
###Here is the part where small edits in the function:
store_list = self.get_stores_2()
###the rest is the same
script_info = {}
running_time = round(time.time() - start,2)
total = str(len(store_list))
script_info['running_time'] = running_time
script_info['total_stores'] = total
print('\n{} total stores : {}'.format(comp_name,script_info['total_stores']), '\n')
print('{} running time : {}'.format(comp_name,script_info['running_time']), '\n')
print('{} runBeforeInsert Done'.format(comp_name), '\n')
print('\n')
return script_info
except Exception as e:
traceback.print_exc()
script_info = {}
script_info['running_time'] = '--'
script_info['total_stores'] = 'error'
return script_info
print(e)
Here is my code with reference to #juanpa.arrivillaga:
class run_pkg_class():
def __init__(self):
pass
def run_before_insert(self, store_function, company_name):
try:
comp_name = company_name
start = time.time()
print('{} runBeforeInsert START'.format(comp_name), '\n')
###
store_list = store_function()
###
script_info = {}
running_time = round(time.time() - start,2)
total = str(len(store_list))
script_info['running_time'] = running_time
script_info['total_stores'] = total
print('\n{} total stores : {}'.format(comp_name,script_info['total_stores']), '\n')
print('{} running time : {}'.format(comp_name,script_info['running_time']), '\n')
print('{} runBeforeInsert Done'.format(comp_name), '\n')
print('\n')
return script_info
except Exception as e:
traceback.print_exc()
script_info = {}
script_info['running_time'] = '--'
script_info['total_stores'] = 'error'
return script_info
print(e)
and import above into hangten crawler class:
def run_before_insert2(self):
rp = run_pkg_class()
rp.run_before_insert(self.get_id())
In this hangTen case, self.get_stores_2() will return a list.
"TypeError: 'list' object is not callable" occur while running.
Not sure for the reason
Python functions are first-class objects. They are like any other attribute. Just pass them directly:
import C
class A:
def x_A (self):
C.x_C(self.u_A)
class B:
def x_B (self):
C.x_C(self.u_B)
And in C, you just call it like so:
unique_function()
Given that C apparently doesn't care about the state in A and B though, I suspect these things shouldn't be classes to begin with.
If I understand correctly, you don't even need to import a module every time. Instead, you can create a basic class from which other classes will inherit the function. For example, classes B and C inherit function "power" from class A.
class A:
""" Example of class A """
def __init__(self):
self.num1 = num1
def power(self, num):
return num**3
class B (A):
""" Example of class B"""
def __init__(self, num1, num2):
super().__init__(num1)
self.num2 = num2
self.power_num1 = self.power(num1)
class C(A):
""" Example of class C"""
def __init__(self, num1, num2, num3):
super().__init__(num1)
self.num2 = num2
self.num3 = num3
def power_total(self):
print(self.power(self.num1) + self.power(self.num2)
+ self.power(self.num3))
Examples of use:
>>> c = C(1, 2, 3)
>>> c.power_total()
36
>>> b = B(2, 4)
>>> b.power_num1
8
TypeError: _slow_trap_ramp() takes 1 positional argument but 2 were given
def demag_chip(self):
coil_probe_constant = float(514.5)
field_sweep = [50 * i * (-1)**(i + 1) for i in range(20, 0, -1)] #print as list
for j in field_sweep:
ramp = self._slow_trap_ramp(j)
def _set_trap_ramp(self):
set_trap_ramp = InstrumentsClass.KeysightB2962A.set_trap_ramp
return set_trap_ramp
def _slow_trap_ramp(self):
slow_trap_ramp = ExperimentsSubClasses.FraunhoferAveraging.slow_trap_ramp
return slow_trap_ramp
The error is straightforward.
ramp = self._slow_trap_ramp(j)
You are calling this method with an argument j, but the method doesn't take an argument (other than self, which is used to pass the object).
Re-define your method to accept an argument if you want to pass it one:
def _slow_trap_ramp(self, j):
It looks like your code extract contains methods of some class, whose full definition is not shown, and you are calling one method from another method (self._slow_trap_ramp(j)). When you call a method, Python automatically passes self before any other arguments. So you need to change def _slow_trap_ramp(self) to def _slow_trap_ramp(self, j).
Update in response to comment
To really help, we would need to see more of the class you are writing, and also some info on the other objects you are calling. But I am going to go out on a limb and guess that your code looks something like this:
InstrumentsClass.py
class KeysightB2962A
def __init__(self):
...
def set_trap_ramp(self):
...
ExperimentsSubClasses.py
class FraunhoferAveraging
def __init__(self):
...
def slow_trap_ramp(self, j):
...
Current version of main.py
import InstrumentsClass, ExperimentsSubClasses
class MyClass
def __init__(self)
...
def demag_chip(self):
coil_probe_constant = float(514.5)
field_sweep = [50 * i * (-1)**(i + 1) for i in range(20, 0, -1)] #print as list
for j in field_sweep:
ramp = self._slow_trap_ramp(j)
def _set_trap_ramp(self):
set_trap_ramp = InstrumentsClass.KeysightB2962A.set_trap_ramp
return set_trap_ramp
def _slow_trap_ramp(self):
slow_trap_ramp = ExperimentsSubClasses.FraunhoferAveraging.slow_trap_ramp
return slow_trap_ramp
if __name__ == "__main__":
my_obj = MyClass()
my_obj.demag_chip()
If this is the case, then these are the main problems:
Python passes self and j to MyClass._slow_trap_ramp, but you've only defined it to accept self (noted above),
you are using class methods from KeysightB2962A and FraunhoferAveraging directly instead of instantiating the class and using the instance's methods, and
you are returning references to the methods instead of calling the methods.
You can fix all of these by changing the code to look like this (see embedded comments):
New version of main.py
import InstrumentsClass, ExperimentsSubClasses
class MyClass
def __init__(self)
# create instances of the relevant classes (note parentheses at end)
self.keysight = InstrumentsClass.KeysightB2962A()
self.fraun_averaging = ExperimentsSubClasses.FraunhoferAveraging()
def demag_chip(self):
coil_probe_constant = float(514.5)
field_sweep = [50 * i * (-1)**(i + 1) for i in range(20, 0, -1)] #print as list
for j in field_sweep:
ramp = self._slow_trap_ramp(j)
def _set_trap_ramp(self):
# call instance method (note parentheses at end)
return self.keysight.set_trap_ramp()
def _slow_trap_ramp(self, j): # accept both self and j
# call instance method (note parentheses at end)
return self.fraun_averaging.slow_trap_ramp(j)
if __name__ == "__main__":
my_obj = MyClass()
my_obj.demag_chip()