I tried to design a workflow using composite pattern, the sample codes looks like this:
class CommandInterface(object):
def __init__(self, name, template=None, tool=None, param : dict={},input=None, output=None ):
self.name = name
self.input = input
self.output = output
self.param = param
self.template = template
def before_invoke(self):
pass # check before invoke
def invoke(self):
pass
def after_invoke(self):
pass # check after invoke
class ShellCommand(CommandInterface):
def render(self):
cmd = self.template.format(input=self.input,
output=self.output,
param=self.param,
)
return cmd
def before_invoke(self):
pass
def invoke(self):
os.system(self.render())
class Workflow(CommandInterface):
def __init__(self, name):
self.name = name
self._input = []
self._output = []
self._commands = []
self._tool = []
def add(self, command : CommandInterface):
self._commands.append(command)
return self
def invoke(self):
for command in self._commands:
command.invoke()
And can be used like this:
workflow = Workflow()
raw_files = ["file1", "file2", "file3"]
for i in raw_files:
head_command = ShellCommand("head {input} {output}",
input = i,
output = i+"_head")
workflow.add(head_command)
merge_command = ShellCommand("cat {input} > {output}",
input=" ".join([i+"_head" for i in rawfiles]),
output="merged")
workflow.add(merge_command)
workflow.invoke()
However, sometimes a command's input might be another command's result, for example:
class PythonCommand(CommandInterface):
def invoke(self):
self._result = self.template(input=self.input, output=self.output, param=self.param)
def a_command(input, output, param):
take_long_time(input, output, param)
return {"a_result": "some result will be used in b_command"}
def b_command(input, output, param):
def need_the_result_of_a_command(input, output, param):
return param["a_result"]
need_the_result_of_a_command(input, output, param)
return "success"
a = PythonCommand(a_command, input_, output_, param_)
a.invoke()
b = PythonCommand(b_command, input_, output_, param= a._result)
b.invoke()
I can't use a workflow to composite these 2 command because b uses a's result as parameter, which is only visiable after a is invoked. However, a workflow manager like the codes before is still necessary in some situations. I think I need something like this:
workflow = Workflow()
a = PythonCommand(a_command, input_, output_, param_)
workflow.add(a)
b = PythonCommand(b_command, input_, output_, param= {"a_result": a.lazy()._result})
workflow.add(b)
workflow.invoke()
Does anyone have ideas about how to design a workflow with .lazy() implementation like this?
Related
Suppose you have three functions:
def a(x, y, z=0, g=10):
pass
def b(*args):
pass
def c(p, q, u=False, *args, **kwargs):
a(...)
b(...)
pass
How do I hydrate c's function signature without copy pasting every argument and keyword argument, their defaults, etc from sub functions a, b, etc and automatically filter the appropriate ones to each function? I am assuming in this case there are no overloaded keyword arguments i.e. if function f(...) has a keyword argument w then function h(...) does not a keyword argument w unless w is used the same in f and h.
Note: I am looking for something like this:
def c(p, q, u=False, *args, **kwargs):
does_a_have_args, does_a_have_kwargs = get_func_sig_info(a)
a_args = get_fn_args(a) if does_a_have_args else []
a_kwargs = get_fn_kwargs(a) if does_a_have_kwargs else []
a_args = get_args_from_self(*args, a_args)
a_kwargs = get_kwargs_from_self(**kwargs, a_kwargs)
a(*a_args, **a_kwargs)
does_b_have_args, does_b_have_kwargs = get_func_sig_info(b)
b_args = get_fn_args(b) if does_b_have_args else []
b_kwargs = get_fn_kwargs(b) if does_b_have_kwargs else []
b_args = get_args_from_self(*args, b_args)
b_kwargs = get_kwargs_from_self(**kwargs, b_kwargs)
b(*b_args, **b_kwargs)
so that I might have a utility function:
def wrangle_function_signature(function, *args, **kwargs):
does_f_have_args, does_f_have_kwargs = get_func_sig_info(a)
f_args = get_fn_args(function) if does_f_have_args else []
f_kwargs = get_fn_kwargs(function) if does_f_have_kwargs else []
f_args = get_args_from_self(*args, f_args)
f_kwargs = get_kwargs_from_self(**kwargs, f_kwargs)
return (*f_args, **f_kwargs)
making the above pseudocode for c:
def c(p, q, u=False, *args, **kwargs):
a_args, a_kwargs = wrangle_function_signature(a, *args, **kwargs)
a(*a_args, **a_kwargs)
b_args, b_kwargs = wrangle_function_signature(b, *args, **kwargs)
b(*b_args, **b_kwargs)
pass
I need to compose information regarding the given information like what parameter the given function takes etc. The example what I would like to do is
#author("Joey")
#parameter("name", type=str)
#parameter("id", type=int)
#returns("Employee", desc="Returns employee with given details", type="Employee")
def get_employee(name, id):
//
// Some logic to return employee
//
Skeleton of decorator could be as follows:
json = {}
def author(author):
def wrapper(func):
def internal(*args, **kwargs):
json["author"] = name
func(args, kwargs)
return internal
return wrapepr
Similarly, parameter decorator could be written as follows:
def parameter(name, type=None):
def wrapper(func):
def internal(*args, **kwargs):
para = {}
para["name"] = name
para["type"] = type
json["parameters"].append = para
func(args, kwargs)
return internal
return wrapepr
Similarly, other handlers could be written. At the end, I can just call one function which would get all formed JSONs for each function.
End output could be
[
{fun_name, "get_employee", author: "Joey", parameters : [{para_name : Name, type: str}, ... ], returns: {type: Employee, desc: "..."}
{fun_name, "search_employee", author: "Bob", parameters : [{para_name : age, type: int}, ... ], returns: {type: Employee, desc: "..."}
...
}
]
I'm not sure how I can maintain the state and know to consolidate the data regarding one function should be handled together.
How can I achieve this?
I don't know if I fully get your use case, but wouldn't it work to add author to your current functions as:
func_list = []
def func(var):
return var
json = {}
json['author'] = 'JohanL'
json['func'] = func.func_name
func.json = json
func_list.append(func.json)
def func2(var):
return var
json = {}
json['author'] = 'Ganesh'
func2.json = json
func_list.append(func2.json)
This can be automated using a decorator as follows:
def author(author):
json = {}
def author_decorator(func):
json['func'] = func.func_name
json['author'] = author
func.json = json
return func
return author_decorator
def append(func_list):
def append_decorator(func):
func_list.append(func.json)
return func
return append_decorator
func_list = []
#append(func_list)
#author('JohanL')
def func(var):
return var
#append(func_list)
#author('Ganesh')
def func2(var):
return var
Then you can access the json dict as func.json and func2.json or find the functions in the func_list. Note that for the decorators to work, you have to add them in the order I have put them and I have not added any error handling.
Also, if you prefer the func_list to not be explicitly passed, but instead use a globaly defined list with an explicit name, the code can be somewhat simplified to:
func_list = []
def author(author):
json = {}
def author_decorator(func):
json['func'] = func.func_name
json['author'] = author
func.json = json
return func
return author_decorator
def append(func):
global func_list
func_list.append(func.json)
return func
#append
#author('JohanL')
def func(var):
return var
#append
#author('Ganesh')
def func2(var):
return var
Maybe this is sufficient for you?
I wanted to run code placed inside of thing.process upon when I hit the command.processCommand object (when i'm looping through all of the commands placed inside of defined[]), is there a way I can achieve this? This aforementioned loop will be executed as is such in myproject.py
command.py
class Command:
global defined
defined = []
def __init__(self, name):
self.name = name
self.description = "This command lacks a description"
self.args = ""
self.process = None
defined.append(self)
eightball.py
def processCommand():
print('hello')
thing = commands.Command('8ball')
thing.description = "Gives you a response from the mighty 8ball."
thing.process = processCommand
myproject.py
# Cogs
import cogs.commands as commands
import cogs.eightball
import cogs.helloworld
def processCommands(message):
if(message.content[:2] == "b#"):
args = message.content.split(' ')
args[0] = args[0][2:]
for command in defined:
if args[0] == command.name:
command.args = args
command.processCommand
for x in defined:
if x.process: # to skip `self.process = None`
x.process()
EDIT: you need process() instead of processCommand
for command in defined:
if args[0] == command.name:
command.args = args
command.process()
I'd like to use the python module timeit to time some functions in my QGIS plugin.
Here, I've called the time it function within a function that I call at the end of the last function. It seems, though, that the plugin is taking even longer to run than usual and I am wondering if i'm calling the timer in the wrong place. Is there a better way to set this up?
class myPluginName:
def firstFunction(self):
...
self.secondFunction()
def secondFunction(self):
...
self.timeThings()
def run(self):
self.firstFunction()
def timeThings(self):
QMessageBox.information(None, 'First Function', 'Time : %s' % timeit.timeit(self.firstFunction,number=1)
QMessageBox.information(None, 'Second Function', 'Time : %s' % timeit.timeit(self.secondFunction,number=1)
UPDATE: After following some advice, i've tried to implement the wrapper in the following way. I get however, a TypeError: firstFunction() takes exactly 1 argument (2 given) on ret = func(**args, **kwargs)
def time_func(func):
try:
name = func.__name__
except:
name = func.f__name
def tf_wrapper(*args, **kwargs):
t = time.time()
ret = func(*args, **kwargs)
QMessageLog.logMessage("{}: {}".format(name, time.time() - t))
return ret
return tf_wrapper
class myPlugin:
def initGui(self):
QObject.connect(self.dlg.ui.comboBox,SIGNAL("currentIndexChanged(int)"), self.firstFunction)
#time_func
def firstFunc(self):
registry = QgsMapLayerRegistry.instance()
firstID = str(self.dlg.ui.firstCombo.itemData(self.dlg.ui.firstCombo.currentIndex()))
secondID = str(self.dlg.ui.secondCombo.itemData(self.dlg.ui.secondCombo.currentIndex()))
self.firstLayer = registry.mapLayer(firstID)
self.secondLayer = registry.mapLayer(secondID)
#time_func
def secondFunc(self):
...
self.thirdFunc()
def thirdFunct(self):
...
def run(self):
self.dlg.ui.firstCombo.clear()
self.dlg.ui.secondCombo.clear()
for layer in self.iface.legendInterface().layers():
if layer.type() == QgsMapLayer.VectorLayer:
self.dlg.ui.firstCombo.addItem(layer.name(), layer.id())
self.dlg.ui.secondCombo.addItem(layer.name(), layer.id())
result = self.dlg.exec_()
if result == 1:
self.secondFunction()
OK, I don't know your exact situation, but I'd set it up though decorators:
import time
def time_func(func):
try:
name = func.__name__
except:
name = func.f_name
def tf_wrapper(*args, **kwargs):
t = time.time()
ret = func(*args, **kwargs)
print("{}: {}".format(name, time.time() - t)) # Or use QMessageBox
return ret
return tf_wrapper
class myPluginName:
#time_func
def firstFunction(self):
pass
#time_func
def secondFunction(self):
pass
def run(self):
self.firstFunction()
myPluginName().firstFunction()
With this code, any function wrapped in time_func will have the time taken to execute the function printed when it returns, along with its name. E.g. running it will print:
firstFunction: 1.430511474609375e-06
For your TypeError, you need to change;
def firstFunction(self):
pass
To:
def firstFunction(self, index):
pass
I am trying to build a Strategy Pattern using S.Lott's response.
Problem is function returns None.
Am using Hickey's Simple vs Easy {what, how, who}-logic.
-[WHAT] I/O
class base_fnc(object):
def fncExc(self,data1,data2):
return
-[HOW] DATA <> queue[where,when] (direct injection)
class stump( base_fnc ):
def fncExc(self, d1, aContext):
return d1
class MAB(base_fnc ):
def fncExc(self, d, aContext ):
return d+10
-[WHO] API
class context( object ):
def __init__(self, alt_how_class ):
self.how = alt_how_class
def exc(self, d ):
return self.how.fncExc(d, self)
if __name__ == "__main__":
c1 = context(MAB())
ss=c1.exc(10)
print ss
ss prints None
You're not returning in exc. You need to do return self.how.fncExc(d, self).