I am working on some project, and we have lots of some code usage like this;
# filtering fields are different from each other, please ignore the similarity below
def function1(self, param):
list_x = Model1.objects.filter(foo=bar, bla=bla).values_list('field', flat=True)
list_y = Model2.objects.filter(foo=bar, bla=bla).values_list('field', flat=True)
lists_to_delete = set(list_x) - set(list_y)
# here is the code line with set() that needed to be method
self._delete(lists_to_delete)
def function2(self, param):
list_z = Model3.objects.filter(foo=bar, bla=bla).values_list('field', flat=True)
list_q = Model4.objects.filter(foo=bar, bla=bla).values_list('field', flat=True).distinct()
list_w = Model5.objects.filter(foo=bar, bla=bla).values_list('field', flat=True)
lists_to_delete = set(list_x) - set(list_y) - set(list_w)
# here is the code line with set() that needed to be method
self._delete(lists_to_delete)
... # other functions continues like above
...
...
So, as you can see we have same usage with set() function. And I need to change this usage with custom method. I tried to write a method like this;
def _get_deleted_lists(self, *args):
value = set()
for arg in args:
value |= set(arg)
return value
and usage will be change like;
lists_to_delete = self._get_deleted_lists(list_x, list_y, ...)
instead of this;
lists_to_delete = set(list_x) - set(list_y)
But my custom method not return same value as before. How can I achieve this?
| operation on sets returns their union. What you want is the difference (-)
def _get_deleted_lists(*lists):
if not lists:
return set()
result = set(lists[0])
for l in lists[1:]:
result -= set(l)
return result
Related
I am trying to create python classes from a program that I wrote in functions only. So the code is working fine with functions but giving an error/errors when I convert the code to python classes.
Functions:
def eachList(index):
Series=df_ideal.iteritems()
bigList = []
for (columnName,item) in Series:
bigList.append(item)
return bigList[index]
def isResult(listing):
mylist=[]
for i in range(len([column_name for column_name in df_ideal])):
result = map(lambda x:x*x,residual(listing,eachList(i)))
Sum = int(sum(result))
mylist.append(Sum)
return(mylist)
With Classes
class Task1:
def __init__(self,train,ideal):
self.train = train
self.ideal = ideal
def __str__(self):
return self.ideal[index]
def residual(self,lst1,lst2):
self.subtracted = []
for item1,item2 in zip(lst1,lst2):
self.subtracted.append(item1-item2)
return self.subtracted
def eachList(self,index):
Series=df_ideal.iteritems()
self.bigList = []
for (columnName,item) in Series:
self.bigList.append(item)
return self.bigList[index]
Inheritance
class Resultant(Task1):
def __init__(self,train,ideal):
super().__init__(train,ideal)
def isResult(self,listing):
mylist=[]
for i in range(len([column_name for column_name in df_ideal])):
result = map(lambda x:x*x,Task1.residual(listing,Task1.eachList(i)))
Sum = int(sum(result))
mylist.append(Sum)
return(mylist)
The error you're hitting (I'm guessing) is that you're calling Task1.residual as if it's a class method, when it's not:
result = map(lambda x:x*x,Task1.residual(listing,Task1.eachList(i)))
For this to work you should be calling self.residual.
However, I think it would be better to just have a simple set of functions, since there's nothing in the implementation of these functions that requires any kind of persistent state to be shared between them. The logic is simple enough that each could be written as a single list comprehension:
def eachList():
return [item for _columnName, item in df_ideal.iteritems()]
def residual(lst1, lst2):
return [item1 - item2 for item1, item2 in zip(lst1, lst2)]
def isResult(listing):
return [sum(x * x for x in residual(listing, item)) for item in eachList()]
I have two functions:
job_status is getting a response from boto3 api.
jobs_detailsis a list comprehension that performs job_status on each element of the input list.
I want to change jobs_details into a decorator of jobs_status but below solutions throws inner() takes 1 positional argument but 2 were given error.
Appreciate any comment/alternative approach to my issue. Thanks!
import boto3
class GlueClient:
def __init__(self):
self.glue_client = boto3.client('glue')
#self.envs = envs
def jobs_list(self):
response = self.glue_client.list_jobs()
result = response["JobNames"]
while "NextToken" in response:
response = self.glue_client.list_jobs(NextToken=response["NextToken"])
result.extend(response["JobNames"])
return [e for e in result if "jobs_xyz" in e]
#WHAT IS CURRENTLY
def job_status(self, job_name):
paginator = self.glue_client.get_paginator('get_job_runs')
response = paginator.paginate(JobName=job_name)
return response
def jobs_details(self, jobs):
return [self.job_status(e) for e in jobs]
#WHAT IS EXPECTED
def pass_by_list_comprehension(func):
def inner(list_of_val):
return [func(value) for value in list_of_val ]
return inner
#pass_by_list_comprehension
def job_status(self, job_name):
paginator = self.glue_client.get_paginator('get_job_runs')
response = paginator.paginate(JobName=job_name)
return response
glue_client = GlueClient()
jobs = glue_client.jobs_list()
jobs_status = glue_client.job_status(jobs)
print(jobs)
You want something like:
import boto3
from typing import Callable
def handle_iterable_input(func):
def inner(self, list_of_val):
return [func(self, value) for value in list_of_val]
return inner
class GlueClient:
def __init__(self):
self.glue_client = boto3.client('glue')
#self.envs = envs
def jobs_list(self):
response = self.glue_client.list_jobs()
result = response["JobNames"]
while "NextToken" in response:
response = self.glue_client.list_jobs(NextToken=response["NextToken"])
result.extend(response["JobNames"])
return [e for e in result if "jobs_xyz" in e]
#handle_iterable_input
def job_status(self, job_name):
paginator = self.glue_client.get_paginator('get_job_runs')
response = paginator.paginate(JobName=job_name)
return response
glue_client = GlueClient()
jobs = glue_client.jobs_list()
jobs_status = glue_client.job_status(jobs)
print(jobs)
This is the most basic way to make your decorator handle methods properly, by explicitly handling the passing of self. Note, it assumes the function being decorated will only take a single argument.
If all you want to do is make job_status iterate through a list of job names instead of operating on just one, something like this should work:
def jobs_status(self, job_names):
paginator = self.glue_client.get_paginator('get_job_runs')
return [paginator.paginate(JobName=job_name) for job_name in job_names]
Using a decorator to change what parameters a method expects seems like a bad idea.
Also, naming your class GlueClient would imply that it is a glue client. The fact that it has an attribute named glue_client makes me suspect you could probably choose a clearer name for one or both of them. (However, I'm not familiar with the package you're using.)
I'm using django 2.2 and in my view I have two functions that do the same thing but only one element changes. I would like to try to improve my code so that I don't repeat the same thing more times, basically do what the vm_schedule_power_on_vm function does and
vm_schedule_power_off_vm into one function. The only thing that will change is the call of vmware_poweron in the vm_schedule_power_on_vm function and vmware_poweroff in the vm_schedule_power_off_vm function.
path('vm/schedule/<int:pk>/powered_on/', vm.vm_schedule_power_on_vm,
name='vm_schedule_power_on_vm'),
path('vm/schedule/<int:pk>/powered_off/', vm.vm_schedule_power_off_vm,
name='vm_schedule_power_off_vm')
def vm_schedule_power_on_vm(request, pk):
sch = VmSchedule.objects.get(pk=pk)
mylistvm, mylist = list(), list()
mydate = time.strftime("%d/%m/%Y")
for i in sch.vms.all():
if i.lock:
return 'locked'
# here the order has importance because
# I try to have the start time and at the end the end time.
mylist.append(mydate)
mylist.append(time.strftime("%H:%M:%S"))
mylist.append(i.name)
mylist.append(i.vmware.hostname)
# only this line changes each time
mylist.append(vmware_poweron(i))
mylist.append(time.strftime("%H:%M:%S"))
mylist.append(sch.schedule)
mylistvm.append(mylist)
mylist = list()
vm_logs_export(mylistvm)
return HttpResponse(json.dumps(mylistvm))
def vm_schedule_power_off_vm(request, pk):
sch = VmSchedule.objects.get(pk=pk)
mylistvm, mylist = list(), list()
mydate = time.strftime("%d/%m/%Y")
for i in sch.vms.all():
if i.lock:
return 'locked'
mylist.append(mydate)
mylist.append(time.strftime("%H:%M:%S"))
mylist.append(i.name)
mylist.append(i.vmware.hostname)
# only this line changes each time
mylist.append(vmware_poweroff(i))
mylist.append(time.strftime("%H:%M:%S"))
mylist.append(sch.schedule)
mylistvm.append(mylist)
mylist = list()
vm_logs_export(mylistvm)
return HttpResponse(json.dumps(mylistvm))
# Example result of vm_schedule_power_on_vm or vm_schedule_power_off_vm
['09/12/2021', '13:54:33', 'API1VTEST11', 'ste1vvcsa', '13:54:33', 'testPowredOn02',
'09/12/2021', '13:54:33', 'API1VTEST12', 'ste1vvcsa', '13:54:33', 'testPowredOn02',
'09/12/2021', '13:54:33', 'API1VTEST2', 'ste1vvcsa', '13:54:33', 'testPowredOn02']
def vmware_poweron(vm):
#return list of something
def vmware_poweroff(vm):
#return list of something
# Example result of vmware_poweron or vmware_poweroff
[["09/12/2021", "13:54:33", "API1VTEST11", "ste1vvcsa", "13:54:33", "testPowredOn02",
"09/12/2021", "13:54:33", "API1VTEST12", "ste1vvcsa", "13:54:33", "testPowredOn02",
"09/12/2021", "13:54:33", "API1VTEST2", "ste1vvcsa", "13:54:33", "testPowredOn02"]
I thought of doing in one fonction like this past code here
The simplest thing to do is to extract the shared operations into a more generically named function, e.g. vm_schedule_power_operation. Give this a third parameter, operation, and pass either vmware_poweron or vmware_poweroff as that parameter.
When you reach the line that changes in the function, modify that line to call operation(i) instead of vmware_poweron or vmware_poweroff directly.
Then, your on/off functions can look like this:
def vm_schedule_power_on_vm(request, pk):
vm_schedule_power_operation(request, pk, vmware_poweron)
def vm_schedule_power_off_vm(request, pk):
vm_schedule_power_operation(request, pk, vmware_poweroff)
Let's assume that we the following ndb model:
class MyModel(ndb.Model):
x = ndb.StringProperty()
y = ndb.StringProperty()
z = ndb.StringProperty(repeated=True)
We have a method that creates a query for the above model, executes it and fetch the results. However, we want this query to be modified my other functions. Specifically, we have the following:
def method_a():
qry = MyModel.query()
values = {'query':qry}
method_b(**values)
entities = qry.fetch()
def method_b(**kwargs):
k = ['a', 'b', 'c']
qry = kwargs['query']
qry.filter(MyModel.z.IN(k))
The problem is that the Query object is immutable, and thus it cannot be modified by method_b. Also, based on the specific architecture of the code, we cannot have method_b to return the new Query to method_a.
Any ideas on how to achieve the aforementioned functionality in another way??
Update: Please check the architecture of my code as presented below:
First, in a configuration file we specify a list of modules and if they are enabled or not. These modules affect the filters of the query we want to execute.
testparams = {
'Test1': True,
'Test2': True,
'Test3': False,
'Test4': True
}
Then, we have a method somewhere in the code that makes a query after the appropriate modules have been executed. Thus, it seems like this:
def my_func():
qry = MyEntity.query()
# modules
query_wrapper = [qry]
values = {'param':'x', 'query_wrapper':query_wrapper} #other values also
execute_modules(**values)
# get query and add some more things, like ordering
entities = query_wrapper[0].fetch()
The execute_modules function is the following:
def execute_modules(**kwargs):
for k in config.testparams:
if config.testparams[k]:
if kwargs['param'] == 'x':
(globals()[k]).x(**kwargs)
elif kwargs['param'] == 'y':
(globals()[k]).y(**kwargs)
Finally, an indicative module is similar to the following:
class Test1():
#classmethod
def x(cls, *args, **kwargs):
qry = kwargs['query_wrapper'][0]
# do some stuff like adding filters
kwargs['query_wrapper'][0] = qry
Any proposals to modify this architecture to a better approach?
I'm not aware of a way to do this without having method_b either return or change a referenced parameter. You should use a technique to pass a variable by reference, like passing a class with parameters.
You can pass in the args in a refrence object such as a dict/list:
def modify_query(kwargs):
kwargs['qry'] = kwargs['qry'].filter(MyModel.z.IN(k))
qry = MyModel.query()
kwargs = {'qry': qry}
modify_query(kwargs)
result = kwargs['qry'].fetch()
It should be noted that this is an extremly dirty way to accomplish what you want to accomplish. Similarly, if you pass in a list with say one object, then you can modify the contents of said list (through assignment) to modify the object:
def modify_query(list_object):
list_object[0] = list_object[0].filter(...)
You can do some hack for replace it object by other. For example:
def f(args):
qry = args[0]
qry_new = qry.filter(Model.a == 2)
args[0] = qry_new
qry = Model.query()
args = [qry]
f(args)
qry = args[0]
I warped a class in this way:
import Queue
import threading
class MyThread():
q = Queue.Queue()
content = []
result = {}
t_num = 0
t_func = None
def __init__ (self, t_num, content, t_func):
for item in content:
self.q.put(item)
self.t_num = t_num
self.t_func = t_func
def start(self):
for i in range(self.t_num):
t = threading.Thread(target=self.worker)
t.daemon = True
t.start()
self.q.join()
return self.result
def worker(self):
while True:
item = self.q.get()
value = self.t_func(item)
self.result[item] = value
self.q.task_done()
x = [5, 6, 7, 8, 9]
def func(i):
return i + 1
m = MyThread(4, x, func)
print m.start()
It works well. If I design the function func with 2 or more parameters, and pass these parameters in a list to the class, how can I call the func function in the function worker properly?
eg.
def __init__ (self, t_num, content, t_func, t_func_p):
for item in content:
self.q.put(item)
self.t_num = t_num
self.t_func = t_func
self.t_func_p = t_func_p
def func(i, j, k):
m = MyThread(4, x, func, [j, k])
You need to use *args and **kwargs to pass any number of parameters to a function.
Here is more info: http://www.saltycrane.com/blog/2008/01/how-to-use-args-and-kwargs-in-python/
Maybe this might help:
def __init__(self, t_num, content, func, *params):
func(*params) # params is a list here [param1, param2, param3....]
def func(param1, param2, param3):
# or
def func(*params): # for arbitrary number of params
m = MyThread(4, x, func, param1, param2, param3....)
As a general rule, if you are going to be passing many parameters to a particular function, you may consider wrapping them into a simple object, the reasons are
If you ever need to add/remove parameters, you just need to modify the object, and the function itself, the method signature (and all its references) will remain untouched
When working with objects, you will always know what your function is receiving (this is specially useful if you are working on a team, where more people will use that function).
Finally, because you control the creation of the object on its constructor, you can ensure that the values associated with the object are correct (for example, in the constructor you can make sure that you have no empty values, or that the types are correct).
If still you want to go with multiple parameters, check the *args and **kwargs, although I personally do not like that, as it may end up forcing people to read the function's source in order to use it.
Good luck :)