I've been wondering recently if there's a way to detect whether a context manager is nested.
I've created Timer and TimerGroup classes:
class Timer:
def __init__(self, name="Timer"):
self.name = name
self.start_time = clock()
#staticmethod
def seconds_to_str(t):
return str(timedelta(seconds=t))
def end(self):
return clock() - self.start_time
def print(self, t):
print(("{0:<" + str(line_width - 18) + "} >> {1}").format(self.name, self.seconds_to_str(t)))
def __enter__(self):
return self
def __exit__(self, exc_type, value, traceback):
self.print(self.end())
class TimerGroup(Timer):
def __enter__(self):
print(('= ' + self.name + ' ').ljust(line_width, '='))
return self
def __exit__(self, exc_type, exc_val, exc_tb):
total_time = self.seconds_to_str(self.end())
print(" Total: {0}".format(total_time).rjust(line_width, '='))
print()
This code prints timings in a readable format:
with TimerGroup("Collecting child documents for %s context" % context_name):
with Timer("Collecting context features"):
# some code...
with Timer("Collecting child documents"):
# some code...
= Collecting child documents for Global context ============
Collecting context features >> 0:00:00.001063
Collecting child documents >> 0:00:10.611130
====================================== Total: 0:00:10.612292
However, when I nest TimerGroups, it messed things up:
with TimerGroup("Choosing the best classifier for %s context" % context_name):
with Timer("Splitting datasets"):
# some code...
for cname, cparams in classifiers.items():
with TimerGroup("%s classifier" % cname):
with Timer("Training"):
# some code...
with Timer("Calculating accuracy on testing set"):
# some code
= Choosing the best classifier for Global context ==========
Splitting datasets >> 0:00:00.002054
= Naive Bayes classifier ===================================
Training >> 0:00:34.184903
Calculating accuracy on testing set >> 0:05:08.481904
====================================== Total: 0:05:42.666949
====================================== Total: 0:05:42.669078
All I need is to do is to indent the nested Timers and TimerGroups somehow. Should I pass any parameters to their constructors? Or can I detect that from inside the class?
There are no special facilities to detect nested context managers, no. You'd have to handle this on your own. You could do this within your own context manager:
import threading
class TimerGroup(Timer):
_active_group = threading.local()
def __enter__(self):
if getattr(TimerGroup._active_group, 'current', False):
raise RuntimeError("Can't nest TimerGroup context managers")
TimerGroup._active_group.current = self
print(('= ' + self.name + ' ').ljust(line_width, '='))
return self
def __exit__(self, exc_type, exc_val, exc_tb):
TimerGroup._active_group.current = None
total_time = self.seconds_to_str(self.end())
print(" Total: {0}".format(total_time).rjust(line_width, '='))
print()
You can then use the TimerGroup._active_group attribute elsewhere to grab the currently active group. I used a thread-local object to ensure that this can be used across multiple threads of execution.
Alternatively, you could make that a stack counter and just increment and decrement in nested __enter__ calls, or a stack list and push self onto that stack, popping it again when you __exit__:
import threading
class TimerGroup(Timer):
_active_group = threading.local()
def __enter__(self):
if not hasattr(TimerGroup._active_group, 'current'):
TimerGroup._active_group.current = []
stack = TimerGroup._active_group.current
if stack:
# nested context manager.
# do something with stack[-1] or stack[0]
TimerGroup._active_group.current.append(self)
print(('= ' + self.name + ' ').ljust(line_width, '='))
return self
def __exit__(self, exc_type, exc_val, exc_tb):
last = TimerGroup._active_group.current.pop()
assert last == self, "Context managers being exited out of order"
total_time = self.seconds_to_str(self.end())
print(" Total: {0}".format(total_time).rjust(line_width, '='))
print()
If all you need to do is adjust an indentation level based on how many nested context managers you're executing in, then have a class attribute called indent_level and adjust it each time you enter and exit a context manager. Something like the following:
class Context:
indent_level = 0
def __init__(self, name):
self.name = name
def __enter__(self):
print(' '*4*self.indent_level + 'Entering ' + self.name)
self.adjust_indent_level(1)
return self
def __exit__(self, *a, **k):
self.adjust_indent_level(-1)
print(' '*4*self.indent_level + 'Exiting ' + self.name)
#classmethod
def adjust_indent_level(cls, val):
cls.indent_level += val
And use it as:
>>> with Context('Outer') as outer_context:
with Context('Inner') as inner_context:
print(' '*inner_context.indent_level*4 + 'In the inner context')
Entering Outer
Entering Inner
In the inner context
Exiting Inner
Exiting Outer
import this:
Explicit is better than implicit
A cleaner design would explicitly allow to specify a group:
with TimerGroup('Doing big task') as big_task_tg:
with Timer('Foo', big_task_tg):
foo_result = foo()
with Timer('Bar', big_task_tg):
bar(baz(foo_result))
On the other hand, you can always use traceback.extract_stack and look for invocations of a particular function upstream. It is very useful for logging and error reporting, and can be moderately useful to ensure that particular functions are only invoked in a certain context. But it tends to create dependencies that are very hard to track.
I would avoid it for grouping timers, though you can try. If you badly need automatic grouping, #Martijn-Pieters's approach is far superior.
Related
Python offers tracing through its trace module. There are also custom solutions like this. But these approaches capture most low-level executions, inside-and-out of most/every library you use. Other than deep-dive debugging this isn't very useful.
It would be nice to have something that captures only the highest-level functions laid out in your pipeline. For example, if I had:
def funct1():
res = funct2()
print(res)
def funct2():
factor = 3
res = funct3(factor)
return(res)
def funct3(factor):
res = 1 + 100*factor
return(res)
...and called:
funct1()
...it would be nice to capture:
function order:
- funct1
- funct2
- funct3
I have looked at:
trace
tracefunc
sys.settrace
trace.py
I am happy to manually mark the functions inside the scripts, like we do with Docstrings. Is there a way to add "hooks" to functions, then track them as they get called?
You can always use a decorator to track which functions are called. Here is an example that allows you to keep track of what nesting level the function is called at:
class Tracker:
level = 0
def __init__(self, indent=2):
self.indent = indent
def __call__(self, fn):
def wrapper(*args, **kwargs):
print(' '*(self.indent * self.level) + '-' + fn.__name__)
self.level += 1
out = fn(*args, **kwargs)
self.level -= 1
return out
return wrapper
track = Tracker()
#track
def funct1():
res = funct2()
print(res)
#track
def funct2():
factor = 3
res = funct3(factor)
return(res)
#track
def funct3(factor):
res = 1 + 100*factor
return(res)
It uses the class variable level to keep track of how many functions have been called and simply prints out the the function name with a space indent. So calling funct1 gives:
funct1()
# prints:
-funct1
-funct2
-funct3
# returns:
301
Depending on how you want to save the output, you can use the logging module for the output
So I have
# my decorator factory
def execute_in(directory): # <-- I want this to be a variable's value which can change
def decorator(function):
def wrapper(*args, **kwargs):
os.chdir(directory)
print(directory) # currently is printing None which is my problem
value = function(*args, **kwargs)
os.chdir(home_dir)
return value
return wrapper
return decorator
and
# a function that runs after assigning General.archive_dir a value
#execute_in(General.archive_dir)
def get_data():
print(General.archive_dir) # will print the correct directory name
with open('data.csv', 'r') as f:
rows = [row for row in csv.reader(f, delimiter=',')]
return rows
My problem is that the decorator factory is using the value of the variable General.archive_dir instantiated at program start when its value is None. I want it to use the value of General.archive_dir at the time the decorated function is called. How can I do this?
I apologize if this question is unclear. If you can, please let me know how I can clarify it if needed.
One solution is calling #execute_in with a lambda.
directory inside wrapper would become a function that, when called, returns the current value.
archive_dir = None
# decorator factory
def execute_in(directory_path_getter):
def decorator(function):
def wrapper(*args, **kwargs):
print('from wrapper:', directory_path_getter()) # Notice the function call
value = function(*args, **kwargs)
return value
return wrapper
return decorator
#execute_in(lambda: archive_dir)
def get_data():
...
archive_dir = 'some directory'
print(get_data())
Prints:
from wrapper: some directory
from get_data: some directory
['some data']
If a decorator isn't strictly required, a context manager can also fulfill the task of temporarily changing directories.
import os
from contextlib import contextmanager
#contextmanager
def execute_in(directory):
orig_dir = os.getcwd()
os.chdir(directory)
try:
yield
finally:
os.chdir(orig_dir)
Using a context manager would allow for changing directories many times in one method, and can be nested.
settings = {
'archive_dir': './dir'
}
def get_data():
print(os.getcwd())
with execute_in(settings['archive_dir']):
print(' ' + os.getcwd())
with execute_in('bin'):
print(' ' + os.getcwd())
print(' ' + os.getcwd())
print(os.getcwd())
And when we run it
>>> get_data()
/home/they4kman/.PyCharm2019.2/config/scratches
/home/they4kman/.PyCharm2019.2/config/scratches/dir
/home/they4kman/.PyCharm2019.2/config/scratches/dir/bin
/home/they4kman/.PyCharm2019.2/config/scratches/dir
/home/they4kman/.PyCharm2019.2/config/scratches
I have the following base class:
class ClientRepo(Repository):
def __init__(self) -> None:
self.__clientList = []
def hasClientWithId(self, clientId):
for client in self.__clientList:
if client.getId() == clientId:
return True
return False
def addClient(self, client):
if type(client).__name__ == 'ClientDAO':
if not self.hasClientWithId(client.getId()):
client.setClientId(self.__maximumIndexInClientList() + 1)
self.__clientList.append(client)
else:
raise ObjectAlreadyInCollectionException
else:
raise TypeError
which basically only holds a list and can add a ClientDAO to it.
And the following, which derives from it:
class ClientFileRepository(ClientRepo):
def __init__(self, fileName) -> None:
super().__init__()
self.__fileName = fileName
self.__file = None
def hasClientWithId(self, clientId):
self.__loadRepo()
hasClientWithId = super().hasClientWithId(clientId)
super().clean()
return hasClientWithId
def addClient(self, client):
self.__loadRepo()
super().addClient(client)
self.__storeRepo()
super().clean()
def __loadFileReadMode(self):
self.__file = open(self.__fileName, "r")
def __loadFileWriteMode(self):
self.__file = open(self.__fileName, "w")
def __closeFile(self):
self.__file.close()
def __loadRepo(self):
self.__loadFileReadMode()
for line in self.__file:
splitLine = line.split()
clientToAdd = ClientDAO(splitLine[1])
clientToAdd.setClientId(int(splitLine[0]))
super().addClientWithId(clientToAdd)
self.__closeFile()
def __storeRepo(self):
self.__loadFileWriteMode()
self.__file.write("")
for client in super().getList():
self.__file.write(self.clientToString(client))
self.__closeFile()
def clientToString(self, clientDAO):
return str(clientDAO.getId()) + " " + clientDAO.getName() + "\n"
a class which should load the list from a file, call addClient from parent, and store the updated list in the file. The problem is that after child class loads the file in addClient, it calls the method in the parent, which calls hasClientWithId, from the child, again. But I want it to call hasClientWithId, from the parent, that is, the context it is in. Can I achieve that?
I can think of several ways to achieve your goal. I ranked them from worst to best
1. Exactly what you asked for
You wanted that ClientRepo.addClient calls ClientRepo.hasClientWithId instead of ClientFileRepository.hasClientWithId. It is possible to enforce that:
class ClientRepo(Repository):
def addClient(self, client):
if type(client).__name__ == 'ClientDAO':
if not ClientRepo.hasClientWithId(self, client.getId()):
client.setClientId(self.__maximumIndexInClientList() + 1)
self.__clientList.append(client)
else:
raise ObjectAlreadyInCollectionException
else:
raise TypeError
This is not a good approach, because it's unintuitive and breaks the principles of OOP. Any other programmer writing a subclass of ClientRepo that overrides hasClientWithId would expect that this will have an effect for every call to hasClientWithId even inside of addClient
2. Let ClientFileRepository decide which function to use
Add a variable
self.__isFileOpen = False
in ClientFileRepository.__init__, set it to True when you open the file and to False when you close the file. Then change the hasClientWithId within ClientFileRepository to
def hasClientWithId(self, clientId):
if not self.__isFileOpen:
self.__loadRepo()
result = super().hasClientWithId(clientId)
super().clean()
return result
else:
return super().hasClientWithId(clientId)
to avoid opening the same file again. This works, but it is pretty difficult to write new functions for this class, because you always need to be aware if the function call is a call from within your class or from somewhere else. Also this seems pretty inefficient, because you read and write the entire file, even when you only add one client.
3. Read the file only once and modify the underlying ClientRepo
class ClientFileRepository(ClientRepo):
def __init__(self, fileName) -> None:
super().__init__()
self.__fileName = fileName
self.__loadRepo()
# No hasClientWithId needed
def addClient(self, client):
super().addClient(client)
self.__storeRepo()
def __loadRepo(self):
with open(self.__filename) as file:
for line in file:
splitLine = line.split()
clientToAdd = ClientDAO(splitLine[1])
clientToAdd.setClientId(int(splitLine[0]))
super().addClientWithId(clientToAdd)
def __storeRepo(self):
with open(self.__filename, "w") as file:
file.write("")
for client in super().getList():
file.write(self.clientToString(client))
This obviously assumes that the file is not changed by someone else between calls to addClient and the program still overwrites the entire file for every addClient. If this is a problem for you it is best to be explicit and make loadRepo and storeRepo public. Then the programmer using this class can decide when loading and saving are necessary and useful. You can use context managers for this.
Extra: Read and save the file for every method
You can use function decorators to use solution 2 without writing the same code for every function:
import functools
def loadAndStore(function):
#functoools.wraps(function)
def wrappedFunction(self, *args, **kwargs):
if self.__isFileOpen:
return function(self, *args, **kwargs)
else:
self.__isFileOpen = True
self.__loadRepo()
try:
return function(self, *args, **kwargs)
except Exception as e: # Only catch expected exceptions
raise
finally:
self.__storeRepo()
self.clear() # some cleanup
self.__isFileOpen = False
return wrappedFunction
class ClientFileRepository(ClientRepo):
def __init__(self, fileName) -> None:
super().__init__()
self.__fileName = fileName
self.__isFileOpen = False
#loadAndStore
def hasClientWithId(self, clientId):
return super().hasClientWithId(clientId)
#loadAndStore
def addClient(self, client):
super().addClient(client)
def __loadRepo(self):
with open(self.__filename) as file:
for line in file:
splitLine = line.split()
clientToAdd = ClientDAO(splitLine[1])
clientToAdd.setClientId(int(splitLine[0]))
super().addClientWithId(clientToAdd)
def __storeRepo(self):
with open(self.__filename, "w") as file:
file.write("")
for client in super().getList():
file.write(self.clientToString(client))
Be careful here, using this is not very intuitive. For example self.__isFileOpen is defined in __init__, but none of the methods below directly use it. Instead its use is hidden in the loadAndStore decorator.
Some quick hints at the end:
type(client).__name__ == 'ClientDAO' is bad practice. Use isinstance(client, ClientDAO) to fully adopt OOP
If this is not part of a bigger project with given naming conventions use the python style guide
Using private variables like __fileName is generally considered unnecessary, just prefix the variable with one underscore to indicate "internal use". The same is true for functions.
I'd like to query each key of a keyboard without using win32api. I have it working using win32api.GetAsyncKeyState(key), but I'd also like to add support for if the module is not installed.
So far I've found one piece of fully working code, though it seems a bit heavyweight as it'd require its own thread, and would need over 1600 separate functions as I want to catch each key no matter of modifiers (there are 14 possible combinations per key).
Here is the code I found, would anyone be able to either suggest an alternative or how to get around the modifier problem?
import ctypes
import ctypes.wintypes
import win32con
class GlobalHotKeys(object):
"""
Register a key using the register() method, or using the #register decorator
Use listen() to start the message pump
Example:
from globalhotkeys import GlobalHotKeys
#GlobalHotKeys.register(GlobalHotKeys.VK_F1)
def hello_world():
print 'Hello World'
GlobalHotKeys.listen()
"""
key_mapping = []
user32 = ctypes.windll.user32
MOD_ALT = win32con.MOD_ALT
MOD_CTRL = win32con.MOD_CONTROL
MOD_CONTROL = win32con.MOD_CONTROL
MOD_SHIFT = win32con.MOD_SHIFT
MOD_WIN = win32con.MOD_WIN
#classmethod
def register(cls, vk, modifier=0, func=None):
"""
vk is a windows virtual key code
- can use ord('X') for A-Z, and 0-1 (note uppercase letter only)
- or win32con.VK_* constants
- for full list of VKs see: http://msdn.microsoft.com/en-us/library/dd375731.aspx
modifier is a win32con.MOD_* constant
func is the function to run. If False then break out of the message loop
"""
# Called as a decorator?
if func is None:
def register_decorator(f):
cls.register(vk, modifier, f)
return f
return register_decorator
else:
cls.key_mapping.append((vk, modifier, func))
#classmethod
def listen(cls):
"""
Start the message pump
"""
for index, (vk, modifiers, func) in enumerate(cls.key_mapping):
if not cls.user32.RegisterHotKey(None, index, modifiers, vk):
raise Exception('Unable to register hot key: ' + str(vk) + ' error code is: ' + str(ctypes.windll.kernel32.GetLastError()))
try:
msg = ctypes.wintypes.MSG()
i = 0
while cls.user32.GetMessageA(ctypes.byref(msg), None, 0, 0) != 0:
if msg.message == win32con.WM_HOTKEY:
(vk, modifiers, func) = cls.key_mapping[msg.wParam]
if not func:
break
func()
cls.user32.TranslateMessage(ctypes.byref(msg))
cls.user32.DispatchMessageA(ctypes.byref(msg))
finally:
for index, (vk, modifiers, func) in enumerate(cls.key_mapping):
cls.user32.UnregisterHotKey(None, index)
#classmethod
def _include_defined_vks(cls):
for item in win32con.__dict__:
item = str(item)
if item[:3] == 'VK_':
setattr(cls, item, win32con.__dict__[item])
#classmethod
def _include_alpha_numeric_vks(cls):
for key_code in (list (range(ord('A'), ord('Z') + 1)) + list(range(ord('0'), ord('9') + 1)) ):
setattr(cls, 'VK_' + chr(key_code), key_code)
GlobalHotKeys._include_defined_vks()
GlobalHotKeys._include_alpha_numeric_vks()
This is an example of how it'd be used to read a:
#GlobalHotKeys.register(ord('A'))
def a():
print 'a'
#GlobalHotKeys.register(ord('A'), GlobalHotKeys.MOD_SHIFT)
def a_shift():
print 'shift + a'
#GlobalHotKeys.register(ord('A'), GlobalHotKeys.MOD_CONTROL | GlobalHotKeys.MOD_SHIFT)
def a_ctrl_shift():
print 'ctrl + shift + a'
...
GlobalHotKeys.listen()
Turned out to be an extremely simple answer, I finally stumbled across it when trying to read the microsoft info for the GetKeyState function.
ctypes.windll.user32.GetKeyState(key)
The state will either be 0 or 1 when not pressed, and increase to something like 60000 when pressed, so to get a True/False result, checking for > 1 seems to do the trick.
GetAsyncKeyState also kinda works, but sometimes results in a negative number, and sometimes doesn't, so I thought it'd be best using the alternative.
I'd like to use the python module timeit to time some functions in my QGIS plugin.
Here, I've called the time it function within a function that I call at the end of the last function. It seems, though, that the plugin is taking even longer to run than usual and I am wondering if i'm calling the timer in the wrong place. Is there a better way to set this up?
class myPluginName:
def firstFunction(self):
...
self.secondFunction()
def secondFunction(self):
...
self.timeThings()
def run(self):
self.firstFunction()
def timeThings(self):
QMessageBox.information(None, 'First Function', 'Time : %s' % timeit.timeit(self.firstFunction,number=1)
QMessageBox.information(None, 'Second Function', 'Time : %s' % timeit.timeit(self.secondFunction,number=1)
UPDATE: After following some advice, i've tried to implement the wrapper in the following way. I get however, a TypeError: firstFunction() takes exactly 1 argument (2 given) on ret = func(**args, **kwargs)
def time_func(func):
try:
name = func.__name__
except:
name = func.f__name
def tf_wrapper(*args, **kwargs):
t = time.time()
ret = func(*args, **kwargs)
QMessageLog.logMessage("{}: {}".format(name, time.time() - t))
return ret
return tf_wrapper
class myPlugin:
def initGui(self):
QObject.connect(self.dlg.ui.comboBox,SIGNAL("currentIndexChanged(int)"), self.firstFunction)
#time_func
def firstFunc(self):
registry = QgsMapLayerRegistry.instance()
firstID = str(self.dlg.ui.firstCombo.itemData(self.dlg.ui.firstCombo.currentIndex()))
secondID = str(self.dlg.ui.secondCombo.itemData(self.dlg.ui.secondCombo.currentIndex()))
self.firstLayer = registry.mapLayer(firstID)
self.secondLayer = registry.mapLayer(secondID)
#time_func
def secondFunc(self):
...
self.thirdFunc()
def thirdFunct(self):
...
def run(self):
self.dlg.ui.firstCombo.clear()
self.dlg.ui.secondCombo.clear()
for layer in self.iface.legendInterface().layers():
if layer.type() == QgsMapLayer.VectorLayer:
self.dlg.ui.firstCombo.addItem(layer.name(), layer.id())
self.dlg.ui.secondCombo.addItem(layer.name(), layer.id())
result = self.dlg.exec_()
if result == 1:
self.secondFunction()
OK, I don't know your exact situation, but I'd set it up though decorators:
import time
def time_func(func):
try:
name = func.__name__
except:
name = func.f_name
def tf_wrapper(*args, **kwargs):
t = time.time()
ret = func(*args, **kwargs)
print("{}: {}".format(name, time.time() - t)) # Or use QMessageBox
return ret
return tf_wrapper
class myPluginName:
#time_func
def firstFunction(self):
pass
#time_func
def secondFunction(self):
pass
def run(self):
self.firstFunction()
myPluginName().firstFunction()
With this code, any function wrapped in time_func will have the time taken to execute the function printed when it returns, along with its name. E.g. running it will print:
firstFunction: 1.430511474609375e-06
For your TypeError, you need to change;
def firstFunction(self):
pass
To:
def firstFunction(self, index):
pass