How to find class parent and subclasses across different modules without running the code (static analysis)
Module Contains __init__.py and 4 files as below
Example first_file.py
class Parent(object):
def __init__(self):
pass
def method1(self):
print('parent')
Example second_file.py
from first_file import Parent
class Child(Parent):
def __init__(self):
pass
def method1(self):
print('child')
Example third_file.py
from first_file import Parent
class Child1(Parent):
def __init__(self):
pass
def method1(self):
print('child1')
Example fourth_file.py
from second_file import Child
class Child2(Child):
def __init__(self):
pass
def method1(self):
print('child2')
I want the way to list down the subclasses of parent given filename and class
example
>>> findclasshierchay first_file.py --class Parent
and it will list down all subclasses with filenames
Here's a stab at using cls.mro().
Disclaimer, I don't know if you consider this to be running the code or not. It's not static analysis, but if you have no great side effects upon module load, it really doesn't do that much either.
file1.py
class File1:
pass
file2.py
from file1 import File1
class File2(File1):
pass
listhier.py
import sys
import copy
import os
from importlib import import_module
from importlib.util import find_spec as importlib_find
#shamelessly copied from django 👇
def import_string(dotted_path):
"""
Import a dotted module path and return the attribute/class designated by the
last name in the path. Raise ImportError if the import failed.
"""
try:
module_path, class_name = dotted_path.rsplit('.', 1)
except ValueError as err:
raise ImportError("%s doesn't look like a module path" % dotted_path) from err
module = import_module(module_path)
try:
return getattr(module, class_name)
except AttributeError as err:
raise ImportError('Module "%s" does not define a "%s" attribute/class' % (
module_path, class_name)
) from err
toload = sys.argv[1]
cls = import_string(toload)
for cls_ in cls.mro():
print("%s.%s" % (cls_.__module__, cls_.__name__))
python listhier.py file2.File2
output:
file2.File2
file1.File1
builtins.object
Related
Call me weird if you like, many have before, but I have a large class which I'd like to make extensible with methods loaded from a plugin directory. Essentially, I'm monkey patching the class. What I have almost works but the method loaded doesn't 'see' the globals defined in __main__. Ideally I'd like a way to tell globals() (or whatever mechanism is actually used to locate global variables) to use that existing in __main__. Here is the code I have (trimmed for the sake of brevity):
#!/usr/bin/env python3
import importlib
import os
import types
main_global = "Hi, I'm in main"
class MyClass:
def __init__(self, plugin_dir=None):
if plugin_dir:
self.load_plugins(plugin_dir, ext="plugin")
def load_plugins(self, plugin_dir, ext):
""" Load plugins
Plugins are files in 'plugin_dir' that have the given extension.
The functions defined within are imported as methods of this class.
"""
cls = self.__class__
# First check that we're not importing the same extension twice into
# the same class.
try:
plugins = getattr(cls, "_plugins")
except AttributeError:
plugins = set()
setattr(cls, "_plugins", plugins)
if ext in plugins:
return
plugins.add(ext)
for file in os.listdir(plugin_dir):
if not file.endswith(ext):
continue
filename = os.path.join(plugin_dir, file)
loader = importlib.machinery.SourceFileLoader("bar", filename)
module = types.ModuleType(loader.name)
loader.exec_module(module)
for name in dir(module):
if name.startswith("__"):
continue
obj = getattr(module, name)
if callable(obj):
obj = obj.__get__(self, cls)
setattr(cls, name, obj)
z = MyClass(plugin_dir="plugins")
z.foo("Hello")
And this is 'foo.plugin' from the plugins directory:
#!/usr/bin/env python3
foo_global = "I am global within foo"
def foo(self, value):
print(f"I am foo, called with {self} and {value}")
print(f"foo_global = {foo_global}")
print(f"main_global = {main_global}")
The output is...
I am foo, called with <__main__.MyClass object at 0x7fd4680bfac8> and Hello
foo_global = I am global within foo
Traceback (most recent call last):
File "./plugged", line 55, in <module>
z.foo("Hello")
File "plugins/foo.plugin", line 8, in foo
print(f"main_global = {main_global}")
NameError: name 'main_global' is not defined
I know it all feels a bit 'hacky', but it's become a challenge so please don't flame me on style etc. If there's another way to achieve this aim, I'm all ears.
Thoughts, learned friends?
You can do what you want with a variation of the technique shown in #Martijn Pieters' answer to the the question: How to inject variable into scope with a decorator? tweaked to inject multiple values into a class method.
from functools import wraps
import importlib
import os
from pathlib import Path
import types
main_global = "Hi, I'm in main"
class MyClass:
def __init__(self, plugin_dir=None):
if plugin_dir:
self.load_plugins(plugin_dir, ext="plugin")
def load_plugins(self, plugin_dir, ext):
""" Load plugins
Plugins are files in 'plugin_dir' that have the given extension.
The functions defined within are imported as methods of this class.
"""
cls = self.__class__
# First check that we're not importing the same extension twice into
# the same class.
try:
plugins = getattr(cls, "_plugins")
except AttributeError:
plugins = set()
setattr(cls, "_plugins", plugins)
if ext in plugins:
return
plugins.add(ext)
for file in Path(plugin_dir).glob(f'*.{ext}'):
loader = importlib.machinery.SourceFileLoader("bar", str(file))
module = types.ModuleType(loader.name)
loader.exec_module(module)
namespace = globals()
for name in dir(module):
if name.startswith("__"):
continue
obj = getattr(module, name)
if callable(obj):
obj = inject(obj.__get__(self, cls), namespace)
setattr(cls, name, obj)
def inject(method, namespace):
#wraps(method)
def wrapped(*args, **kwargs):
method_globals = method.__globals__
# Save copies of any of method's global values replaced by the namespace.
replaced = {key: method_globals[key] for key in namespace if key in method_globals}
method_globals.update(namespace)
try:
method(*args[1:], **kwargs)
finally:
method_globals.update(replaced) # Restore any replaced globals.
return wrapped
z = MyClass(plugin_dir="plugins")
z.foo("Hello")
Example output:
I am foo, called with <__main__.MyClass object at 0x0056F670> and Hello
foo_global = I am global within foo
main_global = Hi, I'm in main
You can approach the problem with a factory function and inheritance. Assuming each of your plugins is something like this, defined in a separate importable file:
class MyPlugin:
foo = 'bar'
def extra_method(self):
print(self.foo)
You can use a factory like this:
def MyClassFactory(plugin_dir):
def search_and_import_plugins(plugin_dir):
# Look for all possible plugins and import them
return plugin_list # a list of plugin classes, like [MyPlugin]
plugin_list = search_and_import_plugins(plugin_dir):
class MyClass(*plugin_list):
pass
return MyClass()
z = MyClassFactory('/home/me/plugins')
How to import classes from all .py in a module with same structure and run by iterating over it. For Example,
module_one:
script_a:
class A:
def __init__(self,**kwargs):
code here
def run(self,**kwargs):
code here
def finish(self,**kwargs):
code here
script_b:
class B:
def __init__(self,**kwargs):
code here
def run(self,**kwargs):
code here
def finish(self,**kwargs):
code here
and so on ...
module_two:
script:
class Run:
def run_all(self,**kwargs):
for class in classes_from_module_one:
c = class()
c.run()
c.finish()
First of all, please note that module refers to python file, while to what you refer as module is usually called a package.
Now, for your problem, you could use some mixture of utilities found in pkgutil, importlib, and inspect or simple dir(). For example, using walk_modules and get_members:
# pack/mod_a.py
class A:
def __init__(self):
pass
def run(self):
print("A run!")
def finish(self):
print("A finished!")
# pack/mod_b.py
class B:
def __init__(self):
pass
def run(self):
print("B run!")
def finish(self):
print("B finished!")
# all.py
from importlib import import_module
from inspect import getmembers
from pkgutil import iter_modules
class Run:
def run_all(self, **kwargs):
modules = iter_modules(["pack"])
for module in modules:
members = getmembers(import_module(f"pack.{module[1]}"))
my_classes = [member for name, member in members if not name.startswith("_")]
for cls in my_classes:
c = cls()
c.run()
c.finish()
if __name__ == '__main__':
run = Run()
run.run_all()
The output is:
A run!
A finished!
B run!
B finished!
However for this solution you have to note, that getmembers will return all members of module - including built-ins, and imported entities to the modules - so you will have to implement your own check to properly filter-out those unwanted (see simple startswith in my example).
Is there a simple way to check in python if a class is defined? Even if you know the location of the module file .py where the class should be defined.
Let's say I have these files, file1.py where I try to check if Class1 is defined in file2.py. And file3.py where I have a second class Class2 defined.
in file1.py, I have this code:
try:
modulePath = os.sep.join([cwd,os.sep.join(factory.split(".")[0:-1])]) + ".py"
moduleName = factory.split(".")[-2]
className = factory.split(".")[-1]
m = imp.load_source(moduleName, modulePath)
c = getattr(m, className)
except:
raise ValueError('Factory Path not correctly specified')
where
factory = <string as path to the class 1> # for example com.Class1
cwd = os.getcwd() # i.e. current working directory
in file2.py
```
from . import Class2
Class1(object):
def __init__(self):
self.object2 = Class2()
in file3.py
```
Class2(object):
def __init__(self):
pass
Basically, as the modules file2.py and file3.py are not installed, the code in file1 raise an error as imp can't find the Class2. I tried with and without relative import, without success...
If you know where the class resides and assuming the module containing the class is in the python path, then you can just wrap an import of that class in a try block
try:
import MyClass
#or
from my_module import MyClass
except ImportError:
#raise an exception or log a warning of some sort
I followed advice here on stackoverflow on how to import a class
from a path outside of the root folder:
How to dynamically load a Python class
Does python have an equivalent to Java Class.forName()?
Unfortunately this raises the error:
ValueError: Empty module name
It is my understanding that import should not return an empty module, like the load_source method from imp. So I do not understand this error nor how to approach it.
What am I implementing wrong here?
Could you tilt me into the right direction here?
Thanks!
Code:
klass = __import__('..folder.module.Class_A')
some_object = klass()
class class_B(some_object):
def run(self):
print ('Test OK!')
Imported class (content of module):
class Class_A():
def __init__(self, arg1, *args):
def run(self):
pass
First of all you are not using import correctly.
Option 1:
According to https://docs.python.org/2/library/functions.html#import you should be doing:
klass = __import__('folder.module', globals(), locals(), ['Class_A'], -1)
Now if you want Class_A itself you should do:
some_object = klass.Class_A
Then you can inherit from it using:
class class_B(some_object):
def run(self):
print ('Test OK!')
Option 2:
from folder.module import Class_A
Then you can inherit from it using:
class class_B(Class_A):
def run(self):
print ('Test OK!')
Note: In folder.module folder should be a python package and module should be a python module
I have the following interface :
class Interface(object):
__metaclass__ = abc.ABCMeta
#abc.abstractmethod
def run(self):
"""Run the process."""
return
I have a collections of modules that are all in the same directory. Each module contains a single class that implements my interface.
For example Launch.py :
class Launch(Interface):
def run(self):
pass
Let's say I have 20 modules, that implements 20 classes. I would like to be able to launch a module that would check if some of the classes do not implement the Interface.
I know I have to use :
issubclass(Launch, ProcessInterface) to know if a certain class implements my process interface.
introspection to get the class that is in my module.
import modules at runtime
I am just not sure how to do that.
I can manage to use issubclass inside a module.
But I cannot use issubclass if I am outside the module.
I need to :
get the list of all modules in the directory
get the class in each module
do issubclass on each class
I would need a draf of a function that could do that.
You're probably looking for something like this:
from os import listdir
from sys import path
modpath = "/path/to/modules"
for modname in listdir(modpath):
if modname.endswith(".py"):
# look only in the modpath directory when importing
oldpath, path[:] = path[:], [modpath]
try:
module = __import__(modname[:-3])
except ImportError:
print "Couldn't import", modname
continue
finally: # always restore the real path
path[:] = oldpath
for attr in dir(module):
cls = getattr(module, attr)
if isinstance(cls, type) and not issubclass(cls, ProcessInterface):
# do whatever