I have a function that takes in an external path that holds some python files and discovers any classes that exist within those files. It returns a map of the class name and the class itself so that I can create an object from it, if I want. The following code below works how I intended it to, but I am trying to do it without appending the sys.path.
def find_plugins(path):
sys.path.append(path)
plugin_map = {}
current_module_name = os.path.splitext(os.path.basename(path))[0]
for file in glob.glob(path + '/*.py'):
name = os.path.splitext(os.path.basename(file))[0]
if name.startswith('__'):
continue
module = importlib.import_module(name, package=current_module_name)
for member in dir(module):
plugin_class = getattr(module, member)
if plugin_class and inspect.isclass(plugin_class):
plugin_map[member] = plugin_class
return plugin_map
If I remove the sys.path line, I get the following stack trace:
ModuleNotFoundError: No module named 'plugin'
where 'plugin' is the plugin.py file in the path provided to the function. Is it even possible to do this without appending the sys.path?
You could do it using the technique shown in the accepted answer to the question How to import a module given the full path which doesn't require manipulating sys.path.
import importlib.util
import inspect
from pathlib import Path
def find_plugins(path):
plugins = {}
for module_path in Path(path).glob('*.py'):
module_name = module_path.stem
if not module_name.startswith('__'):
# Load module.
spec = importlib.util.spec_from_file_location(module_name, module_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module) # Execute module in its own namespace.
# Extract classes defined in module.
for member in dir(module):
plugin_class = getattr(module, member)
if plugin_class and inspect.isclass(plugin_class):
plugins[member] = plugin_class
return plugins
if __name__ == '__main__':
from pprint import pprint
plugins_folder = '/path/to/plugins'
plugins = find_plugins(plugins_folder)
pprint(plugins)
Related
I need to dynamically import modules into my project from another package.
The structure is like:
project_folder/
project/
__init__.py
__main__.py
plugins/
__init__.py
plugin1/
__init__.py
...
plugin2/
__init__.py
...
I made this function to load a module:
import os
from importlib.util import spec_from_file_location, module_from_spec
def load_module(path, name=""):
""" loads a module by path """
try:
name = name if name != "" else path.split(os.sep)[-1] # take the module name by default
spec = spec_from_file_location(name, os.path.join(path, "__init__.py"))
plugin_module = module_from_spec(spec)
spec.loader.exec_module(plugin_module)
return plugin_module
except Exception as e:
print("failed to load module", path, "-->", e)
It works, unless the module uses relative imports:
failed to load module /path/to/plugins/plugin1 --> Parent module 'plugin1' not loaded, cannot perform relative import
What am I doing wrong?
I managed to solve my own issue after a LOT of googling. Turns out I needed to import using relative paths:
>>> from importlib import import_module
>>> config = import_module("plugins.config")
>>> config
<module 'plugins.config' from '/path/to/plugins/config/__init__.py'>
>>>
I had a similar problem not long ago. I added the path of the project folder to the sys.path using the module's absolute path like this:
import sys
import os
sys.path.append(os.path.dirname(os.path.realpath(__file__))+'/..')
This adds the project_folder to the sys.path thus allowing the import statement to find the plugin modules.
In a little project I have following path-structure:
main.py
Data---
|
__init__.py
Actions_2016_01.py
Actions_2016_02.py
Actions_2016_03.py
... and so on...
Every 'Action_date.py' file includes a list called
data = [something_1, something_2, ...]
Now I am trying get all the data-lists of the 'Action_date.py' files to a single list in the 'main.py' file.
I tried something like
files = os.listdir(os.path.join(os.path.dirname(__name__), 'Data'))
all_data = []
for name in files:
if name.startswith('Actions'):
import Data.name
all_data.extend(name.data)
But this doesn't work at all... I'm getting
ImportError: No module named 'Data.name'
as output.
I found a solution. Simply have to use the importlib module.
You could make your Data package's __init_.py do the work:
def _import_modules():
""" Dynamically import certain modules in the package, extract data in each
of them, and store it in a module global named all_data.
"""
from fnmatch import fnmatch
import traceback
import os
global __all__
__all__ = []
global all_data
all_data = []
globals_, locals_ = globals(), locals()
# dynamically import the desired package modules
for filename in os.listdir(os.path.join(os.path.dirname(__name__), 'Data')):
# process desired python files in directory
if fnmatch(filename, 'Actions*.py'):
modulename = filename.split('.')[0] # filename without extension
package_module = '.'.join([__name__, modulename])
try:
module = __import__(package_module, globals_, locals_, [modulename])
except:
traceback.print_exc()
raise
all_data.extend(module.data)
__all__.append('all_data')
_import_modules()
Which would allow you to do this in main.py:
import Data
print(Data.all_data)
This is an adaption of my answer to the question How to import members of modules within a package?
Given a path to a file, I need the package name to pass to importlib.import_module() so that relative imports will work correctly. I can't import it and then check module.__package__ because it won't import successfully.
Here's one rather generic method:
import pathlib
import sys
def get_module_name(path):
f = pathlib.Path(path).resolve()
for i in map(pathlib.Path, sys.path):
try:
f.relative_to(i)
except ValueError:
pass
else:
*parts, fname = f.relative_to(i).parts
return ".".join(parts), [f.stem]
module, fromlist = get_module_name("Programming/Python/kernprof.py")
print(module, fromlist)
imported_module = __import__(module, fromlist=fromlist)
print(imported_module)
print(getattr(imported_module, fromlist[0]))
Outputs:
Programming.Python ['kernprof']
<module 'Programming.Python' (namespace)>
<module 'Programming.Python.kernprof' from '/home/matthew/Programming/Python/kernprof.py'>
This solution can handle import with any path from sys.path, but cannot do relative imports (imports above the sys.path). For the how __import__ is used, see Why does Python's __import__ require fromlist?.
What about this helper ?
import os
def get_parent_package(path):
parent_path = os.path.split(path)[0]
while parent_path != os.sep:
if '__init__.py' in os.listdir(parent_path):
return os.path.basename(parent_path)
parent_path = os.path.split(parent_path)[0]
return None
parent_path != os.sep must be improved if you are under windows.
I want to make a function that loads all the .py files in a directory, and imports them using
__import__(), but I keep getting an ImportError: No module named toolboxtool1.
This is file structure:
project/dirreader.py
project/tools/toolboxtool1.py
project/tools/toolboxtool2.py
project/tools/toolboxtool3.py
What am I doing wrong?
import os
os.chdir(os.getcwd()+"/tools/")
stuff = os.listdir(os.getcwd())
for i in range(0,len(stuff)):
if stuff[i][-3:] == ".py":
stuff[i] = stuff[i][:-3]
else:
pass
modules = map(__import__, stuff)
Try prefixing the module names with "tools."
stuff[i] = 'tools.' + stuff[i][:-3]
because the modules you are trying to import are inside tools module package.
How can I find what directory a module has been imported from, as it needs to load a data file which is in the same directory.
edit:
combining several answers:
module_path = os.path.dirname(imp.find_module(self.__module__)[1])
got me what i wanted
If you want to modules directory by specifying module_name as string i.e. without actually importing the module then use
def get_dir(module_name):
import os,imp
(file, pathname, description) = imp.find_module(module_name)
return os.path.dirname(pathname)
print get_dir('os')
output:
C:\Python26\lib
foo.py
def foo():
print 'foo'
bar.py
import foo
import os
print os.path.dirname(foo.__file__)
foo.foo()
output:
C:\Documents and Settings\xxx\My Documents
foo
This would work:
yourmodule.__file__
and if you want to find the module an object was imported from:
myobject.__module__
The path to the module's file is in module.__file__. You can use that with os.path.dirname to get the directory.
Using the re module as an example:
>>> import re
>>> path = re.__file__
>>> print path
C:\python26\lib\re.pyc
>>> import os.path
>>> print os.path.dirname(os.path.abspath(path))
C:\python26\lib
Note that if the module is in your current working directory ("CWD"), you will get just a relative path e.g. foo.py. I make it a habit of getting the absolute path immediately just in case the CWD gets changed before the module path gets used.