I am having difficulty with a circular import problem. I have 2 class files like so:
--/service
service_module.py
settings.py
service_module imports other various files used throughout the project and acts as a container for various functions throughout the project. I want to assert in my settings.py file that it is properly passed an instance of the service_module parent class. Removing the assert statement in settings.py fixes the issue and I am able to properly call methods in the service_module class. For code completion and error checking it makes my life easier to assert.
I have always struggled with understanding python imports but is this the right direction to handle my particular case?
service_module.py
from PyQt5.QtCore import QObject
from sqlalchemy import *
from sqlalchemy.orm import scoped_session, Session, sessionmaker
from service.logger import logger
from sqlalchemy.orm.exc import NoResultFound
from database.tables import *
from database.load_db import load_db
from service.settings import settings
from service.web_listener import web_listener
from service.character_manager import character_manager
class Service_Module(QObject):
def __init__(self):
super(Service_Module, self).__init__()
load_database = load_db()
self.sc_session: scoped_session = load_database.get_scoped_session()
tb_scopes.make_default_scopes(service_module=self)
self.logger = logger(service_module=self)
self.settings = settings(service_module=self)
self.characters = character_manager(service_module=self)
self.callback_listener: web_listener = web_listener(service_module=self)
self.callback_listener.start()
assert isinstance(self.sc_session, scoped_session)
assert isinstance(self.logger, logger)
assert isinstance(self.settings, settings)
assert isinstance(self.callback_listener, web_listener)
settings.py
from service.service_module import *
class settings(QObject):
def __init__(self, service_module):
super(settings, self).__init__()
self.service = service_module
assert isinstance(self.service, Service_Module) ##raises NameError: name 'Service_Module' is not defined
Edit:
So changing to this solves my issue although I feel like it's kind of hacky and somehow incorrect.
from service.service_module import *
import service.service_module
class settings(QObject):
def __init__(self, service_module):
super(settings, self).__init__()
self.service = service_module
assert isinstance(self.service, service.service_module.Service_Module)
You need to delay the importing of one (or both) of the files, to break the circular import. You can do this by moving the import statement from file-scope (where it is executed as soon as the module is imported) into the execution scope of a function or method, where is not executed until that method is called. Ie)
class setting(QObject):
def __init__(self, service_module):
from service.service_module import Service_Module # <-- import is here
assert isinstance(self.service, Service_Module)
Of course, this may affect other usages of the imported symbols from that module into this file, so you may need to specify the import in more than one place.
Related
I'm asked to develop unit tests for a program which is such badly developed that the tests don't run... but the program does. Thus, I need to explain the reason why and I actually don't know!
Here is a piece of code that intends to represent the code I need to test:
from services import myModule1
from services.spec1 import importedFunc
from services.spec2 import getTool
from services.spec3 import getDict
class myClass(object):
def __init__(self, param1, param2):
self.param1 = param1
self.param2 = param2
self.param3 = 0
self.param4 = 0
def myMethod(self):
try:
myVar1 = globalDict['key1']
myVar2 = globalDict['key2']
newVar = importedFunc(par1=myVar1, par2=myVar2, par3=extVar3)
calcParam = myModule1.methodMod1(self.param1)
self.param3 = calcParam["keyParam3"]
self.param4 = newVar.meth1(self.param2)
globTools.send_message(self.param3, self.param4)
except:
globTools.error_message(self.param3, self.param4)
return
class myClass2(object):
def __init__(self, *myclass2_params):
# some piece of code to intialize dedicated attributes
self.add_objects()
def add_objects(self):
# Some piece of code
my_class = myClass(**necessary_params)
# Some piece of code
return
if __name__ == '__main__':
globTools = getTool("my_program")
globalDict = getDict(some_params)
# Some piece of code
my_class2 = myClass2(**any_params)
# Some piece of code
As you can see, the problem is that the class and its methods uses global variables, defined in the main scope. And it's just a quick summary because it's actually a bit more complicated, but I hope it's enough to give you an overview of the context and help me understand why the unit test fail.
I tried to mock the imported modules, but I did not manage to a successful result, so I first tried to make it simple and just initialize all parameters.
I went to this test file:
import unittest
from my_module import myClass
from services import myModule1
from services.spec1 import importedFunc
from services.spec2 import getTool
from services.spec3 import getDict
def test_myClass(unittest.TestCase):
def setUp(self):
globTools = getTool("my_program")
globalDict = getDict(some_params)
def test_myMethod(self):
test_class = myClass(*necessary_parameters)
test_res = test_class.myMethod()
self.assertIsNotNone(test_res)
if __name__ == '__main__':
unittest.main()
But the test fail, telling me 'globTools is not defined' when trying to instantiate myClass
I also tried to initialize variables directly in the test method, but the result is the same
And to be complete about the technical environment, I cannot run python programs directly and need to launch a docker environment via a Jenkins pipeline - I'm not very familiar with this but I imagine it should not have an impact on the result
I guess the problem comes from the variable's scopes, but I'm not able to explain it in this case: why the test fail where as the method itself works (yes, it actually works, or at least the program globally runs without)
It's not as bad as you think. Your setUp method just needs to define the appropriate top-level globals in your module, rather than local variables.
import unittest
import my_module
from my_module import myClass
from services import myModule1
from services.spec1 import importedFunc
from services.spec2 import getTool
from services.spec3 import getDict
class test_myClass(unittest.TestCase):
def setUp(self):
my_module.globTools = getTool("my_program")
my_module.globalDict = getDict(some_params)
def test_myMethod(self):
test_class = myClass(*necessary_parameters)
test_res = test_class.myMethod()
self.assertIsNotNone(test_res)
if __name__ == '__main__':
unittest.main()
Depending on how the code uses the two globals, setUpClass might be a better place to initialize them, but it's probably not worth worrying about. Once you have tests for the code, you are in a better position to remove the dependency on these globals from the code.
I'm working on a project that dynamically collects classes from another module and creates new classes based off the ones it found. For example, if the source module has a "AdaptiveFilter" class, a new class called "AdaptiveFilterNode" is created that contains an "AdaptiveFilter" instance. I've been adding these generated classes to a dynamically created module (module_from_spec()) and created a MetaPathFinder and Loader to extend the import system. This works perfectly fine when importing the dynamic module, e.g. import nodes.auto_gen or from nodes.auto_gen import AdaptiveFilterNode, but fails when trying to directly import a class, e.g. import nodes.auto_gen.AdaptiveFilterNode. I need to be able to directly import a class because the generated classes need to be pickleable, which requires a direct import. I believe the problem is in the Finder that I made:
class MyNodeFinder(MetaPathFinder):
def find_spec(self, fullname, path=None, target=None):
if fullname == 'nodes.auto_gen':
auto_gen_spec = = ModuleSpec('nodes.auto_gen',
MyLoader())
return auto_gen_spec
return None
From my limited understanding on the import system, there's nothing in the finder or the module spec that tells the import system there's classes contained in the nodes.auto_gen module until the module is already loaded. How can I create a finder/loader that replicates the standard import system for a dynamic module with dynamic classes?
EDIT: Full example, module "dynamic_creator":
dynamic_creator/__init__.py
from importlib.machinery import ModuleSpec
from importlib.util import module_from_spec
import sys
from importlib.abc import MetaPathFinder
from importlib.abc import Loader
__version__ = '0.0.0'
class _MyWrapperClass():
wrapped_func = None
def __init__(self):
super().__init__()
def __call__(self, *args, **kwargs):
return self.wrapped_func(*args, **kwargs)
class MyFinder(MetaPathFinder):
def find_spec(self, fullname, path=None, target=None):
if fullname == 'dynamic_creator.auto_gen':
auto_node_spec = ModuleSpec('dynamic_creator.auto_gen',
MyLoader())
return auto_node_spec
return None
class MyLoader(Loader):
def create_module(self, spec):
return module_from_spec(ModuleSpec('dynamic_creator.auto_gen', None))
def exec_module(self, module):
def create_type(func):
t = type(func.__name__, (_MyWrapperClass, ), {})
t.wrapped_func = func
# Wait, you can do this?
t.__module__ = 'dynamic_creator.auto_gen'
return t
funcs_to_wrap = [
sum,
abs,
pow
]
for func in funcs_to_wrap:
setattr(module, func.__name__, create_type(func))
my_finder = MyFinder()
sys.meta_path.append(my_finder)
Test script:
import dynamic_creator.auto_gen # Works
from dynamic_creator.auto_gen import abs # Works
import dynamic_creator.auto_gen.pow as my_pow # Fails
EDIT: The reported error is as follows:
No module named 'dynamic_creator.auto_gen.pow'; 'dynamic_creator.auto_gen' is not a package
I have config data that should be loaded before another code (because another code use it).
So, for now I see only way to do this is to call the function at the top before rest imports:
from Init.Loaders.InitPreLoader import InitPreLoader
# this is my config loader
InitPreLoader.load()
from World.WorldManager import WorldManager
from Init.Loaders.InitLoader import InitLoader
from Init.Registry.InitRegistry import InitRegistry
from Utils.Debug import Logger
# ...
if __name__ == '__main__':
# ...
InitLoader.load()
Does it possible to do this in more elegant way and avoid to violate pep8 ?
P.S. If I need to share more code please let me know
UPD: All my classes declared in separate files
This is PreLoader:
from Typings.Abstract.AbstractLoader import AbstractLoader
from Init.Registry.InitRegistry import InitRegistry
from Config.Init.configs import main_config
class InitPreLoader(AbstractLoader):
#staticmethod
def load(**kwargs):
InitRegistry.main_config = main_config
This is Registry (where I store all my initialized data):
from Typings.Abstract.AbstractRegistry import AbstractRegistry
class InitRegistry(AbstractRegistry):
main_config = None
login_server = None
world_server = None
world_observer = None
identifier_region_map = None
region_octree_map = None
Parent of all classes (except AbstractRegistry) is AbstractBase class (it contains mixin):
from abc import ABC
from Config.Mixins.ConfigurableMixin import ConfigurableMixin
class AbstractBase(ConfigurableMixin, ABC):
pass
This mixin works with main_config from InitRegistry.
Also, after PreLoader's load was called, I load rest data with my InitLoader.load() (see first code snapshot):
from Typings.Abstract.AbstractLoader import AbstractLoader
from Init.Registry.InitRegistry import InitRegistry
from Server.Init.servers import login_server, world_server
from World.Observer.Init.observers import world_observer
from World.Region.Init.regions import identifier_region_map, region_octree_map
class InitLoader(AbstractLoader):
#staticmethod
def load(**kwargs):
InitRegistry.login_server = login_server
InitRegistry.world_server = world_server
InitRegistry.world_observer = world_observer
InitRegistry.identifier_region_map = identifier_region_map
InitRegistry.region_octree_map = region_octree_map
Well, for now I have found solution: I moved from Init.Loaders.InitPreLoader import InitPreLoader to separate file and called InitPreLoader.load() there. But I not like this solution, because my PyCharm IDE highlights it as unused import:
import Init.Init.preloader
from World.WorldManager import WorldManager
# ...
Maybe it is possible to improve this solution ? Or maybe another (more elegant) solition exists ?
I am using unittest module for writing tests.
I need to test initialization of the object inside a testcase using different inputs.
For this purpose I am importing the class inside setUp(). But when I try to use the class inside test_*() functions, I get this error - NameError: name 'Example' is not defined
Here is my code sample-
import unittest
class TestExample(unittest.TestCase):
def setUp(self):
import Example
def test_sample_function(self):
e = Example(1,2)
I know that I can simply import the class at top of the script. But I do not want to do that. I need to import it only during setup of the testscript.
Looking for some help here.
import unittest
class TestExample(unittest.TestCase):
def setUp(self):
import Example
self.Example = Example
def test_sample_function(self):
e = self.Example(1,2)
There's no reason to import the module in setUp. The module is still available globally in sys.modules, but you've only bound it to a local name that goes away after setUp returns. Just import it globally.
import unittest
import Example
class TestExample(unittest.TestCase):
def test_sample_function(self):
e = Example(1,2)
renegade.py (main before)
import socket, re
from prerequisites import *
import datetime,random,sys,time,os,pickle,urllib
from threading import *
class Renegade():
def __init__(self, settings, main, db):
self.settings = settings
self.main = main
self.fds = fds(self.settings, self.main, self)
self.somedict = {}
webserver.py
import string,cgi,time, threading, logging, sys
from os import curdir, sep
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
import pickle
from renegade import Renegade
class WebServer(BaseHTTPRequestHandler):
#super(WebServer, self).__init__() # if i put it here: NameError: name 'WebServer' is not defined
logger = logging.getLogger('httpd')
def __init__(self, settings, main, renegade):
super(WebServer, self).__init__() # i guess here its right, but that tells me: TypeError: must be type, not classobj
self.settings = settings
self.main = main
self.renegade = renegade
def do_GET(self):
print self.renegade.somedict = {} # want to display this but how do i get somedict of renegade.py (Renegade) class?
Inside webserver.py:
from main import Main
Main.SomeFunction()
You need the import so that webserver knows about main's classes.
With regards to your latest problem, you can do either
BaseHTTPRequestHandler.__init__(self)
or else replace class WebBrowser(BaseHTTPRequestHandler): with
class WebBrowser(BaseHTTPRequestHandler, object):
(The problem is that you have an old-style class in the hierarchy, which is calling the super call to fail. FYI googling the error can often help.)