I am developing a code where I need to create a Manager to synchronize a Queue. In the project I am using hydra and everything works fine until I create a multiprocessing Manager Object. This is the raised error with HYDRA_FULL_ERROR=1:
Primary config module 'config' not found.
Check that it's correct and contains an __init__.py file
Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.
Traceback (most recent call last):
File "mre.py", line 8, in <module>
main()
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\main.py", line 32, in decorated_main
_run_hydra(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\utils.py", line 355, in _run_hydra
run_and_report(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\utils.py", line 207, in run_and_report
return func()
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\utils.py", line 356, in <lambda>
lambda: hydra.run(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\hydra.py", line 107, in run
return run_job(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\core\utils.py", line 125, in run_job
ret.return_value = task_function(task_cfg)
File "mre.py", line 7, in main
m = Manager()
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\multiprocessing\context.py", line 57, in Manager
m.start()
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\multiprocessing\managers.py", line 583, in start
self._address = reader.recv()
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\multiprocessing\connection.py", line 250, in recv
buf = self._recv_bytes()
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\multiprocessing\connection.py", line 305, in _recv_bytes
waitres = _winapi.WaitForMultipleObjects(
KeyboardInterrupt
^C
C:\Users\UX331U\Desktop>SET HYDRA_FULL_ERROR=1
C:\Users\UX331U\Desktop>python mre.py
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\multiprocessing\spawn.py", line 116, in spawn_main
exitcode = _main(fd, parent_sentinel)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\multiprocessing\spawn.py", line 125, in _main
prepare(preparation_data)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\multiprocessing\spawn.py", line 236, in prepare
_fixup_main_from_path(data['init_main_from_path'])
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\multiprocessing\spawn.py", line 287, in _fixup_main_from_path
main_content = runpy.run_path(main_path,
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\runpy.py", line 265, in run_path
return _run_module_code(code, init_globals, run_name,
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\runpy.py", line 97, in _run_module_code
_run_code(code, mod_globals, init_globals,
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\runpy.py", line 87, in _run_code
exec(code, run_globals)
File "C:\Users\UX331U\Desktop\mre.py", line 8, in <module>
main()
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\main.py", line 32, in decorated_main
_run_hydra(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\utils.py", line 355, in _run_hydra
run_and_report(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\utils.py", line 210, in run_and_report
raise ex
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\utils.py", line 207, in run_and_report
return func()
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\utils.py", line 356, in <lambda>
lambda: hydra.run(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\hydra.py", line 100, in run
cfg = self.compose_config(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\hydra.py", line 505, in compose_config
self.config_loader.ensure_main_config_source_available()
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\config_loader_impl.py", line 135, in ensure_main_config_source_available
self.missing_config_error(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\config_loader_impl.py", line 109, in missing_config_error
raise MissingConfigException(
hydra.errors.MissingConfigException: Primary config module 'config' not found.
Check that it's correct and contains an __init__.py file
I also attach a minimal code snippet to reproduce the error.
import hydra
from multiprocessing import Manager
#hydra.main(config_path='config', config_name='config')
def main(cfg):
m = Manager() # The error is in this line
m.Queue()
main()
The config.yaml is correctly created and located since when I don't use the Manager, the entire code works with no errors:
- PROJECT ----- config ---- config.yaml
|
|
----- main.py
As far as I have understood, the problem comes from the fact that hydra changes the directory when initialized and afterwards, when the Manager is created, hydra is called again to be initialized but as the directory has been changed, it can not find the config folder. If this is true, is it a bug or can it be solved?
Thank you so much for your help.
I'm not able to reproduce the error on my machine.
You can stop hydra from changing directories at runtime by passing hydra.job.chdir=False at the command line -- see the docs on how to disable changing current working dir to job's output dir.
python main.py hydra.job.chdir=False
Related
I develop a project with the celery in python,and use the cython to compile the pyd files, every files is good but tasks.pyd, it fails to run. I think perhaps the bug is relate to the #decarator,and I try sererval methons with no solution, who can help me? thanks first.
here is my code in tasks.py
from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)
#app.task
def myTaskFunc(arg1=None, arg2=None):
some code
Traceback (most recent call last):
File "c:\users\en-18\appdata\local\programs\python\python36\lib\runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "c:\users\en-18\appdata\local\programs\python\python36\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\Users\en-18\AppData\Local\Programs\Python\Python36\Scripts\celery.exe\__main__.py", line 9, in <module>
File "c:\users\en-18\appdata\local\programs\python\python36\lib\site-packages\celery\__main__.py", line 16, in main
_main()
File "c:\users\en-18\appdata\local\programs\python\python36\lib\site-packages\celery\bin\celery.py", line 322, in main
cmd.execute_from_commandline(argv)
File "c:\users\en-18\appdata\local\programs\python\python36\lib\site-packages\celery\bin\celery.py", line 495, in execute_from_commandline
super(CeleryCommand, self).execute_from_commandline(argv)))
File "c:\users\en-18\appdata\local\programs\python\python36\lib\site-packages\celery\bin\base.py", line 305, in execute_from_commandline
return self.handle_argv(self.prog_name, argv[1:])
File "c:\users\en-18\appdata\local\programs\python\python36\lib\site-packages\celery\bin\celery.py", line 487, in handle_argv
return self.execute(command, argv)
File "c:\users\en-18\appdata\local\programs\python\python36\lib\site-packages\celery\bin\celery.py", line 419, in execute
).run_from_argv(self.prog_name, argv[1:], command=argv[0])
File "c:\users\en-18\appdata\local\programs\python\python36\lib\site-packages\celery\bin\worker.py", line 223, in run_from_argv
return self(*args, **options)
File "c:\users\en-18\appdata\local\programs\python\python36\lib\site-packages\celery\bin\base.py", line 253, in __call__
ret = self.run(*args, **kwargs)
File "c:\users\en-18\appdata\local\programs\python\python36\lib\site-packages\celery\bin\worker.py", line 258, in run
**kwargs)
File "c:\users\en-18\appdata\local\programs\python\python36\lib\site-packages\celery\worker\worker.py", line 97, in __init__
self.on_before_init(**kwargs)
File "c:\users\en-18\appdata\local\programs\python\python36\lib\site-packages\celery\apps\worker.py", line 93, in on_before_init
trace.setup_worker_optimizations(self.app, self.hostname)
File "c:\users\en-18\appdata\local\programs\python\python36\lib\site-packages\celery\app\trace.py", line 596, in setup_worker_optimizations
app.finalize()
File "c:\users\en-18\appdata\local\programs\python\python36\lib\site-packages\celery\app\base.py", line 538, in finalize
_announce_app_finalized(self)
File "c:\users\en-18\appdata\local\programs\python\python36\lib\site-packages\celery\_state.py", line 54, in _announce_app_finalized
callback(app)
File "c:\users\en-18\appdata\local\programs\python\python36\lib\site-packages\celery\app\base.py", line 411, in cons
return app._task_from_fun(fun, **opts)
File "c:\users\en-18\appdata\local\programs\python\python36\lib\site-packages\celery\app\base.py", line 452, in _task_from_fun
'__header__': staticmethod(head_from_fun(fun, bound=bind)),
File "c:\users\en-18\appdata\local\programs\python\python36\lib\site-packages\celery\utils\functional.py", line 279, in head_from_fun
namespace = {'__name__': fun.__module__}
AttributeError: 'method-wrapper' object has no attribute '__module__'
comtypes > Calling CoUnititialize()
comtypes > CoUnititialize() done.
It's impossible to know without a minimal reproducible example so what follows is a guess.
I suspect #app.task requires the function to have some introspectable attribute that Cython doesn't provide by default. Things to try:
Use the directive #cython: binding=True. This makes Cython functions a custom object type that behave more like a regular function.
Use Cython 3 alpha (although who knows what version of Cython you're using here, since you don't say). It contains some improvements to function introspection (and enables binding=True by default).
Write this bit of code in Python (you could use exec for this for example).
I want alembic to get the current stored procedure from DB compare it to the one in the code. If they are different to update the DB.
I edited the env.py file to check for this, and generate an upgrade versions of the raw SQL.
when running the following :
writer = rewriter.Rewriter()
#writer.rewrites(ops)
def stored_procedure(context, revision, op):
if not change_detected():
print("did not detect difference in stored procedure.")
return op
else:
print("detected difference in stored")
package_sql = open(migration_file_path, "r")
package_sql = migration_file.read()
package_sql = '"""' + package_sql + '"""'
return [
op,
op.get_bind().execute(package_sql),
]
Traceback (most recent call last):
File "c:\Project\.vscode\extensions\ms-python.python-2019.9.34911\pythonFiles\ptvsd_launcher.py", line 43, in <module>
main(ptvsdArgs)
File "c:\Project\.vscode\extensions\ms-python.python-2019.9.34911\pythonFiles\lib\python\ptvsd\__main__.py", line 432, in main
run()
File "c:\Project\.vscode\extensions\ms-python.python-2019.9.34911\pythonFiles\lib\python\ptvsd\__main__.py", line 316, in run_file
runpy.run_path(target, run_name='__main__')
File "C:\Project\Programs\Python\Python37\lib\runpy.py", line 263, in run_path
pkg_name=pkg_name, script_name=fname)
File "C:\Project\Programs\Python\Python37\lib\runpy.py", line 96, in _run_module_code
mod_name, mod_spec, pkg_name, script_name)
File "C:\Project\Programs\Python\Python37\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:/Project/manage.py", line 15, in <module>
manager.run()
...
File "C:\Project\ENV\lib\site-packages\alembic\runtime\migration.py", line 351, in run_migrations
for step in self._migrations_fn(heads, self):
File "C:\Project\ENV\lib\site-packages\alembic\command.py", line 173, in retrieve_migrations
revision_context.run_autogenerate(rev, context)
File "C:\Project\ENV\lib\site-packages\alembic\autogenerate\api.py", line 433, in run_autogenerate
self._run_environment(rev, migration_context, True)
File "C:\Project\ENV\lib\site-packages\alembic\autogenerate\api.py", line 483, in _run_environment
hook(migration_context, rev, self.generated_revisions)
File "migrations\env.py", line 167, in process_revision_directives
stored_procedure(context, revision, directives)
File "migrations\env.py", line 146, in stored_procedure
op.get_bind().execute(package_sql),
AttributeError: 'list' object has no attribute 'get_bind'
I'm using Python 2.7 with SQLAlchemy.
I'm trying to see if I can use Dogpile.Cache.
I've installed dogpile cache using
pip install dogpile.cache
but when I try to run the example given by SQLAlchemy (helloworld.py)
I keep getting ImportError: No module named fcntl. I thought the newest Dogpile.Cache resolved the windows issue...
Could someone please help?
Thank you.
UPDATE
Ok, the example is creating the database and cache. But when it is trying to load data using FromCache it is giving me an error.
people = Session.query(Person).options(FromCache("default")).all()
Here is the Traceback
Traceback (most recent call last):
File "C:\Python27\lib\runpy.py", line 162, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "C:\Python27\lib\runpy.py", line 72, in _run_code
exec code in run_globals
File "C:\Users\JKIM\Downloads\DogpileCacheTest\examples\dogpile_caching\helloworld.py", line 13, in <module>
people = Session.query(Person).options(FromCache("default")).all()
File "build\bdist.win-amd64\egg\sqlalchemy\orm\query.py", line 2613, in all
File "examples\caching_query.py", line 63, in __iter__
return self.get_value(createfunc=lambda: list(Query.__iter__(self)))
File "examples\caching_query.py", line 108, in get_value
expiration_time=expiration_time
File "C:\Python27\lib\site-packages\dogpile\cache\region.py", line 657, in get_or_create
async_creator) as value:
File "C:\Python27\lib\site-packages\dogpile\lock.py", line 154, in __enter__
return self._enter()
File "C:\Python27\lib\site-packages\dogpile\lock.py", line 87, in _enter
value = value_fn()
File "C:\Python27\lib\site-packages\dogpile\cache\region.py", line 610, in get_value
value = self.backend.get(key)
File "C:\Python27\lib\site-packages\dogpile\cache\backends\file.py", line 220, in get
with self._dbm_file(False) as dbm:
File "C:\Python27\lib\contextlib.py", line 17, in __enter__
return self.gen.next()
File "C:\Python27\lib\site-packages\dogpile\cache\backends\file.py", line 212, in _dbm_file
with self._use_rw_lock(write):
File "C:\Python27\lib\contextlib.py", line 17, in __enter__
return self.gen.next()
File "C:\Python27\lib\site-packages\dogpile\cache\backends\file.py", line 207, in _use_rw_lock
with self._rw_lock.read():
File "C:\Python27\lib\contextlib.py", line 17, in __enter__
return self.gen.next()
File "C:\Python27\lib\site-packages\dogpile\cache\backends\file.py", line 327, in read
self.acquire_read_lock(True)
File "C:\Python27\lib\site-packages\dogpile\cache\backends\file.py", line 408, in acquire_read_lock
return self._acquire(wait, os.O_RDONLY, self._module.LOCK_SH)
File "C:\Python27\lib\site-packages\dogpile\util\langhelpers.py", line 63, in __get__
obj.__dict__[self.__name__] = result = self.fget(obj)
File "C:\Python27\lib\site-packages\dogpile\cache\backends\file.py", line 400, in _module
import fcntl
ImportError: No module named fcntl
Setting up the testing framework and I can't seem to get past this error:
Traceback (most recent call last):
File "/usr/local/bin/nosetests", line 8, in <module>
load_entry_point('nose==1.1.2', 'console_scripts', 'nosetests')()
File "/Library/Python/2.7/site-packages/nose-1.1.2-py2.7.egg/nose/core.py", line 118, in __init__
**extra_args)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/unittest/main.py", line 94, in __init__
self.parseArgs(argv)
File "/Library/Python/2.7/site-packages/nose-1.1.2-py2.7.egg/nose/core.py", line 135, in parseArgs
self.config.configure(argv, doc=self.usage())
File "/Library/Python/2.7/site-packages/nose-1.1.2-py2.7.egg/nose/config.py", line 338, in configure
self.plugins.configure(options, self)
File "/Library/Python/2.7/site-packages/nose-1.1.2-py2.7.egg/nose/plugins/manager.py", line 271, in configure
cfg(options, config)
File "/Library/Python/2.7/site-packages/nose-1.1.2-py2.7.egg/nose/plugins/manager.py", line 94, in __call__
return self.call(*arg, **kw)
File "/Library/Python/2.7/site-packages/nose-1.1.2-py2.7.egg/nose/plugins/manager.py", line 162, in simple
result = meth(*arg, **kw)
File "build/bdist.macosx-10.7-intel/egg/nosegae.py", line 80, in configure
File "/usr/local/bin/dev_appserver.py", line 76, in <module>
DIR_PATH = get_dir_path(os.path.join('lib', 'ipaddr'))
File "/usr/local/bin/dev_appserver.py", line 66, in get_dir_path
'file and %s.' % sibling)
ValueError: Could not determine directory that contains both, this file and lib/ipaddr.
It's a correct error coming from dev_appserver.py... there is no lib/ipaddr in the /usr/local/bin directory.
Why is nosetests triggering the raising of this error? Why does dev_appengine try to look for lib/ipaddr in the first place?
Trying to upload simple hello world app, I get the following error:
Traceback (most recent call last):
File "./appcfg.py", line 133, in <module>
run_file(__file__, globals())
File "./appcfg.py", line 129, in run_file
execfile(script_path, globals_)
File "/home/mike/Documents/coder/Go/google_appengine/google/appengine/tools/appcfg.py", line 4083, in <module>
main(sys.argv)
File "/home/mike/Documents/coder/Go/google_appengine/google/appengine/tools/appcfg.py", line 4074, in main
result = AppCfgApp(argv).Run()
File "/home/mike/Documents/coder/Go/google_appengine/google/appengine/tools/appcfg.py", line 2544, in Run
self.action(self)
File "/home/mike/Documents/coder/Go/google_appengine/google/appengine/tools/appcfg.py", line 3830, in __call__
return method()
File "/home/mike/Documents/coder/Go/google_appengine/google/appengine/tools/appcfg.py", line 2991, in Update
self.UpdateVersion(rpcserver, self.basepath, appyaml)
File "/home/mike/Documents/coder/Go/google_appengine/google/appengine/tools/appcfg.py", line 2974, in UpdateVersion
lambda path: self.opener(os.path.join(basepath, path), 'rb'))
File "/home/mike/Documents/coder/Go/google_appengine/google/appengine/tools/appcfg.py", line 2131, in DoUpload
resource_limits = GetResourceLimits(self.rpcserver, self.config)
File "/home/mike/Documents/coder/Go/google_appengine/google/appengine/tools/appcfg.py", line 355, in GetResourceLimits
resource_limits.update(GetRemoteResourceLimits(rpcserver, config))
File "/home/mike/Documents/coder/Go/google_appengine/google/appengine/tools/appcfg.py", line 326, in GetRemoteResourceLimits
version=config.version)
File "/home/mike/Documents/coder/Go/google_appengine/google/appengine/tools/appengine_rpc.py", line 379, in Send
self._Authenticate()
File "/home/mike/Documents/coder/Go/google_appengine/google/appengine/tools/appengine_rpc.py", line 437, in _Authenticate
super(HttpRpcServer, self)._Authenticate()
File "/home/mike/Documents/coder/Go/google_appengine/google/appengine/tools/appengine_rpc.py", line 281, in _Authenticate
auth_token = self._GetAuthToken(credentials[0], credentials[1])
File "/home/mike/Documents/coder/Go/google_appengine/google/appengine/tools/appengine_rpc.py", line 233, in _GetAuthToken
e.headers, response_dict)
File "/home/mike/Documents/coder/Go/google_appengine/google/appengine/tools/appengine_rpc.py", line 94, in __init__
self.reason = args["Error"]
AttributeError: can't set attribute
The e-mail and password are correct. The app works perfectly on my local machine.
OK, finally I found the solution - a very simple one:
it appears that google appp engine is not satisfied with the regular account password, it needs the application-specific one which can be set up here:
http://support.google.com/accounts/bin/static.py?hl=en&page=guide.cs&guide=1056283&answer=185833&rd=3