This is my code (it is a class whose job is to save to a MS Sharepoint folder some Excel files).
class BackupOperation:
def __init__(self, sharepoint: Sharepoint):
self.sharepoint = sharepoint
def run(self, names: list[str) -> None:
pool = multiprocessing.Pool(processes=8)
pool.map(self._backup_name, names)
pool.close()
pool.join()
def _backup_name(self, name: str) -> None:
# do things with Office365-REST-Python-Client
# ...
My code runs smoothly on Windows 11, but now I have deployed it in a container that runs Linux and it outputs the following error: "AttributeError: Can't pickle local object 'ClientRuntimeContext.after_query_execute.._process_response'"
This is the full traceback:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/local/lib/python3.9/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/application/dcr-0.0.0/dcr/__main__.py", line 6, in <module>
Trigger.run()
File "/application/dcr-0.0.0/dcr/tools/trigger/trigger.py", line 47, in run
Pipeline.run()
File "/application/dcr-0.0.0/dcr/controller/pipeline.py", line 55, in run
SharepointBackupper.run()
File "/application/dcr-
0.0.0/dcr/controller/steps/sharepoint_backupper/sharepoint_backupper.py", line 31, in run
backup_operation.run(names=names)
File "/application/dcr-0.0.0/dcr/controller/steps/sharepoint_backupper/backup_operation.py", line 37, in run
pool.map(self._backup_name, names)
File "/usr/local/lib/python3.9/multiprocessing/pool.py", line 364, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "/usr/local/lib/python3.9/multiprocessing/pool.py", line 771, in get
raise self._value
File "/usr/local/lib/python3.9/multiprocessing/pool.py", line 537, in _handle_tasks
put(task)
File "/usr/local/lib/python3.9/multiprocessing/connection.py", line 211, in send
self._send_bytes(_ForkingPickler.dumps(obj))
File "/usr/local/lib/python3.9/multiprocessing/reduction.py", line 51, in dumps
cls(buf, protocol).dump(obj)
AttributeError: Can't pickle local object 'ClientRuntimeContext.after_query_execute.<locals>._process_response'
Why is this happening? From the traceback we can see that the problem arises in the moment it executes pool.map(self._backup_name, names)
Related
I am developing a code where I need to create a Manager to synchronize a Queue. In the project I am using hydra and everything works fine until I create a multiprocessing Manager Object. This is the raised error with HYDRA_FULL_ERROR=1:
Primary config module 'config' not found.
Check that it's correct and contains an __init__.py file
Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.
Traceback (most recent call last):
File "mre.py", line 8, in <module>
main()
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\main.py", line 32, in decorated_main
_run_hydra(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\utils.py", line 355, in _run_hydra
run_and_report(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\utils.py", line 207, in run_and_report
return func()
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\utils.py", line 356, in <lambda>
lambda: hydra.run(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\hydra.py", line 107, in run
return run_job(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\core\utils.py", line 125, in run_job
ret.return_value = task_function(task_cfg)
File "mre.py", line 7, in main
m = Manager()
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\multiprocessing\context.py", line 57, in Manager
m.start()
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\multiprocessing\managers.py", line 583, in start
self._address = reader.recv()
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\multiprocessing\connection.py", line 250, in recv
buf = self._recv_bytes()
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\multiprocessing\connection.py", line 305, in _recv_bytes
waitres = _winapi.WaitForMultipleObjects(
KeyboardInterrupt
^C
C:\Users\UX331U\Desktop>SET HYDRA_FULL_ERROR=1
C:\Users\UX331U\Desktop>python mre.py
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\multiprocessing\spawn.py", line 116, in spawn_main
exitcode = _main(fd, parent_sentinel)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\multiprocessing\spawn.py", line 125, in _main
prepare(preparation_data)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\multiprocessing\spawn.py", line 236, in prepare
_fixup_main_from_path(data['init_main_from_path'])
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\multiprocessing\spawn.py", line 287, in _fixup_main_from_path
main_content = runpy.run_path(main_path,
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\runpy.py", line 265, in run_path
return _run_module_code(code, init_globals, run_name,
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\runpy.py", line 97, in _run_module_code
_run_code(code, mod_globals, init_globals,
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\runpy.py", line 87, in _run_code
exec(code, run_globals)
File "C:\Users\UX331U\Desktop\mre.py", line 8, in <module>
main()
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\main.py", line 32, in decorated_main
_run_hydra(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\utils.py", line 355, in _run_hydra
run_and_report(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\utils.py", line 210, in run_and_report
raise ex
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\utils.py", line 207, in run_and_report
return func()
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\utils.py", line 356, in <lambda>
lambda: hydra.run(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\hydra.py", line 100, in run
cfg = self.compose_config(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\hydra.py", line 505, in compose_config
self.config_loader.ensure_main_config_source_available()
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\config_loader_impl.py", line 135, in ensure_main_config_source_available
self.missing_config_error(
File "C:\Users\UX331U\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\LocalCache\local-packages\Python38\site-packages\hydra\_internal\config_loader_impl.py", line 109, in missing_config_error
raise MissingConfigException(
hydra.errors.MissingConfigException: Primary config module 'config' not found.
Check that it's correct and contains an __init__.py file
I also attach a minimal code snippet to reproduce the error.
import hydra
from multiprocessing import Manager
#hydra.main(config_path='config', config_name='config')
def main(cfg):
m = Manager() # The error is in this line
m.Queue()
main()
The config.yaml is correctly created and located since when I don't use the Manager, the entire code works with no errors:
- PROJECT ----- config ---- config.yaml
|
|
----- main.py
As far as I have understood, the problem comes from the fact that hydra changes the directory when initialized and afterwards, when the Manager is created, hydra is called again to be initialized but as the directory has been changed, it can not find the config folder. If this is true, is it a bug or can it be solved?
Thank you so much for your help.
I'm not able to reproduce the error on my machine.
You can stop hydra from changing directories at runtime by passing hydra.job.chdir=False at the command line -- see the docs on how to disable changing current working dir to job's output dir.
python main.py hydra.job.chdir=False
What I'm trying to do:
In init.py I have a function create_app that needs to call a function called init_tokenizer that uses the global dict g and I get the following errors when I use any flask commands such as flask --version or flask init-db (a command I created and must use):
When I tried to do:
def init_tokenizer(app):
...
with app.app_context():
g.starts = starts
g.ends = ends
g.tokenizer = tokenizer
I get the error:
Traceback (most recent call last):
File "C:\Users\yonik\anaconda3\envs\venv\lib\runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "C:\Users\yonik\anaconda3\envs\venv\lib\runpy.py", line 87, in _run_code
exec(code, run_globals)
File "C:\Users\yonik\anaconda3\envs\venv\Scripts\flask.exe\__main__.py", line 7, in <module>
File "C:\Users\yonik\anaconda3\envs\venv\lib\site-packages\flask\cli.py", line 988, in main
cli.main()
File "C:\Users\yonik\anaconda3\envs\venv\lib\site-packages\flask\cli.py", line 579, in main
return super().main(*args, **kwargs)
File "C:\Users\yonik\anaconda3\envs\venv\lib\site-packages\click\core.py", line 1055, in main
rv = self.invoke(ctx)
File "C:\Users\yonik\anaconda3\envs\venv\lib\site-packages\click\core.py", line 1655, in
invoke
sub_ctx = cmd.make_context(cmd_name, args, parent=ctx)
AttributeError: 'function' object has no attribute 'make_context'
And when I tried to do:
#with_appcontext
def init_tokenizer()
Or:
starts, ends, my_tokenizer = tokenizer.init_tokenizer(app)
with app.app_context():
g.starts = starts
g.ends = ends
g.my_tokenizer = my_tokenizer
I got the infinite recursion:
File "C:\Users\yonik\anaconda3\envs\venv\lib\site-packages\flask\cli.py", line 77, in
find_best_app
app = app_factory()
File "C:\yoni\final_project\web_app\flaskr\__init__.py", line 34, in create_app
tokenizer.init_tokenizer(app)
File "C:\Users\yonik\anaconda3\envs\venv\lib\site-packages\click\decorators.py", line 26, in
new_func
return f(get_current_context(), *args, **kwargs)
File "C:\Users\yonik\anaconda3\envs\venv\lib\site-packages\flask\cli.py", line 426, in
decorator
with __ctx.ensure_object(ScriptInfo).load_app().app_context():
File "C:\Users\yonik\anaconda3\envs\venv\lib\site-packages\flask\cli.py", line 389, in
load_app
app = locate_app(import_name, name)
File "C:\Users\yonik\anaconda3\envs\venv\lib\site-packages\flask\cli.py", line 251, in
locate_app
return find_best_app(module)
File "C:\Users\yonik\anaconda3\envs\venv\lib\site-packages\flask\cli.py", line 77, in
find_best_app
app = app_factory()
I use windows 10, Python 3.9.12, Flask 2.1.2, Werkzeug 2.1.2
Solution:
Turns out that because I already deleted and re-created the database with the command: init-db I can't use the command again before there is something in the database.
So I'm using vininfo to populate information about some units from an external DB using their VIN only and to work with this data I am converting it to a dict using annotate() or annotate().items() but the script breaks down on
TypeError: cannot unpack non-iterable NoneType object
Tried using hasattr(obj, 'attr') but it explodes there with the same error. Cannot iterate with a for..in because the object is not an iterable TypeError: 'NoneType' object is not iterable.
I tried on the CLI version with the same VIN and it spits out the same cannot unpack error.
The thing is this VIN I'm using to test is missing some data .transmission to be precise (but I guess it'll happen with any other field), probably due to a misspelling when filling the registration form so I'm trying to ignore this field but I can't because it doesn't exist!
The VIN is: 1NKZXPEX4GJ109010
The Traceback is the following:
Traceback (most recent call last):
File "c:\program files\python37\lib\runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "c:\program files\python37\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\Program Files\Python37\Scripts\vininfo.exe\__main__.py", line 7, in <module>
File "c:\program files\python37\lib\site-packages\vininfo\cli.py", line 51, in main
entry_point(obj={})
File "c:\program files\python37\lib\site-packages\click\core.py", line 764, in __call__
return self.main(*args, **kwargs)
File "c:\program files\python37\lib\site-packages\click\core.py", line 717, in main
rv = self.invoke(ctx)
File "c:\program files\python37\lib\site-packages\click\core.py", line 1137, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "c:\program files\python37\lib\site-packages\click\core.py", line 956, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "c:\program files\python37\lib\site-packages\click\core.py", line 555, in invoke
return callback(*args, **kwargs)
File "c:\program files\python37\lib\site-packages\vininfo\cli.py", line 35, in show
out(details)
File "c:\program files\python37\lib\site-packages\vininfo\cli.py", line 24, in out
for k, v in annotatable.annotate().items():
File "c:\program files\python37\lib\site-packages\vininfo\common.py", line 17, in annotate
value = getattr(self, attr_name, no_attr)
File "c:\program files\python37\lib\site-packages\vininfo\details\_base.py", line 53, in __get__
return DetailWrapper(instance, self)
File "c:\program files\python37\lib\site-packages\vininfo\details\_base.py", line 20, in __init__
attr_name, attr_idx = detail.source
TypeError: cannot unpack non-iterable NoneType object
You can force transmission to an empty list:
>>> vin = vininfo.Vin("1NKZXPEX4GJ109010")
>>> vin.details.transmission = []
>>> vin.details.annotate()
OrderedDict([('Body', 'E'), ('Engine', 'Z'), ('Model', 'X'), ('Plant', 'J'), ('Serial', '109010'), ('Transmission', '')])
I'm using Python 2.7 with SQLAlchemy.
I'm trying to see if I can use Dogpile.Cache.
I've installed dogpile cache using
pip install dogpile.cache
but when I try to run the example given by SQLAlchemy (helloworld.py)
I keep getting ImportError: No module named fcntl. I thought the newest Dogpile.Cache resolved the windows issue...
Could someone please help?
Thank you.
UPDATE
Ok, the example is creating the database and cache. But when it is trying to load data using FromCache it is giving me an error.
people = Session.query(Person).options(FromCache("default")).all()
Here is the Traceback
Traceback (most recent call last):
File "C:\Python27\lib\runpy.py", line 162, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "C:\Python27\lib\runpy.py", line 72, in _run_code
exec code in run_globals
File "C:\Users\JKIM\Downloads\DogpileCacheTest\examples\dogpile_caching\helloworld.py", line 13, in <module>
people = Session.query(Person).options(FromCache("default")).all()
File "build\bdist.win-amd64\egg\sqlalchemy\orm\query.py", line 2613, in all
File "examples\caching_query.py", line 63, in __iter__
return self.get_value(createfunc=lambda: list(Query.__iter__(self)))
File "examples\caching_query.py", line 108, in get_value
expiration_time=expiration_time
File "C:\Python27\lib\site-packages\dogpile\cache\region.py", line 657, in get_or_create
async_creator) as value:
File "C:\Python27\lib\site-packages\dogpile\lock.py", line 154, in __enter__
return self._enter()
File "C:\Python27\lib\site-packages\dogpile\lock.py", line 87, in _enter
value = value_fn()
File "C:\Python27\lib\site-packages\dogpile\cache\region.py", line 610, in get_value
value = self.backend.get(key)
File "C:\Python27\lib\site-packages\dogpile\cache\backends\file.py", line 220, in get
with self._dbm_file(False) as dbm:
File "C:\Python27\lib\contextlib.py", line 17, in __enter__
return self.gen.next()
File "C:\Python27\lib\site-packages\dogpile\cache\backends\file.py", line 212, in _dbm_file
with self._use_rw_lock(write):
File "C:\Python27\lib\contextlib.py", line 17, in __enter__
return self.gen.next()
File "C:\Python27\lib\site-packages\dogpile\cache\backends\file.py", line 207, in _use_rw_lock
with self._rw_lock.read():
File "C:\Python27\lib\contextlib.py", line 17, in __enter__
return self.gen.next()
File "C:\Python27\lib\site-packages\dogpile\cache\backends\file.py", line 327, in read
self.acquire_read_lock(True)
File "C:\Python27\lib\site-packages\dogpile\cache\backends\file.py", line 408, in acquire_read_lock
return self._acquire(wait, os.O_RDONLY, self._module.LOCK_SH)
File "C:\Python27\lib\site-packages\dogpile\util\langhelpers.py", line 63, in __get__
obj.__dict__[self.__name__] = result = self.fget(obj)
File "C:\Python27\lib\site-packages\dogpile\cache\backends\file.py", line 400, in _module
import fcntl
ImportError: No module named fcntl
Setting up the testing framework and I can't seem to get past this error:
Traceback (most recent call last):
File "/usr/local/bin/nosetests", line 8, in <module>
load_entry_point('nose==1.1.2', 'console_scripts', 'nosetests')()
File "/Library/Python/2.7/site-packages/nose-1.1.2-py2.7.egg/nose/core.py", line 118, in __init__
**extra_args)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/unittest/main.py", line 94, in __init__
self.parseArgs(argv)
File "/Library/Python/2.7/site-packages/nose-1.1.2-py2.7.egg/nose/core.py", line 135, in parseArgs
self.config.configure(argv, doc=self.usage())
File "/Library/Python/2.7/site-packages/nose-1.1.2-py2.7.egg/nose/config.py", line 338, in configure
self.plugins.configure(options, self)
File "/Library/Python/2.7/site-packages/nose-1.1.2-py2.7.egg/nose/plugins/manager.py", line 271, in configure
cfg(options, config)
File "/Library/Python/2.7/site-packages/nose-1.1.2-py2.7.egg/nose/plugins/manager.py", line 94, in __call__
return self.call(*arg, **kw)
File "/Library/Python/2.7/site-packages/nose-1.1.2-py2.7.egg/nose/plugins/manager.py", line 162, in simple
result = meth(*arg, **kw)
File "build/bdist.macosx-10.7-intel/egg/nosegae.py", line 80, in configure
File "/usr/local/bin/dev_appserver.py", line 76, in <module>
DIR_PATH = get_dir_path(os.path.join('lib', 'ipaddr'))
File "/usr/local/bin/dev_appserver.py", line 66, in get_dir_path
'file and %s.' % sibling)
ValueError: Could not determine directory that contains both, this file and lib/ipaddr.
It's a correct error coming from dev_appserver.py... there is no lib/ipaddr in the /usr/local/bin directory.
Why is nosetests triggering the raising of this error? Why does dev_appengine try to look for lib/ipaddr in the first place?