In my windows command prompt "python3 -m scrapy shell "https://scrapy.org"" works as expected and gives a interactive shell to work with as expected.
However if i open windows terminal and type "ubuntu" using windows subsystem for linux and then try to launch scrapy shell with "python3 -m scrapy shell 'https://scrapy.org'" or "scrapy shell 'https://scrapy.org'" i get the following error message:
Traceback (most recent call last):
File "/usr/lib/python3.8/runpy.py", line 193, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 86, in _run_code
exec(code, run_globals)
File "/mnt/e/scrapy-project/miniscrapy/tutorial/env/lib/python3.8/site-packages/scrapy/__main__.py", line 4, in <module>
execute()
File "/mnt/e/scrapy-project/miniscrapy/tutorial/env/lib/python3.8/site-packages/scrapy/cmdline.py", line 145, in execute
_run_print_help(parser, _run_command, cmd, args, opts)
File "/mnt/e/scrapy-project/miniscrapy/tutorial/env/lib/python3.8/site-packages/scrapy/cmdline.py", line 100, in _run_print_help
func(*a, **kw)
File "/mnt/e/scrapy-project/miniscrapy/tutorial/env/lib/python3.8/site-packages/scrapy/cmdline.py", line 153, in _run_command
cmd.run(args, opts)
File "/mnt/e/scrapy-project/miniscrapy/tutorial/env/lib/python3.8/site-packages/scrapy/commands/shell.py", line 68, in run
crawler.engine = crawler._create_engine()
File "/mnt/e/scrapy-project/miniscrapy/tutorial/env/lib/python3.8/site-packages/scrapy/crawler.py", line 101, in _create_engine
return ExecutionEngine(self, lambda _: self.stop())
File "/mnt/e/scrapy-project/miniscrapy/tutorial/env/lib/python3.8/site-packages/scrapy/core/engine.py", line 70, in __init__
self.scraper = Scraper(crawler)
File "/mnt/e/scrapy-project/miniscrapy/tutorial/env/lib/python3.8/site-packages/scrapy/core/scraper.py", line 71, in __init__
self.itemproc = itemproc_cls.from_crawler(crawler)
File "/mnt/e/scrapy-project/miniscrapy/tutorial/env/lib/python3.8/site-packages/scrapy/middleware.py", line 53, in from_crawler
return cls.from_settings(crawler.settings, crawler)
File "/mnt/e/scrapy-project/miniscrapy/tutorial/env/lib/python3.8/site-packages/scrapy/middleware.py", line 35, in from_settings
mw = create_instance(mwcls, settings, crawler)
File "/mnt/e/scrapy-project/miniscrapy/tutorial/env/lib/python3.8/site-packages/scrapy/utils/misc.py", line 156, in create_instance
instance = objcls.from_crawler(crawler, *args, **kwargs)
File "/mnt/e/scrapy-project/miniscrapy/tutorial/env/lib/python3.8/site-packages/scrapy/pipelines/media.py", line 68, in from_crawler
pipe = cls.from_settings(crawler.settings)
File "/mnt/e/scrapy-project/miniscrapy/tutorial/env/lib/python3.8/site-packages/scrapy/pipelines/images.py", line 105, in from_settings
return cls(store_uri, settings=settings)
File "/mnt/e/scrapy-project/miniscrapy/tutorial/env/lib/python3.8/site-packages/scrapy/pipelines/images.py", line 48, in __init__
super(ImagesPipeline, self).__init__(store_uri, settings=settings,
File "/mnt/e/scrapy-project/miniscrapy/tutorial/env/lib/python3.8/site-packages/scrapy/pipelines/files.py", line 361, in __init__
self.store = self._get_store(store_uri)
File "/mnt/e/scrapy-project/miniscrapy/tutorial/env/lib/python3.8/site-packages/scrapy/pipelines/files.py", line 409, in _get_store
store_cls = self.STORE_SCHEMES[scheme]
KeyError: 'e'
in lib/python3.8/site-packages/scrapy/pipelines/files.py i added the following print messages to debug:
which prints:
uri: E:/scrapy-project/miniscrapy/tutorial/images
_get_store pipelines/files scheme: e
i already tried setting my path with "export PATH=$PATH:/usr/local/bin" as well as storing it in ~/.bashrc. All python related libraries are installed with pip inside a virtual environment in the tutorial folder which contains the /images folders. I suppose the os.path.isabs() method functions when just running regular windows command prompt and somehow fails on my non-real ubuntu.
echo $PATH inside virtual environment:
/mnt/e/scrapy-project/miniscrapy/tutorial/env/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games
Related
Trying to run python fabric 2 in windows 10
from fabric import task,SerialGroup,Connection
import os
USERNAME = os.getenv('USERNAME')
my_hosts = ["hostname"]
c = Connection(host="rg#host.com",connect_kwargs={"password":"abcd"})
#task
def test(c):
print(USERNAME)
c.run("echo hello")
Giving the below error
Traceback (most recent call last):
File "c:\users\rg\programs\python\python38\lib\runpy.py", line
194, in _run_module_as_main
return _run_code(code, main_globals, None, File "c:\users\rg\programs\python\python38\lib\runpy.py", line
87, in run_code
exec(code, run_globals) File
"C:\Users\rg\Programs\Python\Python38\Scripts\fab.exe_main.py",
line 7, in File
"c:\users\rg\programs\python\python38\lib\site-packages\invoke\program.py",
line 384, in run
self.execute() File
"c:\users\rg\programs\python\python38\lib\site-packages\invoke\program.py",
line 566, in execute
executor.execute(*self.tasks) File
"c:\users\rg\programs\python\python38\lib\site-packages\invoke\executor.py",
line 129, in execute
result = call.task(*args, **call.kwargs) File
"c:\users\rg\programs\python\python38\lib\site-packages\invoke\tasks.py",
line 127, in call
result = self.body(*args, **kwargs) File
"C:\Users\rg\Documents\Solr\python_scripts\Solr\fabfiles\fabfile.py",
line 10, in test
c.run("echo hello") File
"c:\users\rg\programs\python\python38\lib\site-packages\invoke\context.py",
line 95, in run
return self._run(runner, command, **kwargs) File
"c:\users\rg\programs\python\python38\lib\site-packages\invoke\context.py",
line 102, in _run
return runner.run(command, **kwargs) File
"c:\users\rg\programs\python\python38\lib\site-packages\invoke\runners.py",
line 363, in run
return self._run_body(command, **kwargs) File
"c:\users\rg\programs\python\python38\lib\site-packages\invoke\runners.py",
line 411, in _run_body
self.start(command, self.opts["shell"], self.env) File
"c:\users\rg\programs\python\python38\lib\site-packages\invoke\runners.py",
line 1271, in start
self.process = Popen( File "c:\users\rg\programs\python\python38\lib\subprocess.py",
line 858, in init
self._execute_child(args, executable, preexec_fn, close_fds, File "c:\users\rg\programs\python\python38\lib\subprocess.py",
line 1311, in _execute_child
hp, ht, pid, tid = _winapi.CreateProcess(executable, args, OSError: [WinError 87] The parameter is incorrect
This is a sort-of-a-bug in fabric (as of 2.6.0),
see the discussion here:
https://github.com/fabric/fabric/issues/2142
Apparently, it shows up with Python 3.8 or 3.9, but not with 3.6 or 3.7.
To get around it, you can use c.run("echo hello", replace_env=False).
This is OK for local calls, but can be problematic for remote calls, because it means that the remote session will see all environment variable values (some of which may be sensitive) of your local session.
I was trying to run docker-compose using Fabric, but when I run the docker-compose up command using Fabric I got the following error:
[52577] Failed to execute script docker-compose
Traceback (most recent call last):
File "docker-compose", line 6, in <module>
File "compose/cli/main.py", line 71, in main
File "compose/cli/main.py", line 127, in perform_command
File "compose/cli/main.py", line 1080, in up
File "compose/cli/main.py", line 1076, in up
File "compose/project.py", line 475, in up
File "compose/service.py", line 352, in ensure_image_exists
File "compose/service.py", line 1217, in pull
File "compose/progress_stream.py", line 101, in get_digest_from_pull
File "compose/service.py", line 1182, in _do_pull
File "site-packages/docker/api/image.py", line 381, in pull
File "site-packages/docker/auth.py", line 48, in get_config_header
File "site-packages/docker/auth.py", line 96, in resolve_authconfig
File "site-packages/docker/auth.py", line 127, in _resolve_authconfig_credstore
File "site-packages/dockerpycreds/store.py", line 20, in __init__
File "site-packages/dockerpycreds/utils.py", line 12, in find_executable
File "distutils/spawn.py", line 176, in find_executable
File "os.py", line 669, in __getitem__
KeyError: 'PATH'
Failed to up the env
When I run docker-compose up from the terminal it works fine.
#task
def up(c):
if c.run('docker-compose up', warn=True).failed:
print ("Failed to up the env")
As my comment suggested, it seems docker-composer up when it needs to do a build at least, needs the PATH environment variable to be set. To fix, I just forwarded the PATH from the current session to the child process (in PHP sorry!):
$compose = new Process( 'docker-compose up -d', '', [
'VOLUME' => getcwd(),
'PATH' => getenv( 'PATH' ),
] );
I have referred to this tut to get started with pyspark on windows. These are the steps I followed:
Downloaded spark prebuilt for hadoop 2.7 from here
Extracted spark-2.1.0-bin-hadoop2.7.tgz to the directory set as %SPARK_HOME% in environment variables
Downloaded winutils.exe from here
Pasted winutils.exe in %SPARK_HOME%\bin
Set %HADOOP_HOME% to same directory as %SPARK_HOME%
Set %PYSPARK_DRIVER_PYTHON% to ipython
Set %PYSPARK_DRIVER_PYTHON_OPTS% to notebook
Added ;%SPARK_HOME%\bin to %PATH%
But when I run
> pyspark --master local[2]
I get following error:
[TerminalIPythonApp] WARNING | Subcommand `ipython notebook` is deprecated and will be removed in future versions.
[TerminalIPythonApp] WARNING | You likely want to use `jupyter notebook` in the future
Traceback (most recent call last):
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\runpy.py", line 170, in _run_module_as_main
"__main__", mod_spec)
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "D:\mahesh\Softwares\python\WinPython-64bit-3.4.4.4Qt5\python-3.4.4.amd64\Scripts\ipython.exe\__main__.py", line 9, in <module>
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\site-packages\IPython\__init__.py", line 125, in start_ipython
return launch_new_instance(argv=argv, **kwargs)
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\site-packages\traitlets\config\application.py", line 657, in launch_instance
app.initialize(argv)
File "<decorator-gen-113>", line 2, in initialize
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\site-packages\traitlets\config\application.py", line 87, in catch_config_error
return method(app, *args, **kwargs)
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\site-packages\IPython\terminal\ipapp.py", line 308, in initialize
super(TerminalIPythonApp, self).initialize(argv)
File "<decorator-gen-7>", line 2, in initialize
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\site-packages\traitlets\config\application.py", line 87, in catch_config_error
return method(app, *args, **kwargs)
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\site-packages\IPython\core\application.py", line 450, in initialize
self.parse_command_line(argv)
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\site-packages\IPython\terminal\ipapp.py", line 303, in parse_command_line
return super(TerminalIPythonApp, self).parse_command_line(argv)
File "<decorator-gen-4>", line 2, in parse_command_line
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\site-packages\traitlets\config\application.py", line 87, in catch_config_error
return method(app, *args, **kwargs)
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\site-packages\traitlets\config\application.py", line 514, in parse_command_line
return self.initialize_subcommand(subc, subargv)
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\site-packages\IPython\core\application.py", line 243, in initialize_subcommand
return super(BaseIPythonApplication, self).initialize_subcommand(subc, argv)
File "<decorator-gen-3>", line 2, in initialize_subcommand
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\site-packages\traitlets\config\application.py", line 87, in catch_config_error
return method(app, *args, **kwargs)
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\site-packages\traitlets\config\application.py", line 445, in initialize_subcommand
subapp = import_item(subapp)
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\site-packages\ipython_genutils\importstring.py", line 31, in import_item
module = __import__(package, fromlist=[obj])
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\site-packages\notebook\notebookapp.py", line 31, in <module>
from zmq.eventloop import ioloop
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\site-packages\zmq\eventloop\__init__.py", line 3, in <module>
from zmq.eventloop.ioloop import IOLoop
File "d:\mahesh\softwares\python\winpython-64bit-3.4.4.4qt5\python-3.4.4.amd64\lib\site-packages\zmq\eventloop\ioloop.py", line 21, in <module>
from zmq import (
ImportError: cannot import name 'Poller'
I am correctly able to run spark scala shell with >spark-shell command.
As you can see in the stack trace, I had win-python installed already at path
D:\mahesh\Softwares\python\WinPython-64bit-3.4.4.4Qt5\python-3.4.4.amd64
Thus, my %PYTHON_HOME% is D:\mahesh\Softwares\python\WinPython-64bit-3.4.4.4Qt5.
But my %SPARK_HOME% is D:\mahesh\Programs\spark-2.3.0-bin-hadoop2.7.
Running where pyspark command gives following output:
D:\mahesh\Programs\spark-2.3.0-bin-hadoop2.7\bin\pyspark
D:\mahesh\Programs\spark-2.3.0-bin-hadoop2.7\bin\pyspark.cmd
D:\mahesh\Softwares\python\WinPython-64bit-3.4.4.4Qt5\python-3.4.4.amd64\Scripts\pyspark
D:\mahesh\Softwares\python\WinPython-64bit-3.4.4.4Qt5\python-3.4.4.amd64\Scripts\pyspark.cmd
I believe my issue is some missconfiguration of my windows spark environment. Thats why I gave all above information. So whats going wrong here?
Note that I performed the steps without using Anaconda and GOW (Gnu on windows) as suggested in the tut.
Point your %PYSPARK_DRIVER_PYTHON% to a virtual environment with all dependencies including 'Poller' and then check.
Else you can try installing 'Poller' in ipython environment( which I frankly dont know how!)
Building on top of How to assert output with nosetest/unittest in python? I would like to acieve this inside Pyharm. However, pycharm does not run:
$ python -m tests.test_mymodule --buffer
Rather it does:
$ /usr/bin/python2.7 /opt/helpers/pycharm/utrunner.py \
/home/oz123/PycharmProjects/account/tests/test_mymodule.py true
So, I ran it with:
$ /usr/bin/python2.7 /opt/helpers/pycharm/utrunner.py \
/home/oz123/PycharmProjects/account/tests/test_mymodule.py --buffer true
And it crashes with:
##teamcity[testFinished duration='72' name='test_1_list_files']
Traceback (most recent call last):
File "/opt/helpers/pycharm/utrunner.py", line 151, in <module>
TeamcityTestRunner().run(all, **options)
File "/opt/helpers/pycharm/tcunittest.py", line 249, in run
test(result)
File "/usr/lib/python2.7/unittest/suite.py", line 70, in __call__
return self.run(*args, **kwds)
File "/usr/lib/python2.7/unittest/suite.py", line 108, in run
test(result)
File "/usr/lib/python2.7/unittest/suite.py", line 70, in __call__
return self.run(*args, **kwds)
File "/usr/lib/python2.7/unittest/suite.py", line 108, in run
test(result)
File "/usr/lib/python2.7/unittest/case.py", line 396, in __call__
return self.run(*args, **kwds)
File "/usr/lib/python2.7/unittest/case.py", line 356, in run
result.addError(self, sys.exc_info())
File "/opt/helpers/pycharm/tcunittest.py", line 108, in addError
TestResult.addError(self, test, err)
File "/usr/lib/python2.7/unittest/result.py", line 19, in inner
return method(self, *args, **kw)
File "/usr/lib/python2.7/unittest/result.py", line 116, in addError
self.errors.append((test, self._exc_info_to_string(err, test)))
File "/usr/lib/python2.7/unittest/result.py", line 167, in _exc_info_to_string
output = sys.stdout.getvalue()
AttributeError: 'file' object has no attribute 'getvalue'
Does any one know where does pycharm store the output? How can I access it?
I have my scrapy bot running on two different systems. One of them is working properly while the other one isn't. They're identical copies. When I use -t csv -o data.csv, I get the following traceback.
Traceback (most recent call last):
File "/home/scraper/.python/bin/scrapy", line 4, in <module>
execute()
File "/home/scraper/.python/lib/python2.7/site-packages/scrapy/cmdline.py", line 143, in execute
_run_print_help(parser, _run_command, cmd, args, opts)
File "/home/scraper/.python/lib/python2.7/site-packages/scrapy/cmdline.py", line 89, in _run_print_help
func(*a, **kw)
File "/home/scraper/.python/lib/python2.7/site-packages/scrapy/cmdline.py", line 150, in _run_command
cmd.run(args, opts)
File "/home/scraper/.python/lib/python2.7/site-packages/scrapy/commands/crawl.py", line 50, in run
self.crawler_process.start()
File "/home/scraper/.python/lib/python2.7/site-packages/scrapy/crawler.py", line 92, in start
if self.start_crawling():
File "/home/scraper/.python/lib/python2.7/site-packages/scrapy/crawler.py", line 124, in start_crawling
return self._start_crawler() is not None
File "/home/scraper/.python/lib/python2.7/site-packages/scrapy/crawler.py", line 139, in _start_crawler
crawler.configure()
File "/home/scraper/.python/lib/python2.7/site-packages/scrapy/crawler.py", line 46, in configure
self.extensions = ExtensionManager.from_crawler(self)
File "/home/scraper/.python/lib/python2.7/site-packages/scrapy/middleware.py", line 50, in from_crawler
return cls.from_settings(crawler.settings, crawler)
File "/home/scraper/.python/lib/python2.7/site-packages/scrapy/middleware.py", line 31, in from_settings
mw = mwcls.from_crawler(crawler)
File "/home/scraper/.python/lib/python2.7/site-packages/scrapy/contrib/feedexport.py", line 162, in from_crawler
o = cls(crawler.settings)
File "/home/scraper/.python/lib/python2.7/site-packages/scrapy/contrib/feedexport.py", line 144, in __init__
if not self._storage_supported(self.urifmt):
File "/home/scraper/.python/lib/python2.7/site-packages/scrapy/contrib/feedexport.py", line 214, in _storage_supported
self._get_storage(uri)
File "/home/scraper/.python/lib/python2.7/site-packages/scrapy/contrib/feedexport.py", line 225, in _get_storage
return self.storages[urlparse(uri).scheme](uri)
File "/home/scraper/.python/lib/python2.7/site-packages/scrapy/contrib/feedexport.py", line 70, in __init__
self.path = file_uri_to_path(uri)
File "/home/scraper/.python/lib/python2.7/site-packages/w3lib/url.py", line 141, in file_uri_to_path
uri_path = moves.urllib.parse.urlparse(uri).path
AttributeError: 'Module_six_moves_urllib_parse' object has no attribute 'urlparse'
Looks like your six module is not the required by w3lib.
Try:
pip install -U w3lib six