I want to use Jupyter-matlab notebooks. I've downloaded Anaconda navigator and followed the steps from this web: http://jmlilly.net/jupyter-matlab
Unfortunately, when I try to open a matlab notebook:
Seems like I've installed properly
It happends:
Kernel Error...
Error text (It's very unclear):
Traceback (most recent call last):
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\tornado\web.py", line 1704, in _execute
result = await result
File "C:\Users\lucas\anaconda3\envs\jlab\lib\asyncio\tasks.py", line 328, in __wakeup
future.result()
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\tornado\gen.py", line 769, in run
yielded = self.gen.throw(*exc_info) # type: ignore
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\notebook\services\sessions\handlers.py", line 74, in post
model = yield maybe_future(
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\tornado\gen.py", line 762, in run
value = future.result()
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\tornado\gen.py", line 769, in run
yielded = self.gen.throw(*exc_info) # type: ignore
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\notebook\services\sessions\sessionmanager.py", line 98, in create_session
kernel_id = yield self.start_kernel_for_session(session_id, path, name, type, kernel_name)
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\tornado\gen.py", line 762, in run
value = future.result()
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\tornado\gen.py", line 769, in run
yielded = self.gen.throw(*exc_info) # type: ignore
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\notebook\services\sessions\sessionmanager.py", line 110, in start_kernel_for_session
kernel_id = yield maybe_future(
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\tornado\gen.py", line 762, in run
value = future.result()
File "C:\Users\lucas\anaconda3\envs\jlab\lib\asyncio\futures.py", line 201, in result
raise self._exception
File "C:\Users\lucas\anaconda3\envs\jlab\lib\asyncio\tasks.py", line 256, in __step
result = coro.send(None)
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\notebook\services\kernels\kernelmanager.py", line 176, in start_kernel
kernel_id = await maybe_future(self.pinned_superclass.start_kernel(self, **kwargs))
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\jupyter_client\utils.py", line 26, in wrapped
raise e
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\jupyter_client\utils.py", line 23, in wrapped
return loop.run_until_complete(future)
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\nest_asyncio.py", line 89, in run_until_complete
return f.result()
File "C:\Users\lucas\anaconda3\envs\jlab\lib\asyncio\futures.py", line 201, in result
raise self._exception
File "C:\Users\lucas\anaconda3\envs\jlab\lib\asyncio\tasks.py", line 256, in __step
result = coro.send(None)
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\jupyter_client\multikernelmanager.py", line 207, in _async_start_kernel
starter = ensure_async(km.start_kernel(**kwargs))
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\jupyter_client\utils.py", line 26, in wrapped
raise e
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\jupyter_client\utils.py", line 23, in wrapped
return loop.run_until_complete(future)
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\nest_asyncio.py", line 89, in run_until_complete
return f.result()
File "C:\Users\lucas\anaconda3\envs\jlab\lib\asyncio\futures.py", line 201, in result
raise self._exception
File "C:\Users\lucas\anaconda3\envs\jlab\lib\asyncio\tasks.py", line 256, in __step
result = coro.send(None)
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\jupyter_client\manager.py", line 79, in wrapper
raise e
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\jupyter_client\manager.py", line 71, in wrapper
out = await method(self, *args, **kwargs)
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\jupyter_client\manager.py", line 381, in _async_start_kernel
await ensure_async(self._launch_kernel(kernel_cmd, **kw))
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\jupyter_client\utils.py", line 26, in wrapped
raise e
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\jupyter_client\utils.py", line 23, in wrapped
return loop.run_until_complete(future)
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\nest_asyncio.py", line 89, in run_until_complete
return f.result()
File "C:\Users\lucas\anaconda3\envs\jlab\lib\asyncio\futures.py", line 201, in result
raise self._exception
File "C:\Users\lucas\anaconda3\envs\jlab\lib\asyncio\tasks.py", line 256, in __step
result = coro.send(None)
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\jupyter_client\manager.py", line 301, in _async_launch_kernel
connection_info = await self.provisioner.launch_kernel(kernel_cmd, **kw)
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\jupyter_client\provisioning\local_provisioner.py", line 204, in launch_kernel
self.process = launch_kernel(cmd, **scrubbed_kwargs)
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\jupyter_client\launcher.py", line 169, in launch_kernel
raise ex
File "C:\Users\lucas\anaconda3\envs\jlab\lib\site-packages\jupyter_client\launcher.py", line 157, in launch_kernel
proc = Popen(cmd, **kwargs)
File "C:\Users\lucas\anaconda3\envs\jlab\lib\subprocess.py", line 951, in __init__
self._execute_child(args, executable, preexec_fn, close_fds,
File "C:\Users\lucas\anaconda3\envs\jlab\lib\subprocess.py", line 1420, in _execute_child
hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
FileNotFoundError: [WinError 2] The system cannot find the file specified
I've been trying re-install anaconda, jupyter, matlab-kernel, change environment python version (3.7, 3.8, 3.9) etc. but it doesn't work.
Matlab version: R2021b
First I got this message:
Defaulting to user installation because normal site-packages is not writeable.**
Then I got this error:
ERROR: Exception:
Traceback (most recent call last):
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_vendor\urllib3\response.py", line 438, in _error_catcher
yield
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_vendor\urllib3\response.py", line 519, in read
data = self._fp.read(amt) if not fp_closed else b""
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_vendor\cachecontrol\filewrapper.py", line 62, in read
data = self.__fp.read(amt)
File "c:\program files (x86)\python38-32\lib\http\client.py", line 454, in read
n = self.readinto(b)
File "c:\program files (x86)\python38-32\lib\http\client.py", line 498, in readinto
n = self.fp.readinto(b)
File "c:\program files (x86)\python38-32\lib\socket.py", line 669, in readinto
return self._sock.recv_into(b)
File "c:\program files (x86)\python38-32\lib\ssl.py", line 1241, in recv_into
return self.read(nbytes, buffer)
File "c:\program files (x86)\python38-32\lib\ssl.py", line 1099, in read
return self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\cli\base_command.py", line 173, in _main
status = self.run(options, args)
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\cli\req_command.py", line 203, in wrapper
return func(self, options, args)
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\commands\install.py", line 315, in run
requirement_set = resolver.resolve(
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\resolution\resolvelib\resolver.py", line 94, in resolve
result = self._result = resolver.resolve(
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_vendor\resolvelib\resolvers.py", line 472, in resolve
state = resolution.resolve(requirements, max_rounds=max_rounds)
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_vendor\resolvelib\resolvers.py", line 341, in resolve
self._add_to_criteria(self.state.criteria, r, parent=None)
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_vendor\resolvelib\resolvers.py", line 172, in _add_to_criteria
if not criterion.candidates:
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_vendor\resolvelib\structs.py", line 151, in __bool__
return bool(self._sequence)
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\resolution\resolvelib\found_candidates.py", line 140, in __bool__
return any(self)
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\resolution\resolvelib\found_candidates.py", line 128, in <genexpr>
return (c for c in iterator if id(c) not in self._incompatible_ids)
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\resolution\resolvelib\found_candidates.py", line 32, in _iter_built
candidate = func()
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\resolution\resolvelib\factory.py", line 204, in _make_candidate_from_link
self._link_candidate_cache[link] = LinkCandidate(
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 295, in __init__
super().__init__(
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 156, in __init__
self.dist = self._prepare()
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 227, in _prepare
dist = self._prepare_distribution()
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 305, in _prepare_distribution
return self._factory.preparer.prepare_linked_requirement(
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\operations\prepare.py", line 508, in prepare_linked_requirement
return self._prepare_linked_requirement(req, parallel_builds)
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\operations\prepare.py", line 550, in _prepare_linked_requirement
local_file = unpack_url(
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\operations\prepare.py", line 239, in unpack_url
file = get_http_url(
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\operations\prepare.py", line 102, in get_http_url
from_path, content_type = download(link, temp_dir.path)
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\network\download.py", line 145, in __call__
for chunk in chunks:
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\cli\progress_bars.py", line 144, in iter
for x in it:
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_internal\network\utils.py", line 63, in response_chunks
for chunk in response.raw.stream(
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_vendor\urllib3\response.py", line 576, in stream
data = self.read(amt=amt, decode_content=decode_content)
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_vendor\urllib3\response.py", line 541, in read
raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
File "c:\program files (x86)\python38-32\lib\contextlib.py", line 131, in __exit__
self.gen.throw(type, value, traceback)
File "C:\Users\Marshal\AppData\Roaming\Python\Python38\site-packages\pip\_vendor\urllib3\response.py", line 443, in _error_catcher
raise ReadTimeoutError(self._pool, None, "Read timed out.")
pip._vendor.urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='files.pythonhosted.org', port=443): Read timed out.
I am running newsplease project from https://github.com/fhamborg/news-please/blob/master/newsplease/__main__.py from subprocess
cmd = ["python",r"/media/dhruvagupta/F428F6FA28F6BB26/backup dhruva/python/New folder/training/news-please-master/news-please-master/newsplease/__main__.py","-c /newsplease/config/"]
j = psutil.Popen(cmd)
i am getting this error
Unhandled Error
Traceback (most recent call last):
File "/media/dhruvagupta/F428F6FA28F6BB26/backup dhruva/python/New folder/training/news-please-master/news-please-master/newsplease/single_crawler.py", line 152, in __init__
self.process.start()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/crawler.py", line 293, in start
reactor.run(installSignalHandlers=False) # blocking call
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/twisted/internet/base.py", line 1272, in run
self.mainLoop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/twisted/internet/base.py", line 1281, in mainLoop
self.runUntilCurrent()
--- <exception caught here> ---
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/twisted/internet/base.py", line 902, in runUntilCurrent
call.func(*call.args, **call.kw)
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/utils/reactor.py", line 41, in __call__
return self._func(*self._a, **self._kw)
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/core/engine.py", line 122, in _next_request
if not self._next_request_from_scheduler(spider):
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/core/engine.py", line 149, in _next_request_from_scheduler
request = slot.scheduler.next_request()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/core/scheduler.py", line 71, in next_request
request = self._dqpop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/core/scheduler.py", line 106, in _dqpop
d = self.dqs.pop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/queuelib/pqueue.py", line 43, in pop
m = q.pop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/squeues.py", line 19, in pop
s = super(SerializableQueue, self).pop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/queuelib/queue.py", line 161, in pop
size, = struct.unpack(self.SIZE_FORMAT, self.f.read())
struct.error: unpack requires a string argument of length 4
and another error
Unhandled Error
Traceback (most recent call last):
File "/media/dhruvagupta/F428F6FA28F6BB26/backup dhruva/python/New folder/training/news-please-master/news-please-master/newsplease/single_crawler.py", line 152, in __init__
self.process.start()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/crawler.py", line 293, in start
reactor.run(installSignalHandlers=False) # blocking call
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/twisted/internet/base.py", line 1272, in run
self.mainLoop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/twisted/internet/base.py", line 1281, in mainLoop
self.runUntilCurrent()
--- <exception caught here> ---
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/twisted/internet/base.py", line 902, in runUntilCurrent
call.func(*call.args, **call.kw)
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/utils/reactor.py", line 41, in __call__
return self._func(*self._a, **self._kw)
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/core/engine.py", line 122, in _next_request
if not self._next_request_from_scheduler(spider):
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/core/engine.py", line 149, in _next_request_from_scheduler
request = slot.scheduler.next_request()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/core/scheduler.py", line 71, in next_request
request = self._dqpop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/core/scheduler.py", line 106, in _dqpop
d = self.dqs.pop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/queuelib/pqueue.py", line 43, in pop
m = q.pop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/squeues.py", line 19, in pop
s = super(SerializableQueue, self).pop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/queuelib/queue.py", line 162, in pop
self.f.seek(-size-self.SIZE_SIZE, os.SEEK_END)
exceptions.IOError: [Errno 22] Invalid argument
I am trying to run multiple scrapy spiders at once, each from the command line. For example: command
>> python run_scrapy_spider_1.py
>> python run_scrapy_spider_2.py
However, once I already have 5 spiders running (all on different domains), the next one I try to run hangs with the message:
INFO: Telnet console listening on 127.0.0.1:6028
I don't think its an issue with the 6th site I am trying to run (it works fine when less than 5 spiders are running), but instead the fact that 5 spiders are already running.
Here is my very log output long incase it helps, any ideas how I can get more than 5 spiders running at once?
--- Logging error ---
Traceback (most recent call last):
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 33, in emit
payload = self.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 833, in format
return fmt.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 573, in format
s = self.formatMessage(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 542, in formatMessage
return self._style.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 386, in format
return self._fmt % record.__dict__
KeyError: 'request_id'
Call stack:
File "run_spider_alexa_id.py", line 16, in <module>
SpiderTasks.run_spider_for_alexa_site_id(alexa_site_id)
File "/home/ec2-user/code/green_brick_two/pricecomparison_project/pricecomparison/scripts/spider_scripts/spider_tasks.py", line 18, in run_spider_for_alexa_site_id
process.crawl(MySpider, alexa_site_id=alexa_site_id)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 172, in crawl
return self._crawl(crawler, *args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 176, in _crawl
d = crawler.crawl(*args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1532, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 80, in crawl
self.engine = self._create_engine()
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 105, in _create_engine
return ExecutionEngine(self, lambda _: self.stop())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/core/engine.py", line 70, in __init__
self.scraper = Scraper(crawler)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/core/scraper.py", line 69, in __init__
self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/middleware.py", line 53, in from_crawler
return cls.from_settings(crawler.settings, crawler)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/middleware.py", line 48, in from_settings
extra={'crawler': crawler})
File "/usr/local/lib/python3.6/logging/__init__.py", line 1301, in info
self._log(INFO, msg, args, **kwargs)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1437, in _log
self.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1447, in handle
self.callHandlers(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/sentry_sdk/integrations/logging.py", line 47, in sentry_patched_callhandlers
return old_callhandlers(self, record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1509, in callHandlers
hdlr.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 858, in handle
self.emit(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 38, in emit
self.handleError(record)
Message: 'Enabled %(componentname)ss:\n%(enabledlist)s'
Arguments: {'componentname': 'spider middleware', 'enabledlist': "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n 'scrapy.spidermiddlewares.depth.DepthMiddleware']"}
2019-03-03 22:34:30 [scrapy.middleware] INFO: Enabled spider middlewares:
['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',
'scrapy.spidermiddlewares.referer.RefererMiddleware',
'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',
'scrapy.spidermiddlewares.depth.DepthMiddleware']
--- Logging error ---
Traceback (most recent call last):
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 33, in emit
payload = self.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 833, in format
return fmt.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 573, in format
s = self.formatMessage(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 542, in formatMessage
return self._style.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 386, in format
return self._fmt % record.__dict__
KeyError: 'request_id'
Call stack:
File "run_spider_alexa_id.py", line 16, in <module>
SpiderTasks.run_spider_for_alexa_site_id(alexa_site_id)
File "/home/ec2-user/code/green_brick_two/pricecomparison_project/pricecomparison/scripts/spider_scripts/spider_tasks.py", line 18, in run_spider_for_alexa_site_id
process.crawl(MySpider, alexa_site_id=alexa_site_id)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 172, in crawl
return self._crawl(crawler, *args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 176, in _crawl
d = crawler.crawl(*args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1532, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 80, in crawl
self.engine = self._create_engine()
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 105, in _create_engine
return ExecutionEngine(self, lambda _: self.stop())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/core/engine.py", line 70, in __init__
self.scraper = Scraper(crawler)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/core/scraper.py", line 71, in __init__
self.itemproc = itemproc_cls.from_crawler(crawler)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/middleware.py", line 53, in from_crawler
return cls.from_settings(crawler.settings, crawler)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/middleware.py", line 48, in from_settings
extra={'crawler': crawler})
File "/usr/local/lib/python3.6/logging/__init__.py", line 1301, in info
self._log(INFO, msg, args, **kwargs)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1437, in _log
self.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1447, in handle
self.callHandlers(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/sentry_sdk/integrations/logging.py", line 47, in sentry_patched_callhandlers
return old_callhandlers(self, record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1509, in callHandlers
hdlr.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 858, in handle
self.emit(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 38, in emit
self.handleError(record)
Message: 'Enabled %(componentname)ss:\n%(enabledlist)s'
Arguments: {'componentname': 'item pipeline', 'enabledlist': '[]'}
2019-03-03 22:34:30 [scrapy.middleware] INFO: Enabled item pipelines:
[]
--- Logging error ---
Traceback (most recent call last):
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 33, in emit
payload = self.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 833, in format
return fmt.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 573, in format
s = self.formatMessage(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 542, in formatMessage
return self._style.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 386, in format
return self._fmt % record.__dict__
KeyError: 'request_id'
Call stack:
File "run_spider_alexa_id.py", line 16, in <module>
SpiderTasks.run_spider_for_alexa_site_id(alexa_site_id)
File "/home/ec2-user/code/green_brick_two/pricecomparison_project/pricecomparison/scripts/spider_scripts/spider_tasks.py", line 18, in run_spider_for_alexa_site_id
process.crawl(MySpider, alexa_site_id=alexa_site_id)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 172, in crawl
return self._crawl(crawler, *args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 176, in _crawl
d = crawler.crawl(*args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1532, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 82, in crawl
yield self.engine.open_spider(self.spider, start_requests)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1532, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/core/engine.py", line 256, in open_spider
logger.info("Spider opened", extra={'spider': spider})
File "/usr/local/lib/python3.6/logging/__init__.py", line 1301, in info
self._log(INFO, msg, args, **kwargs)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1437, in _log
self.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1447, in handle
self.callHandlers(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/sentry_sdk/integrations/logging.py", line 47, in sentry_patched_callhandlers
return old_callhandlers(self, record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1509, in callHandlers
hdlr.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 858, in handle
self.emit(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 38, in emit
self.handleError(record)
Message: 'Spider opened'
Arguments: ()
2019-03-03 22:34:30 [scrapy.core.engine] INFO: Spider opened
--- Logging error ---
Traceback (most recent call last):
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 33, in emit
payload = self.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 833, in format
return fmt.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 573, in format
s = self.formatMessage(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 542, in formatMessage
return self._style.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 386, in format
return self._fmt % record.__dict__
KeyError: 'request_id'
Call stack:
File "run_spider_alexa_id.py", line 16, in <module>
SpiderTasks.run_spider_for_alexa_site_id(alexa_site_id)
File "/home/ec2-user/code/green_brick_two/pricecomparison_project/pricecomparison/scripts/spider_scripts/spider_tasks.py", line 18, in run_spider_for_alexa_site_id
process.crawl(MySpider, alexa_site_id=alexa_site_id)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 172, in crawl
return self._crawl(crawler, *args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 176, in _crawl
d = crawler.crawl(*args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1532, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 82, in crawl
yield self.engine.open_spider(self.spider, start_requests)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1532, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/core/engine.py", line 266, in open_spider
yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/signalmanager.py", line 61, in send_catch_log_deferred
return _signal.send_catch_log_deferred(signal, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/utils/signal.py", line 65, in send_catch_log_deferred
*arguments, **named)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 150, in maybeDeferred
result = f(*args, **kw)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/pydispatch/robustapply.py", line 55, in robustApply
return receiver(*arguments, **named)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/extensions/logstats.py", line 35, in spider_opened
self.task.start(self.interval)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/task.py", line 194, in start
self()
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/task.py", line 239, in __call__
d = defer.maybeDeferred(self.f, *self.a, **self.kw)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 150, in maybeDeferred
result = f(*args, **kw)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/extensions/logstats.py", line 48, in log
logger.info(msg, log_args, extra={'spider': spider})
File "/usr/local/lib/python3.6/logging/__init__.py", line 1301, in info
self._log(INFO, msg, args, **kwargs)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1437, in _log
self.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1447, in handle
self.callHandlers(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/sentry_sdk/integrations/logging.py", line 47, in sentry_patched_callhandlers
return old_callhandlers(self, record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1509, in callHandlers
hdlr.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 858, in handle
self.emit(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 38, in emit
self.handleError(record)
Message: 'Crawled %(pages)d pages (at %(pagerate)d pages/min), scraped %(items)d items (at %(itemrate)d items/min)'
Arguments: {'pages': 0, 'pagerate': 0.0, 'items': 0, 'itemrate': 0.0}
2019-03-03 22:34:30 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
--- Logging error ---
Traceback (most recent call last):
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 33, in emit
payload = self.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 833, in format
return fmt.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 573, in format
s = self.formatMessage(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 542, in formatMessage
return self._style.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 386, in format
return self._fmt % record.__dict__
KeyError: 'request_id'
Call stack:
File "run_spider_alexa_id.py", line 16, in <module>
SpiderTasks.run_spider_for_alexa_site_id(alexa_site_id)
File "/home/ec2-user/code/green_brick_two/pricecomparison_project/pricecomparison/scripts/spider_scripts/spider_tasks.py", line 18, in run_spider_for_alexa_site_id
process.crawl(MySpider, alexa_site_id=alexa_site_id)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 172, in crawl
return self._crawl(crawler, *args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 176, in _crawl
d = crawler.crawl(*args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1532, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 83, in crawl
yield defer.maybeDeferred(self.engine.start)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 150, in maybeDeferred
result = f(*args, **kw)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1532, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/core/engine.py", line 78, in start
yield self.signals.send_catch_log_deferred(signal=signals.engine_started)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/signalmanager.py", line 61, in send_catch_log_deferred
return _signal.send_catch_log_deferred(signal, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/utils/signal.py", line 65, in send_catch_log_deferred
*arguments, **named)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 150, in maybeDeferred
result = f(*args, **kw)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/pydispatch/robustapply.py", line 55, in robustApply
return receiver(*arguments, **named)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/extensions/telnet.py", line 74, in start_listening
extra={'crawler': self.crawler})
File "/usr/local/lib/python3.6/logging/__init__.py", line 1301, in info
self._log(INFO, msg, args, **kwargs)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1437, in _log
self.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1447, in handle
self.callHandlers(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/sentry_sdk/integrations/logging.py", line 47, in sentry_patched_callhandlers
return old_callhandlers(self, record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1509, in callHandlers
hdlr.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 858, in handle
self.emit(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 38, in emit
self.handleError(record)
Message: 'Telnet console listening on %(host)s:%(port)d'
Arguments: {'host': '127.0.0.1', 'port': 6028}
2019-03-03 22:34:30 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6028
I got this error while running my app in command line.
Traceback (most recent call last):
File "manage.py", line 10, in <module>
execute_from_command_line(sys.argv)
File "/usr/local/lib/python2.7/dist-packages/django/core/management/__init__.py", line 399, in execute_from_command_line
utility.execute()
File "/usr/local/lib/python2.7/dist-packages/django/core/management/__init__.py", line 392, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "/usr/local/lib/python2.7/dist-packages/django/core/management/base.py", line 242, in run_from_argv
self.execute(*args, **options.__dict__)
File "/usr/local/lib/python2.7/dist-packages/django/core/management/base.py", line 280, in execute
translation.activate('en-us')
File "/usr/local/lib/python2.7/dist-packages/django/utils/translation/__init__.py", line 130, in activate
return _trans.activate(language)
File "/usr/local/lib/python2.7/dist-packages/django/utils/translation/trans_real.py", line 188, in activate
_active.value = translation(language)
File "/usr/local/lib/python2.7/dist-packages/django/utils/translation/trans_real.py", line 177, in translation
default_translation = _fetch(settings.LANGUAGE_CODE)
File "/usr/local/lib/python2.7/dist-packages/django/utils/translation/trans_real.py", line 159, in _fetch
app = import_module(appname)
File "/usr/local/lib/python2.7/dist-packages/django/utils/importlib.py", line 40, in import_module
__import__(name)
File "/usr/local/lib/python2.7/dist-packages/django/contrib/admin/__init__.py", line 6, in <module>
from django.contrib.admin.sites import AdminSite, site
File "/usr/local/lib/python2.7/dist-packages/django/contrib/admin/sites.py", line 4, in <module>
from django.contrib.admin.forms import AdminAuthenticationForm
File "/usr/local/lib/python2.7/dist-packages/django/contrib/admin/forms.py", line 6, in <module>
from django.contrib.auth.forms import AuthenticationForm
File "/usr/local/lib/python2.7/dist-packages/django/contrib/auth/forms.py", line 17, in <module>
from django.contrib.auth.models import User
File "/usr/local/lib/python2.7/dist-packages/django/contrib/auth/models.py", line 48, in <module>
class Permission(models.Model):
File "/usr/local/lib/python2.7/dist-packages/django/db/models/base.py", line 96, in __new__
new_class.add_to_class('_meta', Options(meta, **kwargs))
File "/usr/local/lib/python2.7/dist-packages/django/db/models/base.py", line 264, in add_to_class
value.contribute_to_class(cls, name)
File "/usr/local/lib/python2.7/dist-packages/django/db/models/options.py", line 124, in contribute_to_class
self.db_table = truncate_name(self.db_table, connection.ops.max_name_length())
File "/usr/local/lib/python2.7/dist-packages/django/db/__init__.py", line 34, in __getattr__
return getattr(connections[DEFAULT_DB_ALIAS], item)
File "/usr/local/lib/python2.7/dist-packages/django/db/utils.py", line 199, in __getitem__
conn = backend.DatabaseWrapper(db, alias)
File "/usr/local/lib/python2.7/dist-packages/django/contrib/gis/db/backends/postgis/base.py", line 11, in __init__
self.ops = PostGISOperations(self)
File "/usr/local/lib/python2.7/dist-packages/django/contrib/gis/db/backends/postgis/operations.py", line 158, in __init__
if self.spatial_version < (1, 3, 4):
File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 49, in __get__
res = instance.__dict__[self.func.__name__] = self.func(instance)
File "/usr/local/lib/python2.7/dist-packages/django/contrib/gis/db/backends/postgis/operations.py", line 252, in spatial_version
vtup = self.postgis_version_tuple()
File "/usr/local/lib/python2.7/dist-packages/django/contrib/gis/db/backends/postgis/operations.py", line 426, in postgis_version_tuple
version = self.postgis_lib_version()
File "/usr/local/lib/python2.7/dist-packages/django/contrib/gis/db/backends/postgis/operations.py", line 406, in postgis_lib_version
return self._get_postgis_func('postgis_lib_version')
File "/usr/local/lib/python2.7/dist-packages/django/contrib/gis/db/backends/postgis/operations.py", line 396, in _get_postgis_func
with self.connection.temporary_connection() as cursor:
File "/usr/lib/python2.7/contextlib.py", line 17, in __enter__
return self.gen.next()
File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 515, in temporary_connection
cursor = self.cursor()
File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 157, in cursor
cursor = self.make_debug_cursor(self._cursor())
File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 129, in _cursor
self.ensure_connection()
File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 124, in ensure_connection
self.connect()
File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 115, in connect
self.set_autocommit(True)
File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 333, in set_autocommit
self._set_autocommit(autocommit)
File "/usr/local/lib/python2.7/dist-packages/django/db/backends/postgresql_psycopg2/base.py", line 172, in _set_autocommit
if self.psycopg2_version >= (2, 4, 2):
File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 49, in __get__
res = instance.__dict__[self.func.__name__] = self.func(instance)
File "/usr/local/lib/python2.7/dist-packages/django/db/backends/postgresql_psycopg2/base.py", line 201, in psycopg2_version
return tuple(int(v) for v in version.split('.'))
File "/usr/local/lib/python2.7/dist-packages/django/db/backends/postgresql_psycopg2/base.py", line 201, in <genexpr>
return tuple(int(v) for v in version.split('.'))
I installed the necessary modules. I am using Django version 1.6. The same app runs smoothly in my windows laptop but when I migrated it in a Linux OS, the errors shows.