I am running newsplease project from https://github.com/fhamborg/news-please/blob/master/newsplease/__main__.py from subprocess
cmd = ["python",r"/media/dhruvagupta/F428F6FA28F6BB26/backup dhruva/python/New folder/training/news-please-master/news-please-master/newsplease/__main__.py","-c /newsplease/config/"]
j = psutil.Popen(cmd)
i am getting this error
Unhandled Error
Traceback (most recent call last):
File "/media/dhruvagupta/F428F6FA28F6BB26/backup dhruva/python/New folder/training/news-please-master/news-please-master/newsplease/single_crawler.py", line 152, in __init__
self.process.start()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/crawler.py", line 293, in start
reactor.run(installSignalHandlers=False) # blocking call
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/twisted/internet/base.py", line 1272, in run
self.mainLoop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/twisted/internet/base.py", line 1281, in mainLoop
self.runUntilCurrent()
--- <exception caught here> ---
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/twisted/internet/base.py", line 902, in runUntilCurrent
call.func(*call.args, **call.kw)
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/utils/reactor.py", line 41, in __call__
return self._func(*self._a, **self._kw)
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/core/engine.py", line 122, in _next_request
if not self._next_request_from_scheduler(spider):
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/core/engine.py", line 149, in _next_request_from_scheduler
request = slot.scheduler.next_request()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/core/scheduler.py", line 71, in next_request
request = self._dqpop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/core/scheduler.py", line 106, in _dqpop
d = self.dqs.pop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/queuelib/pqueue.py", line 43, in pop
m = q.pop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/squeues.py", line 19, in pop
s = super(SerializableQueue, self).pop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/queuelib/queue.py", line 161, in pop
size, = struct.unpack(self.SIZE_FORMAT, self.f.read())
struct.error: unpack requires a string argument of length 4
and another error
Unhandled Error
Traceback (most recent call last):
File "/media/dhruvagupta/F428F6FA28F6BB26/backup dhruva/python/New folder/training/news-please-master/news-please-master/newsplease/single_crawler.py", line 152, in __init__
self.process.start()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/crawler.py", line 293, in start
reactor.run(installSignalHandlers=False) # blocking call
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/twisted/internet/base.py", line 1272, in run
self.mainLoop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/twisted/internet/base.py", line 1281, in mainLoop
self.runUntilCurrent()
--- <exception caught here> ---
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/twisted/internet/base.py", line 902, in runUntilCurrent
call.func(*call.args, **call.kw)
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/utils/reactor.py", line 41, in __call__
return self._func(*self._a, **self._kw)
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/core/engine.py", line 122, in _next_request
if not self._next_request_from_scheduler(spider):
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/core/engine.py", line 149, in _next_request_from_scheduler
request = slot.scheduler.next_request()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/core/scheduler.py", line 71, in next_request
request = self._dqpop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/core/scheduler.py", line 106, in _dqpop
d = self.dqs.pop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/queuelib/pqueue.py", line 43, in pop
m = q.pop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/scrapy/squeues.py", line 19, in pop
s = super(SerializableQueue, self).pop()
File "/home/dhruvagupta/.local/lib/python2.7/site-packages/queuelib/queue.py", line 162, in pop
self.f.seek(-size-self.SIZE_SIZE, os.SEEK_END)
exceptions.IOError: [Errno 22] Invalid argument
Related
I created a small program that uses numba on a method containing a 2D array (list of lists). This program runs just fine on Spyder, but when I try to run the exact same program on a remote Ubuntu server, I get a long error saying "unsupported nested memory-managed object" (listed below the program code).
The code:
from numba import jit
#jit(nopython=True)
def test():
num1 = 10
num2 = 5
array = [ [ 1 for i in range(num1) ] for j in range(num2) ]
#array = [2 for i in range(num1) ]
sum = 0
for i in range(0,num1):
for j in range(0,num2):
sum = sum + array[i][j]
print(sum)
test()
The error:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/numba/runtime/context.py", line 186, in _call_incref_decref
meminfo = data_model.get_nrt_meminfo(builder, value)
File "/usr/lib/python3/dist-packages/numba/datamodel/models.py", line 329, in get_nrt_meminfo
"unsupported nested memory-managed object")
NotImplementedError: unsupported nested memory-managed object
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/numba/errors.py", line 243, in new_error_context
yield
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 216, in lower_block
self.lower_inst(inst)
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 263, in lower_inst
self.storevar(val, inst.target.name)
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 928, in storevar
self.decref(fetype, old)
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 982, in decref
self.context.nrt.decref(self.builder, typ, val)
File "/usr/lib/python3/dist-packages/numba/runtime/context.py", line 208, in decref
self._call_incref_decref(builder, typ, typ, value, "NRT_decref")
File "/usr/lib/python3/dist-packages/numba/runtime/context.py", line 179, in _call_incref_decref
funcname, getters + (getter,))
File "/usr/lib/python3/dist-packages/numba/runtime/context.py", line 188, in _call_incref_decref
raise NotImplementedError("%s: %s" % (root_type, str(e)))
NotImplementedError: list(list(int64)): unsupported nested memory-managed object
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "2DArrayTest.py", line 15, in <module>
test()
File "/usr/lib/python3/dist-packages/numba/dispatcher.py", line 307, in _compile_for_args
return self.compile(tuple(argtypes))
File "/usr/lib/python3/dist-packages/numba/dispatcher.py", line 579, in compile
cres = self._compiler.compile(args, return_type)
File "/usr/lib/python3/dist-packages/numba/dispatcher.py", line 80, in compile
flags=flags, locals=self.locals)
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 740, in compile_extra
return pipeline.compile_extra(func)
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 360, in compile_extra
return self._compile_bytecode()
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 699, in _compile_bytecode
return self._compile_core()
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 686, in _compile_core
res = pm.run(self.status)
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 246, in run
raise patched_exception
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 238, in run
stage()
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 621, in stage_nopython_backend
self._backend(lowerfn, objectmode=False)
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 576, in _backend
lowered = lowerfn()
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 563, in backend_nopython_mode
self.flags)
File "/usr/lib/python3/dist-packages/numba/compiler.py", line 858, in native_lowering_stage
lower.lower()
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 135, in lower
self.lower_normal_function(self.fndesc)
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 176, in lower_normal_function
entry_block_tail = self.lower_function_body()
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 201, in lower_function_body
self.lower_block(block)
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 216, in lower_block
self.lower_inst(inst)
File "/usr/lib/python3.6/contextlib.py", line 99, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/lib/python3/dist-packages/numba/errors.py", line 249, in new_error_context
six.reraise(type(newerr), newerr, sys.exc_info()[2])
File "/usr/lib/python3/dist-packages/numba/six.py", line 658, in reraise
raise value.with_traceback(tb)
File "/usr/lib/python3/dist-packages/numba/errors.py", line 243, in new_error_context
yield
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 216, in lower_block
self.lower_inst(inst)
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 263, in lower_inst
self.storevar(val, inst.target.name)
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 928, in storevar
self.decref(fetype, old)
File "/usr/lib/python3/dist-packages/numba/lowering.py", line 982, in decref
self.context.nrt.decref(self.builder, typ, val)
File "/usr/lib/python3/dist-packages/numba/runtime/context.py", line 208, in decref
self._call_incref_decref(builder, typ, typ, value, "NRT_decref")
File "/usr/lib/python3/dist-packages/numba/runtime/context.py", line 179, in _call_incref_decref
funcname, getters + (getter,))
File "/usr/lib/python3/dist-packages/numba/runtime/context.py", line 188, in _call_incref_decref
raise NotImplementedError("%s: %s" % (root_type, str(e)))
numba.errors.LoweringError: Failed at nopython (nopython mode backend)
list(list(int64)): unsupported nested memory-managed object
File "2DArrayTest.py", line 7
[1] During: lowering "$56 = build_list(items=[])" at 2DArrayTest.py (7)
I have seen elsewhere that numba doesn't like 2D arrays and lists of lists because of memory. Is there a way to make it work on the remote server, since it works on Spyder? What is it that Spyder does differently to make it work?
i'm trying to convert the results to json of a postgres database,it works! but doesnt finish and it triggers an error
rows 984288
try:
cur = conn.cursor()
cur.execute("select * from daily")
row_headers = [x[0] for x in cur.description]
rows = cur.fetchall()
json_data = []
for result in rows:
json_data.append(dict(zip(row_headers,result)))
# rows = [dict((cur.description[i][0], value) \
# for i, value in enumerate(row)) for row in cur.fetchall()]
print("the number of parts: ", cur.rowcount)
conn.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
return json.dumps(json_data)
debug:
error:
out of memory for query result
127.0.0.1 - - [03/Aug/2020 02:12:18] "GET /pruebas HTTP/1.1" 500 -
Exception happened during processing of request from ('127.0.0.1', 59545)
Traceback (most recent call last):
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\debug_init_.py", line 304, in debug_application
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\flask\app.py", line 2464, in call
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\flask\app.py", line 2450, in wsgi_app
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\flask\app.py", line 1867, in handle_exception
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\flask_compat.py", line 39, in reraise
raise value
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\flask\app.py", line 2447, in wsgi_app
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\flask\app.py", line 1952, in full_dispatch_request
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\flask\app.py", line 1821, in handle_user_exception
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\flask_compat.py", line 39, in reraise
raise value
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\flask\app.py", line 1950, in full_dispatch_request
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\flask\app.py", line 1936, in dispatch_request
File "C:\Users\iscca\PycharmProjects\covid-uabc\main.py", line 114, in pruebas
cosa = Prueba()
File "C:\Users\iscca\PycharmProjects\covid-uabc\pruebas.py", line 14, in init
self.rows = self.enlistar(conn)
File "C:\Users\iscca\PycharmProjects\covid-uabc\pruebas.py", line 38, in enlistar
return json.dumps(json_data)
UnboundLocalError: local variable 'json_data' referenced before assignment
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\serving.py", line 323, in run_wsgi
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\serving.py", line 314, in execute
for data in application_iter:
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\debug_init_.py", line 344, in debug_application
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\debug\tbtools.py", line 344, in render_full
"summary": self.render_summary(include_title=False),
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\debug\tbtools.py", line 314, in render_summary
frames = [group.render(mark_lib=mark_lib) for group in self.groups]
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\debug\tbtools.py", line 314, in
frames = [group.render(mark_lib=mark_lib) for group in self.groups]
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\debug\tbtools.py", line 435, in render
frame.render(mark_lib=mark_lib),
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\debug\tbtools.py", line 485, in render
"lines": self.render_line_context(),
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\debug\tbtools.py", line 504, in render_line_context
before, current, after = self.get_context_lines()
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\debug\tbtools.py", line 611, in get_context_lines
before = self.sourcelines[self.lineno - context - 1 : self.lineno - 1]
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\utils.py", line 90, in get
value = self.func(obj)
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\debug\tbtools.py", line 608, in sourcelines
return source.decode(charset, "replace").splitlines()
MemoryError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\iscca\AppData\Local\Programs\Python\Python37-32\Lib\socketserver.py", line 650, in process_request_thread
self.finish_request(request, client_address)
File "C:\Users\iscca\AppData\Local\Programs\Python\Python37-32\Lib\socketserver.py", line 360, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "C:\Users\iscca\AppData\Local\Programs\Python\Python37-32\Lib\socketserver.py", line 720, in init
self.handle()
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\serving.py", line 345, in handle
File "C:\Users\iscca\AppData\Local\Programs\Python\Python37-32\Lib\http\server.py", line 426, in handle
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\serving.py", line 379, in handle_one_request
return self.run_wsgi()
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\serving.py", line 340, in run_wsgi
self.server.log("error", "Error on request:\n%s", traceback.plaintext)
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\utils.py", line 90, in get
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\debug\tbtools.py", line 353, in plaintext
return u"\n".join([group.render_text() for group in self.groups])
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\debug\tbtools.py", line 353, in
return u"\n".join([group.render_text() for group in self.groups])
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\debug\tbtools.py", line 446, in render_text
out.append(frame.render_text())
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\debug\tbtools.py", line 500, in render_text
self.current_line.strip(),
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\debug\tbtools.py", line 618, in current_line
return self.sourcelines[self.lineno - 1]
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\utils.py", line 90, in get
value = self.func(obj)
File "C:\Users\iscca.virtualenvs\covid-uabc-YDkkGE3-\lib\site-packages\werkzeug\debug\tbtools.py", line 579, in sourcelines
source = f.read()
MemoryError
this is my error. I don't know what's my error is. And I really do not know what to do right now. I am using PyTorch-lightning with tensorboard. I do not where this error comes from, I can provide more details if you ask. I delete several error information because StackOverflow doesn't allow me to post so much code.
File "/home/jq/PycharmProjects/Unet/Code/Lit_train.py", line 49, in <module>
trainer.fit(model)
File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 859, in fit
self.single_gpu_train(model)
File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/trainer/distrib_parts.py", line 503, in single_gpu_train
self.run_pretrain_routine(model)
File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 1015, in run_pretrain_routine
self.train()
File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py", line 347, in train
self.run_training_epoch()
File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py", line 451, in run_training_epoch
self.run_evaluation(test_mode=self.testing)
File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/trainer/evaluation_loop.py", line 391, in run_evaluation
self.log_metrics(log_metrics, {})
File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/trainer/logging.py", line 74, in log_metrics
self.logger.save()
File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/utilities/distributed.py", line 10, in wrapped_fn
return fn(*args, **kwargs)
File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/loggers/tensorboard.py", line 161, in save
save_hparams_to_yaml(hparams_file, self.hparams)
File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/core/saving.py", line 151, in save_hparams_to_yaml
yaml.dump(hparams, fp)
File "/home/jq/.local/lib/python3.6/site-packages/yaml/__init__.py", line 290, in dump
return dump_all([data], stream, Dumper=Dumper, **kwds)
File "/home/jq/.local/lib/python3.6/site-packages/yaml/__init__.py", line 278, in dump_all
dumper.represent(data)
File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 27, in represent
node = self.represent_data(data)
File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 48, in represent_data
node = self.yaml_representers[data_types[0]](self, data)
File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 207, in represent_dict
return self.represent_mapping('tag:yaml.org,2002:map', data)
File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 118, in represent_mapping
node_value = self.represent_data(item_value)
File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 52, in represent_data
node = self.yaml_multi_representers[data_type](self, data)
File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 343, in represent_object
'tag:yaml.org,2002:python/object:'+function_name, state)
File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 118, in represent_mapping
node_value = self.represent_data(item_value)
File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 52, in represent_data
node = self.yaml_multi_representers[data_type](self, data)
File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 346, in represent_object
return self.represent_sequence(tag+function_name, args)
File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 92, in represent_sequence
node_item = self.represent_data(item)
File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 52, in represent_data
node = self.yaml_multi_representers[data_type](self, data)
File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 343, in represent_object
File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 52, in represent_data
node = self.yaml_multi_representers[data_type](self, data)
File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 317, in represent_object
reduce = data.__reduce_ex__(2)
TypeError: can't pickle _thread.lock objects
Exception ignored in: <object repr() failed>
Traceback (most recent call last):
File "/home/jq/.local/lib/python3.6/site-packages/tqdm/std.py", line 1086, in __del__
File "/home/jq/.local/lib/python3.6/site-packages/tqdm/std.py", line 1293, in close
File "/home/jq/.local/lib/python3.6/site-packages/tqdm/std.py", line 1471, in display
File "/home/jq/.local/lib/python3.6/site-packages/tqdm/std.py", line 1089, in __repr__
File "/home/jq/.local/lib/python3.6/site-packages/tqdm/std.py", line 1433, in format_dict
TypeError: 'NoneType' object is not iterable
I am trying to check whether a certain dataset exists in BigQuery and I get this very strange error I've never had untill yesterday:
ERROR:dsUtils.bq_utils:Could not check if dataset tmp exists.
Traceback (most recent call last):
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\dsUtils\bq_utils.py", line 113, in _get
resp = bq_service.datasets().get(projectId=self.project_id, datasetId=self.id).execute(num_retries=2)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\oauth2client\util.py", line 137, in positional_wrapper
return wrapped(*args, **kwargs)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\googleapiclient\http.py", line 755, in execute
method=str(self.method), body=self.body, headers=self.headers)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\googleapiclient\http.py", line 93, in _retry_request
resp, content = http.request(uri, method, *args, **kwargs)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\oauth2client\client.py", line 598, in new_request
self._refresh(request_orig)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\oauth2client\client.py", line 864, in _refresh
self._do_refresh_request(http_request)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\oauth2client\client.py", line 891, in _do_refresh_request
body = self._generate_refresh_request_body()
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\oauth2client\client.py", line 1597, in _generate_refresh_request_body
assertion = self._generate_assertion()
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\oauth2client\service_account.py", line 318, in _generate_assertion
key_id=self._private_key_id)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\oauth2client\crypt.py", line 97, in make_signed_jwt
signature = signer.sign(signing_input)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\oauth2client\_pycrypto_crypt.py", line 101, in sign
return PKCS1_v1_5.new(self._key).sign(SHA256.new(message))
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\Crypto\Signature\PKCS1_v1_5.py", line 112, in sign
m = self._key.decrypt(em)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\Crypto\PublicKey\RSA.py", line 174, in decrypt
return pubkey.pubkey.decrypt(self, ciphertext)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\Crypto\PublicKey\pubkey.py", line 93, in decrypt
plaintext=self._decrypt(ciphertext)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\Crypto\PublicKey\RSA.py", line 235, in _decrypt
r = getRandomRange(1, self.key.n-1, randfunc=self._randfunc)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\Crypto\PublicKey\RSA.py", line 126, in __getattr__
raise AttributeError("%s object has no %r attribute" % (self.__class__.__name__, attrname,))
AttributeError: _RSAobj object has no '_randfunc' attribute
Has anyone an idea of why I get these errors suddenly?
I'm using django-dynamic-scrapy with scrapyd, and confused with this error, (in the last line, ScrapyDemo is my scrapy project name)
somebody help me please:
2015-08-16 20:18:31+0800 [Launcher,12972/stderr] Unhandled error in Deferred:
2015-08-16 20:18:31+0800 [Launcher,12972/stderr]
Traceback (most recent call last):
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/cmdline.py", line 150, in _run_command
cmd.run(args, opts)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/commands/crawl.py", line 57, in run
self.crawler_process.crawl(spname, **opts.spargs)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/crawler.py", line 153, in crawl
d = crawler.crawl(*args, **kwargs)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/twisted/internet/defer.py", line 1274, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
--- <exception caught here> ---
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/twisted/internet/defer.py", line 1128, in _inlineCallbacks
result = g.send(result)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/crawler.py", line 71, in crawl
self.engine = self._create_engine()
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/crawler.py", line 83, in _create_engine
return ExecutionEngine(self, lambda _: self.stop())
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/core/engine.py", line 67, in __init__
self.scraper = Scraper(crawler)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/core/scraper.py", line 70, in __init__
self.itemproc = itemproc_cls.from_crawler(crawler)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/middleware.py", line 56, in from_crawler
return cls.from_settings(crawler.settings, crawler)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/middleware.py", line 34, in from_settings
mw = mwcls.from_crawler(crawler)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/pipelines/media.py", line 33, in from_crawler
pipe = cls.from_settings(crawler.settings)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/pipelines/images.py", line 57, in from_settings
return cls(store_uri)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/dynamic_scraper/pipelines.py", line 15, in __init__
super(DjangoImagesPipeline, self).__init__(*args, **kwargs)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/pipelines/files.py", line 160, in __init__
self.store = self._get_store(store_uri)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/pipelines/files.py", line 181, in _get_store
return store_cls(uri)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/pipelines/files.py", line 43, in __init__
self._mkdir(self.basedir)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/pipelines/files.py", line 72, in _mkdir
os.makedirs(dirname)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/os.py", line 150, in makedirs
makedirs(head, mode)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/os.py", line 150, in makedirs
makedirs(head, mode)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/os.py", line 150, in makedirs
makedirs(head, mode)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/os.py", line 157, in makedirs
mkdir(name, mode)
exceptions.OSError: [Errno 20] Not a directory: '/private/var/folders/32/2_lmw4t55m5fx_gb5v7d_0vr0000gn/T/ScrapyDemo-1439724559-dDctsA.egg/ScrapyDemo'