Scrapy hanging with any more than five spiders running - python

I am trying to run multiple scrapy spiders at once, each from the command line. For example: command
>> python run_scrapy_spider_1.py
>> python run_scrapy_spider_2.py
However, once I already have 5 spiders running (all on different domains), the next one I try to run hangs with the message:
INFO: Telnet console listening on 127.0.0.1:6028
I don't think its an issue with the 6th site I am trying to run (it works fine when less than 5 spiders are running), but instead the fact that 5 spiders are already running.
Here is my very log output long incase it helps, any ideas how I can get more than 5 spiders running at once?
--- Logging error ---
Traceback (most recent call last):
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 33, in emit
payload = self.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 833, in format
return fmt.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 573, in format
s = self.formatMessage(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 542, in formatMessage
return self._style.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 386, in format
return self._fmt % record.__dict__
KeyError: 'request_id'
Call stack:
File "run_spider_alexa_id.py", line 16, in <module>
SpiderTasks.run_spider_for_alexa_site_id(alexa_site_id)
File "/home/ec2-user/code/green_brick_two/pricecomparison_project/pricecomparison/scripts/spider_scripts/spider_tasks.py", line 18, in run_spider_for_alexa_site_id
process.crawl(MySpider, alexa_site_id=alexa_site_id)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 172, in crawl
return self._crawl(crawler, *args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 176, in _crawl
d = crawler.crawl(*args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1532, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 80, in crawl
self.engine = self._create_engine()
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 105, in _create_engine
return ExecutionEngine(self, lambda _: self.stop())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/core/engine.py", line 70, in __init__
self.scraper = Scraper(crawler)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/core/scraper.py", line 69, in __init__
self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/middleware.py", line 53, in from_crawler
return cls.from_settings(crawler.settings, crawler)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/middleware.py", line 48, in from_settings
extra={'crawler': crawler})
File "/usr/local/lib/python3.6/logging/__init__.py", line 1301, in info
self._log(INFO, msg, args, **kwargs)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1437, in _log
self.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1447, in handle
self.callHandlers(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/sentry_sdk/integrations/logging.py", line 47, in sentry_patched_callhandlers
return old_callhandlers(self, record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1509, in callHandlers
hdlr.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 858, in handle
self.emit(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 38, in emit
self.handleError(record)
Message: 'Enabled %(componentname)ss:\n%(enabledlist)s'
Arguments: {'componentname': 'spider middleware', 'enabledlist': "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n 'scrapy.spidermiddlewares.depth.DepthMiddleware']"}
2019-03-03 22:34:30 [scrapy.middleware] INFO: Enabled spider middlewares:
['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',
'scrapy.spidermiddlewares.referer.RefererMiddleware',
'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',
'scrapy.spidermiddlewares.depth.DepthMiddleware']
--- Logging error ---
Traceback (most recent call last):
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 33, in emit
payload = self.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 833, in format
return fmt.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 573, in format
s = self.formatMessage(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 542, in formatMessage
return self._style.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 386, in format
return self._fmt % record.__dict__
KeyError: 'request_id'
Call stack:
File "run_spider_alexa_id.py", line 16, in <module>
SpiderTasks.run_spider_for_alexa_site_id(alexa_site_id)
File "/home/ec2-user/code/green_brick_two/pricecomparison_project/pricecomparison/scripts/spider_scripts/spider_tasks.py", line 18, in run_spider_for_alexa_site_id
process.crawl(MySpider, alexa_site_id=alexa_site_id)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 172, in crawl
return self._crawl(crawler, *args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 176, in _crawl
d = crawler.crawl(*args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1532, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 80, in crawl
self.engine = self._create_engine()
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 105, in _create_engine
return ExecutionEngine(self, lambda _: self.stop())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/core/engine.py", line 70, in __init__
self.scraper = Scraper(crawler)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/core/scraper.py", line 71, in __init__
self.itemproc = itemproc_cls.from_crawler(crawler)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/middleware.py", line 53, in from_crawler
return cls.from_settings(crawler.settings, crawler)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/middleware.py", line 48, in from_settings
extra={'crawler': crawler})
File "/usr/local/lib/python3.6/logging/__init__.py", line 1301, in info
self._log(INFO, msg, args, **kwargs)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1437, in _log
self.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1447, in handle
self.callHandlers(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/sentry_sdk/integrations/logging.py", line 47, in sentry_patched_callhandlers
return old_callhandlers(self, record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1509, in callHandlers
hdlr.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 858, in handle
self.emit(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 38, in emit
self.handleError(record)
Message: 'Enabled %(componentname)ss:\n%(enabledlist)s'
Arguments: {'componentname': 'item pipeline', 'enabledlist': '[]'}
2019-03-03 22:34:30 [scrapy.middleware] INFO: Enabled item pipelines:
[]
--- Logging error ---
Traceback (most recent call last):
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 33, in emit
payload = self.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 833, in format
return fmt.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 573, in format
s = self.formatMessage(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 542, in formatMessage
return self._style.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 386, in format
return self._fmt % record.__dict__
KeyError: 'request_id'
Call stack:
File "run_spider_alexa_id.py", line 16, in <module>
SpiderTasks.run_spider_for_alexa_site_id(alexa_site_id)
File "/home/ec2-user/code/green_brick_two/pricecomparison_project/pricecomparison/scripts/spider_scripts/spider_tasks.py", line 18, in run_spider_for_alexa_site_id
process.crawl(MySpider, alexa_site_id=alexa_site_id)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 172, in crawl
return self._crawl(crawler, *args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 176, in _crawl
d = crawler.crawl(*args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1532, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 82, in crawl
yield self.engine.open_spider(self.spider, start_requests)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1532, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/core/engine.py", line 256, in open_spider
logger.info("Spider opened", extra={'spider': spider})
File "/usr/local/lib/python3.6/logging/__init__.py", line 1301, in info
self._log(INFO, msg, args, **kwargs)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1437, in _log
self.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1447, in handle
self.callHandlers(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/sentry_sdk/integrations/logging.py", line 47, in sentry_patched_callhandlers
return old_callhandlers(self, record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1509, in callHandlers
hdlr.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 858, in handle
self.emit(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 38, in emit
self.handleError(record)
Message: 'Spider opened'
Arguments: ()
2019-03-03 22:34:30 [scrapy.core.engine] INFO: Spider opened
--- Logging error ---
Traceback (most recent call last):
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 33, in emit
payload = self.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 833, in format
return fmt.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 573, in format
s = self.formatMessage(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 542, in formatMessage
return self._style.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 386, in format
return self._fmt % record.__dict__
KeyError: 'request_id'
Call stack:
File "run_spider_alexa_id.py", line 16, in <module>
SpiderTasks.run_spider_for_alexa_site_id(alexa_site_id)
File "/home/ec2-user/code/green_brick_two/pricecomparison_project/pricecomparison/scripts/spider_scripts/spider_tasks.py", line 18, in run_spider_for_alexa_site_id
process.crawl(MySpider, alexa_site_id=alexa_site_id)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 172, in crawl
return self._crawl(crawler, *args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 176, in _crawl
d = crawler.crawl(*args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1532, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 82, in crawl
yield self.engine.open_spider(self.spider, start_requests)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1532, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/core/engine.py", line 266, in open_spider
yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/signalmanager.py", line 61, in send_catch_log_deferred
return _signal.send_catch_log_deferred(signal, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/utils/signal.py", line 65, in send_catch_log_deferred
*arguments, **named)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 150, in maybeDeferred
result = f(*args, **kw)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/pydispatch/robustapply.py", line 55, in robustApply
return receiver(*arguments, **named)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/extensions/logstats.py", line 35, in spider_opened
self.task.start(self.interval)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/task.py", line 194, in start
self()
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/task.py", line 239, in __call__
d = defer.maybeDeferred(self.f, *self.a, **self.kw)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 150, in maybeDeferred
result = f(*args, **kw)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/extensions/logstats.py", line 48, in log
logger.info(msg, log_args, extra={'spider': spider})
File "/usr/local/lib/python3.6/logging/__init__.py", line 1301, in info
self._log(INFO, msg, args, **kwargs)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1437, in _log
self.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1447, in handle
self.callHandlers(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/sentry_sdk/integrations/logging.py", line 47, in sentry_patched_callhandlers
return old_callhandlers(self, record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1509, in callHandlers
hdlr.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 858, in handle
self.emit(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 38, in emit
self.handleError(record)
Message: 'Crawled %(pages)d pages (at %(pagerate)d pages/min), scraped %(items)d items (at %(itemrate)d items/min)'
Arguments: {'pages': 0, 'pagerate': 0.0, 'items': 0, 'itemrate': 0.0}
2019-03-03 22:34:30 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
--- Logging error ---
Traceback (most recent call last):
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 33, in emit
payload = self.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 833, in format
return fmt.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 573, in format
s = self.formatMessage(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 542, in formatMessage
return self._style.format(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 386, in format
return self._fmt % record.__dict__
KeyError: 'request_id'
Call stack:
File "run_spider_alexa_id.py", line 16, in <module>
SpiderTasks.run_spider_for_alexa_site_id(alexa_site_id)
File "/home/ec2-user/code/green_brick_two/pricecomparison_project/pricecomparison/scripts/spider_scripts/spider_tasks.py", line 18, in run_spider_for_alexa_site_id
process.crawl(MySpider, alexa_site_id=alexa_site_id)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 172, in crawl
return self._crawl(crawler, *args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 176, in _crawl
d = crawler.crawl(*args, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1532, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/crawler.py", line 83, in crawl
yield defer.maybeDeferred(self.engine.start)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 150, in maybeDeferred
result = f(*args, **kw)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1532, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/core/engine.py", line 78, in start
yield self.signals.send_catch_log_deferred(signal=signals.engine_started)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/signalmanager.py", line 61, in send_catch_log_deferred
return _signal.send_catch_log_deferred(signal, **kwargs)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/utils/signal.py", line 65, in send_catch_log_deferred
*arguments, **named)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/Twisted-18.4.0-py3.6-linux-x86_64.egg/twisted/internet/defer.py", line 150, in maybeDeferred
result = f(*args, **kw)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/pydispatch/robustapply.py", line 55, in robustApply
return receiver(*arguments, **named)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/scrapy/extensions/telnet.py", line 74, in start_listening
extra={'crawler': self.crawler})
File "/usr/local/lib/python3.6/logging/__init__.py", line 1301, in info
self._log(INFO, msg, args, **kwargs)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1437, in _log
self.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1447, in handle
self.callHandlers(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/sentry_sdk/integrations/logging.py", line 47, in sentry_patched_callhandlers
return old_callhandlers(self, record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 1509, in callHandlers
hdlr.handle(record)
File "/usr/local/lib/python3.6/logging/__init__.py", line 858, in handle
self.emit(record)
File "/home/ec2-user/MYVENV/lib/python3.6/site-packages/loggly/handlers.py", line 38, in emit
self.handleError(record)
Message: 'Telnet console listening on %(host)s:%(port)d'
Arguments: {'host': '127.0.0.1', 'port': 6028}
2019-03-03 22:34:30 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6028

Related

Paramter Estimation with parmest for differential equations in Pyomo

I have a model and I want to estimate the values of the variables using parmest in pyomo.
Using parmest, I got a runtime error and a TypeError, and I can't figure out why.
Below is my code:
data = pd.read_csv('Test.csv')
#data
def modelfunc(data):
model = m = ConcreteModel()
m.t = ContinuousSet(bounds=(0,1))
# Define parameters
m.Ssu_in = Param(m.t, mutable=True)
m.Saa_in = Param(m.t, mutable=True)
m.Sfa_in = Param(m.t, mutable=True)
m.Q = Param(m.t, mutable=True)
m.V_liq = Param(initialize=3400, within=PositiveReals)
# Variables
m.S_su = Var(m.t, initialize=0.012394, domain=PositiveReals, bounds=(0.001,1))
m.S_aa = Var(m.t, initialize=0.0055432, domain=PositiveReals, bounds=(0,0.1))
m.S_fa = Var(m.t, initialize=0.10741, domain=PositiveReals, bounds=(0.001,2))
# Derivatives
m.dS_su_dt = DerivativeVar(m.S_su, wrt=m.t)
m.dS_aa_dt = DerivativeVar(m.S_aa, wrt=m.t)
m.dS_fa_dt = DerivativeVar(m.S_fa, wrt=m.t)
#initial values
m.S_su[0].fix(0.012394)
m.S_aa[0].fix(0.0055432)
m.S_fa[0].fix(0.10741)
#Discretize model using Finite Difference Method
discretizer = TransformationFactory('dae.finite_difference')
discretizer.apply_to(m,nfe=50,wrt=m.t,scheme='BACKWARD')
# Load data into the following variables
timepoints = list(m.t)
data_timepoints = data['time'].tolist()
data_profiles1 = data['S_su'].tolist()
data_profiles2 = data['S_aa'].tolist()
data_profiles3 = data['S_fa'].tolist()
data_profiles27 = data['Q'].tolist()
# Interpolate the data
interp_Ssu_values = np.interp(timepoints, data_timepoints, data_profiles1)
interp_Saa_values = np.interp(timepoints, data_timepoints, data_profiles2)
interp_Sfa_values = np.interp(timepoints, data_timepoints, data_profiles3)
interp_Q_values = np.interp(timepoints, data_timepoints, data_profiles27)
for i,t in enumerate(timepoints):
m.Ssu_in[t] = interp_Ssu_values[i]
m.Saa_in[t] = interp_Saa_values[i]
m.Sfa_in[t] = interp_Sfa_values[i]
m.Q[t] = interp_Q_values[i]
#Constraints
def S_su_out_bal(m,t):
return m.dS_su_dt[t] == (m.Q[t]/m.V_liq) * (m.Ssu_in[t] - m.S_su[t]) + 0.000662979
m.Ssu_outcon = Constraint(m.t, rule=S_su_out_bal)
def S_aa_out_bal(m,t):
return m.dS_aa_dt[t] == (m.Q[t]/m.V_liq) * (m.Saa_in[t] - m.S_aa[t]) - 0.00202160
m.Saa_outcon = Constraint(m.t, rule=S_aa_out_bal)
def S_fa_out_bal(m,t):
return m.dS_fa_dt[t] == (m.Q[t]/m.V_liq) * (m.Sfa_in[t] - m.S_fa[t]) + 0.005667982
m.Sfa_outcon = Constraint(m.t, rule=S_fa_out_bal)
return model
#Vars to estimate
theta_names = ['m.S_su', 'm.S_aa', 'm.S_fa']
#Sum of squred error
def SSE(m, data):
expr = (float(data['S_su']) - m.S_su)**2 + \
(float(data['S_aa']) - m.S_aa)**2 + \
(float(data['S_fa']) - m.S_fa)**2
return expr
# Create an instance of the Parameter Estimation
pest = parmest.Estimator(modelfunc, data, theta_names, SSE, tee=True)
# Parameter Estimation
obj, theta = pest.theta_est()
I get the following error:
ERROR: Rule failed for Expression 'SecondStageCost' with index None:
TypeError: unsupported operand type(s) for -: 'float' and 'IndexedVar'
ERROR: Constructing component 'SecondStageCost' from data=None failed:
TypeError: unsupported operand type(s) for -: 'float' and 'IndexedVar'
--- Logging error ---
Traceback (most recent call last):
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1085, in emit
msg = self.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 929, in format
return fmt.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 238, in format
return self.standard_formatter.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 107, in format
msg = record.getMessage()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 373, in getMessage
msg = msg % self.args
TypeError: not enough arguments for format string
Call stack:
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\runpy.py", line 87, in _run_code
exec(code, run_globals)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel_launcher.py", line 17, in <module>
app.launch_new_instance()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\traitlets\config\application.py", line 976, in launch_instance
app.start()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelapp.py", line 712, in start
self.io_loop.start()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\tornado\platform\asyncio.py", line 199, in start
self.asyncio_loop.run_forever()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\base_events.py", line 570, in run_forever
self._run_once()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\base_events.py", line 1859, in _run_once
handle._run()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\events.py", line 81, in _run
self._context.run(self._callback, *self._args)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 510, in dispatch_queue
await self.process_one()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 499, in process_one
await dispatch(*args)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 406, in dispatch_shell
await result
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 730, in execute_request
reply_content = await reply_content
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\ipkernel.py", line 383, in do_execute
res = shell.run_cell(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\zmqshell.py", line 528, in run_cell
return super().run_cell(*args, **kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 2975, in run_cell
result = self._run_cell(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3030, in _run_cell
return runner(coro)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\async_helpers.py", line 78, in _pseudo_sync_runner
coro.send(None)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3257, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3473, in run_ast_nodes
if (await self.run_code(code, result, async_=asy)):
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3553, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "C:\Users\mfo21001\AppData\Local\Temp\ipykernel_4860\3065394695.py", line 84, in <cell line: 84>
obj, theta = pest.theta_est()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 687, in theta_est
return self._Q_opt(solver=solver, return_values=return_values,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 432, in _Q_opt
ef = local_ef.create_EF(scen_names,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\create_ef.py", line 88, in create_EF
scen_dict = {
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\create_ef.py", line 89, in <dictcomp>
name: scenario_creator(name, **scenario_creator_kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 143, in _experiment_instance_creation_callback
instance = callback(experiment_number = exp_num, cb_data = cb_data)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 391, in _instance_creation_callback
model = self._create_parmest_model(exp_data)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 344, in _create_parmest_model
logger.warning(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1458, in warning
self._log(WARNING, msg, args, **kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1589, in _log
self.handle(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1599, in handle
self.callHandlers(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1661, in callHandlers
hdlr.handle(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 954, in handle
self.emit(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 250, in emit
super(StdoutHandler, self).emit(record)
Message: 'theta_name[%s] (%s) was not found on the model'
Arguments: ((0, 'm.S_su'),)
--- Logging error ---
Traceback (most recent call last):
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1085, in emit
msg = self.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 929, in format
return fmt.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 238, in format
return self.standard_formatter.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 107, in format
msg = record.getMessage()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 373, in getMessage
msg = msg % self.args
TypeError: not enough arguments for format string
Call stack:
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\runpy.py", line 87, in _run_code
exec(code, run_globals)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel_launcher.py", line 17, in <module>
app.launch_new_instance()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\traitlets\config\application.py", line 976, in launch_instance
app.start()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelapp.py", line 712, in start
self.io_loop.start()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\tornado\platform\asyncio.py", line 199, in start
self.asyncio_loop.run_forever()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\base_events.py", line 570, in run_forever
self._run_once()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\base_events.py", line 1859, in _run_once
handle._run()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\events.py", line 81, in _run
self._context.run(self._callback, *self._args)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 510, in dispatch_queue
await self.process_one()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 499, in process_one
await dispatch(*args)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 406, in dispatch_shell
await result
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 730, in execute_request
reply_content = await reply_content
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\ipkernel.py", line 383, in do_execute
res = shell.run_cell(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\zmqshell.py", line 528, in run_cell
return super().run_cell(*args, **kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 2975, in run_cell
result = self._run_cell(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3030, in _run_cell
return runner(coro)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\async_helpers.py", line 78, in _pseudo_sync_runner
coro.send(None)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3257, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3473, in run_ast_nodes
if (await self.run_code(code, result, async_=asy)):
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3553, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "C:\Users\mfo21001\AppData\Local\Temp\ipykernel_4860\3065394695.py", line 84, in <cell line: 84>
obj, theta = pest.theta_est()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 687, in theta_est
return self._Q_opt(solver=solver, return_values=return_values,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 432, in _Q_opt
ef = local_ef.create_EF(scen_names,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\create_ef.py", line 88, in create_EF
scen_dict = {
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\create_ef.py", line 89, in <dictcomp>
name: scenario_creator(name, **scenario_creator_kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 143, in _experiment_instance_creation_callback
instance = callback(experiment_number = exp_num, cb_data = cb_data)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 391, in _instance_creation_callback
model = self._create_parmest_model(exp_data)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 344, in _create_parmest_model
logger.warning(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1458, in warning
self._log(WARNING, msg, args, **kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1589, in _log
self.handle(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1599, in handle
self.callHandlers(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1661, in callHandlers
hdlr.handle(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 954, in handle
self.emit(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 250, in emit
super(StdoutHandler, self).emit(record)
Message: 'theta_name[%s] (%s) was not found on the model'
Arguments: ((1, 'm.S_aa'),)
--- Logging error ---
Traceback (most recent call last):
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1085, in emit
msg = self.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 929, in format
return fmt.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 238, in format
return self.standard_formatter.format(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 107, in format
msg = record.getMessage()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 373, in getMessage
msg = msg % self.args
TypeError: not enough arguments for format string
Call stack:
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\runpy.py", line 87, in _run_code
exec(code, run_globals)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel_launcher.py", line 17, in <module>
app.launch_new_instance()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\traitlets\config\application.py", line 976, in launch_instance
app.start()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelapp.py", line 712, in start
self.io_loop.start()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\tornado\platform\asyncio.py", line 199, in start
self.asyncio_loop.run_forever()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\base_events.py", line 570, in run_forever
self._run_once()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\base_events.py", line 1859, in _run_once
handle._run()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\asyncio\events.py", line 81, in _run
self._context.run(self._callback, *self._args)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 510, in dispatch_queue
await self.process_one()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 499, in process_one
await dispatch(*args)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 406, in dispatch_shell
await result
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\kernelbase.py", line 730, in execute_request
reply_content = await reply_content
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\ipkernel.py", line 383, in do_execute
res = shell.run_cell(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\ipykernel\zmqshell.py", line 528, in run_cell
return super().run_cell(*args, **kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 2975, in run_cell
result = self._run_cell(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3030, in _run_cell
return runner(coro)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\async_helpers.py", line 78, in _pseudo_sync_runner
coro.send(None)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3257, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3473, in run_ast_nodes
if (await self.run_code(code, result, async_=asy)):
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\IPython\core\interactiveshell.py", line 3553, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "C:\Users\mfo21001\AppData\Local\Temp\ipykernel_4860\3065394695.py", line 84, in <cell line: 84>
obj, theta = pest.theta_est()
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 687, in theta_est
return self._Q_opt(solver=solver, return_values=return_values,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 432, in _Q_opt
ef = local_ef.create_EF(scen_names,
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\create_ef.py", line 88, in create_EF
scen_dict = {
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\create_ef.py", line 89, in <dictcomp>
name: scenario_creator(name, **scenario_creator_kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 143, in _experiment_instance_creation_callback
instance = callback(experiment_number = exp_num, cb_data = cb_data)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 391, in _instance_creation_callback
model = self._create_parmest_model(exp_data)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py", line 344, in _create_parmest_model
logger.warning(
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1458, in warning
self._log(WARNING, msg, args, **kwargs)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1589, in _log
self.handle(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1599, in handle
self.callHandlers(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 1661, in callHandlers
hdlr.handle(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\logging\__init__.py", line 954, in handle
self.emit(record)
File "c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\common\log.py", line 250, in emit
super(StdoutHandler, self).emit(record)
Message: 'theta_name[%s] (%s) was not found on the model'
Arguments: ((2, 'm.S_fa'),)
TypeError Traceback (most recent call last)
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py in _experiment_instance_creation_callback(scenario_name, node_names, cb_data)
142 try:
--> 143 instance = callback(experiment_number = exp_num, cb_data = cb_data)
144 except TypeError:
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py in _instance_creation_callback(self, experiment_number, cb_data)
390 raise RuntimeError(f'Unexpected data format for cb_data={cb_data}')
--> 391 model = self._create_parmest_model(exp_data)
392
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py in _create_parmest_model(self, data)
365 model.FirstStageCost = pyo.Expression(rule=FirstStageCost_rule)
--> 366 model.SecondStageCost = pyo.Expression(rule=_SecondStageCostExpr(self.obj_function, data))
367
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\core\base\block.py in __setattr__(self, name, val)
543 #
--> 544 self.add_component(name, val)
545 else:
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\core\base\block.py in add_component(self, name, val)
1088 try:
-> 1089 val.construct(data)
1090 except:
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\core\base\expression.py in construct(self, data)
368 assert data is None
--> 369 self._construct_from_rule_using_setitem()
370 finally:
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\core\base\indexed_component.py in _construct_from_rule_using_setitem(self)
707 # constant, then only call the rule once.
--> 708 val = rule(block, None)
709 for index in self.index_set():
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\core\base\initializer.py in __call__(self, parent, idx)
372 def __call__(self, parent, idx):
--> 373 return self._fcn(parent)
374
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py in __call__(self, model)
269 def __call__(self, model):
--> 270 return self._ssc_function(model, self._data)
271
~\AppData\Local\Temp\ipykernel_4860\3065394695.py in SSE(m, data)
74 def SSE(m, data):
---> 75 expr = (float(data['S_su']) - m.S_su)**2 + \
76 (float(data['S_aa']) - m.S_aa)**2 + \
TypeError: unsupported operand type(s) for -: 'float' and 'IndexedVar'
During handling of the above exception, another exception occurred:
RuntimeError Traceback (most recent call last)
~\AppData\Local\Temp\ipykernel_4860\3065394695.py in <cell line: 84>()
82
83 # Parameter Estimation
---> 84 obj, theta = pest.theta_est()
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py in theta_est(self, solver, return_values, calc_cov, cov_n)
685 assert cov_n > len(self.theta_names), "The number of datapoints must be greater than the number of parameters to estimate"
686
--> 687 return self._Q_opt(solver=solver, return_values=return_values,
688 bootlist=None, calc_cov=calc_cov, cov_n=cov_n)
689
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py in _Q_opt(self, ThetaVals, solver, return_values, bootlist, calc_cov, cov_n)
430 scenario_creator_kwargs=scenario_creator_options)
431 else:
--> 432 ef = local_ef.create_EF(scen_names,
433 _experiment_instance_creation_callback,
434 EF_name = "_Q_opt",
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\create_ef.py in create_EF(scenario_names, scenario_creator, scenario_creator_kwargs, EF_name, suppress_warnings, nonant_for_fixed_vars)
86 if scenario_creator_kwargs is None:
87 scenario_creator_kwargs = dict()
---> 88 scen_dict = {
89 name: scenario_creator(name, **scenario_creator_kwargs)
90 for name in scenario_names
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\create_ef.py in <dictcomp>(.0)
87 scenario_creator_kwargs = dict()
88 scen_dict = {
---> 89 name: scenario_creator(name, **scenario_creator_kwargs)
90 for name in scenario_names
91 }
c:\users\mfo21001\anaconda5\envs\watertap\lib\site-packages\pyomo\contrib\parmest\parmest.py in _experiment_instance_creation_callback(scenario_name, node_names, cb_data)
143 instance = callback(experiment_number = exp_num, cb_data = cb_data)
144 except TypeError:
--> 145 raise RuntimeError("Only one callback signature is supported: "
146 "callback(experiment_number, cb_data) ")
147 """
RuntimeError: Only one callback signature is supported: callback(experiment_number, cb_data)

spacy.load error: RuntimeError: dictionary changed size during iteration

I am loading a spaCy model as part of a step in my Dataflow streaming pipeline. To load the pre-downloaded spaCy model for a specific language I am using
nlp_model = spacy.load(SPACY_KEYS[lang])
where SPACY_KEYS is a dictionary containing the names of the models for each language (e.g. 'en': 'en_core_web_sm').
This works without any issues for the majority of the jobs run by the pipeline, but for a few iterations I am getting the following error:
Error message from worker: generic::unknown: Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1232, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 752, in apache_beam.runners.common.PerWindowInvoker.invoke_process
File "apache_beam/runners/common.py", line 870, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
File "apache_beam/runners/common.py", line 1368, in apache_beam.runners.common._OutputProcessor.process_outputs
File "/usr/local/lib/python3.7/site-packages/submodules/entities_and_pii_removal.py", line 259, in entities_and_PII
nlp_model = spacy.load(SPACY_KEYS[lang]) # load spacy model
File "/usr/local/lib/python3.7/site-packages/spacy/__init__.py", line 52, in load
name, vocab=vocab, disable=disable, exclude=exclude, config=config
File "/usr/local/lib/python3.7/site-packages/spacy/util.py", line 420, in load_model
return load_model_from_package(name, **kwargs) # type: ignore[arg-type]
File "/usr/local/lib/python3.7/site-packages/spacy/util.py", line 453, in load_model_from_package
return cls.load(vocab=vocab, disable=disable, exclude=exclude, config=config) # type: ignore[attr-defined]
File "/usr/local/lib/python3.7/site-packages/de_core_news_sm/__init__.py", line 10, in load
return load_model_from_init_py(__file__, **overrides)
File "/usr/local/lib/python3.7/site-packages/spacy/util.py", line 621, in load_model_from_init_py
config=config,
File "/usr/local/lib/python3.7/site-packages/spacy/util.py", line 489, in load_model_from_path
return nlp.from_disk(model_path, exclude=exclude, overrides=overrides)
File "/usr/local/lib/python3.7/site-packages/spacy/language.py", line 2042, in from_disk
util.from_disk(path, deserializers, exclude) # type: ignore[arg-type]
File "/usr/local/lib/python3.7/site-packages/spacy/util.py", line 1299, in from_disk
reader(path / key)
File "/usr/local/lib/python3.7/site-packages/spacy/language.py", line 2037, in <lambda>
p, exclude=["vocab"]
File "spacy/pipeline/trainable_pipe.pyx", line 343, in spacy.pipeline.trainable_pipe.TrainablePipe.from_disk
File "/usr/local/lib/python3.7/site-packages/spacy/util.py", line 1299, in from_disk
reader(path / key)
File "spacy/pipeline/trainable_pipe.pyx", line 333, in spacy.pipeline.trainable_pipe.TrainablePipe.from_disk.load_model
File "spacy/pipeline/trainable_pipe.pyx", line 334, in spacy.pipeline.trainable_pipe.TrainablePipe.from_disk.load_model
File "/usr/local/lib/python3.7/site-packages/thinc/model.py", line 593, in from_bytes
return self.from_dict(msg)
File "/usr/local/lib/python3.7/site-packages/thinc/model.py", line 624, in from_dict
loaded_value = deserialize_attr(default_value, value, attr, node)
File "/usr/local/lib/python3.7/functools.py", line 840, in wrapper
return dispatch(args[0].__class__)(*args, **kw)
File "/usr/local/lib/python3.7/site-packages/thinc/model.py", line 804, in deserialize_attr
return srsly.msgpack_loads(value)
File "/usr/local/lib/python3.7/site-packages/srsly/_msgpack_api.py", line 27, in msgpack_loads
msg = msgpack.loads(data, raw=False, use_list=use_list)
File "/usr/local/lib/python3.7/site-packages/srsly/msgpack/__init__.py", line 76, in unpackb
for decoder in msgpack_decoders.get_all().values():
File "/usr/local/lib/python3.7/site-packages/catalogue/__init__.py", line 110, in get_all
for keys, value in REGISTRY.items():
RuntimeError: dictionary changed size during iteration
I have not been able to identify the cause of this problem. Is there a way of getting around it?

google-api-python-client broken by OAuth2?

I am trying to check whether a certain dataset exists in BigQuery and I get this very strange error I've never had untill yesterday:
ERROR:dsUtils.bq_utils:Could not check if dataset tmp exists.
Traceback (most recent call last):
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\dsUtils\bq_utils.py", line 113, in _get
resp = bq_service.datasets().get(projectId=self.project_id, datasetId=self.id).execute(num_retries=2)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\oauth2client\util.py", line 137, in positional_wrapper
return wrapped(*args, **kwargs)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\googleapiclient\http.py", line 755, in execute
method=str(self.method), body=self.body, headers=self.headers)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\googleapiclient\http.py", line 93, in _retry_request
resp, content = http.request(uri, method, *args, **kwargs)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\oauth2client\client.py", line 598, in new_request
self._refresh(request_orig)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\oauth2client\client.py", line 864, in _refresh
self._do_refresh_request(http_request)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\oauth2client\client.py", line 891, in _do_refresh_request
body = self._generate_refresh_request_body()
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\oauth2client\client.py", line 1597, in _generate_refresh_request_body
assertion = self._generate_assertion()
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\oauth2client\service_account.py", line 318, in _generate_assertion
key_id=self._private_key_id)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\oauth2client\crypt.py", line 97, in make_signed_jwt
signature = signer.sign(signing_input)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\oauth2client\_pycrypto_crypt.py", line 101, in sign
return PKCS1_v1_5.new(self._key).sign(SHA256.new(message))
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\Crypto\Signature\PKCS1_v1_5.py", line 112, in sign
m = self._key.decrypt(em)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\Crypto\PublicKey\RSA.py", line 174, in decrypt
return pubkey.pubkey.decrypt(self, ciphertext)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\Crypto\PublicKey\pubkey.py", line 93, in decrypt
plaintext=self._decrypt(ciphertext)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\Crypto\PublicKey\RSA.py", line 235, in _decrypt
r = getRandomRange(1, self.key.n-1, randfunc=self._randfunc)
File "C:\Users\paco\Anaconda3\envs\visitForecastEnv\lib\site-packages\Crypto\PublicKey\RSA.py", line 126, in __getattr__
raise AttributeError("%s object has no %r attribute" % (self.__class__.__name__, attrname,))
AttributeError: _RSAobj object has no '_randfunc' attribute
Has anyone an idea of why I get these errors suddenly?

scrapyd exceptions.OSError: [Errno 20] Not a directory

I'm using django-dynamic-scrapy with scrapyd, and confused with this error, (in the last line, ScrapyDemo is my scrapy project name)
somebody help me please:
2015-08-16 20:18:31+0800 [Launcher,12972/stderr] Unhandled error in Deferred:
2015-08-16 20:18:31+0800 [Launcher,12972/stderr]
Traceback (most recent call last):
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/cmdline.py", line 150, in _run_command
cmd.run(args, opts)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/commands/crawl.py", line 57, in run
self.crawler_process.crawl(spname, **opts.spargs)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/crawler.py", line 153, in crawl
d = crawler.crawl(*args, **kwargs)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/twisted/internet/defer.py", line 1274, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
--- <exception caught here> ---
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/twisted/internet/defer.py", line 1128, in _inlineCallbacks
result = g.send(result)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/crawler.py", line 71, in crawl
self.engine = self._create_engine()
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/crawler.py", line 83, in _create_engine
return ExecutionEngine(self, lambda _: self.stop())
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/core/engine.py", line 67, in __init__
self.scraper = Scraper(crawler)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/core/scraper.py", line 70, in __init__
self.itemproc = itemproc_cls.from_crawler(crawler)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/middleware.py", line 56, in from_crawler
return cls.from_settings(crawler.settings, crawler)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/middleware.py", line 34, in from_settings
mw = mwcls.from_crawler(crawler)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/pipelines/media.py", line 33, in from_crawler
pipe = cls.from_settings(crawler.settings)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/pipelines/images.py", line 57, in from_settings
return cls(store_uri)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/dynamic_scraper/pipelines.py", line 15, in __init__
super(DjangoImagesPipeline, self).__init__(*args, **kwargs)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/pipelines/files.py", line 160, in __init__
self.store = self._get_store(store_uri)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/pipelines/files.py", line 181, in _get_store
return store_cls(uri)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/pipelines/files.py", line 43, in __init__
self._mkdir(self.basedir)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/site-packages/scrapy/pipelines/files.py", line 72, in _mkdir
os.makedirs(dirname)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/os.py", line 150, in makedirs
makedirs(head, mode)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/os.py", line 150, in makedirs
makedirs(head, mode)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/os.py", line 150, in makedirs
makedirs(head, mode)
File "/Users/zhushajun/.pyenv/versions/scrapy_env2/lib/python2.7/os.py", line 157, in makedirs
mkdir(name, mode)
exceptions.OSError: [Errno 20] Not a directory: '/private/var/folders/32/2_lmw4t55m5fx_gb5v7d_0vr0000gn/T/ScrapyDemo-1439724559-dDctsA.egg/ScrapyDemo'

SQLAlchemy - how to avoid an error: UnicodeEncodeError: 'latin-1' codec can't encode characters

Here is model column:
title = Column(Unicode(100))
When I try to add a record to db with cyrillic letters in title I get an error:
UnicodeEncodeError:
latin-1' codec can't encode characters on position ...
If i encode it with utf-8
title = request.POST['title'].encode('utf-8')
I get next error:
ProgrammingError: You must not use 8-bit bytestrings unless you use a text_factory that can interpret 8-bit btystrings (like text_factory = str). It is highly recommended that you instead just switch your application to Unicode strings.
When I decode utf-8
title = request.POST['title'].decode('utf-8')
There is new error
UnicodeDecodeError: 'ascii' codec can't encode characters...
What to do?
EDIT:
I'm using Sqlite3, I thought it may be important.
EDIT 2: (code and traceback)
My code:
title = request.POST['title']
new_model = Model(
...
title = title,
...
)
DBSession.add(new_model)
DBSession.flush()
And Traceback (most recent call last):
File "/home/p/work/SUN/lib/python2.6/site-packages/pyramid_debugtoolbar-0.9.7-py2.6.egg/pyramid_debugtoolbar/panels/performance.py", line 55, in resource_timer_handler
result = handler(request)
File "/home/p/work/SUN/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/tweens.py", line 20, in excview_tween
response = handler(request)
File "/home/p/work/SUN/lib/python2.6/site-packages/pyramid_tm-0.3-py2.6.egg/pyramid_tm/init.py", line 61, in tm_tween
response = handler(request)
File "/home/p/work/SUN/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/router.py", line 164, in handle_request
response = view_callable(context, request)
File "/home/p/work/SUN/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/config/views.py", line 316, in rendered_view
result = view(context, request)
File "/home/p/work/SUN/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/config/views.py", line 426, in _requestonly_view
response = view(request)
File "/home/p/work/SUN/sunviver/sunviver/views/advert.py", line 187, in create
return HTTPFound(location=new_advert.url())
File "/home/p/work/SUN/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/httpexceptions.py", line 444, in init
body_template=body_template, location=location, **kw)
File "/home/p/work/SUN/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/httpexceptions.py", line 213, in init
Response.init(self, status=status, **kw)
File "build/bdist.linux-i686/egg/webob/response.py", line 141, in init
setattr(self, name, value)
File "build/bdist.linux-i686/egg/webob/descriptors.py", line 112, in fset
value = value.encode('latin-1')
And new_advert.url():
def url(self):
return '/%s/%s' % (self.id, self.title.replace(' ', '_'))
#J.F. Sebastian
Traceback (most recent call last):
File "/usr/lib/python2.6/wsgiref/handlers.py", line 93, in run
self.result = application(self.environ, self.start_response)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/router.py", line 187, in call
response = self.handle_request(request)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid_debugtoolbar-0.9.7-py2.6.egg/pyramid_debugtoolbar/toolbar.py", line 157, in toolbar_tween
toolbar.process_response(response)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid_debugtoolbar-0.9.7-py2.6.egg/pyramid_debugtoolbar/toolbar.py", line 55, in process_response
vars, request=request)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/renderers.py", line 81, in render
return helper.render(value, None, request=request)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/renderers.py", line 420, in render
result = renderer(value, system_values)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/mako_templating.py", line 162, in call
reraise(MakoRenderingException(errtext), None, exc_info[2])
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/mako_templating.py", line 154, in call
result = template.render_unicode(**system)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/template.py", line 311, in render_unicode
as_unicode=True)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/runtime.py", line 660, in _render
**_kwargs_for_callable(callable_, data))
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/runtime.py", line 692, in _render_context
_exec_template(inherit, lclcontext, args=args, kwargs=kwargs)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/runtime.py", line 718, in _exec_template
callable_(context, *args, **kwargs)
File "pyramid_debugtoolbar_templates_toolbar_mako", line 117, in render_body
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid_debugtoolbar-0.9.7-py2.6.egg/pyramid_debugtoolbar/panels/headers.py", line 54, in content
vars, self.request)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid_debugtoolbar-0.9.7-py2.6.egg/pyramid_debugtoolbar/panels/init.py", line 24, in render
return render(template_name, vars, request=request)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/renderers.py", line 81, in render
return helper.render(value, None, request=request)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/renderers.py", line 420, in render
result = renderer(value, system_values)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/mako_templating.py", line 162, in call
reraise(MakoRenderingException(errtext), None, exc_info[2])
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/mako_templating.py", line 154, in call
result = template.render_unicode(**system)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/template.py", line 311, in render_unicode
as_unicode=True)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/runtime.py", line 660, in _render
**_kwargs_for_callable(callable_, data))
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/runtime.py", line 692, in _render_context
_exec_template(inherit, lclcontext, args=args, kwargs=kwargs)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/runtime.py", line 718, in _exec_template
callable_(context, *args, **kwargs)
File "pyramid_debugtoolbar_panels_templates_headers_mako", line 34, in render_body
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/MarkupSafe-0.15-py2.6.egg/markupsafe/_native.py", line 21, in escape
return Markup(unicode(s)
MakoRenderingException:
Traceback (most recent call last):
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/mako_templating.py", line 154, in call
result = template.render_unicode(**system)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/template.py", line 311, in render_unicode
as_unicode=True)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/runtime.py", line 660, in _render
**_kwargs_for_callable(callable_, data))
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/runtime.py", line 692, in _render_context
_exec_template(inherit, lclcontext, args=args, kwargs=kwargs)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/runtime.py", line 718, in _exec_template
callable_(context, *args, **kwargs)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid_debugtoolbar-0.9.7-py2.6.egg/pyramid_debugtoolbar/templates/toolbar.mako", line 61, in render_body
${panel.content()|n}
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid_debugtoolbar-0.9.7-py2.6.egg/pyramid_debugtoolbar/panels/headers.py", line 54, in content
vars, self.request)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid_debugtoolbar-0.9.7-py2.6.egg/pyramid_debugtoolbar/panels/init.py", line 24, in render
return render(template_name, vars, request=request)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/renderers.py", line 81, in render
return helper.render(value, None, request=request)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/renderers.py", line 420, in render
result = renderer(value, system_values)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/mako_templating.py", line 162, in call
reraise(MakoRenderingException(errtext), None, exc_info[2])
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/mako_templating.py", line 154, in call
result = template.render_unicode(**system)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/template.py", line 311, in render_unicode
as_unicode=True)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/runtime.py", line 660, in _render
**_kwargs_for_callable(callable_, data))
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/runtime.py", line 692, in _render_context
_exec_template(inherit, lclcontext, args=args, kwargs=kwargs)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/runtime.py", line 718, in _exec_template
callable_(context, *args, **kwargs)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid_debugtoolbar-0.9.7-py2.6.egg/pyramid_debugtoolbar/panels/templates/headers.mako", line 13, in render_body
${value|h}
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/MarkupSafe-0.15-py2.6.egg/markupsafe/_native.py", line 21, in escape
return Markup(unicode(s)
MakoRenderingException:
Traceback (most recent call last):
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid-1.3a3-py2.6.egg/pyramid/mako_templating.py", line 154, in call
result = template.render_unicode(**system)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/template.py", line 311, in render_unicode
as_unicode=True)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/runtime.py", line 660, in _render
**_kwargs_for_callable(callable_, data))
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/runtime.py", line 692, in _render_context
_exec_template(inherit, lclcontext, args=args, kwargs=kwargs)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/Mako-0.5.0-py2.6.egg/mako/runtime.py", line 718, in _exec_template
callable_(context, *args, **kwargs)
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/pyramid_debugtoolbar-0.9.7-py2.6.egg/pyramid_debugtoolbar/panels/templates/headers.mako", line 13, in render_body
${value|h}
File "/home/ponomar/v/ENV/lib/python2.6/site-packages/MarkupSafe-0.15-py2.6.egg/markupsafe/_native.py", line 21, in escape
return Markup(unicode(s)
UnicodeDecodeError: 'ascii' codec can't decode byte 0xd0 in position 7: ordinal not in range(128)
Encoding converts Unicode strings to byte strings. You need to go in the other direction,
decoding: request.POST['title'].decode('utf-8') (assuming request.POST['title'] is a UTF8-encoded str).

Categories