I am making a class to handle operations for a DynamoDB database. It is going to be simple database with only one table.
I am getting a NoRegionError() when running my script as python dynamo.db. However, when I just call the class in a jupyter notebook or qtconsole environment or run each of the commands in the script separately in a python shell, I get no such error.
I have added my credentials to the local .aws folder so I'm not sure why this error is occurring and how to debug it. Here is the class I'm working on:
import boto3
class Dynamo(object):
def __init__(self, **kwargs):
self._db_api = 'dynamodb'
self._session = None
self._client = None
self._database = None
self._region_name = kwargs.get('region_name', 'us-west-2')
self._endpoint_url = kwargs.get('endpoint_url', 'http://localhost:8000')
self.cached_table = {}
self.table = None
def connect(self):
self._session = boto3.Session()
print(self._region_name)
print(self._db_api)
if self._session:
self._database = self._session.resource(self._db_api)
def main():
dynamo = Dynamo()
dynamo.connect()
return dynamo
if __name__ == '__main__':
dynodb = main()
And this is the error I'm getting:
Traceback (most recent call last):
File "test.py", line 39, in <module>
dynodb = main()
File "test.py", line 34, in main
dynamo.connect()
File "test.py", line 28, in connect
self._database = self._session.resource(self._db_api)
File "python2.7/site-packages/boto3/session.py", line 389, in resource
aws_session_token=aws_session_token, config=config)
File "python2.7/site-packages/boto3/session.py", line 263, in client
aws_session_token=aws_session_token, config=config)
File "python2.7/site-packages/botocore/session.py", line 824, in create_client
client_config=config, api_version=api_version)
File "python2.7/site-packages/botocore/client.py", line 69, in create_client
verify, credentials, scoped_config, client_config, endpoint_bridge)
File "python2.7/site-packages/botocore/client.py", line 222, in _get_client_args
verify, credentials, scoped_config, client_config, endpoint_bridge)
File "python2.7/site-packages/botocore/args.py", line 44, in get_client_args
endpoint_url, is_secure, scoped_config)
File "python2.7/site-packages/botocore/args.py", line 101, in compute_client_args
service_name, region_name, endpoint_url, is_secure)
File "python2.7/site-packages/botocore/client.py", line 295, in resolve
service_name, region_name)
File "python2.7/site-packages/botocore/regions.py", line 122, in construct_endpoint
partition, service_name, region_name)
File "python2.7/site-packages/botocore/regions.py", line 135, in _endpoint_for_partition
raise NoRegionError()
botocore.exceptions.NoRegionError: You must specify a region.
You should create a DynamoDB client using boto3 templates:
boto3.client('dynamodb', region_name='us-west-2', endpoint_url='http://localhost:8000')
Also see this question and answer.
boto3.session('dynamodb', region_name='us-west-2', endpoint_url='http://localhost:8000')
This will may helpful for you.
Related
I'm new to python, and I am trying to build a little Reddit bot to get some practice.
My bot works just fine (tested many times) but once I try to move the credentials to a .env file it fails.
I've ran pip3 install python-dotenv
This is my code, just one file, removing irrelevant stuff:
import os
import praw
import firebase_admin
from firebase_admin import credentials
from firebase_admin import db
from dotenv import load_dotenv
load_dotenv()
cred = credentials.Certificate("serviceAccountKey.json")
firebase_admin.initialize_app(cred, {
'databaseURL': 'https://makideat-6a-default-db.firebaseio.com'
})
db_ref = db.reference()
reddit = praw.Reddit(
client_id=os.getenv('CLIENT_ID'),
client_secret=os.getenv('CLIENT_SECRET'),
username=os.getenv('USERNAME'),
password=os.getenv('PASSWORD'),
user_agent=os.getenv('USER_AGENT')
)
def summoned_reply(comment):
DO SOME STUFF...
messages = reddit.inbox.stream() # creates an iterable for your inbox and streams it
for message in messages: # iterates through your messages
try:
if message in reddit.inbox.mentions() and message in reddit.inbox.unread():
summoned_reply(message)
message.mark_read()
except praw.exceptions.APIException:
print("probably a rate limit....")
And this is the error I'm getting (line 56 is for message in messages:):
Traceback (most recent call last):
File "C:\Users\Yanay\Documents\Coding\NiceAndPretty\main.py", line 56, in <module>
for message in messages: # iterates through your messages
File "C:\Users\Yanay\.virtualenvs\NiceAndPretty\lib\site-packages\praw\models\util.py", line 195, in stream_generator
for item in reversed(list(function(limit=limit, **function_kwargs))):
File "C:\Users\Yanay\.virtualenvs\NiceAndPretty\lib\site-packages\praw\models\listing\generator.py", line 63, in __next__
self._next_batch()
File "C:\Users\Yanay\.virtualenvs\NiceAndPretty\lib\site-packages\praw\models\listing\generator.py", line 89, in _next_batch
self._listing = self._reddit.get(self.url, params=self.params)
File "C:\Users\Yanay\.virtualenvs\NiceAndPretty\lib\site-packages\praw\util\deprecate_args.py", line 43, in wrapped
return func(**dict(zip(_old_args, args)), **kwargs)
File "C:\Users\Yanay\.virtualenvs\NiceAndPretty\lib\site-packages\praw\reddit.py", line 634, in get
return self._objectify_request(method="GET", params=params, path=path)
File "C:\Users\Yanay\.virtualenvs\NiceAndPretty\lib\site-packages\praw\reddit.py", line 739, in _objectify_request
self.request(
File "C:\Users\Yanay\.virtualenvs\NiceAndPretty\lib\site-packages\praw\util\deprecate_args.py", line 43, in wrapped
return func(**dict(zip(_old_args, args)), **kwargs)
File "C:\Users\Yanay\.virtualenvs\NiceAndPretty\lib\site-packages\praw\reddit.py", line 941, in request
return self._core.request(
File "C:\Users\Yanay\.virtualenvs\NiceAndPretty\lib\site-packages\prawcore\sessions.py", line 330, in request
return self._request_with_retries(
File "C:\Users\Yanay\.virtualenvs\NiceAndPretty\lib\site-packages\prawcore\sessions.py", line 228, in _request_with_retries
response, saved_exception = self._make_request(
File "C:\Users\Yanay\.virtualenvs\NiceAndPretty\lib\site-packages\prawcore\sessions.py", line 185, in _make_request
response = self._rate_limiter.call(
File "C:\Users\Yanay\.virtualenvs\NiceAndPretty\lib\site-packages\prawcore\rate_limit.py", line 33, in call
kwargs["headers"] = set_header_callback()
File "C:\Users\Yanay\.virtualenvs\NiceAndPretty\lib\site-packages\prawcore\sessions.py", line 283, in _set_header_callback
self._authorizer.refresh()
File "C:\Users\Yanay\.virtualenvs\NiceAndPretty\lib\site-packages\prawcore\auth.py", line 425, in refresh
self._request_token(
File "C:\Users\Yanay\.virtualenvs\NiceAndPretty\lib\site-packages\prawcore\auth.py", line 158, in _request_token
raise OAuthException(
prawcore.exceptions.OAuthException: invalid_grant error processing request
Then, in the same directory I have a .env file in the following structure (changed characters for everything here of course, but haven't changed the structure or added or removed spaces from anywhere):
CLIENT_ID=gGATUW-Ea507LoQ
CLIENT_SECRET=_b6vCCHKBCqVHzlJin3sEQ
USERNAME=Maddeeat
PASSWORD=!Neatat!
USER_AGENT=Maddeeat:1.0.0 (by u/Maddeeat)
I'm getting this error of autoreconnect, and there are around 100 connections in the logs during this call to .objects. Here is the document:
class NotificationDoc(Document):
patient_id = StringField(max_length=32)
type = StringField(max_length=32)
sms_sent = BooleanField(default=False)
email_sent = BooleanField(default=False)
sending_time = DateTimeField(default=datetime.utcnow)
def __unicode__(self):
return "{}_{}_{}".format(self.patient_id, self.type, self.sending_time.strftime('%Y-%m-%d'))
and this is the query call that causes the issue:
docs = NotificationDoc.objects(sending_time__lte=from_date)
This is the complete stacktrace. How can I understand what is going on?
pymongo.errors.AutoReconnect: connection pool paused
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/celery/app/trace.py", line 451, in trace_task
R = retval = fun(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/celery/app/trace.py", line 734, in __protected_call__
return self.run(*args, **kwargs)
File "/app/src/reminder/configurations/app/worker.py", line 106, in schedule_recall
schedule_recall_use_case.execute()
File "/app/src/reminder/domain/notification/recall/use_cases/schedule_recall.py", line 24, in execute
notifications_sent = self.notifications_provider.find_recalled_notifications_for_date(no_recalls_before_date)
File "/app/src/reminder/data_providers/database/odm/repositories.py", line 42, in find_recalled_notifications_for_date
if NotificationDoc.objects(sending_time__lte=from_date).count() > 0:
File "/usr/local/lib/python3.8/site-packages/mongoengine/queryset/queryset.py", line 144, in count
return super().count(with_limit_and_skip)
File "/usr/local/lib/python3.8/site-packages/mongoengine/queryset/base.py", line 423, in count
count = count_documents(
File "/usr/local/lib/python3.8/site-packages/mongoengine/pymongo_support.py", line 38, in count_documents
return collection.count_documents(filter=filter, **kwargs)
File "/usr/local/lib/python3.8/site-packages/pymongo/collection.py", line 1502, in count_documents
return self.__database.client._retryable_read(
File "/usr/local/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1307, in _retryable_read
with self._secondaryok_for_server(read_pref, server, session) as (
File "/usr/local/lib/python3.8/contextlib.py", line 113, in __enter__
return next(self.gen)
File "/usr/local/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1162, in _secondaryok_for_server
with self._get_socket(server, session) as sock_info:
File "/usr/local/lib/python3.8/contextlib.py", line 113, in __enter__
return next(self.gen)
File "/usr/local/lib/python3.8/site-packages/pymongo/mongo_client.py", line 1099, in _get_socket
with server.get_socket(
File "/usr/local/lib/python3.8/contextlib.py", line 113, in __enter__
return next(self.gen)
File "/usr/local/lib/python3.8/site-packages/pymongo/pool.py", line 1371, in get_socket
sock_info = self._get_socket(all_credentials)
File "/usr/local/lib/python3.8/site-packages/pymongo/pool.py", line 1436, in _get_socket
self._raise_if_not_ready(emit_event=True)
File "/usr/local/lib/python3.8/site-packages/pymongo/pool.py", line 1407, in _raise_if_not_ready
_raise_connection_failure(
File "/usr/local/lib/python3.8/site-packages/pymongo/pool.py", line 250, in _raise_connection_failure
raise AutoReconnect(msg) from error
pymongo.errors.AutoReconnect: mongo:27017: connection pool paused
turned out to be celery is leaving connections open, so here is the solution
from mongoengine import disconnect
from celery.signals import task_prerun
#task_prerun.connect
def on_task_init(*args, **kwargs):
disconnect(alias='default')
connect(db, host=host, port=port, maxPoolSize=400, minPoolSize=200, alias='default')
This could be related to the fact that PyMongo is not fork-safe. If you're using a process pool in any way, which includes server software like uWSGI and even some configurations of popular ASGI servers.
Every time you run a query in PyMongo, that MongoClient object becomes fork-unsafe. A MongoClient object that has never run any queries is fork-safe. Created Database and Collection objects will reference back to their parent MongoClient without checking for existence.
I do see you are using MongoEngine, which uses PyMongo underneath. I don't know the semantics of that particular library but I would assume they are the same.
In short: you must re-create your connections as part of your forking process.
pip install -U pymongo; is already fix; see this https://github.com/mongodb/mongo-python-driver/pull/944
I'm using Google cloud functions and the Cloud Billing Budget API to get a list with all of my budgets, but I'm having the following error:
Traceback (most recent call last): File "/env/local/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 67, in error_remapped_callable return callable_(*args, **kwargs) File "/env/local/lib/python3.7/site-packages/grpc/_channel.py", line 946, in __call__ return _end_unary_response_blocking(state, call, False, None) File "/env/local/lib/python3.7/site-packages/grpc/_channel.py", line 849, in _end_unary_response_blocking raise _InactiveRpcError(state) grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with: status = StatusCode.PERMISSION_DENIED details = "The caller does not have permission"
debug_error_string = "{"created":"#9627456.9324530376","description":"Error received from peer ipv4:54.128.19.5:443","file":"src/core/lib/surface/call.cc","file_line":1069,"grpc_message":"The caller does not have permission","grpc_status":7}" > The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/env/local/lib/python3.7/site-packages/google/cloud/functions/worker_v2.py", line 449, in run_background_function _function_handler.invoke_user_function(event_object) File "/env/local/lib/python3.7/site-packages/google/cloud/functions/worker_v2.py", line 268, in invoke_user_function return call_user_function(request_or_event) File "/env/local/lib/python3.7/site-packages/google/cloud/functions/worker_v2.py", line 265, in call_user_function event_context.Context(**request_or_event.context)) File "/user_code/main.py", line 22, in getting_data all_budgets = client.list_budgets(request = {'parent': BILLING_ACCOUNT}) File "/env/local/lib/python3.7/site-packages/google/cloud/billing/budgets_v1/services/budget_service/client.py", line 693, in list_budgets response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,) File "/env/local/lib/python3.7/site-packages/google/api_core/gapic_v1/method.py", line 145, in __call__ return wrapped_func(*args, **kwargs) File "/env/local/lib/python3.7/site-packages/google/api_core/retry.py", line 290, in retry_wrapped_func on_error=on_error, File "/env/local/lib/python3.7/site-packages/google/api_core/retry.py", line 188, in retry_target return target() File "/env/local/lib/python3.7/site-packages/google/api_core/grpc_helpers.py", line 69, in error_remapped_callable six.raise_from(exceptions.from_grpc_error(exc), exc) File "<string>", line 3, in raise_from google.api_core.exceptions.PermissionDenied: 403 The caller does not have permission
What I've done is give the appropriate permissions (billing.budget.list ... billing.budget.get ... etc ) at the organization level to the service account of the function, but it does not work.
My code is this:
#main.py
import os
import get_budgets
from google.cloud.billing import budgets
def getting_data(data, context):
BILLING_ACCOUNT = 'billingAccounts/XXXXXX-XXXXXX-XXXXXX'
client = budgets.BudgetServiceClient()
all_budgets = client.list_budgets(request = {'parent': BILLING_ACCOUNT})
budget_actions.budget_list(all_budgets)
#get_budgets.py
from google.cloud.billing import budgets
from googleapiclient import discovery
#BUDGET LIST
def budget_list(all_budgets):
print('Budget summary')
for budget in all_budgets:
print(f'Name: {budget.display_name}')
b_amount = budget.amount
if 'specified_amount' in b_amount:
print(f'Specified Amount: {b_amount.specified_amount.units} {b_amount.specified_amount.currency_code}')
if 'last_period_amount' in b_amount:
print('Dynamic spend (based on last period)')
print('')
Is there something that I've forgotten?
The solution to my problem was easier than I expected.
What I've done is add the service account as a member of the billing account.
Here is a quick video from Google of how to do it
https://www.youtube.com/watch?v=Vti0OGQfLHQ
And in the comments of my question, you can find additional details.
I use celery 4.3.0 and rabbitmq 3.6.6. There is also a Haproxy for balancing rabbitmq, but right now it only uses one rabbitmq node.
Timeout client/server in Haproxy 24 hours.
Sometimes I get an error and the task execution status is not returned, while the tasks continue to run:
Received 0x00 while expecting 0xce
Traceback (most recent call last):
File "/var/www/serv/app/mod_axmsg/dispatch.py", line 376, in execute
result_value = func_meta.func(*task.args_list, **task.kwargs_dict)
File "/var/www/serv/app/mod_rmc/libs/health.py", line 91, in update_all_health
health.update_all_health()
File "/var/www/serv/app/mod_core/libs/health.py", line 515, in update_all_health
result.join()
File "/usr/local/serv/lib/python3.5/site-packages/celery/result.py", line 765, in join
interval=interval, no_ack=no_ack, on_interval=on_interval,
File "/usr/local/serv/lib/python3.5/site-packages/celery/result.py", line 226, in get
on_message=on_message,
File "/usr/local/serv/lib/python3.5/site-packages/celery/backends/asynchronous.py", line 188, in wait_for_pending
for _ in self._wait_for_pending(result, **kwargs):
File "/usr/local/serv/lib/python3.5/site-packages/celery/backends/asynchronous.py", line 255, in _wait_for_pending
on_interval=on_interval):
File "/usr/local/serv/lib/python3.5/site-packages/celery/backends/asynchronous.py", line 56, in drain_events_until
yield self.wait_for(p, wait, timeout=1)
File "/usr/local/serv/lib/python3.5/site-packages/celery/backends/asynchronous.py", line 65, in wait_for
wait(timeout=timeout)
File "/usr/local/serv/lib/python3.5/site-packages/celery/backends/rpc.py", line 63, in drain_events
return self._connection.drain_events(timeout=timeout)
File "/usr/local/serv/lib/python3.5/site-packages/kombu/connection.py", line 323, in drain_events
return self.transport.drain_events(self.connection, **kwargs)
File "/usr/local/serv/lib/python3.5/site-packages/kombu/transport/pyamqp.py", line 103, in drain_events
return connection.drain_events(**kwargs)
File "/usr/local/serv/lib/python3.5/site-packages/amqp/connection.py", line 505, in drain_events
while not self.blocking_read(timeout):
File "/usr/local/serv/lib/python3.5/site-packages/amqp/connection.py", line 510, in blocking_read
frame = self.transport.read_frame()
File "/usr/local/serv/lib/python3.5/site-packages/amqp/transport.py", line 280, in read_frame
'Received {0:#04x} while expecting 0xce'.format(ch))
amqp.exceptions.UnexpectedFrame: Received 0x00 while expecting 0xce
My code: celery.py
import logging
from app import db
from celery import Celery
LOG = logging.getLogger(__name__)
def make_celery(flask_app):
broker = flask_app.config['AMQP_URL']
celery = Celery('mod_celery', broker=broker, backend="rpc://")
celery.conf.update(flask_app.config)
class ContextTask(celery.Task):
def __call__(self, *args, **kwargs):
with flask_app.app_context():
return self.run(*args, **kwargs)
celery.Task = ContextTask
return celery
def in_tx(task_function):
def tx_wrapper(*args):
try:
task_function(*args)
db.session.commit()
except Exception as ex:
msg = 'Could not execute task {}. Reason: {}.' \
.format(task_function.__name__, str(ex))
LOG.error(msg)
db.session.rollback()
return tx_wrapper
And file tasks.py
from app import celery
from .celery import in_tx
#celery.task(name='app.mod_celery.update_client_health_task')
#in_tx
def update_client_health_task(client_id):
from app.mod_core.libs.health import update_client_health
update_client_health(client_id)
What could be the problem? Please don't offer me redis.
I'm not a developer and I didn't write it the code, but I really need to figure it out. I'm new to python
Thanks.
I'm trying to run two servers using web.py and initiating calls from one to another. Both servers start normally but when I try to call a url the below stack trace is thrown.
import web
urls = (
'/ping', 'Ping',
'/acqlock/+(.*)', 'Acquire',
)
class MSA(web.application):
def run(self, port=8081, *middleware):
func = self.wsgifunc(*middleware)
return web.httpserver.runsimple(func, ('127.0.0.1', port))
app = MSA(urls, globals())
if __name__ == "__main__":
app.run(port=8081)
class Acquire:
def GET(self, resource_name):
print resource_name
response = app.request('http://127.0.0.1:8080/acqlock/' + resource_name, method='GET')
return response
But I keep getting this error after calling the /acqlock.
Traceback (most recent call last):
File "C:\Python27\lib\site-packages\web\wsgiserver\__init__.py", line 1245, in communicate
req.respond()
File "C:\Python27\lib\site-packages\web\wsgiserver\__init__.py", line 775, in respond
self.server.gateway(self).respond()
File "C:\Python27\lib\site-packages\web\wsgiserver\__init__.py", line 2018, in respond
response = self.req.server.wsgi_app(self.env, self.start_response)
File "C:\Python27\lib\site-packages\web\httpserver.py", line 306, in __call__
return self.app(environ, xstart_response)
File "C:\Python27\lib\site-packages\web\httpserver.py", line 274, in __call__
return self.app(environ, start_response)
File "C:\Python27\lib\site-packages\web\application.py", line 279, in wsgi
result = self.handle_with_processors()
File "C:\Python27\lib\site-packages\web\application.py", line 249, in handle_with_processors
return process(self.processors)
File "C:\Python27\lib\site-packages\web\application.py", line 246, in process
raise self.internalerror()
File "C:\Python27\lib\site-packages\web\application.py", line 515, in internalerror
parent = self.get_parent_app()
File "C:\Python27\lib\site-packages\web\application.py", line 500, in get_parent_app
if self in web.ctx.app_stack:
AttributeError: 'ThreadedDict' object has no attribute 'app_stack'
Use requests library for this.
import requests
response = requests.request(method='GET', url ='http://127.0.0.1:8080/acqlock/' + resource_name)
Note: You have used port 8080 in url even though you have hosted the web.py in 8081