Related
I am trying to print output from an ECR image with python boto3. I can get it to print out the imageDigest, but would like to add the imageTag. Can anyone think of a way to add the imageTag? Everyway I have tried has errored out.
import json
import boto3
def get_reponames():
client = boto3.client('ecr')
reponames = [repo['repositoryName'] for repo in client.describe_repositories()['repositories']]
return reponames
def get_imageids(prepo):
client = boto3.client('ecr')
imageids = [img['imageDigest'] for img in client.list_images(repositoryName=prepo,)['imageIds']]
return imageids
def lambda_handler(event, context):
output = get_reponames()
for rn in output:
print(rn)
outputii = get_imageids(rn)
for ii in outputii:
print(ii)
return {
'body': json.dumps("hello world")
}
I'll post the output for list_images below. The above code works to display imageDigest, but I want to add imageTag too.
{
'imageIds': [
{
'imageDigest': 'sha256:764f63476bdff6d83a09ba2a818f0d35757063724a9ac3ba5019c56f74ebf42a',
'imageTag': 'precise',
},
],
'ResponseMetadata': {
'...': '...',
},
}
Your get_imageids function only return imageDigest so you can not access it on lambda_handler function. You need to return imageTag as well to read it on lambda_handler
import json
import boto3
def get_reponames():
client = boto3.client('ecr')
reponames = [repo['repositoryName'] for repo in client.describe_repositories()['repositories']]
return reponames
def get_imageids(prepo):
client = boto3.client('ecr')
imageids = [
{"digest": img['imageDigest'], "tag": img.get('imageTag', None)} for img in
client.list_images(repositoryName=prepo, )['imageIds']
]
return imageids
def lambda_handler(event, context):
output = get_reponames()
for rn in output:
print(rn)
outputii = get_imageids(rn)
for ii in outputii:
print(f"digest : {ii['digest']}, tag: {ii['tag']}")
return {
'body': json.dumps("hello world")
}
Here's my sample code which I want to test.
decorator.py
def token_required(f):
#wraps(f)
def decorated(*args, **kwargs):
data_header = request.headers['Authorization']
token = str.replace(str(data_header), 'Bearer ', '')
if not token:
return jsonify({'Message':'Token is missing'}), 401
try:
data = jwt.decode(token, "mysecret", algorithms=['HS256'])
except jwt.InvalidTokenError:
return make_response({'Message':'Invalid Token!'}, 403)
return f(data, *args, **kwargs)
return decorated
get_user_uid.py
#getuid_bp.route('/users/<uid>', methods=['GET'])
#token_required
def profile_view(data, uid):
profile_data = service_profile(data, uid)
return profile_data
data in profile_view(data, uid) is jwt decode value.
I need to test functionalities present in the decorator such as "Token is missing" & "invalid token".
This is what I've tried.
def test_service_user_profile(self):
data = {
"address": "Ahmedabad",
"dob": "2022-06-07T23:16:20",
"email_address": "testuser58#example.com",
"password": "sha256$WbLrMDWEUHKvLDSC$2a051f8689222fc9815cd00da2c3260e114c313ba20edebf8df17e6d409721ac",
"uid": "75f9d2c5570f4c7bb5e28c5807f92c48",
"user_name": "testuser58"
}
encoded_jwt = jwt.encode(data, "mysecret", algorithm="HS256")
decoded_jwt = jwt.decode(encoded_jwt, "mysecret", algorithms="HS256")
data_header = f'Bearer {encoded_jwt}'
m1 = mock.MagicMock()
m1.headers.return_value = data_header
print("m", m1)
response = profile_view(data,"75f9d2c5570f4c7bb5e28c5807f92c48")
# mock_profile.assert_called_once()
But I'm confused about how to set data in profile_view(data, uid): and how to test the decorator. I've tried to mock the request in decorator but I don't know whether it is right or wrong.
I have a chalice project (1.26.2) which is always exiting with a time out when deployed, chalice local works fine.
This is my project structure:
This is the code in my app.py:
from datetime import datetime
from decimal import Decimal
import boto3
import firebase_admin
from chalice import Chalice, AuthResponse
from chalice.app import AuthRequest
from firebase_admin import credentials, auth
from chalicelib import crud
from chalicelib import models
from chalicelib import schemas
from chalicelib.auth import full_user_from_context, user_from_context
from chalicelib.db import SessionLocal, engine
app = Chalice(app_name='server-aws')
BUCKET = 'meet-app'
s3_client = boto3.client('s3')
cred = credentials.Certificate('chalicelib/serviceAccountKey.json')
firebase_admin.initialize_app(cred)
models.Base.metadata.create_all(bind=engine)
DISTANCE_IN_METERS = 100
_DB = None
def get_db():
global _DB
if _DB is None:
_DB = SessionLocal()
return _DB
#app.lambda_function(name='test-function')
def create_user(event, context):
return {'hello': 'world'}
#app.route('/health')
def health():
return {'status': 'ok'}
#app.authorizer()
def token_authorizer(auth_request: AuthRequest) -> AuthResponse:
token = auth_request.token
try:
decoded_token = auth.verify_id_token(token)
decoded = decoded_token
allowed_routes = [
'/auth',
'/me',
]
if 'permission_verified' in decoded and decoded['permission_verified'] is True:
allowed_routes.append('/me/location')
allowed_routes.append('/nearby')
allowed_routes.append('/me/profile-image')
print('routes', allowed_routes)
return AuthResponse(routes=allowed_routes, principal_id=decoded['sub'], context=decoded)
except Exception as e:
print('error', e)
return AuthResponse(routes=[], principal_id='non-user')
#app.route('/auth', methods=['GET'], authorizer=token_authorizer)
def authorize():
u = user_from_context(app.current_request.context)
user = crud.get_user(get_db(), u['uid'])
if user is None:
user = crud.create_user(get_db(), schemas.UserCreate(
uid=u['uid'],
phone=u['phone_number'],
permission_verified=True # TODO: find verification method
))
token = auth.create_custom_token(user.uid, {
"permission_verified": user.permission_verified,
"uid": user.uid,
"phone": user.phone,
"name": user.name,
"linkedin": user.linkedin,
"instagram": user.instagram,
})
return {
'user': user.__json__(),
'token': token.decode()
}
#app.route('/me', methods=["PUT"], authorizer=token_authorizer)
def update_me():
r = app.current_request
u = full_user_from_context(r.context)
data = r.json_body
u = crud.update_user(get_db(), schemas.UserUpdate(
uid=u.uid,
name=data.get('name'),
phone=data.get('phone'),
instagram=data.get('instagram'),
linkedin=data.get('linkedin'),
))
if u is None: # todo: code
return {
"error": "could not update"
}
return {
"user": u.__json__()
}
#app.route('/me/profile-image', methods=["PUT"], content_types=['application/octet-stream'],
authorizer=token_authorizer)
def update_me():
r = app.current_request
u = full_user_from_context(r.context)
data = r.raw_body
file_name = u.uid
tmp_file_name = '/tmp/' + file_name + '.jpg'
with open(tmp_file_name, 'wb') as tmp_file:
tmp_file.write(data)
key = 'profile-images/' + file_name
try:
s3_client.upload_file(tmp_file_name, BUCKET, key, ExtraArgs={'ACL': 'public-read'})
except Exception as e:
app.log.error(e)
return {
"error": str(e)
}
url = f'https://{BUCKET}.s3.amazonaws.com/{key}'
u = crud.update_user(get_db(), schemas.UserUpdate(
uid=u.uid,
profile_image_url=url
))
return {
"url": url
}
#app.route('/me/location', methods=["PUT"], authorizer=token_authorizer)
def update_me_location():
r = app.current_request
u = full_user_from_context(r.context)
data = r.json_body
lat = Decimal(str(data.get('latitude')))
lng = Decimal(str(data.get('longitude')))
loc = crud.update_user_location(get_db(), schemas.UserLocationUpdate(
uid=u.uid,
latitude=lat,
longitude=lng,
timestamp=datetime.now(),
geo=f'POINT({lng} {lat})'
))
if loc is None:
loc = crud.create_user_location(get_db(), schemas.UserLocationCreate(
uid=u.uid,
latitude=lat,
longitude=lng
))
loc = schemas.UserLocationGet(
uid=u.uid,
user=schemas.UserGet(
uid=u.uid,
name=u.name,
linkedin=u.linkedin,
instagram=u.instagram,
profile_image_url=u.profile_image_url
),
latitude=loc.latitude,
longitude=loc.longitude,
timestamp=loc.timestamp.isoformat()
)
return {
'loc': loc.json()
}
#app.route('/nearby', methods=["GET"], authorizer=token_authorizer)
def nearby():
r = app.current_request
u = full_user_from_context(r.context)
user_location = crud.get_user_location(get_db(), u.uid)
if user_location is None:
return {
"error": "no user location"
} # todo: better errro
nearby_users = crud.get_nearby_users(get_db(), u.uid, schemas.UserLocationGet(
uid=user_location.user_id,
latitude=user_location.latitude,
longitude=user_location.longitude,
user=u,
timestamp=user_location.timestamp.isoformat()
), DISTANCE_IN_METERS)
return {
"nearby_distance_in_meters": DISTANCE_IN_METERS,
"nearby_users": list(map(lambda x: schemas.UserLocationGet(
uid=x.user_id,
latitude=x.latitude,
longitude=x.longitude,
timestamp=x.timestamp.isoformat(),
user=schemas.UserGet(
uid=x.user.uid,
name=x.user.name,
instagram=x.user.instagram,
linkedin=x.user.linkedin,
profile_image_url=x.user.profile_image_url
)
).dict(), nearby_users))
}
This is the response when I invoke the test-function from the AWS console:
This is what I get when I do chalice logs:
Traceback (most recent call last):[ERROR] Runtime.ImportModuleError: Unable to import module 'app': No module named 'crud'
Traceback (most recent call last):[ERROR] Runtime.ImportModuleError: Unable to import module 'app': No module named 'crud'
2021-11-14 19:29:04.197000 88eb68 2021-11-14T19:29:04.197Z c143dcd4-e212-41f1-a786-7d86e4b58e59 Task timed out after 60.06 seconds
This is my requirements.txt:
chalice~=1.26.2
firebase-admin
boto3~=1.18.53
botocore
sqlalchemy
pydantic
# psycopg2
GeoAlchemy2
psycopg2-binary
I need to use psycopg2 so maybe that's the problem.
Every http request results in a 504 time out.
In local mode everything works fine.
Thanks in advance.
your chalicelib package folder's __init__.py (it must be written like this) has an extra underscore at the end of it. So python import system doesn't recognize such folder as a package, hence the error.
Am testing an endpoint that retrieves data using a ModelViewSet, and am passing a param via a URL to it to get data but am getting this error when I run the unit tests:
File "/Users/lutaayaidris/Documents/workspace/project_sample/project_sample/financing_settings/tests.py", line 195, in test_get_blocks
self.block_get_data), content_type='application/json')
File "/Users/lutaayaidris/Documents/workspace/project_sample/lib/python3.6/site-packages/rest_framework/test.py", line 286, in get
response = super().get(path, data=data, **extra)
File "/Users/lutaayaidris/Documents/workspace/project_sample/lib/python3.6/site-packages/rest_framework/test.py", line 194, in get
'QUERY_STRING': urlencode(data or {}, doseq=True),
File "/Users/lutaayaidris/Documents/workspace/project_sample/lib/python3.6/site-packages/django/utils/http.py", line 93, in urlencode
for key, value in query:
ValueError: not enough values to unpack (expected 2, got 1)
This is how I have structured my tests , plus some dummy data for testing :
class TemplateData:
"""Template Mock data."""
step_get_data = {
"param": "step"
}
block_get_data = {
"param": "block"
}
get_no_data = {
"param_": "block"
}
class TemplateViewTests(TestCase, TemplateData):
"""Template Tests (Block & Step)."""
def setUp(self):
"""
Initialize client, Step and Block id and data created.
"""
self.client = APIClient()
self.block_id = 0
self.step_id = 0
self.create_block_step_data()
def create_block_step_data(self):
"""Create ProcessVersion, Step, & Block mock data."""
self.process_version = ProcessVersion.objects.create(
tag="TESTING_TAG",
is_process_template=False,
status="IN EDITING",
attr_map="TESTING_ATTR",
loan_options=None
)
self.step = Step.objects.create(
version=self.process_version,
is_process_template=True,
title="TESTING",
help_text="TESTING",
order=1,
slug="slug",
can_be_duplicated=False,
max_duplicated_number=2,
)
self.step_id = self.step.pk
self.block_id = Block.objects.create(
step=self.step,
is_process_template=True,
title="TESTING",
information_text="This is testing "
"information",
order=1,
depending_field="depending_field",
visibility_value="visibility_value",
slug="slug",
can_be_duplicated=False,
max_duplicated_number=2,
).pk
self.process_version_1 = ProcessVersion.objects.create(
tag="TESTING_TAG",
is_process_template=False,
status="IN EDITING",
attr_map="TESTING_ATTR",
loan_options=None
)
self.step_1 = Step.objects.create(
version=self.process_version_1,
is_process_template=True,
title="TESTING",
help_text="TESTING",
order=1,
slug="slug",
can_be_duplicated=False,
max_duplicated_number=2,
)
self.block_1 = Block.objects.create(
step=self.step,
is_process_template=True,
title="TESTING",
information_text="This is testing "
"information",
order=1,
depending_field="depending_field",
visibility_value="visibility_value",
slug="slug",
can_be_duplicated=False,
max_duplicated_number=2,
).pk
def test_get_blocks(self):
"""Test get list of Block. """
response = self.client.get(
"/api/v1/financing-settings/template/",
data=json.dumps(
self.block_get_data), content_type='application/json')
self.assertEqual(response.status_code, 200)
def test_get_steps(self):
"""Test get list of Step. """
response = self.client.get(
"/api/v1/financing-settings/template/",
data=json.dumps(
self.block_get_data),
content_type='application/json')
self.assertEqual(response.status_code, 200)
def test_no_step_or_block(self):
"""Test get no list of Step or Block. """
response = self.client.get(
"/api/v1/financing-settings/template/",
data=json.dumps(
self.block_get_data),
content_type='application/json')
self.assertEqual(response.status_code, 204)
As you can see above those are my tests, I have already setup the data , now I want to retrieve back the data, but because of the exception above I can't.
Lastly, in my endpoint implementation, I used a Viewset to handle this , below is the code :
class TemplateView(ModelViewSet):
"""ViewSet for Saving Block/ Step template."""
def list(self, request, *args, **kwargs):
"""Get list of Block/Steps with is_process_template is equal to True."""
param = request.data['param']
if param == "block":
_block = Block.objects.filter(is_process_template=True).values()
return JsonResponse({"data": list(_block)}, safe=False, status=200)
elif param == "step":
_step = Step.objects.filter(is_process_template=True)
return JsonResponse({"data": list(_step)}, safe=False, status=200)
return Response(status=status.HTTP_204_NO_CONTENT)
What is causing this , in my understanding I feel like everything should work.
The function Client.get expect a dictionary as data argument and try to encode it in the url using the function urlencode. You could do something like that:
from django.test import Client
c = Client()
block_get_data = {
"param": "block"
}
c.get('path', block_get_data)
block_get_data will be sent in the url as 'param=block'
If you want to send JSON formated data in a GET method, you can use Client.generic function as follow:
from django.test import Client
import json
c = Client()
block_get_data = {
"param": "block"
}
c.generic('GET', 'path', json.dumps(block_get_data), 'application/json')
You are facing this error because this dict
block_get_data = {
"param": "block"
}
you are trying to use it in this way
for key,val in block_get_data
and it will produce the error like
for key,val in block_get_data:
ValueError: too many values to unpack (expected 2)
It will be solved if your loop through dict by using .items() method.
for key,val in block_get_data.items():
I think by passing parameter as self.block_get_data.items() may solve your problem.
I'm trying to use for the first time ElasticSearch 6.4 with an existing web application wrote in Python/Django. I have some issues and I would like to understand why and how I can solve these issues.
###########
# Existing : #
###########
In my application, it's possible to upload document files (.pdf or .doc for example). Then, I have a search function in my application which let to search over documents indexed by ElasticSearch when they are uploaded.
Document title is always written through the same way :
YEAR - DOC_TYPE - ORGANISATION - document_title.extension
For example :
1970_ANNUAL_REPORT_APP-TEST_1342 - loremipsum.pdf
The search function is always done among doc_type = ANNUAL_REPORT. because there are several doc_types (ANNUAL_REPORT, OTHERS, ....).
##################
# My environment : #
##################
This is some data according to my ElasticSearch part. I'm learning ES commands too.
$ curl -XGET http://127.0.0.1:9200/_cat/indices?v
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
yellow open app 5T0HZTbmQU2-ZNJXlNb-zg 5 1 742 2 396.4kb 396.4kb
So my index is app
For the above example, if I search this document : 1970_ANNUAL_REPORT_APP-TEST_1342 - loremipsum.pdf, I have :
$ curl -XGET http://127.0.0.1:9200/app/annual-report/1343?pretty
{
"_index" : "app",
"_type" : "annual-report",
"_id" : "1343",
"_version" : 33,
"found" : true,
"_source" : {
"attachment" : {
"date" : "2010-03-04T12:08:00Z",
"content_type" : "application/pdf",
"author" : "manshanden",
"language" : "et",
"title" : "Microsoft Word - Test document Word.doc",
"content" : "some text ...",
"content_length" : 3926
},
"relative_path" : "app_docs/APP-TEST/1970_ANNUAL_REPORT_APP-TEST_1342.pdf",
"title" : "1970_ANNUAL_REPORT_APP-TEST_1342 - loremipsum.pdf"
}
}
Now, with my search part in my web application, I would like to find this document with this search : 1970.
def search_in_annual(self, q):
try:
response = self.es.search(
index='app', doc_type='annual-report',
q=q, _source_exclude=['data'], size=5000)
except ConnectionError:
return -1, None
total = 0
hits = []
if response:
for hit in response["hits"]["hits"]:
hits.append({
'id': hit['_id'],
'title': hit['_source']['title'],
'file': hit['_source']['relative_path'],
})
total = response["hits"]["total"]
return total, hits
But when q=1970, the result is 0
If I write :
response = self.es.search(
index='app', doc_type='annual-report',
q="q*", _source_exclude=['data'], size=5000)
It returns my document, but many documents too with no 1970 inside the title or the document content.
#################
# My global code : #
#################
This is the global class which manage indexing functions :
class EdqmES(object):
host = 'localhost'
port = 9200
es = None
def __init__(self, *args, **kwargs):
self.host = kwargs.pop('host', self.host)
self.port = kwargs.pop('port', self.port)
# Connect to ElasticSearch server
self.es = Elasticsearch([{
'host': self.host,
'port': self.port
}])
def __str__(self):
return self.host + ':' + self.port
#staticmethod
def file_encode(filename):
with open(filename, "rb") as f:
return b64encode(f.read()).decode('utf-8')
def create_pipeline(self):
body = {
"description": "Extract attachment information",
"processors": [
{"attachment": {
"field": "data",
"target_field": "attachment",
"indexed_chars": -1
}},
{"remove": {"field": "data"}}
]
}
self.es.index(
index='_ingest',
doc_type='pipeline',
id='attachment',
body=body
)
def index_document(self, doc, bulk=False):
filename = doc.get_filename()
try:
data = self.file_encode(filename)
except IOError:
data = ''
print('ERROR with ' + filename)
# TODO: log error
item_body = {
'_id': doc.id,
'data': data,
'relative_path': str(doc.file),
'title': doc.title,
}
if bulk:
return item_body
result1 = self.es.index(
index='app', doc_type='annual-report',
id=doc.id,
pipeline='attachment',
body=item_body,
request_timeout=60
)
print(result1)
return result1
def index_annual_reports(self):
list_docs = Document.objects.filter(category=Document.OPT_ANNUAL)
print(list_docs.count())
self.create_pipeline()
bulk = []
inserted = 0
for doc in list_docs:
inserted += 1
bulk.append(self.index_document(doc, True))
if inserted == 20:
inserted = 0
try:
print(helpers.bulk(self.es, bulk, index='app',
doc_type='annual-report',
pipeline='attachment',
request_timeout=60))
except BulkIndexError as err:
print(err)
bulk = []
if inserted:
print(helpers.bulk(
self.es, bulk, index='app',
doc_type='annual-report',
pipeline='attachment', request_timeout=60))
My document is indexed when he's submitted thanks a Django form with a signal :
#receiver(signals.post_save, sender=Document, dispatch_uid='add_new_doc')
def add_document_handler(sender, instance=None, created=False, **kwargs):
""" When a document is created index new annual report (only) with Elasticsearch and update conformity date if the
document is a new declaration of conformity
:param sender: Class which is concerned
:type sender: the model class
:param instance: Object which was just saved
:type instance: model instance
:param created: True for a creation, False for an update
:type created: boolean
:param kwargs: Additional parameter of the signal
:type kwargs: dict
"""
if not created:
return
# Index only annual reports
elif instance.category == Document.OPT_ANNUAL:
es = EdqmES()
es.index_document(instance)
This is what I've done and it seems to work :
def search_in_annual(self, q):
try:
response = self.es.search(
index='app', doc_type='annual-report', q=q, _source_exclude=['data'], size=5000)
if response['hits']['total'] == 0:
response = self.es.search(
index='app', doc_type='annual-report',
body={
"query":
{"prefix": {"title": q}},
}, _source_exclude=['data'], size=5000)
except ConnectionError:
return -1, None
total = 0
hits = []
if response:
for hit in response["hits"]["hits"]:
hits.append({
'id': hit['_id'],
'title': hit['_source']['title'],
'file': hit['_source']['relative_path'],
})
total = response["hits"]["total"]
return total, hits
It lets to search over title, prefix and content to find my document.