I have created a lambda function to send billing reports to certain email addresses, for that im using my email address and app password. I have saved the app password under secret manager as other api. When i try to retrieve it using the lambda function it shows error.
import boto3
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import ast
import os
import datetime
import base64
import logging
import collections
import json
logger = logging.getLogger()
logger.setLevel(logging.INFO)
def get_secret():
secret_name = "email_app_password"
region_name = "ca-central-1"
# Create a Secrets Manager client
session = boto3.session.Session()
client = session.client(
service_name='secretsmanager',
region_name=region_name
)
# In this sample we only handle the specific exceptions for the 'GetSecretValue' API.
# See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html
# We rethrow the exception by default.
try:
get_secret_value_response = client.get_secret_value(
SecretId=secret_name
)
except ClientError as e:
if e.response['Error']['Code'] == 'DecryptionFailureException':
# Secrets Manager can't decrypt the protected secret text using the provided KMS key.
# Deal with the exception here, and/or rethrow at your discretion.
raise e
elif e.response['Error']['Code'] == 'InternalServiceErrorException':
# An error occurred on the server side.
# Deal with the exception here, and/or rethrow at your discretion.
raise e
elif e.response['Error']['Code'] == 'InvalidParameterException':
# You provided an invalid value for a parameter.
# Deal with the exception here, and/or rethrow at your discretion.
raise e
elif e.response['Error']['Code'] == 'InvalidRequestException':
# You provided a parameter value that is not valid for the current state of the resource.
# Deal with the exception here, and/or rethrow at your discretion.
raise e
elif e.response['Error']['Code'] == 'ResourceNotFoundException':
# We can't find the resource that you asked for.
# Deal with the exception here, and/or rethrow at your discretion.
raise e
else:
# Decrypts secret using the associated KMS key.
# Depending on whether the secret is a string or binary, one of these fields will be populated.
if 'SecretString' in get_secret_value_response:
secret = get_secret_value_response['SecretString']
else:
decoded_binary_secret = base64.b64decode(get_secret_value_response['SecretBinary'])
# Your code goes here.
def send_email(email_body):
"""
Sends email according to environment variables.
:param email_body: Body of email.
:return: None
"""
msg = MIMEMultipart('alternative')
email = os.environ['EMAIL_FROM']
try:
secret_value = get_secret()
print(secret_value[os.environ['SECRET_KEY_NAME']])
except Exception as e:
logger.exception("Exception while trying to get password from Secrets Manager")
return
sec_key = os.environ['SECRET_KEY_NAME']
password = ast.literal_eval(secret_value)[os.environ['SECRET_KEY_NAME']]
msg['Subject'] = os.environ["EMAIL_SUBJECT"]
msg['From'] = email
you = os.environ['EMAIL_TO'].split(',')
msg['To'] = ", ".join(you)
body = email_body
msg.attach(MIMEText(body, 'html'))
try:
smtpObj = smtplib.SMTP('smtp.gmail.com', 587)
smtpObj.starttls()
smtpObj.login(email, password)
smtpObj.sendmail(email, you, msg.as_string())
smtpObj.quit()
logger.info('Email sent')
except smtplib.SMTPException as e:
logger.info("Error: unable to send email due to %s", e)
class FileOpener:
"""
Class to cache file contents.
"""
file_cache = {}
#staticmethod
def open_file(filename):
if filename not in FileOpener.file_cache:
with open(filename) as fp:
FileOpener.file_cache[filename] = fp.read()
return FileOpener.file_cache[filename]
def get_account_cost(account_no, start_day, end_day):
client = boto3.client('ce')
response = client.get_cost_and_usage(
TimePeriod={
'Start': start_day,
'End': end_day
},
Granularity='MONTHLY',
Filter={
"And": [{
"Dimensions": {
"Key": "LINKED_ACCOUNT",
"Values": [account_no]
}
}, {
"Not": {
"Dimensions": {
"Key": "RECORD_TYPE",
"Values": ["Credit", "Refund"]
}
}
}]
},
Metrics=["BlendedCost"],
GroupBy=[
{
'Type': 'DIMENSION',
'Key': 'SERVICE'
},
]
)
cost_dictionary = collections.Counter()
for result in response['ResultsByTime']:
for group in result['Groups']:
servicename = group['Keys'][0]
amount = round(float(group['Metrics']['BlendedCost']['Amount']), 2)
if amount == 0.0:
continue
cost_dictionary[servicename] += amount
return cost_dictionary
def combine_cost_dictionaries(prev_cost_dict, curr_cost_dict):
combined_cost_dict = {}
prev_cost_total = 0.0
curr_cost_total = 0.0
for service, curr_cost in curr_cost_dict.items():
prev_cost = 0.0
if service in prev_cost_dict:
prev_cost = prev_cost_dict[service]
combined_cost_dict[service] = (prev_cost, curr_cost)
prev_cost_total += prev_cost
curr_cost_total += curr_cost
for service, prev_cost in prev_cost_dict.items():
curr_cost = 0.0
if service not in combined_cost_dict:
combined_cost_dict[service] = (prev_cost, curr_cost)
prev_cost_total += prev_cost
curr_cost_total += curr_cost
return combined_cost_dict, prev_cost_total, curr_cost_total
def generate_account_cost_html(account_name, combined_cost_dict, prev_cost_total, curr_cost_total):
prev_date = str((datetime.datetime.now().date() - datetime.timedelta(days=2)))
curr_date = str((datetime.datetime.now().date() - datetime.timedelta(days=1)))
table_rows = ""
sorted_combined_cost = sorted(combined_cost_dict.items(), key=lambda x: x[1][1], reverse=True)
for service, costs in sorted_combined_cost:
table_row = FileOpener.open_file("table_row.html")
prev_cost = round(float(costs[0]), 2)
curr_cost = round(float(costs[1]), 2)
if prev_cost < 0.01:
percentage_change = 'New Charge'
else:
percentage_change = ((curr_cost - prev_cost) / prev_cost) * 100
percentage_change = round(float(percentage_change), 2)
if percentage_change > 0:
percentage_change = "↑ {}%".format(percentage_change)
elif percentage_change == 0.0:
percentage_change = "{}%".format(percentage_change)
else:
percentage_change = "↓ {}%".format(percentage_change)
if percentage_change[0] == '↑':
percentage_css = "background-color: pink;border:darkblue solid thin;"
elif percentage_change == 'New Charge':
percentage_css = "background-color: LightGreen;border:darkblue solid thin;"
else:
percentage_css = "border:lightblue solid thin;"
table_rows += table_row.format(service, prev_cost, curr_cost, percentage_css, percentage_change)
prev_cost_total = round(float(prev_cost_total), 2)
curr_cost_total = round(float(curr_cost_total), 2)
table = FileOpener.open_file("table.html")
table = table.format(account_name, prev_cost_total, curr_cost_total, prev_date, curr_date, table_rows)
return table
def lambda_handler(event, context):
account_names = os.environ['ACCOUNT_NAMES'].split(",")
account_numbers = os.environ['ACCOUNT_NUMBERS'].split(",")
table_body_html = ''
for account_name, account_no in zip(account_names, account_numbers):
day_1 = str((datetime.datetime.now().date() - datetime.timedelta(days=2)))
day_2 = str((datetime.datetime.now().date() - datetime.timedelta(days=1)))
day_3 = str(datetime.datetime.now().date())
prev_cost_dict = get_account_cost(account_no, day_1, day_2)
curr_cost_dict = get_account_cost(account_no, day_2, day_3)
combined_cost_dict, prev_cost_total, curr_cost_total = combine_cost_dictionaries(prev_cost_dict, curr_cost_dict)
table_body = generate_account_cost_html(account_name, combined_cost_dict, prev_cost_total, curr_cost_total)
table_body_html += table_body
email_body = FileOpener.open_file("email_body.html").format(table_body_html)
send_email(email_body)
I get the following error when running the lambda function. I have even tried removing the print line. Then the error goes to password = ast.literal_eval(secret_value)[os.environ['SECRET_KEY_NAME']]
[ERROR] 2022-08-18T06:02:03.968Z 2ae88ceb-39a6-4feb-aa7c-2cbb17ec655c Exception while trying to get password from Secrets Manager
Traceback (most recent call last):
File "/var/task/lambda_function.py", line 88, in send_email
print(secret_value[os.environ['SECRET_KEY_NAME']])
TypeError: 'NoneType' object is not subscriptableEND RequestId: 2ae88ceb-39a6-4feb-aa7c-2cbb17ec655c
Your get_secret function does not return the value to the calling code. Please edit the else clause:
else:
# Decrypts secret using the associated KMS key.
# Depending on whether the secret is a string or binary, one of these fields will be populated.
if 'SecretString' in get_secret_value_response:
return get_secret_value_response['SecretString']
else:
return base64.b64decode(get_secret_value_response['SecretBinary'])
Related
Full disclosure: I am very new to coding and I really don't know what I'm doing. Currently trying to send multiple transactional emails using SendinBlue SMTP API based on the one I pick. I have already made the templates on SendinBlue, but how do I call a specific template?
My code
from __future__ import print_function
import sib_api_v3_sdk
import smtplib
from sib_api_v3_sdk.rest import ApiException
from pprint import pprint
from email.mime.text import MIMEText
from flask import request, render_template
from sql_helper import *
# Configuration of API key authorization:
api_config = sib_api_v3_sdk.Configuration()
api_config.api_key['api-key'] = 'api_key'
#If Partnership Account:
#api_config = sib_api_v3_sdk.Configuration()
#api_config.api_key['partner-key'] = 'API_Key'
api_instance = sib_api_v3_sdk.TransactionalEmailsApi(sib_api_v3_sdk.ApiClient(api_config))
subject = "My Subject"
html_content = "<html><body><h1>This is my first transactional email </h1></body></html>"
sender = {"name":"company name","email":"admin#company.com"}
to = [{"email":"example#example.com","name":"Jane Doe"}]
# cc = [{"email":"example2#example2.com","name":"Janice Doe"}]
# bcc = [{"name":"John Doe","email":"example#example.com"}]
reply_to = {"email":"admin#company.com","name":"John Doe"}
headers = {"Some-Custom-Name":"unique-id-1234"}
params = {"parameter":"My param value","subject":"New Subject"}
send_smtp_email = sib_api_v3_sdk.SendSmtpEmail(to=to, reply_to=reply_to, headers=headers, html_content=html_content, sender=sender, subject=subject)
def send_email(template, recipient, custom_message = None):
if template == "approve":
template_id = 1
elif template == "deny":
template_id = 2
elif template == "skip":
template_id = 3
try:
#send transactional email
api_response = api_instance.send_transac_email(send_smtp_email)
pprint(api_response)
except ApiException as e:
print("Exception when calling SMTPApi -> send_transac_email: %s\n" % e)
Any help would be appreciated.
I really am not quite sure what I'm doing, so anything would help. I'm sorry if the code is bad, I'm just starting out.
##################################################################################################
# TO INSTALL THIS DEMO SSE CLIENT for Python 3:
# pip install sseclient
# pip install PyJWT
# pip install requests
##################################################################################################
from sseclient import SSEClient
from datetime import datetime
from requests.auth import AuthBase
from pyspark.context import SparkContext
from pyspark.sql.functions import explode, to_date, col, current_date, lit
from delta.tables import *
import jwt # to install: pip install PyJWT
import requests # to install: pip install requests
import time
import boto3
import json
import gc
##################################################################################################
# Authorization
##################################################################################################
class Oauth2(AuthBase):
def __init__(self, client_id, client_secret, oauth2_url):
self.client_id = client_id
self.client_secret = client_secret
self.oauth2_url = oauth2_url
self.token = None
self.expires = None
def __call__(self, r):
"""
If we don't have an access token in hand, or it has expired, get a new token from the auth service.
Then set the access token in the request's Authorization header.
"""
now = time.time()
if not self.token or now > self.expires:
self.token = self.get_fresh_token()
self.expires = now + self.token['expires_in']
r.headers['Authorization'] = 'Bearer ' + self.token['access_token']
return r
def get_fresh_token(self):
"""
Get an authorization header that contains a valid access token for the client
"""
# The token used to sign the grant request itself
jwt_token = jwt.encode({'iat': int(time.time())}, self.client_secret, algorithm='HS256')
# Make the request to the auth service to get an access token for the client
resp = requests.post(
self.oauth2_url,
data = {'grant_type': 'client_credentials', 'client_id': self.client_id, 'client_secret': jwt_token},
verify = False,
allow_redirects = False
)
json_resp = resp.json()
if 'access_token' in json_resp:
return json_resp
elif 'error' in json_resp:
raise Exception("OAuth failed: %s: %s" % (json_resp['error'], json_resp.get('error_description')))
else:
raise Exception("OAuth failed: %s" % (str(json_resp)))
##################################################################################################
# Insert account_field_assignments
##################################################################################################
def insert_account_field_assignments(df):
account_field_assignments_DF = None
account_field_assignments_DF = df.select(
col("account_field_object"),
col("account_no").alias("account_fields_identifier"),
explode("account_fields").alias("account_fields_explode"),
col("created_tstamp"),
col("event_date")
).select(
col("account_field_object").cast("string"),
col("account_fields_identifier").cast("bigint"),
col("account_fields_explode").account_field.ids.field_no.alias("account_field_no").cast("bigint"),
col("account_fields_explode").account_field.ids.field_name.alias("account_field_name").cast("string"),
col("account_fields_explode").field_values.alias("account_field_value").cast("string"),
col("created_tstamp").cast("string"),
col("event_date").cast("string")
)
# Insert into tss_plan table_rate_schedule_assignments
account_field_assignments_DF.write.mode("append") \
.format("delta").partitionBy("event_date").saveAsTable("test.tss_account_field_assignments")
del df
del account_field_assignments_DF
##################################################################################################
# Insert Account
##################################################################################################
def insert_account(data):
print("Data" + data + "\n")
df = spark.read.json(sc.parallelize([data])) \
.withColumn("event_date", current_date().cast("string"))
account_DF = None
try:
df = df.withColumn("status_no", col("status").ids.status_no)
except:
df = df.withColumn("status_no", lit(None))
try:
df = df.withColumn("currency_cd", col("currency").ids.currency_cd)
except:
df = df.withColumn("currency_cd", lit(None))
#....some code here
#....some code here
account_DF = df.select(
col("account_no").cast("bigint"),
col("userid").cast("string"),
col("client_account_id").cast("string"),
col("balance").cast("float"),
col("status_no").cast("integer"),
col("status_tstamp").cast("string"),
col("created_tstamp").cast("string"),
col("updated_tstamp").cast("string"),
col("is_test_account").cast("boolean"),
col("currency_cd").cast("string"),
col("updated_receipt_id").cast("string"),
col("created_receipt_id").cast("string"),
col("sequence_functional_group_id").cast("string"),
col("start_date").cast("string"),
col("is_anonymized").cast("boolean"),
col("notify_method_id").cast("integer"),
col("notification_template_group_id").cast("string"),
col("event_date").cast("string")
)
# Insert into tss_plan table
account_DF.write.mode("append").format("delta").partitionBy("event_date").saveAsTable("test.tss_account")
testing = df.collect()[0]
# Insert into tss_account_field_assignments
try:
if testing.__getitem__("account_fields"):
insert_account_field_assignments(df)
except:
None
del df
del account_DF
del testing
del data
gc.collect()
###################################################################################################################
# Update Account
###################################################################################################################
def update_account(data, header):
print("Data" + data + "\nHeader" + header + "\n")
jsonData = json.loads(data)
jsonCond = json.loads(header)
# prepare condition to update the table test.tss_account
cond = 'account_no=' + str(jsonCond["ref"]["ids"]["account_no"]) + ' and userid="' + str(jsonCond["ref"]["ids"]["userid"]) + '"'
# prepare data to update the table test.tss_account
updateJson = '{'
try:
if jsonData["client_account_id"] != None:
updateJson = updateJson + '"client_account_id":"\'' + str(jsonData["client_account_id"]) + '\'",'
else:
updateJson = updateJson + '"client_account_id":"null",'
except:
None
try:
if jsonData["balance"] != None:
updateJson = updateJson + '"balance":"' + str(jsonData["balance"]) + '",'
else:
updateJson = updateJson + '"balance":"null",'
except:
None
#....some code here
#....some code here
updateJson = updateJson[:-1] + '}'
# update table test.tss_account if not all elements are empty
if updateJson != '}':
try:
deltaTable = DeltaTable.forName(spark, "test.tss_account")
deltaTable.update(
condition = cond,
set = json.loads(updateJson)
)
except Exception as e:
print(e.message, e.args)
del updateJson
del cond
df = spark.read.json(sc.parallelize([data])) \
.withColumn("event_date", current_date().cast("string"))
# prepare data to update the table test.tss_account_field_assignments
# (Basically delete rows matching to account_no from table and insert fresh data back into the table) => only when account_fields is present and non-empty
try:
if jsonData["account_fields"]:
# delete row of table test.tss_account_field_assignments
deleteData = 'account_fields_identifier=' + str(jsonCond["ref"]["ids"]["account_no"])
deltaTable = DeltaTable.forName(spark, "test.tss_account_field_assignments")
deltaTable.delete(deleteData)
deleteData = None
# Insert fresh data into the table test.tss_account_field_assignments
insert_account_field_assignments(df)
except:
None
del jsonCond
del jsonData
del df
del data
del header
gc.collect()
print("Account update complete")
##################################################################################################
# Stream event Handlers (Insert)
##################################################################################################
def insert_messages(data, event):
# Get the message_type of the event
line = data.split("\n", 2)
message_type = (json.loads(line[0]))['ref']['type']
# switch case of message_type
if message_type == "Account":
insert_account(line[1])
##################################################################################################
# Stream event Handlers (Update)
##################################################################################################
def update_messages(data, event):
# Get the message_type of the event
line = data.split("\n", 2)
message_type = (json.loads(line[0]))['ref']['type']
# switch case of message_type
if message_type == "Account":
update_account(line[1], line[0])
##################################################################################################
# Stream event Handlers (Handler Class)
##################################################################################################
class SseHandlers(object):
def __init__(self):
# Map event type to handler
self.event_handlers = {
"create" : insert_messages,
"update" : update_messages,
"delete" : delete_messages,
"load" : insert_messages,
"message" : handle_heartbeat
}
def handleMsg(self, msg):
# Get the handler for the event type. Call that handler with the event's data
self.event_handlers.get(msg.event)(msg.data, msg.event)
##################################################################################################
# A store for a stream's last event id.
##################################################################################################
class StreamStatusStore(object):
def __init__(self):
self.saved_last_id = None
def save_last_id(self, last_id):
self.saved_last_id = last_id
def get_last_id(self):
return self.saved_last_id
##################################################################################################
# Main program
##################################################################################################
if __name__ == "__main__":
# Set all input parameter
sse_service_url = dbutils.widgets.get("sse_service_url")
client_id = dbutils.widgets.get("client_id")
oauth2_url = dbutils.widgets.get("oauth2_url")
client_secret = dbutils.widgets.get("client_secret")
# Create Status store, for fault tolarance
status_store = StreamStatusStore()
# Make an SSE message stream
messages = SSEClient(
sse_service_url, # URL of SSE service
auth = Oauth2(client_id, client_secret, oauth2_url), # authenticator
last_id = status_store.get_last_id() # if a last id was stored locally, start there
)
# Handle messages as they come in off the stream
handlers = SseHandlers()
for msg in messages:
# Handle each message as it comes in
handlers.handleMsg(msg)
# Call the status store with the latest event id. If we crash, we will restart at this point.
status_store.save_last_id(messages.last_id)
###############################################################################################
/*
Explaination
============
=> We are using databricks notebook here
=> The SSEClient loops through each message and then gets stuck at certain point and gives
GC(Allocation Failure)
=> Streaming data gets connected to this SSEClient and then we get the message
=> This is message contains headers and the data
=> We have mentioned the example of 'Create Event' below
header:{"ref":{"ids":{"account_no":XXXXX,"userid":"QA-XXXXX"},"type":"Account"},"tick":202546873272,"tstamp":"2022-04-13T02:51:46-05:00"}
data:{"account_no":XXXXX,"userid":"QA-XXXXX","client_account_id":null,"balance":0,"status":{"ids":{"status_no":1,"enum":1},"type":"Account_Status"},"status_tstamp":"2022-04-13T02:51:48-05:00",
"parent_account":null,"created_tstamp":"2022-04-13T02:51:48-05:00","updated_tstamp":"2022-04-13T02:51:48-05:00","secret_question":null,"secret_question_answer":null,"is_test_account":XXXX,
"is_invoice_approval_required":XXX,"currency":{"ids":{"currency_cd":"usd","code":"usd"},"type":"Currency"},"updated_receipt_id":null,"created_receipt_id":null,
"sequence_functional_group":{"ids":{"group_no":10035487,"client_group_id":"AAMFAG"},"type":"Functional_Account_Group"},"start_date":"XXXX-XX-XX",
"admin_contact":{"ids":{"contact_no":XXXXXXXX},"type":"Contact"},"is_consumer_account":false,"locale":{"ids":{"locale_no":XXXXX},"type":"XXXXX"},"legal_entity":null,"is_anonymized":null,
"notify_method":{"ids":{"method_no":2,"enum":2},"type":"Notify_Method"},"functional_account_groups":[{"ids":{"group_no":10035487,"client_group_id":"XXXXXX"},"type":"Functional_Account_Group"}],"collection_account_groups":[{"ids":{"group_no":XXXXXXXX,"client_group_id":"USChase-USD_Arlo"},"type":"Collection_Account_Group"}],
"account_fields":[{"account_field":{"ids":{"field_no":XXXX,"field_name":"Legal Entity"},"type":"Account_Field"},"field_values":["Arlo Technologies Inc"]},{"account_field":{"ids":{"field_no":XXXX,"field_name":"GeoIP Country"},"type":"Account_Field"},"field_values":["US"]},{"account_field":{"ids":{"field_no":2998,"field_name":"Calling Application"},"type":"Account_Field"},"field_values":["XXXX"]},{"account_field":{"ids":{"field_no":XXXX,"field_name":"Company Code"},"type":"Account_Field"},"field_values":["XXXX"]}],
"tax_exemption_level":null,"notification_overrides":[],"notification_template_group":{"ids":{"notification_template_group_no":XXXX,"client_notification_template_group_id":"XXXXXXXXX"},"type":"Notification_Template_Group"}
,"surcharges":[],"revrec_profile_number":null,"purchase_order_number":null,"cc_email_addresses":[],"bcc_email_addresses":[]}
event:create
##################################################################################################
Below is the response message after running the code
======================================================
2022-05-19T09:24:05.599+0000: [GC (Allocation Failure) [PSYoungGen: 6264108K->70234K(6417920K)] 6630117K->436251K(19642880K), 0.0477541 secs] [Times: user=0.16 sys=0.00, real=0.05 secs]
*/
When I am trying to run my python code in lambda passing the handler to the function.module getting the below error, any suggestions how i could resolve this?
the below file test_client_visitor is triggered to call the client_visitor and send an email to the clients accordingly, when i run thd python file test_client_visitor in my local i get the email triggered successfully but in lambda facing the issue.
file_name: test_client_visitor
function = __import__('client_visitor')
handler = function.scan_clients
class TestFunction(unittest.TestCase):
def test_function(self):
file = open('event.json', 'rb')
try:
ba = bytearray(file.read())
event = jsonpickle.decode(ba)
print('## EVENT')
print(jsonpickle.encode(event))
context = {'requestid': '1234'}
result = handler(event, context)
print(result)
self.assertTrue(result, 'Emails could not be sent!')
finally:
file.close()
file.close()
if __name__ == '__main__':
unittest.main()
file_name: client_visitor.py
import datetime
import boto3
from aws_ses import send_bulk_templated_email
# boto3.set_stream_logger('botocore', level='DEBUG')
from mongodb import get_mongo_db
def process_clients(clients, developers, clients_to_be_notified, days):
if not clients:
pass
check_date = datetime.datetime.now() + datetime.timedelta(days)
for client in clients:
client_id_ = client['client_id']
if 'developer_id' in client:
developers[client_id_] = client['developer_id']
else:
if 'secrets' in client:
secrets = client['secrets']
for secret in secrets:
if 'not_on_or_after' in secret and secret['not_on_or_after'] < check_date.timestamp():
clients_to_be_notified.append({'client_id': client_id_,
'expiration_date': datetime.datetime.fromtimestamp(
secret['not_on_or_after']).strftime('%m/%d/%Y')})
print("adding client to notify List", client_id_, ":", client['sort'])
def notify_clients(clients_to_be_notified, developers):
developer_id_list = []
for client_secret in clients_to_be_notified:
developer_id_list.append(developers[client_secret['client_id']])
if developer_id_list:
db = get_mongo_db()
if db:
users = list(db.users.find({'guid': {'$in': developer_id_list}}, {'email', 'guid'}))
need_to_send_email = False
for user in users:
for client_secret in clients_to_be_notified:
if developers[client_secret['client_id']] == user['guid']:
client_secret['email'] = user['email']
need_to_send_email = True
break
if need_to_send_email:
return send_bulk_templated_email(clients_to_be_notified)
else:
return False
return True
def scan_clients(event, context):
local = False
if 'local' in event:
local = event['local'] == 'True'
if local:
dynamodb = boto3.resource('dynamodb', endpoint_url="http://localhost:8000")
else:
dynamodb = boto3.resource('dynamodb')
days = 30
if 'days' in event:
days = int(event['days'])
print(f"Scanning Clients with {days} or less to secret expiration")
table = dynamodb.Table('****')
scan_kwargs = {
'ProjectionExpression': 'client_id, sort, developer_id, secrets, approved'
}
test = False
if 'test' in event:
test = event['test'] == 'True'
done = False
start_key = None
developers = {}
clients_to_be_notified = []
if test:
developers['idm-portal1'] = '***'
clients_to_be_notified = [{'client_id': 'idm-portal1', 'expiration_date': '04/17/2021'}]
while not done:
if start_key:
scan_kwargs['ExclusiveStartKey'] = start_key
response = table.scan(**scan_kwargs)
process_clients(response.get('Items', []), developers, clients_to_be_notified, days)
start_key = response.get('LastEvaluatedKey', None)
done = start_key is None
print("total developers ", len(developers), " total clients_to_be_notified ", len(clients_to_be_notified))
return notify_clients(clients_to_be_notified, developers)
if __name__ == '__main__':
scan_clients(event={'days': 30, 'local': False, 'test': True}, context=None)
Response
{
"errorMessage": "Unable to import module 'test_client_visitor': No module named 'test_client_visitor'",
"errorType": "Runtime.ImportModuleError",
"stackTrace": []
}
Your file must be named test_client_visitor.py. The way lambda runs the code is by trying to import the main file and call the handler function. See the AWS docs to set up a handler for Python.
The reason you didn't run into this issue locally is because I assume you are calling python directly on the command line — python test_client_visitor. When you import a module in Python, the file has to end in the .py extension.
Able to fix this issue with right packaging of the contents to zip, avoided the creation of extra folder with the below command.
Command:
cd folder; zip -r ../filename.zip *
Thankyou everyone for your inputs.
New to cloud, Can anyone help to correct this code
This module is to list the regions and delete the complete default vpc via a lambda function.
Getting below error while testing this:
Syntax error in module 'lambda function': unindent does not match any outer indentation level
Please help on this
Removed other function like vpc, sc as the block looks very big here in the post just added the igw for understanding..
Need assistance
def lambda_handler(event, context):
# TODO implement
#for looping across the regions
regionList=[]
region=boto3.client('ec2')
regions=region.describe_regions()
#print('the total region in aws are : ',len(regions['Regions']))
for r in range(0,len(regions['Regions'])):
regionaws=regions['Regions'][r]['RegionName']
regionList.append(regionaws)
#print(regionList)
#regionsl=['us-east-1']
#sending regions as a parameter to the remove_default_vps function
res=remove_default_vpcs(regionList)
return {
'status':res
}
def get_default_vpcs(client):
vpc_list = []
vpcs = client.describe_vpcs(
Filters=[
{
'Name' : 'isDefault',
'Values' : [
'true',
],
},
]
)
vpcs_str = json.dumps(vpcs)
resp = json.loads(vpcs_str)
data = json.dumps(resp['Vpcs'])
vpcs = json.loads(data)
for vpc in vpcs:
vpc_list.append(vpc['VpcId'])
return vpc_list
def del_igw(ec2, vpcid):
""" Detach and delete the internet-gateway """
vpc_resource = ec2.Vpc(vpcid)
igws = vpc_resource.internet_gateways.all()
if igws:
for igw in igws:
try:
print("Detaching and Removing igw-id: ", igw.id) if (VERBOSE == 1) else ""
igw.detach_from_vpc(
VpcId=vpcid
)
igw.delete(
)
except boto3.exceptions.Boto3Error as e:
print(e)
def remove_default_vpcs():
for region in res:
try:
client = boto3.client('ec2', region_name = region)
ec2 = boto3.resource('ec2', region_name = region)
vpcs = get_default_vpcs(client)
except boto3.exceptions.Boto3Error as e:
print(e)
exit(1)
else:
for vpc in vpcs:
print("\n" + "\n" + "REGION:" + region + "\n" + "VPC Id:" + vpc)
del_igw(ec2, vpc)
print(completed)
It looks to me a code indentation issue. Please try with this
def lambda_handler(event, context):
# TODO implement
#for looping across the regions
regionList=[]
region=boto3.client('ec2')
regions=region.describe_regions()
#print('the total region in aws are : ',len(regions['Regions']))
for r in range(0,len(regions['Regions'])):
regionaws=regions['Regions'][r]['RegionName']
regionList.append(regionaws)
#print(regionList)
#regionsl=['us-east-1']
#sending regions as a parameter to the remove_default_vps function
res=remove_default_vpcs(regionList)
return {
'status':res
}
def get_default_vpcs(client):
vpc_list = []
vpcs = client.describe_vpcs(
Filters=[
{
'Name' : 'isDefault',
'Values' : [
'true',
],
},
]
)
vpcs_str = json.dumps(vpcs)
resp = json.loads(vpcs_str)
data = json.dumps(resp['Vpcs'])
vpcs = json.loads(data)
for vpc in vpcs:
vpc_list.append(vpc['VpcId'])
return vpc_list
def del_igw(ec2, vpcid):
""" Detach and delete the internet-gateway """
vpc_resource = ec2.Vpc(vpcid)
igws = vpc_resource.internet_gateways.all()
if igws:
for igw in igws:
try:
print("Detaching and Removing igw-id: ", igw.id) if (VERBOSE == 1) else ""
igw.detach_from_vpc(
VpcId=vpcid
)
igw.delete(
)
except boto3.exceptions.Boto3Error as e:
print(e)
def remove_default_vpcs():
for region in res:
try:
client = boto3.client('ec2', region_name = region)
ec2 = boto3.resource('ec2', region_name = region)
vpcs = get_default_vpcs(client)
except boto3.exceptions.Boto3Error as e:
print(e)
exit(1)
else:
for vpc in vpcs:
print("\n" + "\n" + "REGION:" + region + "\n" + "VPC Id:" + vpc)
del_igw(ec2, vpc)
print(completed)
Howdie do,
So I have this API that does the following:
1) It receives a XML request. It first parses that request to ensure the request is in the correct format.
2) If the XML is in the correct format, it checks to ensure that the correct user is authenicating.
3) If authenication is successful, it then retreives a URI from a database that the API will send a get request to.
4) If the response is successful, meaning it's a XML reply, it will use XSLT to transform the request into a format
5) It then adds the request to the database and returns the transformed XML to the user that queried the API
I have error handling involved at each step, but the issue is, I've had to nest 5 if else statements to accomplish this.
I know there has to be a better way to rewrite this error handling logic without so many nested if statements, but I'm not sure how. The subsequent steps rely on the previous to ensure that if any error occurs, it's returned properly to the user.
Below is my main Flask-API that I've created. The second file is a module that I've created which does a lot of the error processing. Those functions return a state(True/False) and the response to the main Flask-API.
Can someone give me some ideas on how to rewrite this API without the nesting? The API works 100% and does what it should for catching errors, but I just know there's a better way
Main API:
#dbConnect.app.route('/services/tracking/getShipmentStatus', methods=['POST'])
def parsexml2():
parseStatus, returnValues = func.parseNWRequest(request.data, 'SS')
if parseStatus:
authStatus, authResponse = func.checkAuthorization(returnValues['bu'], request.headers['Authorization'], 'SS')
if authStatus:
getURIStatus, uriResponse = func.getURI('SS')
if getURIStatus:
search = {'bu': returnValues['bu'], 'starttime': returnValues['start'], 'endtime': returnValues['end'],
'requestid': returnValues['requestid'], 'pagesize': returnValues['page']}
responseStatus, depascoResponse = func.sendDepascoRequest(uriResponse, search, 'SS')
if responseStatus:
nakedResponse = func.transformXML(depascoResponse.content, 'transformTracking.xsl')
# Write Request to db
file_name = 'WS_SS_' + returnValues['bu']
request_file_size = request.headers['Content-Length']
if func.addToDb(file_name, 'text/xml', 'SS.Request', returnValues['bu'], 'Y', request.data,
request_file_size) or func.addToDb(file_name, 'text/xml', 'SS.Result',
returnValues['bu'], 'Y', nakedResponse):
pass
return Response(nakedResponse, mimetype='text/xml')
else:
return Response(depascoResponse, mimetype='text/xml')
else:
return Response(uriResponse, mimetype='text/xml')
else:
return Response(authResponse, mimetype='text/xml')
else:
return Response(returnValues, mimetype='text/xml')
Imported module functions:
def transformXML(response, xsl):
xml = ET.tostring(ET.fromstring(response))
xslt = ET.XSLT(ET.parse(xsl))
transformedXML = xslt(ET.XML(xml))
return ET.tostring(transformedXML, pretty_print=True)
def addToDb(filename, mime, docType, customer_code, activeState, document_blob, filesize=None):
try:
response = dbConnect.Documents(file_name=filename, mime_type=mime, file_size=filesize, doc_type=docType,
customer_code=customer_code, is_active=activeState, document_blob=document_blob)
dbConnect.db.session.add(response)
dbConnect.db.session.commit()
dbConnect.db.session.close()
except exc.SQLAlchemyError:
return False
else:
return True
def generateXMLErrorResponse(errorMessage, api):
E = ElementMaker()
if api == 'SS':
GETSHIPMENTSTATUSRESPONSE = E.getShipmentStatusResponse
GETSHIPMENTSTATUSRESULT = E.getShipmentStatusResult
OUTCOME = E.outcome
RESULT = E.result
ERROR = E.error
xml_error = GETSHIPMENTSTATUSRESPONSE(
GETSHIPMENTSTATUSRESULT(
OUTCOME(
RESULT('Failure'),
ERROR(errorMessage)
)
)
)
return ET.tostring(xml_error, pretty_print=True)
elif api == 'IS':
GETINVENTORYSTATUSRESPONSE = E.getInventoryStatusResponse
GETINVENTORYSTATUSRESULT = E.getInventoryStatusResult
OUTCOME = E.outcome
RESULT = E.result
ERROR = E.error
xml_error = GETINVENTORYSTATUSRESPONSE(
GETINVENTORYSTATUSRESULT(
OUTCOME(
RESULT('Failure'),
ERROR(errorMessage)
)
)
)
return ET.tostring(xml_error, pretty_print=True)
def checkAuthorization(bu, headers, status):
error = 'Invalid clientCode for account type'
auth_search = re.search('username="(.*?)"', headers)
auth_user = auth_search.group(1)
if auth_user.upper() != "ADMIN":
if (str(bu).upper() != auth_user.upper()) and status == 'SS':
return False, Response(generateXMLErrorResponse(error, status), mimetype='text/xml')
elif (str(bu).upper() != auth_user.upper()) and status == 'IS':
return False, Response(generateXMLErrorResponse(error, status), mimetype='text/xml')
else:
return True, 'User authenticated'
else:
return True, 'User authenticated'
def getURI(api):
try:
if api == 'SS':
tracking = dbConnect.db.session.query(dbConnect.AppParam).\
filter(dbConnect.AppParam.name == 'TRACKING_WEB_SERVICE_URI').first()
return True, tracking.value
elif api == 'IS':
inventory = dbConnect.db.session.query(dbConnect.AppParam).\
filter(dbConnect.AppParam.name == 'INVENTORY_WEB_SERVICE_URI').first()
return True, inventory.value
except exc.OperationalError:
sendEmail()
return False, generateXMLErrorResponse('Service Unavailable', api)
def sendEmail():
msg = MIMEText('Unable to connect to DB')
msg['Subject'] = "Database server down!"
msg['From'] = ''
msg['To'] = ''
s = smtplib.SMTP('localhost')
s.sendmail(msg['From'], msg['To'], msg.as_string())
s.quit()
return True
def parseNWRequest(nwRequest, api):
returnValues = {}
if api == 'SS':
try:
xml = xmltodict.parse(nwRequest)
returnValues['start'] = xml['getShipmentStatus']['getShipmentStatusRequest']['startTime']
returnValues['end'] = xml['getShipmentStatus']['getShipmentStatusRequest']['endTime']
if validateDate(returnValues['start'], returnValues['end']):
returnValues['bu'] = xml['getShipmentStatus']['getShipmentStatusRequest']['clientCode']
returnValues['page'] = xml['getShipmentStatus']['getShipmentStatusRequest']['pageSize']
returnValues['requestid'] = xml['getShipmentStatus']['getShipmentStatusRequest']['requestId']
return True, returnValues
else:
return False, generateXMLErrorResponse('Invalid startDate/endDate', api)
except xmltodict.expat.ExpatError:
return False, generateXMLErrorResponse('Invalid Formed XML', api)
elif api == 'IS':
try:
xml = xmltodict.parse(request.data)
returnValues['bu'] = xml['getInventoryStatus']['getInventoryStatusRequest']['clientCode']
returnValues['facility'] = xml['getInventoryStatus']['getInventoryStatusRequest']['facility']
return True, returnValues
except xmltodict.expat.ExpatError:
return False, generateXMLErrorResponse('Invalid Formed XML', api)
def validateDate(startDate, endDate):
try:
datetime.strptime(startDate, '%Y-%m-%dT%H:%M:%S')
datetime.strptime(endDate, '%Y-%m-%dT%H:%M:%S')
except ValueError:
return False
else:
return True
def sendDepascoRequest(uri, search, api):
auth=('', '')
depascoResponse = requests.get(uri, auth=auth, params=search)
try:
depascoResponse.raise_for_status()
except requests.exceptions.HTTPError:
return False, generateXMLErrorResponse(depascoResponse.content, api)
else:
return True, requests.get(uri, auth=auth, params=search)
******* UPDATE **********
Thanks to the accepted answer, I removed all layers of the nested if statements. I have my functions just raise a custom exception which is handled in the main program.
__author__ = 'jw1050'
from functions import parseNWRequest, checkAuthorization, getURI, sendDepascoRequest, transformXML, addToDb, APIError
from flask import request
from flask import Response
import dbConnect
import lxml.etree as ET
from sqlalchemy import text
#dbConnect.app.route('/services/inventory/getInventoryStatus', methods=['POST'])
def parsexml():
try:
returnValues = parseNWRequest(request.data, 'IS')
checkAuthorization(returnValues['bu'], request.headers['Authorization'], 'IS')
uri = getURI('IS')
search = {'bu': returnValues['bu'], 'facility': returnValues['facility']}
depascoResponse = sendDepascoRequest(uri, search, 'IS')
s = text("Select sku, allocated from fgw_allocated_sku_count where client_code = :c and fulfillment_location = :t")
result = dbConnect.db.engine.execute(s, c=returnValues['bu'], t=returnValues['facility']).fetchall()
root = ET.fromstring(depascoResponse.content)
for row in result:
for element in root.iter('Item'):
xmlSKU = element.find('SKU').text
if xmlSKU == row[0]:
newQonOrder = int(element.find('QuantityOnOrder').text) + row[1]
element.find('QuantityOnOrder').text = str(newQonOrder)
newQAvailable = int(element.find('QuantityAvailable').text) - newQonOrder
element.find('QuantityAvailable').text = str(newQAvailable)
nakedResponse = transformXML(ET.tostring(root), 'transformInventory.xsl')
# Write Request to db
file_name = 'WS_IS_' + returnValues['bu']
request_file_size = request.headers['Content-Length']
addToDb('SS', file_name, 'text/xml', 'IS.Req', returnValues['bu'], 'Y', request.data, request_file_size)
addToDb('SS', file_name, 'text/xml', 'IS.Result', returnValues['bu'], 'Y', nakedResponse)
return Response(nakedResponse, mimetype='text/xml')
except APIError as error:
return Response(error.errorResponse, mimetype='text/xml')
#dbConnect.app.route('/services/tracking/getShipmentStatus', methods=['POST'])
def parsexml2():
try:
returnValues = parseNWRequest(request.data, 'SS')
checkAuthorization(returnValues['bu'], request.headers['Authorization'], 'SS')
uri = getURI('SS')
search = {'bu': returnValues['bu'], 'starttime': returnValues['start'], 'endtime': returnValues['end'],
'requestid': returnValues['requestid'], 'pagesize': returnValues['page']}
depascoResponse = sendDepascoRequest(uri, search, 'SS')
nakedResponse = transformXML(depascoResponse.content, 'transformTracking.xsl')
# Write Request to db
file_name = 'WS_SS_' + returnValues['bu']
request_file_size = request.headers['Content-Length']
addToDb('SS', file_name, 'text/xml', 'SS.Request', returnValues['bu'], 'Y', request.data, request_file_size)
addToDb('SS', file_name, 'text/xml', 'SS.Result', returnValues['bu'], 'Y', nakedResponse)
return Response(nakedResponse, mimetype='text/xml')
except APIError as error:
return Response(error.errorResponse, mimetype='text/xml')
if __name__ == '__main__':
dbConnect.app.run(host='localhost', port=int("5010"), debug=True)
My custom exception class:
class APIError(Exception):
def __init__(self, errorResponse):
self.errorResponse = errorResponse
An example of a function that raises my custom exception:
def validateDate(startDate, endDate):
try:
datetime.strptime(startDate, '%Y-%m-%dT%H:%M:%S')
datetime.strptime(endDate, '%Y-%m-%dT%H:%M:%S')
except ValueError:
raise APIError(generateXMLErrorResponse('Invalid format for startDate/endDate', 'SS'))
else:
return True
Another way is to raise exceptions and handle them.
import myexceptions
try:
# a complicated hunk of stuff that raises exceptions when
# things don't work out. The exceptions can be raised down
# inside functions, if breaking this lump into functions
# makes it easier to follow.
except MyExceptionA:
return Response( ...) # appropriate response for error A
except MyExceptionB
return Response( ...) # appropriate response for error B
except ...
I like to leave my methods ASAP, so in your case, it would look like:
if !X:
return...
if !Y:
return...
if !Z:
return...
though generally, I prefer
if X:
return...
if Y:
return...