This question already has answers here:
TypeError: string argument without an encoding
(3 answers)
Closed 3 years ago.
I'm trying to save password string encrypted in DynamoDb, I get this error.
Response:
{
"errorMessage": "string argument without an encoding",
"errorType": "TypeError",
"stackTrace": [
" File \"/var/task/lambda_function.py\", line 25, in lambda_handler\n encrypted_password = encrypt(session, plain_text_password, key_alias)\n",
" File \"/var/task/lambda_function.py\", line 11, in encrypt\n Plaintext=bytes(secret)\n"
]
}
This is the code I'm trying to work with.
import boto3
import base64
from botocore.exceptions import ClientError
def encrypt(session, secret, alias):
client = session.client('kms')
ciphertext = client.encrypt(
KeyId=alias,
Plaintext=bytes(secret)
)
return base64.b64encode(ciphertext["CiphertextBlob"])
def lambda_handler(event, context):
plain_text_password = event['password']
username = event['username']
key_alias = 'alias/ProjectKey'
table_name = 'Authentication'
session = boto3.session.Session()
table = boto3.resource('dynamodb').Table(table_name)
encrypted_password = encrypt(session, plain_text_password, key_alias)
print('ENCRYPTED STRING: ' + encrypted_password)
item = {
'username':username,
'password':encrypted_password
}
#check if item with the username already exists; if so, update password; else create new item
entry = table.get_item(TableName=table_name, Key={'username':username})
# if an entry with that username already exists, then update its corresponding password
if 'Item' in entry:
print('Item found. Updating password.')
print("entry['Item']" + str(entry['Item']))
response = table.update_item(
Key={
'username': username
},
UpdateExpression="set password = :p",
ExpressionAttributeValues={
':p': encrypted_password
},
ReturnValues="UPDATED_NEW"
)
else:
#if an entry with that username doesn't already exist, then create it
print('Adding new item to table.')
table.put_item(Item=item)
new_entry = table.get_item(TableName=table_name, Key={'username':username})
if 'Item' in new_entry:
print('A new item was inserted in the table.')
else:
print('Failed to insert new item in table')
return 'Function succeeded!'
I tried to run in python 2.7 and python 3 but no go.
I have added Lambda full access and dynamodb full access roles for Lambda and DB respectively and for KMS I have given the same accessess to administrate and key usage.
could you provide more informations (type, ...) on ciphertext["CiphertextBlob"] ?
maybe you just need to convert to bytes, e.g
base64.b64encode(bytes("yourstring", 'utf-8'))
or another way
base64.b64encode(ciphertext["CiphertextBlob"].encode('utf-8'))
Related
call to http()URl & download the file in S3 bucket. its working. then in 2nd part i am calling guardduty & give location of s3 file to create threat intel set. while running code i am getting below error:-
Response
{
"errorMessage": "'BadRequestException' object has no attribute 'message'",
"errorType": "AttributeError",
"requestId": "bec541eb-a315-4f65-9fa9-3f1139e31f86",
"stackTrace": [
" File \"/var/task/lambda_function.py\", line 38, in lambda_handler\n if \"name already exists\" in error.message:\n"
]
}
i want to create threat intel set using the file which is in S3--(downloaded from the URl)
code:-
import boto3
from datetime import datetime
import requests.packages.urllib3 as urllib3
def lambda_handler(event, context):
url='https://rules.emergingthreats.net/blockrules/compromised-ips.txt' # put your url here
bucket = 'awssaflitetifeeds-security' #your s3 bucket
key = 'GDfeeds/compromised-ips.csv' #your desired s3 path or filename
s3=boto3.client('s3')
http=urllib3.PoolManager()
s3.upload_fileobj(http.request('GET', url,preload_content=False), bucket, key)
#------------------------------------------------------------------
# Guard Duty
#------------------------------------------------------------------
location = "https://s3://awssaflitetifeeds-security/GDfeeds/compromised-ips.csv"
timeStamp = datetime.now()
name = "TF-%s"%timeStamp.strftime("%Y%m%d")
guardduty = boto3.client('guardduty')
response = guardduty.list_detectors()
if len(response['DetectorIds']) == 0:
raise Exception('Failed to read GuardDuty info. Please check if the service is activated')
detectorId = response['DetectorIds'][0]
try:
response = guardduty.create_threat_intel_set(
Activate=True,
DetectorId=detectorId,
Format='FIRE_EYE',
Location=location,
Name=name
)
except Exception as error:
if "name already exists" in error.message:
found = False
response = guardduty.list_threat_intel_sets(DetectorId=detectorId)
for setId in response['ThreatIntelSetIds']:
response = guardduty.get_threat_intel_set(DetectorId=detectorId, ThreatIntelSetId=setId)
if (name == response['Name']):
found = True
response = guardduty.update_threat_intel_set(
Activate=True,
DetectorId=detectorId,
Location=location,
Name=name,
ThreatIntelSetId=setId
)
break
if not found:
raise
#-------------------------------------------------------------------
# Update result data
#------------------------------------------------------------------
result = {
'statusCode': '200',
'body': {'message': "You requested: %s day(s) of /view/iocs indicators in CSV"%environ['DAYS_REQUESTED']}
}
except Exception as error:
logging.getLogger().error(str(error))
responseStatus = 'FAILED'
reason = error.message
result = {
'statusCode': '500',
'body': {'message': error.message}
}
finally:
#------------------------------------------------------------------
# Send Result
#------------------------------------------------------------------
if 'ResponseURL' in event:
send_response(event, context, responseStatus, responseData, event['LogicalResourceId'], reason)
The reason you are getting that error message is because the exception being returned from guardduty.create_threat_intel_set does not have the message attribute directly on the exception. I think you want either error.response['Message'] or error.response['Error']['Message'] for this exception case.
A couple of other suggestions:
you should replace the except Exception which is matching the exception showing an already-existing name with something more targeted. I'd recommend looking at what exceptions the guardduty client can throw for the particular operation and catch just the one you care about.
it is likely better to check that error.response['Error']['Code'] is exactly the error you want rather than doing a partial string match.
I am using the instances.list method from cloud sql admin-api to retrieve the information about instances in project. I am using the example code provided by google and it provides information about the instance. I am retrieving the name of the instance and then I need to update the instance labels if the label is matching the provided 'RC_PlatformCode'. This needs to be done to all sql instances in project matching the specific label. How can this be achieved as my code is not working. Or is there an easier way to do this in Python?
from config import Config, log, get_secret
from botocore.exceptions import ClientError
from typing import Dict, Iterable
from pprint import pprint
from googleapiclient import discovery
import json
import os
config = Config()
os.environ["GOOGLE_APPLICATION_CREDENTIALS"]="creds.json"
def updateSqlLabels(data, account):
log.info("-----")
log.info("updating Cloud Storage labels")
RC_PlatformCode = data['NewImage']['PlatformCode']['S']
platformcode_gcp = 'rc_platformcode'
tagKey = data['NewImage']['Key']['S']
tagValue = data['NewImage']['Value']['S']
char_to_replace = {
'#': '_at_',
'.': '_'
}
tagKey = tagKey.lower()
for key, value in char_to_replace.items():
tagValue = tagValue.replace(key, value)
service = discovery.build('sqladmin', 'v1beta4')
project = account # TODO: Update placeholder value.
request = service.instances().list(project=project)
while request is not None:
response = request.execute()
for database_instance in response['items']:
# TODO: Change code below to process each `database_instance` resource:
log.info("db_name = " + database_instance['name'])
update_tag = False
try:
labels = database_instance['settings']['userLabels']
log.info("tags -> " + str(labels))
except ClientError:
continue
for label in labels:
if labels["rc_platformcode"] == RC_PlatformCode:
log.info(
f"RC_PlatformCode [{RC_PlatformCode}] present for instance [{database_instance['name']}]")
update_tag = True
break
if update_tag:
create_tag = True
log.info("processing instance -> " + database_instance['name'])
log.info("setting tag Key -> " + tagKey)
log.info("setting tag Value -> " + tagValue)
for label in labels:
log.info("checking tag -> " + label)
labels[tagKey] = tagValue
instance_labels = labels
database_instance_body = {
'settings': {
'userLabels': instance_labels
}
}
log.info("project = " + project)
log.info("instance = " + database_instance['name'])
log.info("labels = " + str(database_instance_body))
request = service.instances().patch(project=project, instance=database_instance['name'], body=database_instance_body)
response = request.execute
break
I am receiving following error :
Response
{
"errorMessage": "'items'",
"errorType": "KeyError",
"stackTrace": [
" File \"/var/task/handler.py\", line 63, in handler\n updateSqlLabels(data, account)\n",
" File \"/var/task/cloudSql.py\", line 32, in updateSqlLabels\n for database_instance in response['items']:\n"
]
}
Any tips and help would be appreciated
I am creating a SAM web app, with the backend being an API in front of a Python Lambda function with a DynamoDB table that maintains a count of the number of HTTP calls to the API. The API must also return this number. The yaml code itself loads normally. My problem is writing the Lambda function to iterate and return the count. Here is my code:
def lambda_handler(event, context):
dynamodb = boto3.resource("dynamodb")
ddbTableName = os.environ["databaseName"]
table = dynamodb.Table(ddbTableName)
# Update item in table or add if doesn't exist
ddbResponse = table.update_item(
Key={"id": "VisitorCount"},
UpdateExpression="SET count = count + :value",
ExpressionAttributeValues={":value": Decimal(context)},
ReturnValues="UPDATED_NEW",
)
# Format dynamodb response into variable
responseBody = json.dumps({"VisitorCount": ddbResponse["Attributes"]["count"]})
# Create api response object
apiResponse = {"isBase64Encoded": False, "statusCode": 200, "body": responseBody}
# Return api response object
return apiResponse
I can get VisitorCount to be a string, but not a number. I get this error: [ERROR] TypeError: lambda_handler() missing 1 required positional argument: 'cou response = request_handler(event, lambda_context)le_event_request
What is going on?
[UPDATE] I found the original error, which was that the function was not properly received by the SAM app. Changing the name fixed this, and it is now being read. Now I have to troubleshoot the actual Python. New Code:
import json
import boto3
import os
dynamodb = boto3.resource("dynamodb")
ddbTableName = os.environ["databaseName"]
table = dynamodb.Table(ddbTableName)
Key = {"VisitorCount": { "N" : "0" }}
def handler(event, context):
# Update item in table or add if doesn't exist
ddbResponse = table.update_item(
UpdateExpression= "set VisitorCount = VisitorCount + :val",
ExpressionAttributeValues={":val": {"N":"1"}},
ReturnValues="UPDATED_NEW",
)
# Format dynamodb response into variable
responseBody = json.dumps({"VisitorCount": ddbResponse["Attributes"]["count"]})
# Create api response object
apiResponse = {"isBase64Encoded": False, "statusCode": 200,"body": responseBody}
# Return api response object
return apiResponse
I am getting a syntax error on Line 13, which is
UpdateExpression= "set VisitorCount = VisitorCount + :val",
But I can't tell where I am going wrong on this. It should update the DynamoDB table to increase the count by 1. Looking at the AWS guide it appears to be the correct syntax.
Not sure what the exact error is but ddbResponse will be like this:
ddbResponse = table.update_item(
Key={
'key1': aaa,
'key2': bbb
},
UpdateExpression= "set VisitorCount = VisitorCount + :val",
ExpressionAttributeValues={":val": Decimal(1)},
ReturnValues="UPDATED_NEW",
)
Specify item to be updated with Key (one item for one Lambda call)
Set Decimal(1) for ExpressionAttributeValues
https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GettingStarted.Python.03.html#GettingStarted.Python.03.04
I'm trying to get the recipient's email using the ID that I have retrieved. Below is the code.
messages = service.users().threads().list(userId='me', q='to:').execute().get('threads', [])
for message in messages:
#print(message)
# print(dir(message))
# print(message.get())
if search in message['snippet']:
#print(message['id'])
message_id = message['id']
# print(message_id)
full_message = service.users().messages().get(userId='me', id=message_id, format="raw").execute()
#print(full_message)
msg_headers = full_message['payload']['headers']
#print(msg_headers)
msg_str = base64.urlsafe_b64decode(full_message['raw'].encode('ASCII'))
mime_msg = email.message_from_bytes(msg_str)
#print(mime_msg)
x = re.findall('id=(\w+)\'', str(mime_msg))
print(x)
if len(x)> 0:
return x[0]
if msg_headers[x]['name'] == "To":
return msg_headers[x[0]]['value']
else:
return None
I get the error below. How do I resolve this?
Traceback (most recent call last):
File "C:\test\test2.py", line 102, in <module>
test = get_email_with("Test string")
File "C:\test\test2.py", line 55, in get_email_with
msg_headers = full_message['payload']['headers']
KeyError: 'payload'
Issue:
When the format is set to RAW, the field payload is not populated in the response.
From the API official docs:
RAW: Returns the full email message data with body content in the raw field as a base64url encoded string; the payload field is not used.
Solution:
Use either FULL or METADATA as format when calling messages().get, instead of RAW (format is an option parameter, and if it is not specified, FULL is used as default).
So you should change this:
full_message = service.users().messages().get(userId='me', id=message_id, format="raw").execute()
To this:
full_message = service.users().messages().get(userId='me', id=message_id).execute()
Or, alternatively, use format="raw" anyway and decode the base-64 encoded string full_message["raw"]. Edit: In this case, though, you'd have to remove this line: msg_headers = full_message['payload']['headers'].
Edit:
So, if you want to retrieve the recipient, you can do the following:
full_message = service.users().messages().get(userId='me', id=message_id).execute()
msg_headers = full_message['payload']['headers']
for header in msg_headers:
if header["name"] == "To":
return header["value"]
Reference:
API reference > Format
The response from Gmail can be a little confusing. You are on the right track with
msg_headers = full_message['payload']['headers']
One of the headers is called has a name of "Delivered-To" the value will be the email address that the mail was sent to. So basiclly you need to search your msg_headers and find the one called "Delivered-To"
Unfortunately i am not a python dev i cant help you with the code. Python quickstart may give you some ideas on how to get the response out of the object.
response
"payload": {
"partId": "",
"mimeType": "multipart/alternative",
"filename": "",
"headers": [
{
"name": "Delivered-To",
"value": "xxxx#gmail.com"
},
I'm trying to add an attachment to a testcaseresult using pyral like this:
testCaseResult = rally.create('TestCaseResult', {'TestCase': tc.ref , 'Build': revision,
'Verdict': verdict[test.result], 'Date': resultdate, 'Notes': note,
'Tester': tester, 'Duration': runtime })
res = rally.addAttachment(testCaseResult.oid, file);
The TestaseResult is successfully created but res is False.
What am I doing wrong? Should I not be using the oid? I've tried passing testCaseResult, testCaseResult.oid and "TestCaseResult/" + testCaseResult.oid, and none seem to work...
UPDATED:
Based on Mark's answer below (pyral does not directly support adding attachments to testcaseresults), I wrote the following subroutine:
def addTestCaseResultAttachment(testCaseResult, filename, contentType='text/plain'):
if not os.path.exists(filename):
raise Exception('Named attachment filename: %s not found' % filename)
if not os.path.isfile(filename):
raise Exception('Named attachment filename: %s is not a regular file' % filename)
attachment_file_name = os.path.basename(filename)
attachment_file_size = os.path.getsize(filename)
if attachment_file_size == 0:
raise Exception('Cannot attach zero length file')
if attachment_file_size > 5000000:
raise Exception('Attachment file size too large, unable to attach to Rally Artifact')
contents = ''
with open(filename, 'r') as af:
contents = base64.encodestring(af.read())
# create an AttachmentContent item
ac = rally.create('AttachmentContent', {"Content" : contents}, project=None)
if not ac:
raise RallyRESTAPIError('Unable to create AttachmentContent for %s' % attachment_file_name)
attachment_info = { "Name" : attachment_file_name,
"Content" : ac.ref, # ref to AttachmentContent
"ContentType" : contentType,
"Size" : attachment_file_size, # must be size before encoding!!
"User" : 'user/%s' % me.oid,
"TestCaseResult" : testCaseResult.ref
}
# and finally, create the Attachment
attachment = rally.create('Attachment', attachment_info, project=None)
if not attachment:
raise RallyRESTAPIError('Unable to create Attachment for %s' % attachment_file_name)
The issue is that the addAttachment method for pyral's restapi, sets a ref to the "Artifact" attribute of the Attachment object, as shown following (lines 1241 thru 1251 of restapi):
attachment_info = { "Name" : attachment_file_name,
"Content" : ac.ref, # ref to AttachmentContent
"ContentType" : mime_type,
"Size" : attachment_file_size, # must be size before encoding!!
"User" : 'user/%s' % self.contextHelper.user_oid,
#"Artifact" : artifact.ref # (Artifact is an 'optional' field)
}
# While it's actually possible to have an Attachment not linked to an Artifact,
# in most cases, it'll be far more useful to have the linkage to an Artifact than not.
if artifact:
attachment_info["Artifact"] = artifact.ref
Thus, the addAttachment method will actually only work for objects that inherit from Artifact, i.e. Stories, Defects, Tasks, TestCases, etc. As seen in WSAPI Docs, to associate an Attachment to a TestCaseResult, the needed attribute is actually "TestCaseResult". This syntax was chosen since a TestCaseResult is actually a WorkspaceDomainObject, and not an Artifact.
Here's an example that creates a new TestCaseResult and adds an Attachment
#!/usr/bin/env python
#################################################################################################
#
# create_tcr_and_attachment.py -- Create a New TestCaseResult and attach a file to it
#
USAGE = """
Usage: py create_tcr_and_attachment.py <TestCaseFormatedID> <filename>
"""
#################################################################################################
import sys, os
import re
import string
import base64
from pyral import Rally, rallySettings
#################################################################################################
errout = sys.stderr.write
ATTACHMENT_ATTRIBUTES = ['oid', 'ObjectID', '_type', '_ref', '_CreatedAt', 'Name',
'CreationDate', 'Description',
'Content', 'ContentType', 'Size',
'Subscription',
'Workspace',
'Artifact',
'User'
]
ATTACHMENT_IMPORTANT_ATTRS = """
Subscription ref (supplied at creation)
Workspace ref (supplied at creation)
Name STRING Required (name of the file, like foo.txt or ThePlan.doc)
User ref to User Required Settable (User who added the object)
Content ref to AttachmentContent
Size INTEGER Required
ContentType STRING Required
Artifact ref to Artifact (optional field)
Description TEXT Optional
"""
#################################################################################################
def main(args):
options = [opt for opt in args if opt.startswith('--')]
args = [arg for arg in args if arg not in options]
server = "rally1.rallydev.com"
user = "user#company.com"
password = "topsecret"
workspace = "My Workspace"
project = "My Project"
print " ".join(["|%s|" % item for item in [server, user, '********', workspace, project]])
rally = Rally(server, user, password, workspace=workspace, version="1.43") # specify the Rally server and credentials
rally.enableLogging('rally.hist.create_tcr_and_attachment') # name of file you want logging to go to
if len(args) != 2:
errout('ERROR: You must supply a Test Case FormattedID and an attachment file name')
errout(USAGE)
sys.exit(1)
targetTCID, filename = args
me = rally.getUserInfo(username=user).pop(0)
print "%s user oid: %s" % (user, me.oid)
target_project = rally.getProject()
target_tc = rally.get('TestCase', query='FormattedID = %s' % targetTCID, instance=True)
datestring = "2014-05-01"
tcr_info = {
"TestCase" : target_tc.ref,
"Build" : "master-91321",
"Date" : datestring,
"Verdict" : "Pass",
"Notes" : "Honeycomb harvest project."
}
print "Creating Test Case Result ..."
tcr = rally.put('TestCaseResult', tcr_info)
print "Created TCR: %s" % (tcr.oid)
print "Creating AttachmentContent"
if not os.path.exists(filename):
raise Exception('Named attachment filename: %s not found' % filename)
if not os.path.isfile(filename):
raise Exception('Named attachment filename: %s is not a regular file' % filename)
attachment_file_name = os.path.basename(filename)
attachment_file_size = os.path.getsize(filename)
if attachment_file_size > 5000000:
raise Exception('Attachment file size too large, unable to attach to Rally Artifact')
contents = ''
with open(filename, 'r') as af:
contents = base64.encodestring(af.read())
# create an AttachmentContent item
ac = rally.create('AttachmentContent', {"Content" : contents}, project=None)
if not ac:
raise RallyRESTAPIError('Unable to create AttachmentContent for %s' % attachment_file_name)
attachment_info = { "Name" : attachment_file_name,
"Content" : ac.ref, # ref to AttachmentContent
"ContentType" : "image/jpeg",
"Size" : attachment_file_size, # must be size before encoding!!
"User" : 'user/%s' % me.oid,
"TestCaseResult" : tcr.ref
}
# and finally, create the Attachment
attachment = rally.create('Attachment', attachment_info, project=None)
if not attachment:
raise RallyRESTAPIError('Unable to create Attachment for %s' % attachment_file_name)
#################################################################################################
#################################################################################################
if __name__ == '__main__':
main(sys.argv[1:])
The function addAttachment in Rally API has been updated to support Test Case result:
# While it's actually possible to have an Attachment not linked to an Artifact,
# in most cases, it'll be far more useful to have the linkage to an Artifact than not.
# A special case is where the "Artifact" is actually a TestCaseResult, which is not a
# subclass of Artifact in the Rally data model, but the WSAPI has been adjusted to permit
# us to associate an Attachment with a TestCaseResult instance.
if artifact:
attachment_info["Artifact"] = artifact.ref
if artifact._type == 'TestCaseResult':
del attachment_info["Artifact"]
attachment_info["TestCaseResult"] = artifact.ref
The function call rally.addAttachment(testCaseResult.oid, file); should now work.