We Keep Coding

python script to create the cloud function

We have written python script to create the cloud function , the trigger is Https. We need to invoke fetch the output of the function , So for that we are using the environment variables but some how that is not getting stored ?
def generate_config(context):
""" Entry point for the deployment resources. """
name = context.properties.get('name', context.env['name'])
project_id = context.properties.get('project', context.env['project'])
region = context.properties['region']
resources = []
resources.append(
{
'name': 'createfunction',
'type': 'gcp-types/cloudfunctions-v1:projects.locations.functions',
'properties':
{
'function': "licensevalidation",
'parent': 'projects//locations/',
'sourceArchiveUrl': 'gs://path',
'entryPoint':'handler',
'httpsTrigger': {"url": "https://.cloudfunctions.net/licensevalidation","securityLevel": "SECURE_ALWAYS"},
'timeout': '60s',
'serviceAccountEmail' : '.iam.gserviceaccount.com',
'availableMemoryMb': 256,
'runtime': 'python37' ,
'environmentvaiable' :
}
}
)
call ={
'type': 'gcp-types/cloudfunctions-v1:cloudfunctions.projects.locations.functions.call',
'name': 'call',
'properties':
{
'name':'/licensevalidation',
'data': '{""}'
},
'metadata': {
'dependsOn': ['createfunction']
}
}
resources.append(call)
return{
'resources': resources,
'outputs':
[
{
'name': 'installationtoken',
'value': 'os.environ.get(environment_variable)'
},
]
}

Mock a boto3 (botocore) ClientError for a dynamodb Table update_item call

I am writing a unit test for a python function that does a update_item in dynamodb via the table resource. The normal unit test works but raising a ClientError is something I cannot ge to work..
The tests get executed but I alwys:
E Failed: DID NOT RAISE <class 'botocore.exceptions.ClientError'>
What I am trying to do is simulate the case when I would get a Internal Server Error of AWS or a RequestLimitExceeded error.
Thanks for pointing me in the correct direction!
My functional code:
def _updateCardConfiguration(identifier, cardType, layout):
try:
table.update_item(
Key={
'identifier': identifier,
'cardType': cardType
},
UpdateExpression="SET layout = :layout",
ExpressionAttributeValues={
':layout': layout
}
)
except ClientError as e:
logger.fatal(
f"The update of the cardconfiguration for the niche: {identifier} has failed with the following message: {e.response['Error']['Message']}")
raise
return True
def lambda_handler(event, context):
# Check the cardtype
cardType = 'SearchresultCard' if (
event['detail']['action'] == 'updateSearchresultCard') else 'DetailCard'
# Update the card configuration
_updateCardConfiguration(
event['detail']['identifier'], cardType, event['detail']["layout"])
return True
My test code:
#pytest.fixture(scope='function')
def cardConfigurationsTable(aws_credentials):
with mock_dynamodb2():
#client = boto3.client('dynamodb', region_name='eu-west-1')
dynamodb = boto3.resource('dynamodb')
dynamodb.create_table(
TableName='CardConfigurations',
KeySchema=[
{
'AttributeName': 'identifier',
'KeyType': 'HASH'
},
{
'AttributeName': 'cardType',
'KeyType': 'RANGE'
}
],
AttributeDefinitions=[
{
'AttributeName': 'identifier',
'AttributeType': 'S'
},
{
'AttributeName': 'cardType',
'AttributeType': 'S'
},
],
StreamSpecification={
'StreamEnabled': True,
'StreamViewType': 'NEW_AND_OLD_IMAGES'
},
BillingMode='PAY_PER_REQUEST',
SSESpecification={
'Enabled': True
},
GlobalSecondaryIndexes=[
{
'IndexName': 'byIdentifier',
'KeySchema': [
{
'AttributeName': 'identifier',
'KeyType': 'HASH'
},
{
'AttributeName': 'cardType',
'KeyType': 'RANGE'
}
],
'Projection': {
'ProjectionType': 'ALL'
}
}
])
yield dynamodb.Table('CardConfigurations')
def test_updateItem_raises_clientExecption(cardConfigurationsTable, env_vars):
""" Unit test for testing the lambda that updates the cardconfiguration
Scenario: No detail card is configured for the niche
:param fixture cardConfigurationsTable: The fixture that mocks the dynamodb CardConfigurations table
:param fixture env_vars: The fixture with the env vars
"""
# Prepare the event
with open('tests/unit/event-flows/dataconfiguration-state-transition/events/updateSearchresultCard.json') as json_file:
event = json.load(json_file)
stubber = Stubber(cardConfigurationsTable.meta.client)
stubber.add_client_error(
"update_item",
service_error_code="InternalServerError",
service_message="Internal Server Error",
http_status_code=500,
)
stubber.activate()
# Import the lambda handler
lambdaFunction = importlib.import_module(
"event-flows.dataconfiguration-state-transition.updateCardconfigurationInDb.handler")
with pytest.raises(ClientError) as e_info:
# Execute the handler
response = lambdaFunction.lambda_handler(event, {})

I would start by creating a default table that your _updateCardConfiguration function can use.
def get_table():
table = dynamodb.Table("CardConfigurations")
return table
default_table = get_table()
You can then pass it as a default argument to the function that needs it. This makes it easier to test the function. To do so, modify your _updateCardConfiguration function header to look like this :
def _updateCardConfiguration(identifier, cardType, layout, table=default_table):
Doing this gives us the ability to replace the default table with a mocked table for testing. The complete modified function looks like this:
def _updateCardConfiguration(identifier, cardType, layout, table=default_table):
try:
table.update_item(
Key={"identifier": identifier, "cardType": cardType},
UpdateExpression="SET layout = :layout",
ExpressionAttributeValues={":layout": layout},
)
except ClientError as e:
logger.fatal(
f"The update of the cardconfiguration for the niche: {identifier} has failed with the following message: {e.response['Error']['Message']}"
)
raise
return True
As you can see, all that changes is the header.
Then in your test_updateItem_raises_clientExecption test function file, I would change the lines
with pytest.raises(ClientError) as e_info:
# Execute the handler
response = lambdaFunction.lambda_handler(event, {})
to:
with pytest.raises(ClientError, match="InternalServerError"):
# Execute the _updateCardConfiguration function
lambdaFunction._updateCardConfiguration(
None, None, None, cardConfigurationsTable
)
By passing in your cardConfigurationsTable fixture, your function wiil operate against it and you should get the behaviour you expect.

elasticsearch.exceptions.RequestError: RequestError(400, 'mapper_parsing_exception', 'No handler for type [string] declared on field [texts]')

I use elasticsearch python api to create mappings, but it went some wrong:
es = Elasticsearch("localhost:9200")
request_body = {
"settings": {
"number_of_shards": 5,
"number_of_replicas": 1
},
'mappings': {
'examplecase': {
'properties': {
'tbl_id': {'index': 'not_analyzed', 'type': 'string'},
'texts': {'index': 'analyzed', 'type': 'string'},
}
}
}
}
es.indices.create(index='example_index', body=request_body)
it shows elasticsearch.exceptions.RequestError: RequestError(400, 'mapper_parsing_exception', 'No handler for type [string] declared on field [texts]'), and I find some solution that they say: use text instead of string in the field type, but it also went wrong: elasticsearch.exceptions.RequestError: RequestError(400, 'mapper_parsing_exception', 'Failed to parse mapping [examplecase]: Could not convert [texts.index] to boolean'). The elasticsearch version iselasticsearch-6.5.4. How can I deal with it?

this
'index': 'analyzed' OR 'index': 'not_analyzed'
is an older elasticsearch version mapping and not needed.
All you need to do is use 'text' for analyzed string fields and 'keyword' for not_analyzed text fields, like this:
es = Elasticsearch("localhost:9200")
request_body = {
"settings": {
"number_of_shards": 5,
"number_of_replicas": 1
},
'mappings': {
'examplecase': {
'properties': {
'tbl_id': {'type': 'keyword'},
'texts': {'type': 'text'},
}
}
}
}
es.indices.create(index='example_index', body=request_body)
see reference in Elastic docs here: https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html

See this. The index setting in your mapping is incorrectly configured. It is a mapping parameter and can be set to true or false only. You cannot set this within the properties parameter.

Google DLP: "ValueError: Protocol message Value has no "stringValue" field."

I have a method where I build a table for multiple items for Google's DLP inspect API which can take either a ContentItem, or a table of values
Here is how the request is constructed:
def redact_text(text_list):
dlp = google.cloud.dlp.DlpServiceClient()
project = 'my-project'
parent = dlp.project_path(project)
items = build_item_table(text_list)
info_types = [{'name': 'EMAIL_ADDRESS'}, {'name': 'PHONE_NUMBER'}]
inspect_config = {
'min_likelihood': "LIKELIHOOD_UNSPECIFIED",
'include_quote': True,
'info_types': info_types
}
response = dlp.inspect_content(parent, inspect_config, items)
return response
def build_item_table(text_list):
rows = []
for item in text_list:
row = {"values": [{"stringValue": item}]}
rows.append(row)
table = {"table": {"headers": [{"name": "something"}], "rows": rows}}
return table
When I run this I get back the error ValueError: Protocol message Value has no "stringValue" field. Even though the this example and the docs say otherwise.
Is there something off in how I build the request?
Edit: Here's the output from build_item_table
{
'table':
{
'headers':
[
{'name': 'value'}
],
'rows':
[
{
'values':
[
{
'stringValue': 'My name is Jenny and my number is (555) 867-5309, you can also email me at anemail#gmail.com, another email you can reach me at is email#email.com. '
}
]
},
{
'values':
[
{
'stringValue': 'Jimbob Doe (555) 111-1233, that one place down the road some_email#yahoo.com'
}
]
}
]
}
}

Try string_value .... python uses the field names, not the type name.

Error when scripting the parameters for executing BigQuery via a Python script

I'm trying to adapt the asynch_query.py script found at https://github.com/GoogleCloudPlatform/bigquery-samples-python/tree/master/python/samples for use in executing a query and having the output go to a BigQuery table. The JSON section of the script as I've created it for seting the parameters is as follows:
job_data = {
'jobReference': {
'projectId': project_id,
'job_id': str(uuid.uuid4())
},
'configuration': {
'query': {
'query': queryString,
'priority': 'BATCH' if batch else 'INTERACTIVE',
'createDisposition': 'CREATE_IF_NEEDED',
'defaultDataset': {
'datasetId': 'myDataset'
},
'destinationTable': {
'datasetID': 'myDataset',
'projectId': project_id,
'tableId': 'testTable'
},
'tableDefinitions': {
'(key)': {
'schema': {
'fields': [
{
'description': 'eventLabel',
'fields': [],
'mode': 'NULLABLE',
'name': 'eventLabel',
'type': 'STRING'
}]
}
}
}
}
}
}
When I run my script I get an error message that a "Required parameter is missing". I've been through the documentation at https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.query trying to figure out what is missing, but attempts at various configurations have failed. Can anyone identify what is missing and how I would fix this error?

Not sure what's going on. To insert the results of a query into another table I use this code:
def create_table_from_query(connector, query,dest_table):
body = {
'configuration': {
'query': {
'destinationTable': {
'projectId': your_project_id,
'tableId': dest_table,
'datasetId': your_dataset_id
},
'writeDisposition': 'WRITE_TRUNCATE',
'query': query,
},
}
}
response = connector.jobs().insert(projectId=self._project_id,
body=body).execute()
wait_job_completion(response['jobReference']['jobId'])
def wait_job_completion(connector, job_id):
while True:
response = connector.jobs().get(projectId=self._project_id,
jobId=job_id).execute()
if response['status']['state'] == 'DONE':
return
where connector is build('bigquery', 'v2', http=authorization)
Maybe you could start from there and keep adding new fields as you wish (notice that you don't have to define the schema of the table as it's already contained in the results of the query).