credentials = ServicePrincipalCredentials(client_id=client_id, secret=secret, tenant=tenant)
adf_client = DataFactoryManagementClient(credentials, subscription_id)
run_response = adf_client.pipelines.create_run(rg_name, df_name, pipeline_nm,{})
# Monitor the pipeline run
pipeline_run = adf_client.pipeline_runs.get(rg_name, df_name, run_response.run_id)
while (pipeline_run.status == 'InProgress' or pipeline_run.status == 'Queued'):
#print("[INFO]:Pipeline run status: {}".format(pipeline_run.status))
time.sleep(statuschecktime)
pipeline_run = adf_client.pipeline_runs.get(rg_name, df_name, run_response.run_id)
print("[INFO]:Pipeline run status: {}".format(pipeline_run.status))
print('')
activity_runs_paged = list(adf_client.activity_runs.list_by_pipeline_run(rg_name, df_name, pipeline_run.run_id, datetime.now() - timedelta(1), datetime.now() + timedelta(1)))
Activity run is different from the pipeline run, if you want to fetch the pipelines run details, follow the steps below.
1.Register an application with Azure AD and create a service principal
2.Get tenant and app ID values for signing in and Create a new application secret and save it
3.Navigate to the datafactory in the portal -> Access control (IAM) -> Add role assignment -> add your application as a role e.g. Contributor, details follow this.
4.Install the packages.
pip install azure-mgmt-resource
pip install azure-mgmt-datafactory
5.Then use the code below to query pipeline runs in the factory based on input filter conditions.
from azure.common.credentials import ServicePrincipalCredentials
from azure.mgmt.datafactory import DataFactoryManagementClient
from azure.mgmt.datafactory.models import *
from datetime import datetime, timedelta
subscription_id = "<subscription-id>"
rg_name = "<resource-group-name>"
df_name = "<datafactory-name>"
tenant_id = "<tenant-id>"
client_id = "<application-id (i.e client id)>"
client_secret = "<client-secret>"
credentials = ServicePrincipalCredentials(client_id=client_id, secret=client_secret, tenant=tenant_id)
adf_client = DataFactoryManagementClient(credentials, subscription_id)
filter_params = RunFilterParameters(last_updated_after=datetime.now() - timedelta(1), last_updated_before=datetime.now() + timedelta(1))
pipeline_runs = adf_client.pipeline_runs.query_by_factory(resource_group_name=rg_name, factory_name=df_name, filter_parameters = filter_params)
for pipeline_run in pipeline_runs.value:
print(pipeline_run)
You can also get the specific pipeline run with the Run ID.
specific_pipeline_run = adf_client.pipeline_runs.get(resource_group_name=rg_name,factory_name=df_name,run_id= "xxxxxxxx")
print(specific_pipeline_run)
Related
I would like to be able to access GKE (kubernetes) cluster in GCP from python kubernetes client.
I cant authenticate and connect to my cluster and i dont find the reason.
Here is what i tried so far.
from google.auth import compute_engine
from google.cloud.container_v1 import ClusterManagerClient
from kubernetes import client
def test_gke(request):
project_id = "myproject"
zone = "myzone"
cluster_id = "mycluster"
credentials = compute_engine.Credentials()
cluster_manager_client = ClusterManagerClient(credentials=credentials)
cluster = cluster_manager_client.get_cluster(name=f'projects/{project_id}/locations/{zone}/clusters/{cluster_id}')
configuration = client.Configuration()
configuration.host = f"https://{cluster.endpoint}:443"
configuration.verify_ssl = False
configuration.api_key = {"authorization": "Bearer " + credentials.token}
client.Configuration.set_default(configuration)
v1 = client.CoreV1Api()
print("Listing pods with their IPs:")
pods = v1.list_pod_for_all_namespaces(watch=False)
for i in pods.items:
print("%s\t%s\t%s" % (i.status.pod_ip, i.metadata.namespace, i.metadata.name))
I'd like to get the configuration working I have it work where, the code is running off cluster and it produces the kubectl config file for itself. (see update at end)
Original
The first solution assumes (!) you've the cluster configured in your local (~/.kube/config and probably adjusted by KUBE_CONFIG) config.
from google.cloud.container_v1 import ClusterManagerClient
from kubernetes import client,config
config.load_kube_config()
api_instance = client.CoreV1Api()
resp = api_instance.list_pod_for_all_namespaces()
for i in resp.items:
print(f"{i.status.pod_ip}\t{i.metadata.namespace}\t{i.metadata.name}")
NOTE
Assumes you've run gcloud containers clusters get-credentials to set the ~/.kube/config file for the current cluster (and has a current-context set.
Uses your user credentials in the ~/.kube/config file so no additional credentials are needed.
Update
Okay, I have it working. Here's the code that will generate a kubectl config and connect to the cluster. This code uses Application Default Credentials to provide a Service Account key to the code (usually export GOOGLE_APPLICATION_CREDENTIALS=/path/to/key.json)
import os
import google.auth
import base64
from google.cloud.container_v1 import ClusterManagerClient
from kubernetes import client,config
from ruamel import yaml
PROJECT = os.getenv("PROJECT")
ZONE = os.getenv("ZONE")
CLUSTER = os.getenv("CLUSTER")
# Get Application Default Credentials
# `project_id` is the Service Account's
# This may differ to the cluster's `PROJECT`
credentials, project_id = google.auth.default()
# Get the cluster config from GCP
cluster_manager_client = ClusterManagerClient(credentials=credentials)
name=f"projects/{PROJECT}/locations/{ZONE}/clusters/{CLUSTER}"
cluster = cluster_manager_client.get_cluster(name=name)
SERVER = cluster.endpoint
CERT = cluster.master_auth.cluster_ca_certificate
configuration = client.Configuration()
# Create's a `kubectl` config
NAME="freddie" # arbitrary
CONFIG=f"""
apiVersion: v1
kind: Config
clusters:
- name: {NAME}
cluster:
certificate-authority-data: {CERT}
server: https://{SERVER}
contexts:
- name: {NAME}
context:
cluster: {NAME}
user: {NAME}
current-context: {NAME}
users:
- name: {NAME}
user:
auth-provider:
name: gcp
config:
scopes: https://www.googleapis.com/auth/cloud-platform
"""
# The Python SDK doesn't directly support providing a dict
# See: https://github.com/kubernetes-client/python/issues/870
kubeconfig = yaml.safe_load(CONFIG)
loader = config.kube_config.KubeConfigLoader(kubeconfig)
loader.load_and_set(configuration)
api_client= client.ApiClient(configuration)
api_instance = client.CoreV1Api(api_client)
# Enumerate e.g. Pods
resp = api_instance.list_pod_for_all_namespaces()
for i in resp.items:
print(f"{i.status.pod_ip}\t{i.metadata.namespace}\t{i.metadata.name}")
I have a project to implement where I need to communicate with a Step Function that another department created. I'm new to Step Functions so please bear with me if I'm missing anything.
We have a UI where a user can request their data or have their data deleted. This request is sent to an API Gateway and then to a Step Function which creates multiple workers/subscriptions. My task is to create an Azure Function (using Python) to process the tasks and link to the relevant places where we have data and either delete it or return it to an S3 bucket. I thus have the below script:
import datetime
import logging
import boto3
import os
import json
workerName = creds['workerName']
region_name = creds['region_name']
activityArn = creds['activityArn']
aws_access_key_id = creds['aws_access_key_id']
aws_secret_access_key = creds['aws_secret_access_key']
bucket = creds['bucket']
sfn_client = boto3.client(
service_name='stepfunctions',
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
region_name=region_name
)
activity = sfn_client.get_activity_task(
activityArn = activityArn,
workerName = workerName
)
task_token, task = activity['taskToken'], json.loads(activity['input'])
# TODO Process Task
I notice that every time I run activity = ... I get a new task instead of a list and have read through the documentation that I need to use the send_task_failure(), send_task_heartbeat(), and send_task_success() methods which is fine. Since it returns one activity per run I was planning to run a loop until I have no more activities but when I get to the end (or when there are no activities to run) the script just hangs until timeout.
Is there a way to get a count of unstarted activities only so I can use that to loop through or is there a better approach to this?
Ok so after reading through the documentation I found that I had to add a read_timeout > than the default...I think the default is 60s so I added a 65s timeout as per below
import datetime
import logging
import boto3
from botocore.client import Config
import os
import json
connect_timeout = creds['connect_timeout'] + 5
read_timeout = creds['read_timeout'] + 5
workerName = creds['workerName']
region_name = creds['region_name']
activityArn = creds['activityArn']
aws_access_key_id = creds['aws_access_key_id']
aws_secret_access_key = creds['aws_secret_access_key']
bucket = creds['bucket']
cfg = creds['cfg']
config = Config(
connect_timeout=connect_timeout,
read_timeout=read_timeout,
retries={'max_attempts': 0}
)
sfn_client = boto3.client(
service_name='stepfunctions',
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
region_name=region_name,
config=config
)
while True:
activity_task = sfn_client.get_activity_task(
activityArn = activityArn,
workerName = workerName
)
if 'input' not in activity_task.keys() or 'taskToken' not in activity_task.keys():
print(f"No more activity tasks")
break
taskToken, task = activity_task['taskToken'], json.loads(activity_task['input'])
On the final pass through it returns a JSON with the same keys as all other activities but without the input and taskToken
We are going to get some metrics from Azure Monitor.
I Installed the libraries and made a script to do that.
The script is following.
#! /usr/bin/env python
import datetime
from azure.mgmt.monitor import MonitorManagementClient
from azure.identity import ClientSecretCredential
subscription_id = 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
resource_group_name = 'xxxx-xxxxx'
vm_name = 'xxxxxxxxxx'
resource_id = (
"subscriptions/{}/"
"resourceGroups/{}/"
"providers/Microsoft.Compute/virtualMachines/{}"
).format(subscription_id, resource_group_name, vm_name)
TENANT_ID = 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
CLIENT = 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
KEY = 'xxxxxxxxx'
credentials = ServicePrincipalCredentials(
client_id = CLIENT,
secret = KEY,
tenant = TENANT_ID
)
client = MonitorManagementClient(
credentials,
subscription_id
)
today = datetime.datetime.now()
nexttime = today - datetime.timedelta(minutes=1)
metrics_data = client.metrics.list(
resource_id,
timespan="{}/{}".format(nexttime, today),
interval='PT1M',
metricnames='Percentage CPU',
aggregation='average'
)
for item in metrics_data.value:
for timeserie in item.timeseries:
for data in timeserie.data:
print("{}".format(data.average))
when I run this script, the result shows sometimes 'None',regardless of I can see the right value on Azure-Monitor.
When I run the script that getting metric from Azure Load Balancer, the script returns the right value.
This URL (https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-supported#microsoftcomputevirtualmachines) says the aggregation type of Percentage CPU is Average, so the script seems to be correct.
I don't know why this script of getting Percentage CPU doesn't return the same value as Azure-Monitor.
Does anyone know the solution?
Azure monitor logs metrics by UTC time, so you should use today = datetime.datetime.utcnow() to define your timespan.
Based on your code, you are querying the latest 1 min VM CPU Percentage, it will be some latency that Azure monitors log metrics, so maybe you can't get this value due to latency, so maybe you can try to get i,e last 5th min CPU Percentage.
Just try the code snippet below:
today = datetime.datetime.utcnow()
nexttime = today - datetime.timedelta(minutes=5)
query_timespan = "{}/{}".format(nexttime, today - datetime.timedelta(minutes=4))
print(query_timespan)
metrics_data = client.metrics.list(
resource_id,
timespan=query_timespan,
interval='PT1M',
metricnames='Percentage CPU',
aggregation='average'
)
for item in metrics_data.value:
for timeserie in item.timeseries:
for data in timeserie.data:
print("{}".format(data.average))
Result on my side:
Portal display(my timezone is UTC+8):
I have an app running on Google App Engine, and an AI running on Google Compute Engine. I'm triggering the VM instance to start on a change in a Google Cloud Storage bucket, and have a start-up script that I attempt to store in the metadata of the GCE instance. My cloud functions looks like this:
import os
from googleapiclient.discovery import build
def start(event, context):
file = event
print(file["id"])
string = file["id"]
new_string = string.split('/')
user_id = new_string[1]
payment_id = new_string[2]
name = new_string[3]
print(name)
if name == "uploadcomplete.txt":
startup_script = """ #! /bin/bash
sudo su username
cd directory/directory
python analysis.py -- gs://location/{userId}/{paymentId}
""".format(userId=user_id, paymentId=payment_id)
# initialize compute api
service = build('compute', 'v1', cache_discovery=False)
print('VM Instance starting')
project = 'zephyrd'
zone = 'us-east1-c'
instance = 'zephyr-a'
# get metadata fingerprint in order to set new metadata
metadata = service.instances().get(project=project, zone=zone, instance=instance)
metares = metadata.execute()
fingerprint = metares["metadata"]["fingerprint"]
# set new metadata
bodydata = {"fingerprint": fingerprint,
"items": [{"key": "startup-script", "value": startup_script}]}
meta = service.instances().setMetadata(project=project, zone=zone, instance=instance,
body=bodydata).execute()
print(meta)
# confirm new metdata
instanceget = service.instances().get(project=project, zone=zone, instance=instance).execute()
print("'New Metadata:", instanceget['metadata'])
print(instanceget)
# start VM
request = service.instances().start(project=project, zone=zone, instance=instance)
response = request.execute()
print('VM Instance started')
print(response)
The VM starts, but the startup script does not run. The script has been simplified for the purposes of the question, but this is just a basic command I'm trying to run. I would add the script directly to the metadata in the console, but I use values from the cloud function trigger to run commands in the VM. What am I missing?
I've attempted to set the metadata in two ways:
"items": [{"key": "startup-script", "value": startup_script}]
as well as:
"items": [{"startup-script" : startup_script}]
Neither work. The commands run beautifully if I manually type them in the shell.
Look into your logs to determine why its not executing.
https://cloud.google.com/compute/docs/startupscript#viewing_startup_script_logs
Probably you the issue is that you are trying to execute a python script instead of a bash script.
Your startup script should be something like:
#! /bin/bash
# ...
python3 paht/to/python_script.py
I'm creating a container engine cluster using API python client for google cloud platform.I have done with container creation successfully, Now I need to apply some yaml configurations but before applying any kubernetes yaml configurations the cluster should be provisioned otherwise kubernetes API not available.
I need to do both(Container creation & Apply yaml configs) things in a single request.
How can i get the provision status of a cluster using api?
Here's what i have tried:
After cluster creation:
From views.py:
print('Fetching Cluster configs ....')
cc = subprocess.call(
'gcloud container clusters get-credentials ' + deployment.deploymentName.lower() + ' --zone ' + deployment.region + ' --project ' + deployment.project,
shell=True)
print(cc)
while cc == 1:
cc = subprocess.call(
'gcloud container clusters get-credentials ' + deployment.deploymentName.lower() + ' --zone ' + deployment.region + ' --project ' + deployment.project,
shell=True)
print(cc)
Help me, please!
Thanks in Advance!
This is how I do in my code:
"""
If you have a credentials issue, run:
gcloud beta auth application-default login
"""
import time
import googleapiclient.discovery
service = googleapiclient.discovery.build('container', 'v1')
clusters_resource = service.projects().zones().clusters()
operations_resource = service.projects().zones().operations()
def create_cluster(project_id, zone, config, async=False):
req = clusters_resource.create(projectId=project_id, zone=zone, body=config)
operation = req.execute()
if async:
return
while operation['status'] == 'RUNNING':
time.sleep(1)
req = operations_resource.get(projectId=project_id, zone=zone, operationId=operation['name'])
operation = req.execute()
return operation['status'] == 'DONE'
What you are looking for is the status of the operation whose ID is returned from the create cluster call. You then need to get the operation (via the container API, not the compute API) and check the status of the operation to see if it is DONE. Once it is done, you can determine if there was an error by looking at the status message in the operation. If it is blank, then the create cluster API call succeeded. If it is non-empty then the call failed and the status message will tell you why. Once the operation for creating the cluster is done the get-credentials call will succeed.