We Keep Coding

how to retrieve data from google ads api reports(geo_performance_report)

I am trying to get the reports such as geo_performance_report, keywords_perfromance_report etc. but I am unable to figure how do to this with the new version of google-ads api.
I tried this way trying to use the new google-ads but was not successful.
is there any other ways to automate this process using Python.
def googleads_report(client, client_id, report_type, columns, start_date, end_date):
client.SetClientCustomerId(client_id)
report_downloader = googleads_client.GetReportDownloader(version="v201809")
report = {
'reportName': 'report-google-campaign-performance',
'dateRangeType': 'CUSTOM_DATE',
'reportType': report_type,
'downloadFormat': 'CSV',
'selector': {
'fields': columns,
'dateRange': {'min': start_date, 'max': end_date}
}
}
file = io.StringIO(report_downloader.DownloadReportAsString(
report,
skip_report_header=True,
skip_column_header=True,
skip_report_summary=True,
include_zero_impressions=False)
)
df = pd.read_csv(file, names=columns)
return df
def main(client, customer_id):
keyword_columns = [
'Date',
'AccountDescriptiveName',
'AdGroupId',
'AdGroupName',
'AdGroupStatus',
'CampaignId',
'CampaignName',
'CampaignStatus',
'CpcBid',
'Criteria',
'CriteriaDestinationUrl',
'ExternalCustomerId',
'FirstPageCpc',
'FirstPositionCpc',
'Id',
'KeywordMatchType',
'Labels',
'QualityScore',
'SearchImpressionShare',
'Status',
'TopOfPageCpc',
'Clicks',
'Conversions',
'Cost',
'ConversionValue',
'Impressions',
'ViewThroughConversions'
]
report_types = [
'KEYWORDS_PERFORMANCE_REPORT'
]
for report in report_types:
base_df = pd.DataFrame()
if report == 'CAMPAIGN_PERFORMANCE_REPORT':
table_suffix = 'campaigns'
#columns = campaign_columns
elif report == 'KEYWORDS_PERFORMANCE_REPORT':
table_suffix = 'keywords'
columns = keyword_columns
elif report == 'AD_PERFORMANCE_REPORT':
table_suffix = 'ads'
#columns = ad_columns
start_date = '2019-01-01'
df = googleads_report(client,customer_id, report, columns, start_date, yesterday)
df = df.applymap(str)
# Powershell output
print(df.head())
# csv output
df.to_csv('my_path' + table_suffix + '.csv')
if __name__ == "__main__":
# GoogleAdsClient will read the google-ads.yaml configuration file in the
# home directory if none is specified.
googleads_client = GoogleAdsClient.load_from_storage(path="mypath")
today = datetime.now().date()
yesterday = today - timedelta(days=1)
thirty_days_ago = today - timedelta(days=30)
try:
main( googleads_client, "#######")
except GoogleAdsException as ex:
print(
f'Request with ID "{ex.request_id}" failed with status '
f'"{ex.error.code().name}" and includes the following errors:'
)
for error in ex.failure.errors:
print(f'\tError with message "{error.message}".')
if error.location:
for field_path_element in error.location.field_path_elements:
print(f"\t\tOn field: {field_path_element.field_name}")
sys.exit(1)

Based upon your version='v201809' you are not using the most up to date version of the google ads api. That version of the API is scheduled for deprecation in spring 2022.
The newest version of the google ads api now uses a query language for their reporting examples.
Google ads provides a mapping for common reports into the fields required in their query language.
Once your client is authenticated with a newer version of the API, you can post the Google Ads query to the client.
from google.ads.googleads.client import GoogleAdsClient
client = GoogleAdsClient.load_from_storage("creds")
service = client.get_service("GoogleAdsService", version="v9")
#query below pulls all accounts that your MCC has access to
query = """
SELECT
customer_client.client_customer,
customer_client.level,
customer_client.manager,
customer_client.id
FROM customer_client
WHERE customer_client.manager != True
"""
search_request = client.get_type("SearchGoogleAdsStreamRequest")
search_request.customer_id = "1234adswordscustomer_id"
search_request.query = query
response = service.search_stream(search_request)

AWS Lambda limiting Python output

I have a lambda which i want to run every morning showing the users with AccessKeys older than 90 days. What i am noticing is that the return of users is limited
import boto3, json, time, datetime, sys, pprint
sns = boto3.client('sns')
usernames = []
mylist = []
sts_client = boto3.client('sts')
assumed_role_object=sts_client.assume_role(
RoleArn="arn:aws:iam::11111111:role/lambda_role",
RoleSessionName="AssumedRoleSession4"
)
credentials=assumed_role_object['Credentials']
client=boto3.client(
'iam',
aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_session_token=credentials['SessionToken'],
)
def lambda_handler(event, context):
users = client.list_users()
for key in users['Users']:
a = str(key['UserName'])
usernames.append(a)
for username in usernames:
try:
res = client.list_access_keys(UserName=username)
accesskeydate = res['AccessKeyMetadata'][0]['CreateDate']
accesskeydate = accesskeydate.strftime("%Y-%m-%d %H:%M:%S")
currentdate = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
accesskeyd = time.mktime(datetime.datetime.strptime(accesskeydate, "%Y-%m-%d %H:%M:%S").timetuple())
currentd = time.mktime(datetime.datetime.strptime(currentdate, "%Y-%m-%d %H:%M:%S").timetuple())
active_days = (currentd - accesskeyd)/60/60/24 ### We get the data in seconds. converting it to days
if 90 < active_days:
a = str(username)
c = int(int(round(active_days)))
mylist.append(a)
mylist.append(c)
except:
f = str('')
print(mylist)
finallist = ''.join(str(mylist))
finallist = finallist
sns_message = (finallist)
response = sns.publish(
TopicArn='arn:aws:sns:eu-west-2:111111:sns',
Message= sns_message,
Subject='Access Keys which need rotating',
)
How do i ensure all the users are captured in the python query and then emailed out via SNS?
Thanks

If you read the IAM list_users() documentation, you will see that results can be truncated and you need to paginate. The response you get back will include IsTruncated=True in which case you need to re-issue the list_users() call, supplying Marker, which you retrieve from the previous response.
Alternatively, and probably preferably, you can use a built-in paginator.

Alert for Old Access Keys in AWS

I didn't get an answer to my original question, but I've tweaked my code and I'm leaving this here in case anyone is trying to figure this out in the future. I'm new to Python, but this worked. This is a Python script I created for a Lambda function that checks the users in your AWS account and publishes a notification to an SNS topic. I've scheduled a CloudWatch rule with a cron expression to run it every day.
import boto3, json, time, datetime, sys, re
iam_client = boto3.client('iam')
sns_client = boto3.client('sns')
users = iam_client.list_users()
user_list = []
for key in users['Users']:
user_list = key['UserName']
accesskeys = iam_client.list_access_keys(UserName=key['UserName'])
for items in user_list.split('\n'):
for key in accesskeys['AccessKeyMetadata']:
accesskeydate = accesskeys['AccessKeyMetadata'][0]['CreateDate']
accesskeydate = accesskeydate.strftime("%Y-%m-%d %H:%M:%S")
currentdate = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
accesskeyd = time.mktime(datetime.datetime.strptime(accesskeydate, "%Y-%m-%d %H:%M:%S").timetuple())
currentd = time.mktime(datetime.datetime.strptime(currentdate, "%Y-%m-%d %H:%M:%S").timetuple())
active_days = (currentd - accesskeyd)/60/60/24
message = (key['UserName'],int(round(active_days))),
message = re.sub(r'[^a-zA-Z0-9 ]', "", str(message))
message = re.sub(r' ', ' is ', str(message))
if active_days >= 90:
sns_client.publish(
TopicArn='arn:aws:sns:us-west-2:xxxxxxxxxxxxx:topic-name',
Subject='User with Old Access Key Detected',
Message="The access key for " + str(message) + " days old. This user access key should be replaced ASAP.",
)

What causes memory leak in this function

Memory leak
I have this function volumeCounter() that is in a loop (in function looper())that executes every 10 sec. It is receiving data via api. After that it stores it into a sqlite database. When ran for a few hours, the ram will rise. It is especially noticeable when there are 100 threads or more of this function running (one thread for each market like BTC-ETH). After commenting it out, memory usage did not rise.
My question is, what inside this function is causing memory to rise.
def volumeCounter(self):
pass_to_table = api.get_market_history(self.market, 100)["result"] #This is in json format.
for i in pass_to_table:
id_i = i['Id']
time_i = i['TimeStamp'].replace("T", " ")
type_i = i['OrderType']
total = i['Total']
price = i["Price"]
self.cursor.execute(
"""INSERT OR IGNORE INTO {} (id, price, total, type,time) VALUES(?,?,?,?,?)""".format(
self.table_prices), (id_i, price, total, type_i, time_i))
self.connection.commit()
Function looper() that is looping volumeCounter(). It is executed in __init__() at start.
Script looks like this:
key = 'public_key'
secret = 'secret_key'
api = Bittrex(key, secret)
class CryptoScanner:
def __init__(self, market, interval, mailAdrr, mailPass, volume):
# Spremenljivke
self.market = market
self.timeInterval = interval
self.path = "ann{}/{}".format(interval, market)
self.path_2 = self.market.replace('-', '_')
self.database = "{}/{}.db".format(self.path, self.path_2)
self.mailAdrr = mailAdrr
self.mailPass = mailPass
self.sez_length = 0
self.volume = volume
self.scan_mode = False
# Table names
self.table_prices = '{}_prices'.format(self.path_2)
self.table_prices_sum = '{}_prices_sum'.format(self.path_2)
self.table_volume = '{}_volume'.format(self.path_2)
self.table_volume_change = '{}_volume_change'.format(self.path_2)
self.looper()
Then the class object is made for every market with multithreading.
for i in api.get_market_summaries()['result']:
if "BTC" in i['MarketName'] and i['BaseVolume'] >= lower_boundary and i['BaseVolume'] <= upper_boundary:
string = i['MarketName']
d["{}".format(string[4:])] = "{}".format(i['MarketName'])
volume["{}".format(string[4:])] = i['BaseVolume']
threads = []
for i in d:
t = threading.Thread(target=CryptoScanner, args=(d[i], interval_sec, mail, password, volume[i]))
threads.append(d)
t.start()
print("Created thread {}".format(d[i]))
Edit
Added bittrex api library details
class Bittrex(object):
"""
Used for requesting Bittrex with API key and API secret
"""
def __init__(self, api_key, api_secret):
self.api_key = str(api_key) if api_key is not None else ''
self.api_secret = str(api_secret) if api_secret is not None else ''
def api_query(self, method, options=None):
"""
Queries Bittrex with given method and options
:param method: Query method for getting info
:type method: str
:param options: Extra options for query
:type options: dict
:return: JSON response from Bittrex
:rtype : dict
"""
if not options:
options = {}
nonce = str(int(time.time() * 1000))
method_set = 'public'
if method in MARKET_SET:
method_set = 'market'
elif method in ACCOUNT_SET:
method_set = 'account'
request_url = (BASE_URL % method_set) + method + '?'
if method_set != 'public':
request_url += 'apikey=' + self.api_key + "&nonce=" + nonce + '&'
request_url += urlencode(options)
return requests.get(
request_url,
headers={"apisign": hmac.new(self.api_secret.encode(), request_url.encode(), hashlib.sha512).hexdigest()}
).json()
def get_market_history(self, market, count):
"""
Used to retrieve the latest trades that have occurred for a
specific market.
/market/getmarkethistory
:param market: String literal for the market (ex: BTC-LTC)
:type market: str
:param count: Number between 1-100 for the number of entries to return (default = 20)
:type count: int
:return: Market history in JSON
:rtype : dict
"""
return self.api_query('getmarkethistory', {'market': market, 'count': count})

Google App Engine: Using cron to expire (or 'unpublish') entities

I would like to mimic the 'published/unpublished' functionality of common CMS platforms like Wordpress or Drupal.
So I have this Job(ndb.Model):
class Job(ndb.Model):
title = ndb.StringProperty()
published = ndb.StringProperty(default = "on")
created = ndb.DateTimeProperty(auto_now_add = True)
expire = ndb.DateTimeProperty()
The NewJob handler looks like this:
class NewJob(JobHandler):
def get(self):
self.render('new-job.html')
def post(self):
title = self.request.get('title')
published = "on"
expire = datetime.datetime.now() + datetime.timedelta(days = 30)
if title:
j = Job(
title = title,
published = published,
expire = expire,
created = created)
j.put()
self.redirect('/job/%s' % str(j.key.id()))
else:
self.redirect('/login')
And the saved entity looks something like this:
Job(key=Key('Job', 5910974510923776), created=datetime.datetime(2014, 1, 17, 19, 0, 52, 12379), expire=datetime.datetime(2014, 2, 17, 19, 1, 52, 12174), published=u'on', title=u'Sous Chef')
What I am aiming to do is to set all Job entities to 'publish == "off"' when the their expire time is today (now)
So I've set up a task in the cron.yaml
cron:
- description: expire job entities after 30 days
url: /cron/job-expire
schedule: every day 00:00
...and the /cron/job-expire url is handled by:
class CronJobExpire(BaseHandler):
def get(self):
jobs = Job.query(Job.published == "on").fetch()
now = datetime.datetime.now()
for job in jobs:
if job.expire < now or job.expire == now:
job.published = "off"
The aim of the CronJobExpire handler above is to:
Check through the list of Job entities which are currently published == "on", then
check if their expire dates are 'now' or '< now', and if this is True, set published == "off".
This doesn't work. I am following the documentation. Any help would be appreciated - thank you.

You need to save the jobs that you changed:
class CronJobExpire(BaseHandler):
def get(self):
jobs = Job.query(Job.published == "on").fetch()
now = datetime.datetime.now()
for job in jobs:
if job.expire <= now:
job.published = "off"
job.put()
Also, I suggest fetching only the expired jobs:
now = datetime.datetime.now()
jobs = Job.query(Job.published == "on", Job.expire <= now).fetch()
for job in jobs:
job.published = "off"
job.put()
Or, to reduce API calls and therefore improve speed:
now = datetime.datetime.now()
jobs = Job.query(Job.published == "on", Job.expire <= now).fetch()
for job in jobs:
job.published = "off"
ndb.put_multi(jobs)
Lastly, consider making Job.published a boolean:
published = ndb.BooleanProperty(default=True)