We Keep Coding

Python while loop for API offsets

I did my best writing this API, but I'm now stuck on the while loop.
My main goal is to scan batch by batch the API results and write it in the database.
The way the code is written is bringing me straight the results as offset=4000
Don't know what am I doing wrong.
Follow my code
#!/bin/env python
from asyncio.windows_events import NULL
from re import X
from typing import ItemsView
import requests
import json
import hashlib
import base64
import time
import hmac
import pandas as pd
import datetime
import pyodbc
#Account Info
AccessId = ''
AccessKey = ''
Company = ''
#Request Info
httpVerb ='GET'
resourcePath = '/alert/alerts'
offset = 0
while offset < 5000:
# Query parameters
queryParams ='?size=1000&offset=' + str(offset) + '&sort=-startEpoch&filter=cleared:*,rule:critical'
offset += 1000
data = ''
#Construct URL
url = 'https://'+ Company +'.logicmonitor.com/santaba/rest' + resourcePath + queryParams
print(url)
#Get current time in milliseconds
epoch = str(int(time.time() * 1000))
#Concatenate Request details
requestVars = httpVerb + epoch + data + resourcePath
#Construct signature
hmac1 = hmac.new(AccessKey.encode(),msg=requestVars.encode(),digestmod=hashlib.sha256).hexdigest()
signature = base64.b64encode(hmac1.encode())
#Construct headers
auth = 'LMv1 ' + AccessId + ':' + signature.decode() + ':' + epoch
headers = {'Content-Type':'application/json','Authorization':auth}
#Make request
response = requests.get(url, data=data, headers=headers)
data = response.json()
alerts_df = pd.DataFrame(data['data']['items'])
alerts_df = alerts_df[['id','internalId','rule','monitorObjectName','startEpoch','endEpoch','cleared','resourceTemplateName']]
alerts_df['startEpoch'] = pd.to_datetime(alerts_df['startEpoch'],unit='s')
alerts_df['endEpoch'] = alerts_df['endEpoch'].apply(lambda x: pd.to_datetime(x,unit='s') if x !=0 else x)
print(alerts_df)

You are overwriting the variable constantly before even making one request.
Also, the while loop should be <= 5000 if you want to include the 5000 in the loop.
There are two ways to achieve it.
append the results to a list for later use
make the request after you generated the string

Create a HMAC-SHA1 base 64 signature for API get request

The Problem:
I am trying to get a requests.get to print from a website and it needs a signature. From what I read on the website it says that it needs to take my secret key and make a signature out of it using HMAC-SHA1 base 64.
I know I need to import hmac into my python script but other than that I am not sure if my signature will be what the website is looking for.
This is a link to how the authentication is suppose to be setup: https://www.ninjarmm.com/dev-api/#the_authentication_header
On the website it gives an example of how the signature should be but I don't know how to apply this in python:
### How do I apply this in my python script's header?
Signature = Base64( HMAC-SHA1( YourSecretAccessKeyID, Base64( UTF-8- Encoding-Of( StringToSign ) ) ) ); StringToSign = HTTP-Verb + "n" + Content-MD5 + "n" + Content-Type + "n" + Date + "n" + CanonicalizedResource;
### How do I apply this in my python script's header?
Here is my Code:
import requests
import os
import json
from datetime import date
from email.utils import formatdate
ninjaapi = "some data"
ninjasak = "some data"
ninjaurl = "https://api.ninjarmm.com/v1/devices"
ninjaheaders = {"authorization":ninjaapi + ninjasak, "Date":now,}
ninjastatusresponse = requests.get(ninjaurl, headers=ninjaheaders)
ninja_json = ninjastatusresponse.json()
print(ninja_json)
Here is the results I get so far:
{'error': 'invalid_header', 'error_description': "Invalid 'Authorization' header", 'error_code': 1}
If anyone has any ideas that I can test let me know!
Thank you for your time,

So I found out there was a nice person who made a module package for everything already. I used his code in my program and it worked for building the signature.
Its called ninjarmm_api; here is the link: https://bitbucket.org/raptus-it/py-ninjarmm-api-client/src/master/ninjarmm_api/
Here is the link from the ninjadojo community.
https://ninjarmm.zendesk.com/hc/en-us/community/posts/360037165831-Python-client-library-for-API
Specifically, here is the code for the his auth.py which is what I was trying to figure out how to do.
from requests.auth import AuthBase
from email.utils import formatdate
import base64
import hmac
import hashlib
class auth(AuthBase):
NINJA_HDR_AUTH = "Authorization"
NINJA_HDR_DATE = "Date"
NINJA_ENCODING = "utf-8"
def __init__(self, accessKeyId, secretAccessKey):
self.access_key_id = accessKeyId
self.secret_access_key = secretAccessKey
self.tstamp = formatdate(timeval=None, localtime=False, usegmt=True)
def __call__(self, request):
sts_clear = request.method + "\n" # HTTP verb
sts_clear += "\n" # Content MD5
sts_clear += "\n" # Content type
sts_clear += self.tstamp + "\n" # Date
sts_clear += request.path_url # Canonicalized resource
sts_base64 = base64.b64encode(sts_clear.encode(self.NINJA_ENCODING))
sts_digest = hmac.new(self.secret_access_key.encode(self.NINJA_ENCODING), sts_base64, hashlib.sha1)
signature = base64.b64encode(sts_digest.digest())
request.headers[self.NINJA_HDR_AUTH] = "NJ " + self.access_key_id + ":" + signature.decode(self.NINJA_ENCODING)
request.headers[self.NINJA_HDR_DATE] = self.tstamp
return request

Python request error

I am trying to make a request using headers I have URL encoded. When I run, i get error 500 for some reason. If the code was working we should get back an oauth_token and oauth_token_secret.
following these instructions : http://developer.trademe.co.nz/api-overview/authentication/example-oauth-flow/
import hashlib
import hmac
import base64
import time
from rauth.service import OAuth1Service
import oauth
import rauth
import requests
import oauth2 as oauth1
from urllib.parse import parse_qs
import urllib
nonce = oauth1.generate_nonce()
time = int(time.time())
consumer_key = "5C82CC6BC7C6472154FBC9CAB24A29A2"
New_base_string ="POST&https%3A%2F%2Fsecure.tmsandbox.co.nz%2FOauth%2FRequestToken&oauth_callback%3Dhttp%253A%252F%252Fwww.website-tm-access.co.nz%252Ftrademe-callback%26oauth_consumer_key%" + str(consumer_key) +"3DC74CD73FDBE37D29BDD21BAB54BC70E422%26oauth_nonce%3" + str(nonce) + "%26oauth_signature_method%3DHMAC-SHA1%26oauth_timestamp%3" + str(time) + "%26oauth_version%3D1.0%26scope%3DMyTradeMeRead%252CMyTradeMeWrite"
New_base_string= New_base_string.encode()
KEY = b"5C82CC6BC7C6472154FBC9CAB24A29A2&"
digest = hmac.new(KEY, New_base_string, hashlib.sha1).digest()
signature = (base64.b64encode(digest))
print(base64.b64encode(digest))
url = 'https://secure.tmsandbox.co.nz/Oauth/RequestToken?scope=MyTradeMeRead,MyTradeMeWrite '
headers = {'oauth_callback': "http://www.website-tm-access.co.nz%2Ftrademe-callback",
'oauth_consumer_key' : "C74CD73FDBE37D29BDD21BAB54BC70E422" ,
'oauth_version': "1.0",
'oauth_timestamp': time, #int(time.time()),
'oauth_nonce' : nonce,
'oauth_signature_method' : "HMAC-SHA1",
'oauth_signature' : signature
}
authorization = '5C82CC6BC7C6472154FBC9CAB24A29A2 ' + ', '.join([key + '="' + urllib.parse.quote_plus(str(value)) + '"' for key, value in headers.items()])
http_headers = {'Authorization': authorization}
print(authorization)
r = requests.get(url, headers=http_headers)
*ERROR: * Error 500

Twitter OAuth fails to validate my Python command line tool

I have spent hours in frustration and now I have this:
import requests, json, urllib
import time
import string, random
from hashlib import sha1
import hmac, binascii
def twitterIDGenerator(length):
toRet = ""
for i in range(0, length):
toRet = toRet + random.choice(string.hexdigits)
return toRet
def twitterSignatureGenerator(baseString, keyString):
hashed = hmac.new(keyString, baseString, sha1)
return binascii.b2a_base64(hashed.digest()).rstrip('\n')
OAUTH_CONSUMER_KEY = ''
OAUTH_NONCE = twitterIDGenerator(32)
OAUTH_SIGNATURE_METHOD = 'HMAC-SHA1'
OAUTH_TIMESTAMP = str(int(time.time()))
OAUTH_VERSION = '1.0'
# Get request token from Twitter
request_tokenURL = 'https://api.twitter.com/oauth/request_token'
request_tokenParameterString = ("oauth_consumer_key=" + OAUTH_CONSUMER_KEY +
"&oauth_nonce=" + OAUTH_NONCE + "&oauth_signature_method=" +
OAUTH_SIGNATURE_METHOD + "&oauth_timestamp=" + OAUTH_TIMESTAMP +
"&oauth_version=" + OAUTH_VERSION)
request_tokenSigBaseString = ("POST&https%3A%2F%2Fapi.twitter.com%2Foauth%2Frequest_token&" +
urllib.quote(request_tokenParameterString))
request_tokenSignature = twitterSignatureGenerator(request_tokenSigBaseString,
'[REDACTED consumer secret key]')
request_tokenHeaders = {'oauth_nonce': OAUTH_NONCE,
'oauth_callback': 'oob',
'oauth_signature_method': OAUTH_SIGNATURE_METHOD,
'oauth_timestamp': OAUTH_TIMESTAMP,
'oauth_consumer_key': OAUTH_CONSUMER_KEY,
'oauth_signature': urllib.quote(request_tokenSignature),
'oauth_version': OAUTH_VERSION}
request_tokenResponse = requests.post(request_tokenURL, headers=request_tokenHeaders)
print request_tokenResponse.text
So far, it is supposed to return a request_token so I can have my user go to the PIN website so I can get the access_token. But I just get "Failed to validate oauth signature and token" from Twitter.
A possible reason for this is wrong URL encoding. I see that Twitter needs RFC3986 encoding. Is there a way to do this in Python? If yes, should I do it only at the two locations I am currently using urllib.quote? Is my oauth_signature generated correctly?
The documentation is annoyingly convoluted.

Access Denied with Amazon CloudFront private distribution

I'm am trying to setup CloudFront for private content distribution but I keep getting Access Denied errors when I follow the generated URL. To be clear, I have already created the CloudFront distribution, marked it private, created an Origin Access ID which has been given read permission to all the relevant files.
I'v written a simple Python script to generate the URLs using the examples presented on the Amazon webpage for signing URLs and am including the text below:
import os, time
def GetCloudFrontURL(file, expires=86400):
resource = "http://mydistribution.cloudfront.net/" + file
exptime = int(time.time()) + expires
epochtime = str(exptime)
policy = '{"Statement":[{"Resource":"' + resource + '","Condition":{"DateLessThan":{"AWS:EpochTime":' + epochtime + '}}}]}'
pk = "MY-PK-GOES-HERE"
signature = os.popen("echo '" + policy + "' | openssl sha1 -sign /path/to/file/pk-" + pk + ".pem | openssl base64 | tr '+=/' '-_~'").read()
signature = signature.replace('\n','')
url = resource + "&Expires=" + epochtime + "&Signature=" + signature + "&Key-Pair-Id=" + pk
return url
Can anybody see anything obviously wrong with what I am doing? I've verified that when I sign the digest using the private key that I can verify it with the public key (provided I do the verification before feeding it through base64 and the translation step).
Thanks.

Running your method I get an ampersand before the first keywork, Expires.
>>> GetCloudFrontURL('test123')
http://mydistribution.cloudfront.net/test123&Expires=1297954193&Signature=&Key-Pair-Id=MY-PK-GOES-HERE
Don't know if it solves your whole problem but I suspect you need question mark in the URL to get the params to parse properly. Try something like this:
url = resource + "?Expires=" + epochtime + "&Signature=" + signature + "&Key-Pair-Id=" + pk
Aside, the urllib.urlencode method will convert a dictionary of params into a URL for you. http://docs.python.org/library/urllib.html#urllib.urlencode

Based on this, I was able to get it working with a few tweaks.
Also, see boto's set_all_permissions function that will set you S3 ACL's automatically for you.
from OpenSSL.crypto import *
import base64
import time
from django.conf import settings
ALT_CHARS = '-~'
def get_cloudfront_url(file, expires=86400):
resource = "https://" + settings.AWS_CLOUDFRONT_URL + "/" + file
exptime = int(time.time()) + expires
epochtime = str(exptime)
policy = '{"Statement":[{"Resource":"' + resource + '","Condition":{"DateLessThan":{"AWS:EpochTime":' + epochtime + '}}}]}'
f = open(settings.AWS_PRIVATE_KEY, 'r')
private_key = load_privatekey(FILETYPE_PEM, f.read())
f.close()
signature = base64.b64encode(sign(private_key, policy, 'sha1'), ALT_CHARS)
signature = signature.replace('=', '_')
url = resource + "?Expires=" + epochtime + "&Signature=" + signature + "&Key-Pair-Id=" + settings.AWS_CLOUDFRONT_KEY_PAIR_ID
return url

Here's how you can generate signed URLs without having to os.popen to openssl. This uses the excellent M2Crypto python library
This code is based loosely on the PHP example code provided by Amazon in the CloudFront documentation.
from M2Crypto import EVP
import base64
import time
def aws_url_base64_encode(msg):
msg_base64 = base64.b64encode(msg)
msg_base64 = msg_base64.replace('+', '-')
msg_base64 = msg_base64.replace('=', '_')
msg_base64 = msg_base64.replace('/', '~')
return msg_base64
def sign_string(message, priv_key_string):
key = EVP.load_key_string(priv_key_string)
key.reset_context(md='sha1')
key.sign_init()
key.sign_update(message)
signature = key.sign_final()
return signature
def create_url(url, encoded_signature, key_pair_id, expires):
signed_url = "%(url)s?Expires=%(expires)s&Signature=%(encoded_signature)s&Key-Pair-Id=%(key_pair_id)s" % {
'url':url,
'expires':expires,
'encoded_signature':encoded_signature,
'key_pair_id':key_pair_id,
}
return signed_url
def get_canned_policy_url(url, priv_key_string, key_pair_id, expires):
#we manually construct this policy string to ensure formatting matches signature
canned_policy = '{"Statement":[{"Resource":"%(url)s","Condition":{"DateLessThan":{"AWS:EpochTime":%(expires)s}}}]}' % {'url':url, 'expires':expires}
#now base64 encode it (must be URL safe)
encoded_policy = aws_url_base64_encode(canned_policy)
#sign the non-encoded policy
signature = sign_string(canned_policy, priv_key_string)
#now base64 encode the signature (URL safe as well)
encoded_signature = aws_url_base64_encode(signature)
#combine these into a full url
signed_url = create_url(url, encoded_signature, key_pair_id, expires);
return signed_url
def encode_query_param(resource):
enc = resource
enc = enc.replace('?', '%3F')
enc = enc.replace('=', '%3D')
enc = enc.replace('&', '%26')
return enc
#Set parameters for URL
key_pair_id = "APKAIAZVIO4BQ" #from the AWS accounts CloudFront tab
priv_key_file = "cloudfront-pk.pem" #your private keypair file
# Use the FULL URL for non-streaming:
resource = "http://34254534.cloudfront.net/video.mp4"
#resource = 'video.mp4' #your resource (just object name for streaming videos)
expires = int(time.time()) + 300 #5 min
#Create the signed URL
priv_key_string = open(priv_key_file).read()
signed_url = get_canned_policy_url(resource, priv_key_string, key_pair_id, expires)
print(signed_url)
#Flash player doesn't like query params so encode them if you're using a streaming distribution
#enc_url = encode_query_param(signed_url)
#print(enc_url)
Make sure that you set up your distribution with a TrustedSigners parameter set to the account holding your keypair (or "Self" if it's your own account)
See Getting started with secure AWS CloudFront streaming with Python for a fully worked example on setting this up for streaming with Python