I have written below custom collector which pulls the data from a rest api and adds page view metrics. Each payload has 5 metrics so I am adding timestamp to it. It successfully publishes it to the http listener but the metrics never expires. How can I add the expiry to these metrics?
#!/usr/bin/env python3
import argparse
import re
import sys
import time
import datetime
import urllib3
import requests
import aniso8601
import pytz
import json
from prometheus_client import start_http_server
from prometheus_client.core import GaugeMetricFamily, REGISTRY
class HttpCollector(object):
def __init__(self):
self.url = "my_endpoint"
self.username = 'my_userid'
self.password = 'my_pass'
self.labels = ['app_name', 'url_host' ]
self.page_views_metrics = GaugeMetricFamily('pageviews', 'Page Views', labels=self.labels)
def collect(self):
headers = {'Accept': '*/*', }
auth = (self.username, self.password)
urllib3.disable_warnings()
result = requests.get(self.url, headers=headers, auth=auth, timeout=60, verify=False)
if not result.ok:
# Log error
print("No results")
return
json_result = result.json()
for record in json_result['records']:
timestamp_epoch = covert_date_to_epoch(record["timestamp'])
label_values = ["testapp", "testhost"]
self.page_views_metrics.add_metric(label_values, record["page_views"], timestamp=timestamp_epoch)
yield self.page_views_metrics
Making the self.page_views_metrics as local variable to collect method solved the problem.
import re
import sys
import time
import datetime
import urllib3
import requests
import aniso8601
import pytz
import json
from prometheus_client import start_http_server
from prometheus_client.core import GaugeMetricFamily, REGISTRY
class HttpCollector(object):
def __init__(self):
self.url = "my_endpoint"
self.username = 'my_userid'
self.password = 'my_pass'
self.labels = ['app_name', 'url_host' ]
def collect(self):
headers = {'Accept': '*/*', }
auth = (self.username, self.password)
urllib3.disable_warnings()
result = requests.get(self.url, headers=headers, auth=auth, timeout=60, verify=False)
if not result.ok:
# Log error
print("No results")
return
json_result = result.json()
page_views_metrics = GaugeMetricFamily('pageviews', 'Page Views', labels=self.labels)
for record in json_result['records']:
timestamp_epoch = covert_date_to_epoch(record["timestamp'])
label_values = ["testapp", "testhost"]
page_views_metrics.add_metric(label_values, record["page_views"], timestamp=timestamp_epoch)
yield page_views_metrics
Related
I am having trouble with the code in that every time I go to run it, it receive an error as follows:
File "/Users/richard/Desktop/Python/gdaxauth.py", line 27
request.headers.update({ ^
SyntaxError: invalid syntax
So it seems to relate to something in the request.headers.update section. I have tried researching this but have hit a total brick wall. Can anyone advise as to what the correct syntax should be? I am using Python 3.7.
import json
import hmac
import hashlib
import time
import requests
import base64
import urllib
from requests.auth import AuthBase
# Tracking execution time
start = time.time()
# Create custom authentication for Exchange
class CoinbaseExchangeAuth(AuthBase):
def __init__(self, api_key, secret_key, passphrase):
self.api_key = api_key
self.secret_key = secret_key
self.passphrase = passphrase
def __call__(self, request):
timestamp = str(time.time())
message = timestamp + request.method + request.path_url + (request.body or '')
hmac_key = base64.b64decode(self.secret_key)
signature = hmac.new(hmac_key, message.encode('utf-8'), hashlib.sha256)
signature_b64 = base64.b64encode(signature.digest().rstrip('\n')
request.headers.update({
'CB-ACCESS-SIGN': signature_b64,
'CB-ACCESS-TIMESTAMP': timestamp,
'CB-ACCESS-KEY': self.api_key,
'CB-ACCESS-PASSPHRASE': self.passphrase,
'Content-type': 'application/json'
})
return request
# Credentials - ADD YOUR API KEY CREDS IN THIS SECTION
API_KEY = "xxxxxxxxx"
SECRET_KEY = "xxxxxxxxxx"
API_PASSPHRASE = "xxxxxxxxxx"
# Get accounts
api_url = 'https://api.gdax.com/' #'https://api.pro.coinbase.com'
auth = CoinbaseExchangeAuth(API_KEY,SECRET_KEY,API_PASSPHRASE)
r = requests.get(api_url + 'accounts', auth=auth)
# Output account data and code execution time
print(json.dumps(r.json(),indent=4))
I have 2 models. First is House. Second is HouseImage
Therefore I have to submit the image with ForeigneKey
I can upload by using the REST normally, but failed to do unittest.
The reason why I keep doing unittest in here because I have a more specifications waiting for me and I would not do hand-test for sure.
django==1.11.5
djangorestframework==3.6.4
python3.6.2
PostgreSQL 9.6.5 on x86_64-apple-darwin14.5.0, compiled by Apple LLVM version 7.0.0 (clang-700.1.76), 64-bit
Here is my additional source code.
https://gist.github.com/elcolie/a013be9c3b7ab5f0cc130e320b19da4b
import tempfile
from PIL import Image
from django.contrib.auth.models import User
from model_mommy import mommy
from rest_framework import status
from rest_framework.reverse import reverse
from rest_framework.test import APITestCase, APIClient
from soken_web.apps.houses.models import House
class HouseImageTest(APITestCase):
def setUp(self):
self.client = APIClient()
self.user = mommy.make(User, username='Pan')
self.house = mommy.make(House, location="100.00, 100.00")
def test_post_image(self):
self.client.force_authenticate(user=self.user)
image = Image.new('RGB', (100, 100))
tmp_file = tempfile.NamedTemporaryFile(suffix='.jpg')
image.save(tmp_file)
data = {
'image': tmp_file,
'house': self.house.id,
}
response = self.client.post(reverse('api:house_images-list'), data, format='multipart')
self.assertEqual(status.HTTP_201_CREATED, response.status_code)
Problem:
Server raises appliation/json type to me
Attempts:
1. Replace format=multipart with content_type/multipart. Same error
1. User both format=mulipart and content_type/multipart. It is not permitted by DRF
Solution:
#zaidfazil Thank you very much. You are right. I have to use real-file
import tempfile
from django.conf import settings
from django.contrib.auth.models import User
from django.core.files import File
from django.core.files.uploadedfile import SimpleUploadedFile
from model_mommy import mommy
from rest_framework import status
from rest_framework.reverse import reverse
from rest_framework.test import APITestCase, APIClient
from soken_web.apps.houses.models import House
class HouseImageTest(APITestCase):
def setUp(self):
self.client = APIClient()
self.user = mommy.make(User, username='Pan')
self.house = mommy.make(House, location="100.00, 100.00")
settings.MEDIA_ROOT = tempfile.mkdtemp()
def test_post_image(self):
file = File(open('static/rest_framework/img/grid.png', 'rb'))
uploaded_file = SimpleUploadedFile('new_image.jpg', file.read(), content_type='multipart/form-data')
data = {
'image': uploaded_file,
'houses': self.house.id,
}
self.client.force_authenticate(user=self.user)
response = self.client.post(reverse('api:house_images-list'), data, format='multipart')
response.render()
self.assertEqual(status.HTTP_201_CREATED, response.status_code)
References:
How can I test binary file uploading with django-rest-framework's test client?
http://www.django-rest-framework.org/api-guide/testing/
You may need to convert the file into uploaded file format before posting to the url,
from django.core.files.uploadedfile import SimpleUploadedFile
from django.core.files import File
class HouseImageTest(APITestCase):
def setUp(self):
self.client = APIClient()
self.user = mommy.make(User, username='Pan')
self.house = mommy.make(House, location="100.00, 100.00")
settings.MEDIA_ROOT = tempfile.mkdtemp()
def test_post_image(self):
image = Image.new('RGB', (100, 100))
tmp_file = tempfile.NamedTemporaryFile(suffix='.jpg')
image.save(tmp_file)
file = File(tmp_file)
uploaded_file = SimpleUploadedFile('new_image.jpg', file.read(), content_type='multipart/form-data')
data = {
'image': uploaded_file,
'houses': self.house.id,
}
self.client.force_authenticate(user=self.user)
response = self.client.post(reverse('api:house_images-list'), data, format='multipart')
response.render()
self.assertEqual(status.HTTP_201_CREATED, response.status_code)
I am trying to make a basic authenticated api call to their new v2 api and getting an invalid api key error returned.
I reissued the api key just to verify, same error.
from time import time
import urllib.request
import urllib.parse
import hashlib
import hmac
APIkey = b'myapikeyyouarenotsupposedtosee'
secret = b'myceeeeecretkeyyyy'
url = 'https://api.bitfinex.com/v2/auth/r/wallets'
payload = {
#'request':'/auth/r/wallets',
'nonce': int(time() * 1000),
}
paybytes = urllib.parse.urlencode(payload).encode('utf8')
print(paybytes)
sign = hmac.new(secret, paybytes, hashlib.sha512).hexdigest()
print(sign)
headers = {
'Key': APIkey,
'Sign': sign
}
req = urllib.request.Request(url, headers=headers, data=paybytes)
with urllib.request.urlopen(req) as response:
the_page = response.read()
print(the_page)
How do I make an authenticated api call to the new v2 API for bitfinex?
Your headers are wrong. I was also trying to do this and tried using the example code from the bitfinex v2 api docs, however their example contained a bug in that they needed to encode the strings into UTF-8 byte arrays first. So I've fixed it and posting the entire example below.
#
# Example Bitfinex API v2 Auth Python Code
#
import requests # pip install requests
import json
import base64
import hashlib
import hmac
import os
import time #for nonce
class BitfinexClient(object):
BASE_URL = "https://api.bitfinex.com/"
KEY = "API_KEY_HERE"
SECRET = "API_SECRET_HERE"
def _nonce(self):
# Returns a nonce
# Used in authentication
return str(int(round(time.time() * 10000)))
def _headers(self, path, nonce, body):
secbytes = self.SECRET.encode(encoding='UTF-8')
signature = "/api/" + path + nonce + body
sigbytes = signature.encode(encoding='UTF-8')
h = hmac.new(secbytes, sigbytes, hashlib.sha384)
hexstring = h.hexdigest()
return {
"bfx-nonce": nonce,
"bfx-apikey": self.KEY,
"bfx-signature": hexstring,
"content-type": "application/json"
}
def req(self, path, params = {}):
nonce = self._nonce()
body = params
rawBody = json.dumps(body)
headers = self._headers(path, nonce, rawBody)
url = self.BASE_URL + path
resp = requests.post(url, headers=headers, data=rawBody, verify=True)
return resp
def active_orders(self):
# Fetch active orders
response = self.req("v2/auth/r/orders")
if response.status_code == 200:
return response.json()
else:
print('error, status_code = ', response.status_code)
return ''
# fetch all your orders and print out
client = BitfinexClient()
result = client.active_orders()
print(result)
Why not use one of the open source api clients out there ? and you can compare to your work .
https://github.com/scottjbarr/bitfinex
https://github.com/tuberculo/bitfinex
I know this has been asked before .. but my code doesn't seem to work for users other than the one using whom accesstoken is created
.. i.e I am able to access the profile of only one user.(using whom I created accesstoken)
from urllib2 import urlopen
from json import load
import json
import json
import sys
import time
import requests
userid="userid_of_some_instagram_user"
access_token="mytoken"
url = "https://api.instagram.com/v1/users/%s/?access_token=%s"%(userid,access_token)
response=urlopen(url)
print(response)
jsonobj=load(response)
print "name:",jsonobj["data"]["username"]
print "follows ",jsonobj["data"]["counts"]["follows"]," people"
print "followed by ",jsonobj["data"]["counts"]["followed_by"]," people"
Error I am getting when I am trying to access other profiles is
HTTPError: HTTP Error 400: BAD REQUEST
Edit : **Voila ! I have found the bug . It is the access key. I have tried with a different access key . It worked**
If you don't wanna fight the new InstagramAPI you can use a workaround like this:
import requests
import json
import time
import random
user = 'google'
url = 'https://www.instagram.com/'
url_user = '%s%s%s' % (url, user, '/')
url_login = 'https://www.instagram.com/accounts/login/ajax/'
s = requests.Session()
s.cookies.update ({'sessionid' : '', 'mid' : '', 'ig_pr' : '1',
'ig_vw' : '1920', 'csrftoken' : '',
's_network' : '', 'ds_user_id' : ''})
login_post = {'username' : 'your_login',
'password' : 'your_pw'}
s.headers.update ()
r = s.get(url)
s.headers.update({'X-CSRFToken' : r.cookies['csrftoken']})
time.sleep(5 * random.random())
login = s.post(url_login, data=login_post,
allow_redirects=True)
s.headers.update({'X-CSRFToken' : login.cookies['csrftoken']})
if login.status_code == 200:
r = s.get('https://www.instagram.com/')
finder = r.text.find('your_login')
r = s.get(url_user)
text = r.text
finder_text_start = ('<script type="text/javascript">'
'window._sharedData = ')
finder_text_start_len = len(finder_text_start)-1
finder_text_end = ';</script>'
all_data_start = text.find(finder_text_start)
all_data_end = text.find(finder_text_end, all_data_start + 1)
json_str = text[(all_data_start + finder_text_start_len + 1) \
: all_data_end]
user_info = json.loads(json_str)
follower_count = user_info['entry_data']['ProfilePage'][0]['user']["followed_by"]['count']
print(follower_count)
Try this:
import json
import requests
access_token = "mytoken"
url = "https://api.instagram.com/v1/users/self/?access_token={0}".format(access_token)
response = requests.get(url)
j = json.loads(response.text)
if 'data' in j:
print("handle correct response")
else:
print("handle incorrect response")
I have a piece of code which was written in Python 3.5 and uses urllib module. Now, I tried to convert this so that it will work with Python 2.7, but I get some errors from the urllib() module.
E.g:
Traceback (most recent call last):
File "alert.py", line 13, in <module>
import urllib.request as urllib
ImportError: No module named request
Now, I know that urllib is deprecated in Python 2.7 so I'm coming here to ask for some help with the lines that use urllib.
import urllib.request as urllib
from http.cookiejar import CookieJar
from os.path import isfile
from os.path import join as joinPath
from sys import exc_info
from traceback import print_tb
from urllib.parse import urlencode
# constant
APPLICATION_PATH = '/srv/path/'
ALERT_POINT_PATH = joinPath(APPLICATION_PATH, 'alert_contact')
URL_REQUEST_TIMEOUT = 42
SMS_BOX_URL = 'xx.xxxx.xxx.xxx'
def initWebConnection(): # init web connection
response = 0
initUrlLibResponse = initUrlLib() # init urllib
if initUrlLibResponse:
response = 1
return response
def initUrlLib(): # init urllib
response = 0
try:
cookieJar = CookieJar() # cookies
opener = urllib.build_opener(urllib.HTTPCookieProcessor(cookieJar))
urllib.install_opener(opener)
except Exception as e:
response = 1
# ex_type, ex, tb = exc_info()
return response
def urlRequest(url, data=None): # make url request
contentResponse = None
try:
request = None
if data:
dataRequest = urlencode(data)
dataRequest = dataRequest.encode('UTF-8')
request = urllib.Request(url, dataRequest)
else:
request = urllib.Request(url)
response = urllib.urlopen(url=request, timeout=URL_REQUEST_TIMEOUT) # make request
# get response
contentResponse = response.read()
except Exception as e:
contentResponse = None
# ex_type, ex, tb = exc_info()
return contentResponse
try:
evt.data = 'Some name'
# check production state
isInProduction = False
if evt.prodState == 1000:
isInProduction = True
if isInProduction:
initWebConnection()
# check alert point'
if isfile(ALERT_POINT_PATH):
alertContactContent = None
with open(ALERT_POINT_PATH, 'r') as alertContactFile:
alertContactContent = alertContactFile.read()
alertContactContent = alertContactContent.splitlines()
if alertContactContent:
evt.summary = '#[ DNS: ALERT ]# {}'.format(evt.summary)
for alertContactContentLine in alertContactContent:
webRequestData = dict(
## TO DO: set the url parameters appropriately
phone=alertContactContentLine,
message='NEW ALERT: {}'.format(evt.ipAddress),
)
webRequestResponse = urlRequest(SMS_BOX_URL, webRequestData)
else:
evt.summary = '#[ ERROR: SMS ALERT NO CONTACT ]# {}'.format(evt.summary)
except Exception as e:
ex_type, ex, tb = exc_info()
print('\n #[ERROR]#exception: {ex}\n'.format(ex=e))
print('\n #[ERROR]#exception traceback: {trace}\n'.format(trace=print_tb(tb)))
evt.summary = '#[ DNS:ERROR traceback in event message ]# {}'.format(evt.summary)
evt.message = '#[ DNS:ERROR ex_type:\n {} \nex: {} \n traceback:\n {} \n]# {}'.format(ex_type, ex,
print_tb(tb),
evt.message)
You can change the import lines from
import urllib.request as urllib
from http.cookiejar import CookieJar
from urllib.parse import urlencode
to
import urllib2 as urllib
from cookielib import CookieJar
from urllib import urlencode
for Python 2.7