I need to validate services and their dependencies using around 500+ URL's and I already have a python code that does it.
But the problem is that some of the URL's take a minute each to give a response [due to some known dependencies being down].
As each URL is hosted on a different server, is there a way to access multiple URL's at once using the requests module?
Below is my entire code I use in pycharm:
import requests
import json
import pandas
import datetime
from requests.auth import HTTPBasicAuth
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
def validate():
line = "---------------------------------------------------------------------------------------------------"
print("Validation started for:", datetime.datetime.now().strftime("%d-%B-%Y at %H:%M:%S"), "\n" + line)
username = 'username'
password = 'password'
mydata = pandas.read_excel(r'C:\mydata.xlsx', sheet_name='Sheet1')
for i in mydata.index:
srno = str(mydata['Sr No'][i])
service = mydata['Service Name'][i]
machine = mydata['Machine Name'][i]
url = mydata['Node'][i]
alwaysdownservice = ['service1', 'service2']
paydown = ['dependency1', 'dependency2', 'dependency3']
otherdown = ['dependency3']
def get():
response = requests.get(url, verify=False, auth=HTTPBasicAuth(username, password))
data = json.loads(response.text)
status = data['Success']
if not status:
response = requests.get(url, verify=False, auth=HTTPBasicAuth(username, password))
data = json.loads(response.text)
status = data['Success']
if not status:
for j in list(data['Dependencies']):
dependency = j['DependencyName']
d_status = j['Success']
if not d_status:
if service in alwaysdownservice:
if dependency not in paydown:
print(Dependency, "down on", machine, "for", service.)
else:
if dependency not in otherdown:
print(Dependency, "down on", machine, "for", service.)
try:
get()
except Exception as e:
print(line, "\n", e, "\n", srno, "| Below URL is not accessible: \n", url, "\n" + line)
validate()
You can use threads (using the threading library in Python) to call multiple URL's at once. To do that you can use the following code:
import requests
import json
import pandas
import datetime
from requests.auth import HTTPBasicAuth
from requests.packages.urllib3.exceptions import InsecureRequestWarning
import threading
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
threads = []
def validate():
line = "---------------------------------------------------------------------------------------------------"
print("Validation started for:", datetime.datetime.now().strftime("%d-%B-%Y at %H:%M:%S"), "\n" + line)
username = 'username'
password = 'password'
mydata = pandas.read_excel(r'C:\mydata.xlsx', sheet_name='Sheet1')
for i in mydata.index:
srno = str(mydata['Sr No'][i])
service = mydata['Service Name'][i]
machine = mydata['Machine Name'][i]
url = mydata['Node'][i]
alwaysdownservice = ['service1', 'service2']
paydown = ['dependency1', 'dependency2', 'dependency3']
otherdown = ['dependency3']
def get():
response = requests.get(url, verify=False, auth=HTTPBasicAuth(username, password))
data = json.loads(response.text)
status = data['Success']
if not status:
response = requests.get(url, verify=False, auth=HTTPBasicAuth(username, password))
data = json.loads(response.text)
status = data['Success']
if not status:
for j in list(data['Dependencies']):
dependency = j['DependencyName']
d_status = j['Success']
if not d_status:
if service in alwaysdownservice:
if dependency not in paydown:
print(Dependency, "down on", machine, "for", service)
else:
if dependency not in otherdown:
print(Dependency, "down on", machine, "for", service)
try:
t = threading.Thread(target=get) # Using threading over here
t.start()
threads.append(t)
except Exception as e:
print(line, "\n", e, "\n", srno, "| Below URL is not accessible: \n", url, "\n" + line)
validate()
for thread in threads:
thread.join()
For people who need the solution. I found this from #Yurii Kramarenko. Which worked perfectly and now my script finishes its run in 30 seconds instead of 10-11 minutes.
My Script -
def validate():
alwaysdownservice = ['service1', 'service2']
paydown = ['dependency1', 'dependency2', 'dependency3']
otherdown = ['dependency3']
username = 'username'
password = 'password'
mydata = pandas.read_excel(r'C:\mydata.xlsx', sheet_name='Sheet1')
urls = mydata['urls']
line = "---------------------------------------------------------------------------------------------------"
print("Validation started for:", datetime.datetime.now().strftime("%d-%B-%Y at %H:%M:%S"), "\n" + line)
async def fetch(session, url):
async with session.get(url, auth=aiohttp.BasicAuth(username, password), ssl=False) as response:
data = await response.text()
data = json.loads(data)
status = data['Success']
if not status:
for j in list(data['Dependencies']):
dependency = j['DependencyName']
d_status = j['Success']
if not d_status:
if service in alwaysdownservice:
if dependency not in paydown:
print("Dependency -",
"\'" + dependency + "\'", "down on", "\nURL -", url, "\n" + line)
else:
if dependency not in otherdown:
("Dependency -",
"\'" + dependency + "\'", "down on", "\nURL -", url, "\n" + line)
print(url, "validated at:", datetime.datetime.now().strftime("%H:%M:%S"))
async def fetch_all(urls, loop):
async with aiohttp.ClientSession(loop=loop) as session:
results = await asyncio.gather(*[fetch(session, url) for url in urls], return_exceptions=True)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
htmls = loop.run_until_complete(fetch_all(urls, loop))
print("Validation completed for:",
datetime.datetime.now().strftime("%d-%B-%Y at %H:%M:%S"), "\n" + line, "\n" + line,)
validate()
Related
The below is a result of this question How to sign an OKEx API request? and some of the answers:
import hmac
import base64
import requests
import datetime
import json
from config import KEY, SECRET, PASS, ROOT_URL
def get_time():
now = datetime.datetime.utcnow()
t = now.isoformat("T", "milliseconds")
return t + "Z"
def signature(timestamp, request_type, endpoint, body, secret):
if body != '':
body = json.dumps(body)
message = str(timestamp) + str.upper(request_type) + endpoint + body
print(message)
mac = hmac.new(bytes(secret, encoding='utf-8'), bytes(message, encoding='utf-8'), digestmod='sha256')
d = mac.digest()
return base64.b64encode(d)
def get_header(request_type, endpoint, body):
time = get_time()
header = dict()
header['CONTENT-TYPE'] = 'application/json'
header['OK-ACCESS-KEY'] = KEY
header['OK-ACCESS-SIGN'] = signature(time, request_type, endpoint, body, SECRET)
header['OK-ACCESS-TIMESTAMP'] = str(time)
header['OK-ACCESS-PASSPHRASE'] = PASS
return header
def get(endpoint, body=''):
url = ROOT_URL + endpoint
header = get_header('GET', endpoint, body)
return requests.get(url, headers=header)
def post(endpoint, body=''):
url = ROOT_URL + endpoint
header = get_header('POST', endpoint, body)
return requests.post(url, headers=header)
where KEY, SECRET, PASS are the API key, secret key, and pass phrase respectively; The ROOT_URL is 'https://www.okex.com'.
The Problem
GET requests work absolutely fine, so when I run the following, there are no issues:
ENDPOINT = '/api/v5/account/balance'
BODY = ''
response = get(ENDPOINT)
response.json()
However, when I try to place an order via a POST request, like so:
ENDPOINT = '/api/v5/trade/order'
BODY = {"instId":"BTC-USDT",
"tdMode":"cash",
"side":"buy",
"ordType":"market",
"sz":"1"}
response = post(ENDPOINT, body=BODY)
response.json()
I get the following output, i.e. it won't accept the signature:
{'msg': 'Invalid Sign', 'code': '50113'}
Related Questions
In this one Can't figure out how to send a signed POST request to OKEx an answer was provided, but it does not work for me as I was already using the suggested URL. More or less the same question was asked here Unable to send a post requests OKEX Invalid Signature, but no activity likely due to the format, so I thought I would repost and elaborate.
OKEX Docs
The docs simply specify that The API endpoints of Trade require authentication (https://www.okex.com/docs-v5/en/?python#rest-api-authentication-signature). But they make no reference to there being any difference between the two methods. Away from that, I am including all required parameters in the body of the post request as far as I can see.
I would appreciate any input on this.
Many thanks!
I ran into the same POST problem and figured it out. I used new domain name okex.com. Here is my code.
def set_userinfo(self):
position_path = "/api/v5/account/set-position-mode"
try:
self.get_header("POST", position_path, {"posMode":"net_mode"})
resp = requests.post(url=self.base_url+position_path, headers=self.headers, json={"posMode":"long_short_mode"}).json()
except Exception as e:
log.error("OK set_userinfo error={} type={}".format(f'{e}', f'{type(e)}'))
def get_header(self, request_type, endpoint, body=''):
timestamp = self.get_time()
self.headers["OK-ACCESS-TIMESTAMP"] = timestamp
self.headers["OK-ACCESS-SIGN"] = self.signature(timestamp, request_type, endpoint, body)
def signature(self, timestamp, request_type, endpoint, body):
if body != '':
body = json.dumps(body)
message = str(timestamp) + str.upper(request_type) + endpoint + body
mac = hmac.new(bytes(self.secret_key, encoding='utf-8'), bytes(message, encoding='utf-8'), digestmod='sha256').digest()
return base64.b64encode(mac)
I have fix the same problem.
Both of the 'body' in signature() and in get_header() should be json.
So you should add following code:
if str(body) == '{}' or str(body) == 'None':
body = ''
else:
body = json.dumps(body)
I ran into the same problem and solved it using below code snippet, the idea is from https://stackoverflow.com/a/68115787/20497127, but I modified a little by adding POST functionality
APIKEY = "" # input key
APISECRET = "" #input secret
PASS = "" #input passphrase
BASE_URL = 'https://www.okx.com'
def send_signed_request(http_method, url_path, payload={}):
def get_time():
return dt.datetime.utcnow().isoformat()[:-3]+'Z'
def signature(timestamp, method, request_path, body, secret_key):
if str(body) == '{}' or str(body) == 'None':
body = ''
message = str(timestamp) + str.upper(method) + request_path + str(body)
mac = hmac.new(bytes(secret_key, encoding='utf8'), bytes(message, encoding='utf-8'), digestmod='sha256')
d = mac.digest()
return base64.b64encode(d)
# set request header
def get_header(request='GET', endpoint='', body:dict=dict()):
cur_time = get_time()
header = dict()
header['CONTENT-TYPE'] = 'application/json'
header['OK-ACCESS-KEY'] = APIKEY
header['OK-ACCESS-SIGN'] = signature(cur_time, request, endpoint , body, APISECRET)
header['OK-ACCESS-TIMESTAMP'] = str(cur_time)
header['OK-ACCESS-PASSPHRASE'] = PASS
# demo trading: need to set x-simulated-trading=1, live trading is 0
header['x-simulated-trading'] = '1'
return header
url = BASE_URL + url_path
header = get_header(http_method, url_path, payload)
print(url)
print(header)
if http_method == 'GET':
response = requests.get(url, headers=header)
elif http_method == 'POST':
response = requests.post(url, headers=header, data=payload)
return response.json()
# this will run get requests
res = send_signed_request("GET", "/api/v5/account/balance", payload={})
# this will run post requests
data = {
"instId": "BTC-USDT",
"tdMode": "cross",
"side": "sell",
"ccy":"USDT",
"ordType": "limit",
"px": "100000",
"sz": "0.01"
}
res = send_signed_request("POST", "/api/v5/trade/order", payload=json.dumps(data))
I am able to use haveibeenpwned to search for 1 account compromise. However, I could not find an option to use the API key to search for compromise of all the email accounts on a domain. (For example. if the domain is xyz.com, I want to search for the compromise of abc#xyz.com, peter.charlie#xyz.com and so on). I am aware of the notification email that I can sign up for. But, that is a lengthy process and I prefer using the API.
So, I wrote a script to search against haveibeenpwned for all the email address of my domain, but it takes very long. I searched through a couple of Github projects, but I did not find any such implementation. Has anyone tried this before?
I have added the code below. I am using Multi threading approach, but still it takes very long, is there any other Optimization strategy I can use? Please help. Thank you.
import requests, json
import threading
from time import sleep
import datetime
import splunklib.client as client
import splunklib.results as results
date = datetime.datetime.now()
from itertools import islice
import linecache
import sys
def PrintException():
exc_type, exc_obj, tb = sys.exc_info()
f = tb.tb_frame
lineno = tb.tb_lineno
filename = f.f_code.co_filename
linecache.checkcache(filename)
line = linecache.getline(filename, lineno, f.f_globals)
print 'EXCEPTION IN ({}, LINE {} "{}"): {}'.format(filename, lineno, line.strip(), exc_obj)
class myThread (threading.Thread):
def __init__(self, threadID, name, list_emails):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
self.list_emails = list_emails
def run(self):
i=0
print "Starting " + self.name
for email in self.list_emails:
print i
i=i+1
result = check_pasteaccount(email)
print email
print result
print result
print "Exiting " + self.name
def check_pasteaccount(account):
account = str(account)
result = ""
URL = "https://haveibeenpwned.com/api/v3/pasteaccount/%s?truncateResponse=false" % (account)
# print(URL)
headers= {'hibp-api-key':api_key}
result = ""
try:
r = requests.get(url=URL,headers=headers)
# sleep(2)
status_code = r.status_code
if status_code == 200:
data = r.text
result = []
for entry in json.loads(data.decode('utf8')):
if int((date - datetime.datetime.strptime(entry['Date'], '%Y-%m-%dT%H:%M:%SZ')).days) > 120:
pass
else:
result.append(['Title: {0}'.format(entry['Title']), \
'Source: {0}'.format(['Source']), \
'Paste ID: {0}'.format(entry['Id'])])
if len(result) == 0:
result = "No paste reported for given account and time frame."
else:
paste_result = ""
for entry in result:
for item in entry:
paste_result += str(item) + "\r\n"
paste_result += "\r\n"
result = paste_result
elif status_code == 404:
result = "No paste for the account"
else:
if status_code == 429:
sleep(5)
# print "Limit exceeded, sleeping"
result = check_pasteaccount(account)
else:
result = "Exception"
print status_code
except Exception as e:
result = "Exception"
PrintException()
pass
return result
def split_every(n, iterable):
iterable = iter(iterable)
for chunk in iter(lambda: list(islice(iterable, n)), []):
yield chunk
def main():
print datetime.datetime.now()
# Fetching the list of email addresses from Splunk
list_emails = connect_splunk()
print datetime.datetime.now()
i=0
list_split = split_every(1000,list_emails)
threads=[]
for list in list_split:
i=i+1
thread_name = "Thread" + str(i)
thread = myThread(1, thread_name, list)
thread.start()
threads.append(thread)
# Wait for all the threads to complete
for t in threads:
t.join()
print "Completed Search"
Here's a shorter and maybe more efficient version of your script using the standard multiprocessing library instead of a hand-rolled thread system.
You'll need Python 3.6+ since we're using f-strings.
You'll need to install the tqdm module for fancy progress bars.
You can adjust the number of concurrent requests with the pool size parameter.
Output is written in machine-readable JSON Lines format into a timestamped file.
A single requests session is shared (per-worker), which means less time spent connecting to HIBP.
import datetime
import json
import multiprocessing
import random
import time
import requests
import tqdm
HIBP_PARAMS = {
"truncateResponse": "false",
}
HIBP_HEADERS = {
"hibp-api-key": "xxx",
}
sess = requests.Session()
def check_pasteaccount(account):
while True:
resp = sess.get(
url=f"https://haveibeenpwned.com/api/v3/pasteaccount/{account}",
params=HIBP_PARAMS,
headers=HIBP_HEADERS,
)
if resp.status_code == 429:
print("Quota exceeded, waiting for a while")
time.sleep(random.uniform(3, 7))
continue
if resp.status_code >= 400:
return {
"account": account,
"status": resp.status_code,
"result": resp.text,
}
return {
"account": account,
"status": resp.status_code,
"result": resp.json(),
}
def connect_splunk():
# TODO: return emails
return []
def main():
list_emails = [str(account) for account in connect_splunk()]
datestamp = datetime.datetime.now().isoformat().replace(":", "-")
output_filename = f"accounts-log-{datestamp}.jsonl"
print(f"Accounts to look up: {len(list_emails)}")
print(f"Output filename: {output_filename}")
with multiprocessing.Pool(processes=16) as p:
with open(output_filename, "a") as f:
results_iterable = p.imap_unordered(
check_pasteaccount, list_emails, chunksize=20
)
for result in tqdm.tqdm(
results_iterable,
total=len(list_emails),
unit="acc",
unit_scale=True,
):
print(json.dumps(result, sort_keys=True), file=f)
if __name__ == "__main__":
main()
I'm using an API for ENDPOINT: https://brazilsouth.api.cognitive.microsoft.com/sts/v1.0/issuetoken
and I trying to implement a tutorial present in this site: https://learn.microsoft.com/en-us/azure/cognitive-services/speech-service/rest-text-to-speech
but I don't able to make it work. When I get a listVoices don't show a pt-BR-FranciscaNeural but in the docs it says that the voice is available.
import requests
import time
from xml.etree import ElementTree
try:
input = input
except NameError:
pass
class TextToSpeech(object):
def __init__(self, subscription_key):
self.subscription_key = subscription_key
self.tts = input("What would you like to convert to speech: ")
self.timestr = time.strftime("%Y%m%d-%H%M")
self.access_token = None
def get_token(self):
fetch_token_url = "https://brazilsouth.api.cognitive.microsoft.com/sts/v1.0/issueToken"
headers = {
'Ocp-Apim-Subscription-Key': self.subscription_key
}
response = requests.post(fetch_token_url, headers=headers)
self.access_token = str(response.text)
def save_audio(self):
base_url = 'https://brazilsouth.tts.speech.microsoft.com/'
path = 'cognitiveservices/v1'
constructed_url = base_url + path
headers = {
'Authorization': 'Bearer ' + self.access_token,
'Content-Type': 'application/ssml+xml',
'X-Microsoft-OutputFormat': 'riff-24khz-16bit-mono-pcm',
'User-Agent': 'YOUR_RESOURCE_NAME'
}
xml_body = ElementTree.Element('speak', version='1.0')
xml_body.set('{http://www.w3.org/XML/1998/namespace}lang', 'pt-br')
voice = ElementTree.SubElement(xml_body, 'voice')
voice.set('{http://www.w3.org/XML/1998/namespace}lang', 'pt-BR')
voice.set(
'name', 'Microsoft Server Speech Text to Speech Voice (pt-BR, FranciscaNeural)')
voice.text = self.tts
body = ElementTree.tostring(xml_body)
response = requests.post(constructed_url, headers=headers, data=body)
if response.status_code == 200:
with open('sample-' + self.timestr + '.wav', 'wb') as audio:
audio.write(response.content)
print("\nStatus code: " + str(response.status_code) +
"\nYour TTS is ready for playback.\n")
else:
print("\nStatus code: " + str(response.status_code) +
"\nSomething went wrong. Check your subscription key and headers.\n")
if __name__ == "__main__":
subscription_key = "put-here-a-keycode"
app = TextToSpeech(subscription_key)
app.get_token()
app.save_audio()
I solved the problem, I changed the location of the server, because I found that what I used did not meet the neural language.
I managed to use Francisca's voice using another server:
fetch_token_url="https://eastus.api.cognitive.microsoft.com/sts/v1.0/issueToken"
it's working perfectly
Pls refer to this thread :
https://github.com/MicrosoftDocs/azure-docs/issues/52032
As per thread, there is a known issue that is being worked actively and pt-BR-FranciscaNeural has been removed intentionally until fix.
I'm new to python and I want this code to run only once and stops, not every 30 seconds
because I want to run multiple codes like this with different access tokens every 5 seconds using the command line.
and when I tried this code it never jumps to the second one because it's a while true:
import requests
import time
api_url = "https://graph.facebook.com/v2.9/"
access_token = "access token"
graph_url = "site url"
post_data = { 'id':graph_url, 'scrape':True, 'access_token':access_token }
# Beware of rate limiting if trying to increase frequency.
refresh_rate = 30 # refresh rate in second
while True:
try:
resp = requests.post(api_url, data = post_data)
if resp.status_code == 200:
contents = resp.json()
print(contents['title'])
else:
error = "Warning: Status Code {}\n{}\n".format(
resp.status_code, resp.content)
print(error)
raise RuntimeWarning(error)
except Exception as e:
f = open ("open_graph_refresher.log", "a")
f.write("{} : {}".format(type(e), e))
f.close()
print(e)
time.sleep(refresh_rate)
From what I understood you're trying to execute the piece of code for multiple access tokens. To make your job simple, have all your access_tokens as lists and use the following code. It assumes that you know all your access_tokens in advance.
import requests
import time
def scrape_facebook(api_url, access_token, graph_url):
""" Scrapes the given access token"""
post_data = { 'id':graph_url, 'scrape':True, 'access_token':access_token }
try:
resp = requests.post(api_url, data = post_data)
if resp.status_code == 200:
contents = resp.json()
print(contents['title'])
else:
error = "Warning: Status Code {}\n{}\n".format(
resp.status_code, resp.content)
print(error)
raise RuntimeWarning(error)
except Exception as e:
f = open (access_token+"_"+"open_graph_refresher.log", "a")
f.write("{} : {}".format(type(e), e))
f.close()
print(e)
access_token = ['a','b','c']
graph_url = ['sss','xxx','ppp']
api_url = "https://graph.facebook.com/v2.9/"
for n in range(len(graph_url)):
scrape_facebook(api_url, access_token[n], graph_url[n])
time.sleep(5)
So I am trying to request a definition from oxford dictionary through their API and when the user type !define (users word) the bot returns the definition. I am having a few issues see the code below.
#client.command()
async def define(word_id):
app_id = '************'
app_key = '***********'
language = 'en'
url = 'https://od-api.oxforddictionaries.com:443/api/v1/entries/' +
language + '/' + word_id.lower()
r = requests.get(url, headers={'app_id': app_id, 'app_key': app_key})
await client.say("The definition is " + ("text \n" + r.text))
The error I'm getting is as follows:
discord.errors.HTTPException: BAD REQUEST (status code: 400) + discord.ext.commands.errors.CommandInvokeError: Command raised an exception: HTTPException: BAD REQUEST (status code: 400)
Here is what they expect me to use:
import requests
import json
# TODO: replace with your own app_id and app_key
app_id = '****'
app_key = '******'
language = 'en'
word_id = 'Ace'
url = 'https://od-api.oxforddictionaries.com:443/api/v1/entries/' +
language + '/' + word_id.lower()
r = requests.get(url, headers = {'app_id': app_id, 'app_key':
app_key})
print("code {}\n".format(r.status_code))
print("text \n" + r.text)
print("json \n" + json.dumps(r.json()))