I am trying to fetch product data from an api.
By default this api returns 20 products and in a single request the api can return max 500 products if we use api's parameter Limit=500.
So for fetching all products we need to use one more parameter with Limit- Offset(Number of products to skip).
I have written following function to achieve this but in case of full data my function is not working well and it's giving me error like- Login failed, Signature mismatching.
def get_data(userid, api_key, action, pagination=True):
timeformat = datetime.datetime.now().replace(microsecond=0).isoformat() + '+08:00'
endpoint = 'https://example.com'
page_json = {}
# set required parameters for this api
parameters = {
'UserID': userid,
'Version': '1.0',
'Action': action,
'Format': 'JSON',
'Timestamp': timeformat
}
if pagination:
page = 0
parameters['Limit'] = 500
while True:
parameters['Offset'] = 500 * page
# set the required cryptographic signature
concatenated = urllib.parse.urlencode(sorted(parameters.items()))
parameters['Signature'] = HMAC(api_key, concatenated.encode('utf-8'), sha256).hexdigest()
page += 1
try:
response = requests.get(endpoint, params=parameters)
page_json = response.json()
except requests.exceptions.ConnectionError:
print("Connection refused!")
sleep(5)
else:
try:
concatenated = urllib.parse.urlencode(sorted(parameters.items()))
# set the required cryptographic signature
parameters['Signature'] = HMAC(api_key, concatenated.encode('utf-8'), sha256).hexdigest()
response = requests.get(endpoint, params=parameters)
page_json = response.json()
except requests.exceptions.ConnectionError:
print("Connection refused!")
sleep(5)
return page_json
It looks like I am not fitting my signature parameter line correctly in case of full data.I printed the value of concatenated and it looks like-
page is 1
concatenated:: Action=GetProducts&Format=JSON&Limit=500&Offset=500&Signature=3d9cd320a4bf816aeea828b9392ed2d5a27cd584b3a337338909c0ab161a101e&Timestamp=2018-05-26T12%3A58%3A38%2B08%3A00&UserID=contact%40example.com.sg&Version=1.0
try: {'ErrorResponse': {'Head': {'ErrorCode': '7', 'ErrorMessage': 'E7:Login failed. Signature mismatching', 'ErrorType': 'Sender', 'RequestAction': 'GetProducts', 'RequestId': '0bb606c015273197313552686ec46f'}}}
page is 2
concatenated:: Action=GetProducts&Format=JSON&Limit=500&Offset=1000&Signature=c1bda1a5ab21c4e4182cc82ca7ba87cb9fc6c5f24c36f9bb006f9da906cf7083&Timestamp=2018-05-26T12%3A58%3A38%2B08%3A00&UserID=contact%40example.com.sg&Version=1.0
try: {'ErrorResponse': {'Head': {'ErrorCode': '7', 'ErrorMessage': 'E7:Login failed. Signature mismatching', 'ErrorType': 'Sender', 'RequestAction': 'GetProducts', 'RequestId': '0bb606c015273197321748243ec3a5'}}}
Can you please look into my function and help me to find out what I have written wrong and what it should be like?
Try this please:
if pagination:
page = 0
parameters['Limit'] = 500
while True:
parameters['Offset'] = 500 * page
# set the required cryptographic signature
concatenated = urllib.parse.urlencode(sorted(parameters.items()))
parameters['Signature'] = HMAC(api_key, concatenated.encode('utf-8'), sha256).hexdigest()
page += 1
try:
response = requests.get(endpoint, params=parameters)
page_json = response.json()
except requests.exceptions.ConnectionError:
print("Connection refused!")
sleep(5)
del parameters['Signature']
Related
capabilities = DesiredCapabilities.CHROME
capabilities["goog:loggingPrefs"] = {"performance": "ALL"} # newer: goog:loggingPrefs
driver = webdriver.Chrome(
desired_capabilities=capabilities, executable_path="C:/Users/admin/Downloads/chromedriver_win32 (5)/chromedriver.exe"
)
driver.get('url')
time.sleep(4)
email = driver.find_element_by_xpath('//*[#id="app"]/div/div[2]/div[1]/div[3]/div/div/form/div[1]/div/input').send_keys('email')
password = driver.find_element_by_xpath('//*[#id="app"]/div/div[2]/div[1]/div[3]/div/div/form/div[2]/div/input').send_keys('pw')
time.sleep(0.4)
submit = driver.find_element_by_xpath('//*[#id="app"]/div/div[2]/div[1]/div[3]/div/div/form/div[4]/button').click()
time.sleep(2)
logs_raw = driver.get_log("performance")
logs = [json.loads(lr["message"])["message"] for lr in logs_raw]
def log_filter(log_):
return (
# is an actual response
log_["method"] == "Network.responseReceived"
# and json
and "json" in log_["params"]["response"]["mimeType"]
)
for log in filter(log_filter, logs):
request_id = log["params"]["requestId"]
resp_url = log["params"]["response"]["url"]
print(f"Caught {resp_url}")
info = driver.execute_cdp_cmd("Network.getResponseBody", {request_id})
print(info)
_selluuid = driver.get_cookie("_selluuid")
_gat = driver.get_cookie("_gat")
_gid = driver.get_cookie("_gid")
_ga = driver.get_cookie("_ga")
__cf_bm = driver.get_cookie("__cf_bm")
fcookies = {
'__cf_bm': __cf_bm['value'],
'_ga': _ga['value'],
'_gid': _gid['value'],
'_gat': _gat['value'],
'_selluuid': _selluuid['value'],
}
i need to get the cookies AND response body from a request made within the chrome driver. when i print it it seems to add some extra data compared to the standard response i get manually doing it and I cannot parse it entirely to get 1 specific data.
The data:
{'base64Encoded': False, 'body':
gets added and i can't seem to parse the response.json to get a piece of data `Parse error on line 1:
{'base64Encoded': Fal
Expecting 'STRING', '}', got 'undefined'`
can't use selinumwire because recaptcha always blocks it unless there is a way around that
I'm brand new to python and api as well.
I'm trying to use a endpoint we have at work.
We have an API we are using a lot, we also have an UI. But using the UI we can only extract 10.000 records at the time.
There is no limit on the api.
I have found a small piece of code - but i need to add a nextpagetoken.
My code looks like this:
login_url = 'https://api.ubsend.io/v1/auth/login'
username = 'xxxxx'
password = 'xxxxx'
omitClaims = "true"
session = requests.Session()
session.headers['Accept'] = "application/json; charset=UTF-8"
response = session.post(
login_url,
json={'username': username, 'password': password},
headers={'VERSION': '3'},
)
response.raise_for_status()
response_data = response.json()
print(response_data)
This gives me the AccessToken.
Then I call:
getevents = 'https://api.ubsend.io/v1/reporting/shipments?'
data ={'client_id': 13490, 'created_after': '2020-05-01T00:00', 'created_before': '2021-05-02T00:00'} req.prepare_url(getevents, data)
events = requests.get(req.url, headers={'Authorization' : 'Bearer ' + response_data['accessToken'], Content-Type': 'application/json'})
events.json()
Which returns:
'nextPageToken': 'NjA4ZDc3YzNkMjBjODgyYjBhMWVkMTVkLDE2MTk4ODM5NzA3MDE='}
So I want to loop my script - until nextPageToken is blank ....
Any thoughts?
Edit thanks for the update. I think this might be the solution we're looking for. You might have to do some poking around to figure out exactly what the name of the page_token URL parameter should be.
has_next = True
getevents = 'https://api.ubsend.io/v1/reporting/shipments?'
token = None
while has_next:
data ={'client_id': 13490, 'created_after': '2020-05-01T00:00', 'created_before': '2021-05-02T00:00'}
if token:
# I don't know the proper name for this URL parameter.
data['page_token'] = token
req.prepare_url(getevents, data)
events = requests.get(req.url, headers={'Authorization' : 'Bearer ' + response_data['accessToken'], Content-Type: 'application/json'})
token = events.json().get('nextPageToken')
if not token:
has_next = False
I made a slight typo. It should be events.json().get('nextPageToken') I believe.
Let me know if this works.
I grab the site-key using a some-what hardcoded method, but since the len is always the same it's ok.
After that I use the 2captcha API Documentation so that I can POST the key and GET the token for the captcha back. I face two major problems:
1) I always get the site-key wrong error, but the sitekey is correct according to their example (their sitekey is 40chars long and my sitekeys are 40 too)
2) I tried creating a POST function externally and trying it out seeing if it is a bug, but using the Py2Captcha documentation I always get the following error:
This is captcha key grabbing.
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Captcha Key~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
url=driver.find_element_by_css_selector("iframe[role='presentation']").get_attribute('src')
print(url)
keygoogle = url[53:93]
print('Site Key = ', keygoogle)
This is the captcha key solving block:
answer = ""
answer_id = 0
api_key = '--------------------------------'
data_post = {'key': api_key, 'method': 'userrecaptcha', 'googlekey': keygoogle, "pageurl": mainurl}
response = requests.post(url = 'https://2captcha.com/in.php', data = data_post )
print(response)
print("Waiting for server response.")
for x in range(15):
time.sleep(1)
if x == 5:
print('Downloading info..')
elif x == 10:
print('Processing info..')
elif x == 14:
print('Solving captcha..')
data_request = {
'key': api_key,
'action': answer,
'id': answer_id,
}
requests.get(url ='https://2captcha.com/res.php', data=data_request)
print(answer)
def captcha():
google_key = keygoogle
url = mainurl
client = TwoCaptchaClient(client_key=api_key)
task = GoogleReCaptchaV2Task(googlekey=google_key, pageurl=mainurl)
job = client.create_task(task)
token = job.get_solution_response()
return token
print(captcha())
What I haven't included is the part where the token gets posted into the answer field, I am not sure how to do that yet but I will find a way for sure!
EDIT:
This is the value I get from printing this:
response = requests.post(url = 'https://2captcha.com/in.php', data = data_post )
And this is the value I get from print('Site Key = ', keygoogle)
Site Key = Lc3HAsUAAAAACsN7CgY9MMVxo2M09n_e4heJEiZ&
This is my way of grabbing the correct key:
url=driver.find_element_by_css_selector("iframe[role='presentation']").get_attribute('src')
keygoogle = url[52:92]
And this is my POST function:
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Captcha Solve~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
answer = ""
answer_id = 0
data_post = {'key': api_key, 'method': 'userrecaptcha', 'googlekey': keygoogle, "pageurl": mainurl}
response = requests.post(url = 'https://2captcha.com/in.php', data = data_post )
response = response.text[3:]
print("Waiting for server response.")
for x in range(30):
time.sleep(1)
if x == 8:
print('Downloading info..')
elif x == 15:
print('Processing info..')
data_request = {'key': api_key,'id': int(response),'action': 'get'}
response = requests.get(url='https://2captcha.com/res.php', params=data_request)
token = response.text.split('|')[0]
while response.text == 'CAPCHA_NOT_READY':
print('Waiting for Capcha..')
time.sleep(5)
response = requests.get(url='https://2captcha.com/res.php', params=data_request)
token = response
print(token)
The & on the end of that sitekey definitely isn't supposed to be there. Use a regex instead of what you're doing there with the indexes.
I write a python script to do GET and PUT method in zendesk API and successfully get the data I wanted and do some updates to the tickets.
below method resulting this ticket number "6442" and put method is intended to remove the tags
from urllib.parse import urlencode
import json
import requests
# Set the credentials
credentials = 'some email', 'some password'
session = requests.Session()
session.auth = credentials
# Set the GET parameters
params_noreply_window = {
'query': 'type:ticket tags:test status<closed',
}
params_oustide_businesshour = {
'query': 'type:ticket tags:send_whatsapp_obh status:new',
}
url_search1 = 'https://propertypro.zendesk.com/api/v2/search.json?' + \
urlencode(params_noreply_window)
url_search2 = 'https://propertypro.zendesk.com/api/v2/search.json?' + \
urlencode(params_oustide_businesshour)
response_noreply_window = session.get(url_search1)
response_oustide_businesshour = session.get(url_search2)
# -----------------------------------------------------------------------------
if response_noreply_window.status_code != 200 | response_oustide_businesshour.status_code != 200:
print('Status 1:', response_noreply_window.status_code + 'Status 2:', response_oustide_businesshour.status_code,
'Problem with the request. Exiting.')
exit()
# Print the subject of each ticket in the results
data_noreply_window = response_noreply_window.json()
data_oustide_businesshour = response_oustide_businesshour.json()
# Ticket to update
# Create a list containing the values of the id field
# for each dictionary that is an element of the list data
id_merged1 = [result['id'] for result in data_noreply_window['results']]
print(type(id_merged1))
print(id_merged1)
id_merged2 = [result['id'] for result in data_oustide_businesshour['results']]
print(type(id_merged2))
print(id_merged2)
# Join value of list by using comma separated
id_merged1_joined = ','.join(map(str, id_merged1))
print(id_merged1_joined)
id_merged2_joined = ','.join(map(str, id_merged2))
print(id_merged2_joined)
# Package the data in a dictionary matching the expected JSON
data_comment1 = {"ticket":
{
"remove_tags": ["test"]
}
}
data_comment2 = {"ticket":
{
"remove_tags": ["send_whatsapp_obh"]
}
}
# Encode the data to create a JSON payload
payload1 = json.dumps(data_comment1)
payload2 = json.dumps(data_comment2)
print("**Start**")
# Set the request parameters
url_put_comments1 = 'https://propertypro.zendesk.com/api/v2/tickets/update_many.json?' +\
'ids=' + id_merged1_joined
url_put_comments2 = 'https://propertypro.zendesk.com/api/v2/tickets/update_many.json?' +\
'ids=' + id_merged2_joined
user = 'some email'
pwd = 'some password'
headers = {'content-type': 'application/json'}
# Do the HTTP put request
response_request_noreply = requests.put(url_put_comments1, data=payload1,
auth=(user, pwd), headers=headers)
response_request_obh = requests.put(url_put_comments2, data=payload2,
auth=(user, pwd), headers=headers)
# Check for HTTP codes other than 200
if response_request_noreply.status_code != 200 | response_request_obh.status_code != 200:
print('Status 1:', response_request_noreply.status_code +
'Status 1:', response_request_obh.status_code,
'Problem with the request. Exiting.')
exit()
# Report success
print('Successfully added comment to tickets')
However, after running my python code and do another GET method, the same ticket number still appears and I need to wait in random time to get the result I intend which is return 'null' since I have updated the ticket by using PUT method.
Can anyone explain me how does the Zendesk API works? and my apology for my incorrect sentences in explaining my concern.
Trying to check for none 200 Response in the current_track() function. What could be a problem? It throwing JSONDecodeError error. But if I understood raise_for_ status correctly it should have prevented the function from trying to load a JSON from a faulty web-page? If I run the script without this check and with uncommenting lines check_playback() it successfully catches JSONDecodeError.
The script is fetching data from Spotify and putting it to the status on vk.com.
import config
import webbrowser
import requests
import furl
import secrets
import string
import time
import os
import simplejson as json
URL_CODE_BASE_VK = 'https://oauth.vk.com/authorize'
URL_CODE_BASE_SP = 'https://accounts.spotify.com/authorize'
URL_TOKEN_VK = 'https://oauth.vk.com/access_token'
URL_TOKEN_SP = 'https://accounts.spotify.com/api/token'
URL_TRACK = 'https://api.spotify.com/v1/me/player/currently-playing'
URL_STATUS = 'https://api.vk.com/method/status.set'
EXP_IN_TOKEN_SP = 3400
EXP_IN_TOKEN_VK = 86400
FILE_TOKEN_VK = 'vk_token.json'
FILE_TOKEN_SP = 'sp_token.json'
def get_auth_code_vk():
url_code_params = {
'client_id': config.CLIENT_ID_VK,
'response_type': 'code',
'redirect_uri': 'https://oauth.vk.com/blank.html',
'v': 5.92,
'scope': 'status',
'state': gen_state(),
'display': 'page'
}
code = url_open(URL_CODE_BASE_VK, url_code_params)
return parse_code(code)
def get_auth_code_sp():
url_code_params = {
'client_id': config.CLIENT_ID_SP,
'response_type': 'code',
'redirect_uri': 'https://www.spotify.com/',
'scope': 'user-read-currently-playing',
'state': gen_state()
}
code = url_open(URL_CODE_BASE_SP, url_code_params)
return parse_code(code)
def gen_state():
symbols = string.ascii_lowercase + string.digits
return ''.join(secrets.choice(symbols) for _ in range(12))
def url_open(url_base, url_params):
url_code_full = furl.furl(url_base).add(url_params).url
webbrowser.open_new_tab(url_code_full)
input_url = input('Enter the whole URL, that you have been redirected on: ')
return input_url
def parse_code(url):
return (url.split("code=")[1]).split("&state=")[0]
def get_token_vk():
data = {
'grant_type': 'authorization_code',
'code': get_auth_code_vk(),
'redirect_uri': 'https://oauth.vk.com/blank.html',
'client_id': 6782333,
'client_secret': config.CLIENT_SECRET_VK
}
response = requests.post(url=URL_TOKEN_VK, data=data).json()
write_file(FILE_TOKEN_VK, response)
def get_token_sp():
data = {
'grant_type': 'authorization_code',
'code': get_auth_code_sp(),
'redirect_uri': 'https://www.spotify.com/',
'client_id': config.CLIENT_ID_SP,
'client_secret': config.CLIENT_SECRET_SP
}
response = requests.post(url=URL_TOKEN_SP, data=data).json()
write_file(FILE_TOKEN_SP, response)
def write_file(tkn_file, response):
dict = {}
dict['token'] = response["access_token"]
dict['time'] = time.time()
with open(tkn_file, 'w') as file:
file.write(json.dumps(dict))
def load_file(tkn_file):
with open(tkn_file) as file:
data = json.load(file)
return data
def set_status():
params = {
'v': 5.92,
'access_token': load_file(FILE_TOKEN_VK)['token'],
'text': current_track()
}
set_status = requests.get(url=URL_STATUS, params=params)
def track_data():
tkn_file = load_file(FILE_TOKEN_SP)['token']
headers = {
'Accept': 'application/json',
'Authorization': f'Bearer {tkn_file}'
}
return requests.get(url=URL_TRACK, headers=headers)
def current_track():
response = track_data()
print(response)
try:
response.raise_for_status()
except requests.exceptions.HTTPError as e:
return "Error: " + str(e)
# data = track_data().json()
data = response.json()
artist = data['item']['artists'][0]['name']
track = data['item']['name']
return(f'{artist} - {track}')
def check_playback():
set_status()
print(current_track())
# try:
# set_status()
# print(current_track())
# except json.decoder.JSONDecodeError:
# print('Not playing')
def token_missing(file):
return not os.path.isfile(file)
def token_expired(file, exp_in):
return time.time() - load_file(file)['time'] > exp_in
def token_not_valid(file, exp_in):
return token_missing(file) or token_expired(file, exp_in)
def run_script():
if token_not_valid(FILE_TOKEN_VK, EXP_IN_TOKEN_VK):
get_token_vk()
if token_not_valid(FILE_TOKEN_SP, EXP_IN_TOKEN_SP):
get_token_sp()
check_playback()
if __name__ == "__main__":
run_script()
Error screen
raise_for_status() will only raise an exception if the server reported an error to you (and even then, only if it actually followed the HTTP spec and returned a HTTP error code).
There is no way for the library to know that the response is incorrect. Even if it was correctly formatted JSON, it can't know what schema you expect it to follow (what fields should be present, and what types those fields should have). Even if it knew the schema and had verified it, there is no way for it to know that the data is actually correct and not made up on the spot.