What I am trying to do is perform a search on Splunk's API using python, I am able to get a session key but thats it. I'm new to both python and splunk so im a bit out-of-depth and any help would be really appreciated.
The error:
Traceback (most recent call last):
File "splunkAPI.py", line 31, in <module>
sid = minidom.parseString(r.text).getElementsByTagName('sid')[0].firstChild.nodeValue
IndexError: list index out of range
python:
import time # need for sleep
from xml.dom import minidom
import json, pprint
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
base_url = 'https://___________:8089'
username = '______'
password = '______'
search_query = "____________"
#-------------------------get session token------------------------
r = requests.get(base_url+"/servicesNS/admin/search/auth/login",
data={'username':username,'password':password}, verify=False)
session_key = minidom.parseString(r.text).getElementsByTagName('sessionKey')[0].firstChild.nodeValue
print ("Session Key:", session_key)
#-------------------- perform search -------------------------
r = requests.post(base_url + '/services/search/jobs/', data=search_query,
headers = { 'Authorization': ('Splunk %s' %session_key)},
verify = False)
sid = minidom.parseString(r.text).getElementsByTagName('sid')[0].firstChild.nodeValue
done = False
while not done:
r = requests.get(base_url + '/services/search/jobs/' + sid,
headers = { 'Authorization': ('Splunk %s' %session_key)},
verify = False)
response = minidom.parseString(r.text)
for node in response.getElementsByTagName("s:key"):
if node.hasAttribute("name") and node.getAttribute("name") == "dispatchState":
dispatchState = node.firstChild.nodeValue
print ("Search Status: ", dispatchState)
if dispatchState == "DONE":
done = True
else:
time.sleep(1)
r = requests.get(base_url + '/services/search/jobs/' + sid + '/results/',
headers = { 'Authorization': ('Splunk %s' %session_key)},
data={'output_mode': 'json'},
verify = False)
pprint.pprint(json.loads(r.text))
Hmm... that code looks awfully familiar :P Unfortunately, error checking wasn't that important when I wrote it.
The issue you see occurs if the search_query is not defined properly. It must start with search=. Also note that you need to include an initial search command if doing a standard Splunk search,
For example, search=search index=* will work, search=index=* will not work.
If you need to include quotes in your search string, I suggest you use something like the following format.
search_query = """search=search index=* "a search expression" | stats count"""
Tried this but did not give needed result not sure what is missing
import urllib
import httplib2 #import library
import json
import pprint
import time
import re
from xml.dom import minidom
searchquery = 'search index="movable_in" sourcetype="movable:in:assets" | stats avg(exposure_score)'
myhttp = httplib2.Http()
baseurl = 'https://xxxx.splunkxxx.com:8089'
usernamesp = 'xxxx'
passwordsp = 'xxxx'
def get_splunk_result(searchquery):
# Step 1: Get a session key
servercontent = myhttp.request(f'{baseurl}/services/auth/login', 'POST', headers={},
body=urllib.parse.urlencode({'username': usernamesp, 'password': passwordsp}))[1]
sessionkey = minidom.parseString(servercontent).getElementsByTagName('sessionKey')[0].childNodes[0].nodeValue
# print ("====>sessionkey: %s <====" % sessionkey)
sid = ''
# ------------------
if not searchquery.startswith('search'):
searchquery = f'search {searchquery}'
# Step 2: Get a sid with the search query
i = 0
while True:
time.sleep(1)
try:
searchjob = myhttp.request(f'{baseurl}/services/search/jobs', 'POST',
headers={F'Authorization': F'Splunk %s' % sessionkey},
body=urllib.parse.urlencode({'search': searchquery}))[1]
sid = minidom.parseString(searchjob).getElementsByTagName('sid')[0].childNodes[0].nodeValue
break
except:
i = i + 1
# print(i)
if (i > 30): break
# print("====>SID: %s <====" % sid)
# Step 3: Get search status
myhttp.add_credentials(usernamesp, passwordsp)
servicessearchstatusstr = '/services/search/jobs/%s/' % sid
isnotdone = True
while isnotdone:
searchstatus = myhttp.request(f'{baseurl}{servicessearchstatusstr}', 'GET')[1]
isdonestatus = re.compile('isDone">(0|1)')
strstatus = str(searchstatus)
isdonestatus = isdonestatus.search(strstatus).groups()[0]
if (isdonestatus == '1'):
isnotdone = False
# Step 4: Get the search result
services_search_results_str = '/services/search/jobs/%s/results?output_mode=json_rows&count=0' % sid
searchresults = myhttp.request(f'{baseurl}{services_search_results_str}', 'GET')[1]
searchresults = json.loads(searchresults)
# searchresults = splunk_result(searchresults)
return searchresults
output = get_splunk_result(searchquery)
print(output)
Related
I’ve quite recently found this feature on Bitget which enables users to essentially copy other ranked traders. This feature comes with a corresponding api documentation. But after going through it im more confused than ever. Firstly, im trying to obtain the historical data trading data of specific traders which are available data on their “orders tab” from the website (shown in excerpt above). I reckon this is possible from the following get request from the documentation: “GET /api/mix/v1/trace/waitProfitDateList”.
Based on the above http request from i have produced the following python code below. The request response is 403. Help a fellow novice
import requests
import hmac
import base64
import hashlib
import json
import time
def sign(message, secret_key):
mac = hmac.new(bytes(secret_key, encoding='utf8'), bytes(message, encoding='utf-8'), digestmod='sha256')
d = mac.digest()
return base64.b64encode(d).decode('utf-8')
def pre_hash(timestamp, method, request_path, query_string, body):
return str(timestamp) + str.upper(method) + request_path + query_string + body
if __name__ == '__main__':
params = {
"pageSize": 10,
"pageNo": 1
}
rest_url = "https://api.bitget.com"
secret_key = ""
api_key = ""
passphrase = ""
timestamp = int(time.time_ns() / 1000000);
query_string = '&'.join([f'{k}={v}' for k, v in params.items()])
message = pre_hash(timestamp, 'GET', '/api/mix/v1/trace/waitProfitDateList', "?"+query_string,"")
sign = sign(message, secret_key)
headers = {
"ACCESS-KEY":api_key,
"ACCESS-SIGN":sign,
"ACCESS-TIMESTAMP":str(timestamp),
"ACCESS-PASSPHRASE":passphrase,
"Content-Type":"application/json",
"locale":"en-US"
}
response = requests.get(rest_url, headers=headers, params=params)
if response.status_code == 200:
result = response.json()
print(result)
else:
print(response.status_code)
import sys
import urllib.request
import json
if __name__ == '_main_':
# [CODE1]
page_name = "jtbnews"
app_id = "[App ID]"
app_secret = "[App Secret Code]"
access_token = app_id + "I" +app_secret
# [CODE2]
# http://graph.facebook.com/v2.8/[page_id]/?access_token=[App_ID]I[Secret_Key]
# 형식의 문자열을 만들어 낸다.
base = "http://graph.facebook.com/v2.8"
node = "/" + page_name
parameters = "/?access_token=%s" % access_token
url = base + node + parameters
# [CODE3]
req = urllib.request.Request(url)
# [CODE4]
try:
response = urllib.request.urlopen(req)
if response.getcode() == 200:
data = json.loads(response.read().decode('utf-8'))
page_id = data['id']
print("%s Facebook numeric ID : %s" % (page_name, page_id))
except Exception as e: print(e)
I'm a 100% beginner of programming
I follow the code in a book.
I can't figure out what is the problem.
it just says:
"RESTART: C:\python\python ex\facebook EX.py"
I know this has been asked before .. but my code doesn't seem to work for users other than the one using whom accesstoken is created
.. i.e I am able to access the profile of only one user.(using whom I created accesstoken)
from urllib2 import urlopen
from json import load
import json
import json
import sys
import time
import requests
userid="userid_of_some_instagram_user"
access_token="mytoken"
url = "https://api.instagram.com/v1/users/%s/?access_token=%s"%(userid,access_token)
response=urlopen(url)
print(response)
jsonobj=load(response)
print "name:",jsonobj["data"]["username"]
print "follows ",jsonobj["data"]["counts"]["follows"]," people"
print "followed by ",jsonobj["data"]["counts"]["followed_by"]," people"
Error I am getting when I am trying to access other profiles is
HTTPError: HTTP Error 400: BAD REQUEST
Edit : **Voila ! I have found the bug . It is the access key. I have tried with a different access key . It worked**
If you don't wanna fight the new InstagramAPI you can use a workaround like this:
import requests
import json
import time
import random
user = 'google'
url = 'https://www.instagram.com/'
url_user = '%s%s%s' % (url, user, '/')
url_login = 'https://www.instagram.com/accounts/login/ajax/'
s = requests.Session()
s.cookies.update ({'sessionid' : '', 'mid' : '', 'ig_pr' : '1',
'ig_vw' : '1920', 'csrftoken' : '',
's_network' : '', 'ds_user_id' : ''})
login_post = {'username' : 'your_login',
'password' : 'your_pw'}
s.headers.update ()
r = s.get(url)
s.headers.update({'X-CSRFToken' : r.cookies['csrftoken']})
time.sleep(5 * random.random())
login = s.post(url_login, data=login_post,
allow_redirects=True)
s.headers.update({'X-CSRFToken' : login.cookies['csrftoken']})
if login.status_code == 200:
r = s.get('https://www.instagram.com/')
finder = r.text.find('your_login')
r = s.get(url_user)
text = r.text
finder_text_start = ('<script type="text/javascript">'
'window._sharedData = ')
finder_text_start_len = len(finder_text_start)-1
finder_text_end = ';</script>'
all_data_start = text.find(finder_text_start)
all_data_end = text.find(finder_text_end, all_data_start + 1)
json_str = text[(all_data_start + finder_text_start_len + 1) \
: all_data_end]
user_info = json.loads(json_str)
follower_count = user_info['entry_data']['ProfilePage'][0]['user']["followed_by"]['count']
print(follower_count)
Try this:
import json
import requests
access_token = "mytoken"
url = "https://api.instagram.com/v1/users/self/?access_token={0}".format(access_token)
response = requests.get(url)
j = json.loads(response.text)
if 'data' in j:
print("handle correct response")
else:
print("handle incorrect response")
I've found this post on the Work Etc. forums for a Python REST client and the forum used didn't include the indentation for the code nor did the author include them, so what I've done is input what I believe to be the correct indentation and have gotten the following:
import sys, json, urllib
from httplib2 import Http
class WORKetcRESTClient():
session_key = None
connector_hash = None
def __init__(self,url):
if not "http://" in url and not "https://" in url:
url = "http://%s" % url
self.base_url = url
else:
self.base_url = url
def authenticate(self,user,password):
args = { "email" : user,"pass" : password,}
res = self.request("AuthenticateWebSafe", args)
if res["Code"] == 1:
self.session_key = res["SessionKey"]
self.user = res["User"]
return True
else:
return False
def request(self,service,args):
url = "%s/%s" % (self.base_url, service)
if not self.session_key is None:
url = "%s?VeetroSession=%s" %(url,self.session_key)
p = {}
p['data'] = json.dumps(args)
h = Http()
r,c = h.request(url, body=json.dumps(args), method="POST", headers = {'contentType' : 'application/json; charset=utf-8'})
if r.status == 200:
jsondata = json.loads(c)
return jsondata
else:
print r,c
return {}
client = WORKetcRESTClient('http://company.worketc.com')
client.authenticate('User#company.com', 'pAsSwOrD')
result = client.request('FindCompanies',{'keywords':'customer'})
print result
To be 100% honest, if this were to run without any errors, I'm not sure what I would get printed to the console, but the errors I'm getting are keeping me from finding out:
Traceback (most recent call last):
File "worketc.py", line 42, in <module>
File "worketc.py", line 17, in authenticate
res = self.request("AuthenticateWebSafe", args)
File "worketc.py", line 34, in request
if r.status == 200:
UnboundLocalError: local variable 'r' referenced before assignment
It's telling me that the variable 'r' is getting called before it is assigned/created, but I'm not sure where it needs to be created, or moved to, considering it's location in the current script. Can anybody shed some light onto this?
if not self.session_key is None:
url = "%s?VeetroSession=%s" %(url,self.session_key)
p = {}
p['data'] = json.dumps(args)
h = Http()
r,c = h.request(url, body=json.dumps(args), method="POST", headers = {'contentType' : 'application/json; charset=utf-8'})
is the correct indentation
I am currently using OAuth 1 for auth with the Reporting API with GData and Python. Is it possible to use OAuth 2, I can't find a reference that this is doable?
I wasn't able to find any reference for the OAuth 2 and the Reporting api but by following samples for the GData libraries (http://code.google.com/p/gdata-python-client/source/browse/#hg%2Fsamples%2Fapps) I was able to cobble this together:
#!/usr/bin/python
import sys
import os
import time
import gdata.gauth
import gdata.client
import httplib2
import oauth2client.file
import oauth2client.tools
REPORTING_URI = 'https://www.google.com/hosted/services/v1.0/reports/ReportingData'
REPORTING_XML_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
<rest xmlns="google:accounts:rest:protocol"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<type>Report</type>
<domain>%s</domain>
<date>%s</date>
<page>%s</page>
<reportType>daily</reportType>
<reportName>%s</reportName>
</rest>'''
OAUTH2FILENAME = 'oauth_20.dat'
OAUTH2JSONFILE = 'client_secrets.json'
OAUTH2SCOPES = 'https://www.google.com/hosted/services/v1.0/reports/ReportingData'
OAUTH2USERAGENT = 'REPORTING'
CLIENTSOURCE = 'REPORTING'
MISSING_OAUTHJSON_FILE_MESSAGE = """
WARNING: Please configure OAuth 2.0
To continue you will need to populate the client_secrets.json file:
%s
with information from the APIs Console <https://code.google.com/apis/console>.
""" % os.path.join(os.path.dirname(__file__), OAUTH2JSONFILE)
### Reporting
def RunReport (http_object, domain, report=None, date=None):
if date is None:
now = time.time()
report_time = time.gmtime(now)
date = time.strftime("%Y-%m-%d",report_time)
if report is None:
report='accounts'
report_data = RequestReport(http_object,domain=domain,report=report,date=date)
if not report_data:
print 'No report data'
return report_data
def RequestReport (http_object, domain=None, report=None, date=None):
"""Retrieves a report
Args:
domain: string
report: string: accounts, activity, disk_space, email_clients, summary
date: string: YYYY-MM-DD
Returns:
String, the report data
"""
report_data = ''
uri = REPORTING_URI
if not report or report is None:
return report_data
if not date or date is None:
return report_data
if not domain or domain is None:
domain = self.domain
page = 1
while True:
report_xml = REPORTING_XML_TEMPLATE %(domain, date, page, report)
response = ''
report_page = ''
try:
response, report_page = http_object.request(
uri,method='POST',body=report_xml)
except Exception, rexcept:
print 'Exception: ',rexcept
report_page = ''
break
if response.status != 200:
print 'Error: ',response.status
report_page = ''
break
if not report_page or report_page == 'End-Of-Report':
break
else:
report_data += report_page
page = page + 1
return report_data
scopes = OAUTH2SCOPES
user_agent = OAUTH2USERAGENT
client_source = CLIENTSOURCE
str_oauth2file = OAUTH2FILENAME
str_oauthjsonfile = OAUTH2JSONFILE
domain = 'somedomain'
report_name = 'accounts'
client_id = 'string'
client_secret = 'string'
report_data = ''
oauth2_flow = ''
now = time.time()
report_time = time.gmtime(now)
report_date = time.strftime("%Y-%m-%d",report_time)
if not os.path.isfile(str_oauth2file):
token = gdata.gauth.OAuth2Token(client_id=client_id,
client_secret=client_secret, scope=scopes, user_agent=user_agent)
uri = token.generate_authorize_url()
print 'Please visit this URL to authorize the application:'
print uri
# Get the verification code from the standard input.
code = raw_input('What is the verification code? ').strip()
token.get_access_token(code)
oauth2_flow = oauth2client.client.flow_from_clientsecrets(str_oauthjsonfile,
scope=scopes,message=MISSING_OAUTHJSON_FILE_MESSAGE)
storage = oauth2client.file.Storage(str_oauth2file)
oauth2_credentials = storage.get()
if oauth2_credentials is None or oauth2_credentials.invalid:
if not oauth2_flow:
oauth2_flow = oauth2client.client.flow_from_clientsecrets(str_oauthjsonfile,
scope=scopes,message=MISSING_OAUTHJSON_FILE_MESSAGE)
print '\nYou must authorize access to the request APIS.\n'
# Save the credentials in storage to be used in subsequent runs.
oauth2_credentials = oauth2client.tools.run(oauth2_flow, storage)
http_oauth2_object = httplib2.Http()
http_oauth2_object = oauth2_credentials.authorize(http_oauth2_object)
report_data = RunReport(
http_oauth2_object,domain,report=report_name,date=report_date)
if report_data:
print report_data
sys.exit(0)