We Keep Coding

Search Splunk API using python

What I am trying to do is perform a search on Splunk's API using python, I am able to get a session key but thats it. I'm new to both python and splunk so im a bit out-of-depth and any help would be really appreciated.
The error:
Traceback (most recent call last):
File "splunkAPI.py", line 31, in <module>
sid = minidom.parseString(r.text).getElementsByTagName('sid')[0].firstChild.nodeValue
IndexError: list index out of range
python:
import time # need for sleep
from xml.dom import minidom
import json, pprint
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
base_url = 'https://___________:8089'
username = '______'
password = '______'
search_query = "____________"
#-------------------------get session token------------------------
r = requests.get(base_url+"/servicesNS/admin/search/auth/login",
data={'username':username,'password':password}, verify=False)
session_key = minidom.parseString(r.text).getElementsByTagName('sessionKey')[0].firstChild.nodeValue
print ("Session Key:", session_key)
#-------------------- perform search -------------------------
r = requests.post(base_url + '/services/search/jobs/', data=search_query,
headers = { 'Authorization': ('Splunk %s' %session_key)},
verify = False)
sid = minidom.parseString(r.text).getElementsByTagName('sid')[0].firstChild.nodeValue
done = False
while not done:
r = requests.get(base_url + '/services/search/jobs/' + sid,
headers = { 'Authorization': ('Splunk %s' %session_key)},
verify = False)
response = minidom.parseString(r.text)
for node in response.getElementsByTagName("s:key"):
if node.hasAttribute("name") and node.getAttribute("name") == "dispatchState":
dispatchState = node.firstChild.nodeValue
print ("Search Status: ", dispatchState)
if dispatchState == "DONE":
done = True
else:
time.sleep(1)
r = requests.get(base_url + '/services/search/jobs/' + sid + '/results/',
headers = { 'Authorization': ('Splunk %s' %session_key)},
data={'output_mode': 'json'},
verify = False)
pprint.pprint(json.loads(r.text))

Hmm... that code looks awfully familiar :P Unfortunately, error checking wasn't that important when I wrote it.
The issue you see occurs if the search_query is not defined properly. It must start with search=. Also note that you need to include an initial search command if doing a standard Splunk search,
For example, search=search index=* will work, search=index=* will not work.
If you need to include quotes in your search string, I suggest you use something like the following format.
search_query = """search=search index=* "a search expression" | stats count"""

Tried this but did not give needed result not sure what is missing
import urllib
import httplib2 #import library
import json
import pprint
import time
import re
from xml.dom import minidom
searchquery = 'search index="movable_in" sourcetype="movable:in:assets" | stats avg(exposure_score)'
myhttp = httplib2.Http()
baseurl = 'https://xxxx.splunkxxx.com:8089'
usernamesp = 'xxxx'
passwordsp = 'xxxx'
def get_splunk_result(searchquery):
# Step 1: Get a session key
servercontent = myhttp.request(f'{baseurl}/services/auth/login', 'POST', headers={},
body=urllib.parse.urlencode({'username': usernamesp, 'password': passwordsp}))[1]
sessionkey = minidom.parseString(servercontent).getElementsByTagName('sessionKey')[0].childNodes[0].nodeValue
# print ("====>sessionkey: %s <====" % sessionkey)
sid = ''
# ------------------
if not searchquery.startswith('search'):
searchquery = f'search {searchquery}'
# Step 2: Get a sid with the search query
i = 0
while True:
time.sleep(1)
try:
searchjob = myhttp.request(f'{baseurl}/services/search/jobs', 'POST',
headers={F'Authorization': F'Splunk %s' % sessionkey},
body=urllib.parse.urlencode({'search': searchquery}))[1]
sid = minidom.parseString(searchjob).getElementsByTagName('sid')[0].childNodes[0].nodeValue
break
except:
i = i + 1
# print(i)
if (i > 30): break
# print("====>SID: %s <====" % sid)
# Step 3: Get search status
myhttp.add_credentials(usernamesp, passwordsp)
servicessearchstatusstr = '/services/search/jobs/%s/' % sid
isnotdone = True
while isnotdone:
searchstatus = myhttp.request(f'{baseurl}{servicessearchstatusstr}', 'GET')[1]
isdonestatus = re.compile('isDone">(0|1)')
strstatus = str(searchstatus)
isdonestatus = isdonestatus.search(strstatus).groups()[0]
if (isdonestatus == '1'):
isnotdone = False
# Step 4: Get the search result
services_search_results_str = '/services/search/jobs/%s/results?output_mode=json_rows&count=0' % sid
searchresults = myhttp.request(f'{baseurl}{services_search_results_str}', 'GET')[1]
searchresults = json.loads(searchresults)
# searchresults = splunk_result(searchresults)
return searchresults
output = get_splunk_result(searchquery)
print(output)

Getting Unbound local Error: local variable 'x' referenced below assignment

I have a python flask app that is receiving webhook from another application. When it receives the webhook, it responds back by carry out a task (looking up someone's availability) and responding back to the web application with a response. I am getting an unbound local error local variable 'response' referenced below assignment when sending a response back. It looks like calling response at that level is causing issues.
Any help would be greatly appreciated.
from flask import Flask
from flask import request
from flask import make_response
import logging
import json
import random
import os
import importlib
import win32com.client
import pywintypes
import datetime
import pythoncom
from gevent.pywsgi import WSGIServer
from gevent import monkey; monkey.patch_all()
import string
pythoncom.CoInitialize()
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(filename)s - %(funcName)s - %(message)s')
app = Flask(__name__)
#app.route('/webhook', methods=['POST'])
def webhook():
req = request.get_json(silent=True, force=True)
logger.info("Incoming request: %s", req)
intent = get_intent_from_req(req)
logger.info('Detected intent %s', intent)
if intent == "Check Schedule Next":
pythoncom.CoInitialize()
emailparam = req.get('queryResult').get('parameters').get('email')
datetime1 = req.get('queryResult').get('parameters').get('date-time').get("date_time")
datetime2=datetime1.replace('T',' ')
datetime3=datetime2.replace("-04:00", "")
print(datetime3)
pythoncom.CoInitialize()
class MeetingRoom:
def __init__(self, inputDate, duration, locationMail):
self.inputDate = inputDate
self.oOutlook = win32com.client.Dispatch("Outlook.Application")
self.bookings = self.oOutlook.CreateItem(1)
self.bookings.Start = inputDate
self.bookings.Duration = duration
self.bookings.Subject = 'Follow Up Meeting'
self.bookings.MeetingStatus = 1
self.roomRecipient = self.bookings.Recipients.Add(locationMail)
def checkRoomAvailability(self):
bookingDateTime = datetime.datetime.strptime(self.inputDate, '%Y-%m-%d %H:%M:%S')
self.roomRecipient.resolve
myDate = bookingDateTime.date()
pywintypeDate = pywintypes.Time(myDate)
availabilityInfo = self.roomRecipient.FreeBusy(pywintypeDate, self.bookings.Duration, True)
timeAvailability = []
newTime = pywintypeDate
# print(newTime)
currentTime = datetime.datetime.now()
for isAvailable in availabilityInfo:
# print(newTime, " :: ", isAvailable)
if isAvailable == "0" and newTime > currentTime:
timeAvailability.append(newTime)
newTime = newTime + datetime.timedelta(minutes=self.bookings.Duration)
# print(availabilityInfo)
# for value in timeAvailability:
# print(value)
try:
index = timeAvailability.index(bookingDateTime)
print(emailparam, "is available")
response = {
'fulfillmentText': emailparam
}
# self.bookings.Save()
# self.bookings.Send()
except ValueError:
for timestamp in timeAvailability:
if bookingDateTime <= timestamp:
break
print("I dont see availability for", emailparam, "at", bookingDateTime, " but next available time is ", timestamp)
x = ("I dont see availability for", emailparam, "at", bookingDateTime, " but next available time is ", timestamp)
response = {
'fulfillmentText': x
}
# def bookMeetingRoom():
if __name__ == '__main__':
meetingRoomObj = MeetingRoom(datetime3, 15, emailparam)
meetingRoomObj.checkRoomAvailability()
#response = {
# 'fulfillmentText': emailparam
#}
res = create_response(response)
return res
def get_intent_from_req(req):
try:
intent_name = req['queryResult']['intent']['displayName']
except KeyError:
return None
return intent_name
def get__from_req(req):
try:
intent_name = req['queryResult']['intent']['displayName']
except KeyError:
return None
return intent_name
def create_response(response):
res = json.dumps(response, indent=4)
logger.info(res)
r = make_response(res)
r.headers['Content-Type'] = 'application/json'
return r
if __name__ == '__main__':
LISTEN = ('0.0.0.0',8080)
http_server = WSGIServer( LISTEN, app )
http_server.serve_forever()

This line references response before it's initialized:
res = create_response(response)
Perhaps make sure all code paths initialize the response varaiable?

Solution
It seems you've created your response object in the wrong scope, remove it from the function checkRoomAvailability.
Inside the function checkRoomAvailability after you've created the response object, return it like so
response = {
'fulfillmentText': x
}
return response #ADD THIS LINE
Remove these lines
if __name__ == '__main__':
meetingRoomObj = MeetingRoom(datetime3, 15, emailparam)
meetingRoomObj.checkRoomAvailability()
Then add back the object creation and call right before you create your response like so
meetingRoomObj = MeetingRoom(datetime3, 15, emailparam)
response = meetingRoomObj.checkRoomAvailability()
res = create_response(response)
return res
Suggestion
You are lacking some fundamental understanding about how python or scope works so I suggest taking a read friend
https://docs.python.org/3.3/reference/executionmodel.html

Pymongo, TypeError : expected a character buffer object

I'm trying to connect and to read data in a MongoDB database, to learn Python. I'm using Pymongo and I have this error :
Traceback (most recent call last):
File "secondtest.py", line 107, in <module>
user_info = dbco.find({})
TypeError: expected a character buffer object
This is my database.ini :
[postgresql]
host=monhostname
database=monpass
port=15000
user=monuser
password=monpass
[mongodb]
hostname=127.0.0.1
database=Mydatabase
username=Myname
password=Myn#me!
collection=measure
port=27017
And my code using it :
# -*- coding: utf-8 -*-
# secondtest.py
import psycopg2
import sys
import pymongo
from urllib import quote_plus
from pymongo import MongoClient
from configparser import ConfigParser
# Connection information in database.ini
params = ''
mongollection = ''
# Variables to connect to a database, to use a cursor object and to fetch all results from a query
mongoClient = ''
pgsqlClient = ''
pgsqlCursor = ''
pgsqlRecords = ''
mongoRecords = ''
dbco = ''
# Retrieve connection information from ini file
def dbConfig(section, filename='database.ini'):
# Create a parser
parser = ConfigParser()
# Read config file
parser.read(filename)
# Get section, depending on the database engine
db = {}
if parser.has_section(section):
params = parser.items(section)
for param in params:
db[param[0]] = param[1]
else:
raise Exception('Section {0} not found in the {1} file'.format(section, filename))
# Return data or directly as a string
if section == 'postgresql':
return db
elif section == 'mongodb':
host = ''
username = ''
passwd = ''
port = ''
dbmongo = ''
connectstring = ''
# Make a string to connect to MongoDB
for key, value in db.iteritems():
if key == 'hostname':
host = value.encode("utf-8")
elif key == 'username':
username = value
elif key == 'password':
passwd = value
elif key == 'database':
dbmongo = value
elif key == 'collection':
mongollection = value
elif key == 'port':
port = value
connectstring = "mongodb://" + username + ":" + quote_plus(passwd) + "#" + host + ":" + port
print("Internal test = " + connectstring)
return connectstring.encode('iso-8859-1')
# Connection to MongoDB
def connectToMongoDb():
# The f-string is only available in Python >= 3.6
params = dbConfig('mongodb')
print("Parameters : " + params)
mongoClient = MongoClient(params)
try:
# print("Connection to database")
dbco = mongoClient.mongollection
print("Here")
print("Test dbco : " + dbco)
print("Connected to MongoDB !")
return dbco
except:
return "Error : can't connect to MongoDB !"
# Close MongoDB connection
def closeMongoDbConnection():
# try:
mongoClient.close()
return 'Connection closed'
# except:
# return "Can't close the connection. See if you already had one or if you didn't mispell its name."
# Make a query in MongoDB
def mongoDbQuery():
#mongocursor = mongoClient.mongollection.find()
#for document in cursor:
#print(document)
mongoClient.database_names()
if __name__ == '__main__':
dataconnect = connectToMongoDb()
print("Connection\n")
#mongoDbQuery()
#collec = mongoClient.measure
user_info = dbco.find({})
print(user_info)
print(closeMongoDbConnection())
Could you help me with this problem ? I think quote_plus() or even dbco = mongoClient.mongollection is what makes this error occurs. But I'm not 100% sure and I don't see, even with the documentation, how could I resolve this.
Thank you.

I'm reading your code. I see in the beginning of the program, you create the dbco variable to point to an empty string:
dbco = ''
following after that, I see you define a couple functions, then you call find() on that string:
user_info = dbco.find({})
You're passing {} (empty dict) as parameter to the method... but as you can see in the documentation here, this method needs another string as first parameter. That causes the error you see.
Now I'm not entirely sure how to fix it because I don't know what you mean. Maybe you mean to use the dataconnect variable, since that is the one that gets the results of the connectToMongoDb function:
dataconnect = connectToMongoDb()

I made it again and changed some little things. Now, it works. Here is the code, for people who'd need it in the future.
import sys
import pymongo
from urllib import quote_plus
from pymongo import MongoClient
from configparser import ConfigParser
client = MongoClient()
connected = ''
# Retrieve connection information from ini file
def dbConfig(section, filename='database.ini'):
# Keep result in global variable when the function is finished
global client
# Create a parser
parser = ConfigParser()
# Read config file
parser.read(filename)
# Get section, depending on the database engine
db = {}
if parser.has_section(section):
params = parser.items(section)
for param in params:
db[param[0]] = param[1]
else:
raise Exception('Section {0} not found in the {1} file'.format(section, filename))
# Return data or directly as a string
if section == 'postgresql':
return db
elif section == 'mongodb':
# Variables for the connection
host = ''
username = ''
passwd = ''
port = ''
connectstring = ''
# Make a string to connect to MongoDB
for key, value in db.iteritems():
if key == 'hostname':
host = value.encode("utf-8")
elif key == 'username':
username = value
elif key == 'password':
passwd = value
elif key == 'database':
dbmongo = value
elif key == 'collection':
mongollection = value
elif key == 'port':
port = value
# Make the URI needed for the connection to Mongo DB
passwing = "mongodb://" + username + ":" + quote_plus(passwd) + "#" + host + ":" + port
client = MongoClient(passwing)
return client
# Close MongoDB connection
def closeMongoDbConnection():
# Try to close the connection to Mongo DB
try:
client.close()
return 'Connection closed'
except:
return "Can't close the connection. See if you already had one or if you didn't mispell its name."
# Connection to MongoDB
def connectToMongoDb(mydb):
db = client.get_database(mydb)
return db.measure
# Make a query in MongoDB
def mongoDbQuery():
docs = connected.find().count()
#for document in docs:
#print(document)
print(docs)
if __name__ == '__main__':
connected = connectToMongoDb('neocampus')
#docs = connected.find()
# print(test)
#for document in docs:
#print(document)
mongoDbQuery()
# Show if the connection to Mongo DB is a success or not
print(closeMongoDbConnection())
The problems were :
- about global variables in and out of functions
- the database variable empty (because of that)
- a first call of MongoClient()

My python webhook isn't giving me results

I've been working on trying to edit a webhook that was originally meant to be used for a weather API to get to be used with a postcode/zipcode API. The original file is here: https://github.com/dialogflow/fulfillment-webhook-weather-python/blob/master/app.py
I can't understand where mine is different, I thought I had solved it when I replaced urlencode with quote but alas, it wasn't enough.
The problem is very unlikely to do with the source json request that collects the postcode in postcodeValue(). The api url comes out correct when you enter it into a browser and is presented quite simply.
https://api.getaddress.io/find/SW11%201th?api-key=I98umgPiu02GEMmHdmfg3w12959
Is it in the correct format? Maybe I need to convert it to become even more JSON then it already is. This question is essentially an end of day brain dump that I I'm hoping that someone can save me with.
from __future__ import print_function
from future.standard_library import install_aliases
install_aliases()
from urllib.parse import urlparse, urlencode, quote
from urllib.request import urlopen, Request
from urllib.error import HTTPError
import json
import os
from flask import Flask
from flask import request
from flask import make_response
# Flask app should start in global layout
app = Flask(__name__)
#this line is just naming conventions I reckon with a reference to expect to receive data as POST
#app.route('/webhook', methods=['POST'])
def webhook():
req = request.get_json(silent=True, force=True)
#who knows where this is getting printed
print("Request:")
print(json.dumps(req, indent=4))
res = processRequest(req)
res = json.dumps(res, indent=4)
# print(res)
r = make_response(res)
r.headers['Content-Type'] = 'application/json'
return r
def processRequest(req):
if req.get("result").get("action") != "yahooWeatherForecast":
return {}
baseurl = "https://api.getaddress.io/find/"
apikey = "?api-key=I98umgPiu02GEMmHdmfg3w12959"
yql_query = postcodeValue(req)
if yql_query is None:
return {}
#this line is the actual api request
yql_url = baseurl + quote(yql_query) + apikey
result = urlopen(yql_url).read()
data = json.loads(result)
res = makeWebhookResult(data)
return res
#this function extracts an individual parameter and turns it into a string
def postcodeValue(req):
result = req.get("result")
parameters = result.get("parameters")
postcode = parameters.get("postcode")
if postcode is None:
return None
return postcode
#def housenoValue(req):
# result = req.get("result")
#parameters = result.get("parameters")
#houseno = parameters.get("houseno")
#if houseno is None:
# return None
#return houseno
def makeWebhookResult(data):
longitude = data.get("longitude")
if longitude is None:
return {}
#def makeWebhookResult(data):
# query = data.get('query')
# if query is None:
# return {}
# result = query.get('results')
# if result is None:
# return {}
# channel = result.get('channel')
# if channel is None:
# return {}
# item = channel.get('item')
# location = channel.get('location')
# units = channel.get('units')
# if (location is None) or (item is None) or (units is None):
# return {}
# condition = item.get('condition')
# if condition is None:
# return {}
# print(json.dumps(item, indent=4))
speech = "Sausage face " + longitude
print("Response:")
print(speech)
return {
"speech": speech,
"displayText": speech,
# "data": data,
# "contextOut": [],
"source": "apiai-weather-webhook-sample"
}
#More flask specific stuff
if __name__ == '__main__':
port = int(os.getenv('PORT', 5000))
print("Starting app on port %d" % port)
app.run(debug=False, port=port, host='0.0.0.0')

Here is a bit cleaner version of your code:
from urllib.request import urlopen
import os
from flask import Flask
app = Flask(__name__)
#app.route('/webhook', methods=['GET'])
def webhook():
res = processRequest()
return res
def processRequest():
try:
result = urlopen("https://api.getaddress.io/find/SW11%201th?api-key=I98umgPiu02GEMmHdmfg3w12959").read()
return result
except:
return "Error fetching data"
if __name__ == '__main__':
port = int(os.getenv('PORT', 5000))
print("Starting app on port %d" % port)
app.run(debug=False, port=port, host='0.0.0.0')
Open your browser and go to http://localhost:5000/webhook and you should see a response.

Unicode-objects must be encoded before hashing Python 3.6 Vuforia

I'm trying to get my targets from vuforia's API, but I can't pass the last value of the header "Authorization" which is an encoded data, the error that I'm getting is this:
Unicode-objects must be encoded before hashing
this is in try snippet of the code, I'm following the vuforia's documentation but still, something is wrong with my code and I don't have a clue what it is
import base64
import hashlib
import hmac
import requests
from flask import Flask, request
from email.utils import formatdate
import logging
app = Flask(__name__)
#app.route('/')
def hello_world():
try:
import http.client as http_client
except ImportError:
# Python 2
import httplib as http_client
http_client.HTTPConnection.debuglevel = 1
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
requests_log = logging.getLogger("requests.packages.urllib3")
requests_log.setLevel(logging.DEBUG)
requests_log.propagate = True
url = 'https://vws.vuforia.com/targets'
req = requests.Request('GET', url)
req.headers = setHeaders(req)
resp = requests.Session().send(req.prepare())
return resp.text
def compute_md5_hex(data):
"""Return the hex MD5 of the data"""
h = hashlib.md5()
h.update(data)
return h.hexdigest()
def compute_hmac_base64(key, data):
"""Return the Base64 encoded HMAC-SHA1 using the provide key"""
h = hmac.new(key, None, hashlib.sha1)
h.update(data)
return base64.b64encode(h.digest())
def setHeaders(request):
date = formatdate(None, localtime=False, usegmt=True)
accessKey = "ce1500fhfth429279173fd839f9d414532014a3da"
secret_key = b"5d3fdawd7211447c35be607ae5a08ec794a09d71d"
headers = {'Date': date, 'Authorization': "VWS " + accessKey + ":" + tmsSignature(request, secret_key)}
return headers
def tmsSignature(request, secretKey):
method = request.method
contentType = ""
hexDigest = "d41d8cd98f00b204e9800998ecf8427e"
if method == "GET" or method == "POST":
pass
# Do nothing because the strings are already set correctly
elif method == "POST" or method == "PUT":
contentType = "application/json"
# If this is a POST or PUT the request should have a request body
hexDigest = compute_md5_hex(request)
else:
print("ERROR: Invalid content type passed to Sig Builder")
# Date in the header and date used to calculate the hash must be the same
dateValue = formatdate(None, localtime=False, usegmt=True)
requestPath = str(request.url)
components_to_sign = list()
components_to_sign.append(method)
components_to_sign.append(str(hexDigest))
components_to_sign.append(str(contentType))
components_to_sign.append(str(dateValue))
components_to_sign.append(str(requestPath))
string_to_sign = "\n".join(components_to_sign)
shaHashed = ""
try:
shaHashed = compute_hmac_base64(secretKey, string_to_sign)
except Exception as e:
print("ERROR ", e)
return shaHashed
if __name__ == '__main__':
app.run()

Looking into your hmac_base64_key function, this particular call is the cause:
h.update(data)
Since that is the update function from the hmac library, that requires the input to be byte instead of string/unicode (check out the documentation on hmac which refers to hashlib for its update signature).
So it seems like the fix is simply:
h.update(data.encode("utf8")) # or other encoding you want to use
Note that you'll need to change the return value of compute_hmac_base64 (shaHashed) to string again since you're concatenating it with a string in setHeaders.
(I'm assuming a Python 3 code even though you have a check for Python 2 in your code by the way, since you've tagged this Python 3).