I have created a custom class, which push my logs to splunk, but somehow it is not working. Here is the class.
class Splunk(logging.StreamHandler):
def __init__(self, url, token):
super().__init__()
self.url = url
self.headers = {f'Authorization': f'Splunk {token}'}
self.propagate = False
def emit(self, record):
mydata = dict()
mydata['sourcetype'] = 'mysourcetype'
mydata['event'] = record.__dict__
response = requests.post(self.url, data=json.dumps(mydata), headers=self.headers)
return response
I call the class from my logger class, somehow like this (adding additional handler), so that it can log on console along with send to splunk
if splunk_config is not None:
splunk_handler = Splunk(splunk_config["url"], splunk_config["token"])
self.default_logger.addHandler(splunk_handler)
But somehow, I am not able to see any logs in splunk. Though I can see the logs in console.
When I try to run the strip down version of above logic from python3 terminal, it is successful.
import requests
import json
url = 'myurl'
token = 'mytoken'
headers = {'Authorization': 'Splunk mytoken'}
propagate = False
mydata = dict()
mydata['sourcetype'] = 'mysourcetype'
mydata['event'] = {'name': 'root', 'msg': 'this is a sample message'}
response = requests.post(url, data=json.dumps(mydata), headers=headers)
print(response.text)
Things I have already tried, making my dictionary data as JSON serializable using below link but it didn't helped.
https://pynative.com/make-python-class-json-serializable/
Any other things to try ?
I've successfully used this Python Class for Sending Events to Splunk HTTP Event Collector instead of writing a dedicated class
https://github.com/georgestarcher/Splunk-Class-httpevent
Advantage is that it implements batchEvent() and flushBatch() methods to submit multiple events at once across multiple threads.
The example here should get you started:
https://github.com/georgestarcher/Splunk-Class-httpevent/blob/master/example.py
If this answers your question, take a moment to accept the answer. This can be done by clicking on the check mark beside the answer to toggle it from greyed out to filled in!
Related
I'm trying to work with a third party API and I am having problems with sending the request when using the requests or even urllib.request.
Somehow when I use http.client I am successful sending and receiving the response I need.
To make life easier for me, I created an API class below:
class API:
def get_response_data(self, response: http.client.HTTPResponse) -> dict:
"""Get the response data."""
response_body = response.read()
response_data = json.loads(response_body.decode("utf-8"))
return response_data
The way I use it is like this:
api = API()
rest_api_host = "api.app.com"
connection = http.client.HTTPSConnection(rest_api_host)
token = "my_third_party_token"
data = {
"token":token
}
payload = json.loads(data)
headers = {
# some headers
}
connection.request("POST", "/some/endpoint/", payload, headers)
response = connection.getresponse()
response_data = api.get_response_data(response) # I get a dictionary response
This workflow works for me. Now I just want to write a test for the get_response_data method.
How do I instantiate a http.client.HTTPResponse with the desired output to be tested?
For example:
from . import API
from unittest import TestCase
class APITestCase(TestCase):
"""API test case."""
def setUp(self) -> None:
super().setUp()
api = API()
def test_get_response_data_returns_expected_response_data(self) -> None:
"""get_response_data() method returns expected response data in http.client.HTTPResponse"""
expected_response_data = {"token": "a_secret_token"}
# I want to do something like this
response = http.client.HTTPResponse(expected_response_data)
self.assertEqual(api.get_response_data(response), expected_response_data)
How can I do this?
From the http.client docs it says:
class http.client.HTTPResponse(sock, debuglevel=0, method=None, url=None)
Class whose instances are returned upon successful connection. Not instantiated directly by user.
I tried looking at socket for the sock argument in the instantiation but honestly, I don't understand it.
I tried reading the docs in
https://docs.python.org/3/library/http.client.html#http.client.HTTPResponse
https://docs.python.org/3/library/socket.html
Searched the internet on "how to test http.client.HTTPResponse" but I haven't found the answer I was looking for.
For example with TestCase
I can check login post and so on with self.client
class TestMyProj(TestCase):
response = self.client.login(username="user#example.com", password="qwpo1209")
response = self.client.post('/cms/content/up',
{'name': 'test', '_content_file': fp},
follow=True)
However now I want to use this in script not in test case.
because this is very useful to make initial database.
I want to do like this.
def run():
response = client.login(username="user#example.com", password="qwpo1209")
with open('_material/content.xlsx','rb') as fp:
response = client.post('/cms/content/up',
{'name': 'faq_test', 'scenario_content_file': fp})
What is equivalent to TestCase.client in normal script??
More details
If there is not file upload, I can make database directory from model.
However, I want to upload file and parse then put into database, same as user does. (via form_valid and so on)
so, I want to use post for url from script.
My Solution
Use from django.test.client import Client as Willem Van Onsem mensioned.
somehow client.login returns True not response.
So, I use post to login
def run():
client = Client()
#response = client.login(username="guest#guest.com", password="guest")# it doesn't work some how.
response = client.post('/login/', {'username': 'guest#guest.com', 'password': 'guest'},follow=True)
with open('_material/content.xlsx','rb') as fp:
response = client.post('/cms/content/up',
{'name': 'test','is_all':"True", '_content_file': fp})
It is a Client object [Django-doc], so:
from django.test.client import Client
def run():
client = Client()
response = client.login(username="user#example.com", password="qwpo1209")
response = client.post(
'/cms/content/up',
{'name': 'test', '_content_file': fp},
follow=True
)
The documentation discusses the parameters that can be passed when constructing a Client object.
Generally I recommend to create command to make initial database instead of using the Client class that we can use to test the application.
https://docs.djangoproject.com/en/4.0/howto/custom-management-commands/
You can also take advantage of bulk methods to create many instances in a single query.
I am looking into to moving my multi-threaded python script to locust.
A simple explanation of what my script does is:
Create a thread per user
In each thread authenticates user and get auth cookie
With that auth cookie perform various api calls at a set interval
When i started looking into locust, I have noticed that the only way to perform each task at its own specific interval, I would need to create a taskset per task.
This brought up an issue of how do i share the auth cookie for the given spawned user between task sets? Since in the long run I also need to share response data between taskset for the given spawned user as it differs between spawned users.
In the sample code below, all of the users spawned by locust, share the same "storage.cookie". Is there a way to keep storage.cookie unique per user, share it with all tasks sets for the given spawned user by locust ? Does locust report on which user is currently executing the task?
from __future__ import print_function
from locust import Locust, TaskSet, task, HttpLocust
import json
def auth(l):
payload = {"username":"some_username","password":"some_password"}
resp = l.client.post('/auth', data = json.dumps(payload))
storage.cookie = # get auth cookie from resp
def do_i_auth(l):
if len(storage.cookie) == 0:
auth(l)
class storage(object):
cookie == ''
class first_call(TaskSet):
def on_start(self):
do_i_auth(self)
#task
def get_api_a(self):
headers = {"Cookie":storage.cookie}
self.client.get('/api_a', headers)
class second_call(TaskSet):
def on_start(self):
do_i_auth(self)
#task
def get_api_b(self):
headers = {"Cookie":storage.cookie}
self.client.get('/api_b', headers)
class api_A(HttpLocust):
task_set = first_call
min_wait = 5000
max_wait = 5000
class api_B(HttpLocust):
task_set = second_call
min_wait = 10000
max_wait = 10000
You can try have your authorization function return the cookie and have it stored in each class separately. Something like this:
from __future__ import print_function
from locust import Locust, TaskSet, task, HttpLocust
import json
def auth(l):
payload = {"username":"some_username","password":"some_password"}
resp = l.client.post('/auth', data = json.dumps(payload))
cookie = # get auth cookie from resp
return cookie
class first_call(TaskSet):
cookie = ""
def on_start(self):
self.cookie = auth(self)
#task
def get_api_a(self):
headers = {"Cookie":self.cookie}
self.client.get('/api_a', headers)
I think the solution here is to not have separate classes for each call, but instead to have the calls as methods on a single class. That way you can store the cookies on the objects (referenced via self.cookie).
This worked for me:
https://gist.github.com/MatrixManAtYrService/1d83abd54adc9d4181f9ebb98b9799f7
I recently implemented cookies in a DotNet application load test script.
Cookies should be passed using dictionary objects.
cookiedict={}
cookiedict['Key1'] = 'Value1'
cookiedict['Key2'] = 'Value2'
#Auth API
self.auth_response = self.gettoken(cookiedict)
self.token = self.auth_response.cookies['RequestVerificationToken']
self.cookies = self.auth_response.cookies
#Login API
cookiedict['RequestVerificationToken'] = self.token
`
self.login_response=self.login(self.user_name,self.password,self.token,cookiedict)
Also note that you need to use HttpSession as well
from locust.clients import HttpSession
self.requests = HttpSession(consumer_cfg.rest_api_url)
executor = self.requests.post
if method == 'PUT':
executor = self.requests.put
elif method == 'GET':
executor = self.requests.get
self._request_proceed(method='GET', url=url, data=formdata,catch_response=catch_response, cookies = CookiesSent,allow_redirects = True)
I've got a piece of code that I can't figure out how to unit test! The module pulls content from external XML feeds (twitter, flickr, youtube, etc.) with urllib2. Here's some pseudo-code for it:
params = (url, urlencode(data),) if data else (url,)
req = Request(*params)
response = urlopen(req)
#check headers, content-length, etc...
#parse the response XML with lxml...
My first thought was to pickle the response and load it for testing, but apparently urllib's response object is unserializable (it raises an exception).
Just saving the XML from the response body isn't ideal, because my code uses the header information too. It's designed to act on a response object.
And of course, relying on an external source for data in a unit test is a horrible idea.
So how do I write a unit test for this?
urllib2 has a functions called build_opener() and install_opener() which you should use to mock the behaviour of urlopen()
import urllib2
from StringIO import StringIO
def mock_response(req):
if req.get_full_url() == "http://example.com":
resp = urllib2.addinfourl(StringIO("mock file"), "mock message", req.get_full_url())
resp.code = 200
resp.msg = "OK"
return resp
class MyHTTPHandler(urllib2.HTTPHandler):
def http_open(self, req):
print "mock opener"
return mock_response(req)
my_opener = urllib2.build_opener(MyHTTPHandler)
urllib2.install_opener(my_opener)
response=urllib2.urlopen("http://example.com")
print response.read()
print response.code
print response.msg
It would be best if you could write a mock urlopen (and possibly Request) which provides the minimum required interface to behave like urllib2's version. You'd then need to have your function/method which uses it able to accept this mock urlopen somehow, and use urllib2.urlopen otherwise.
This is a fair amount of work, but worthwhile. Remember that python is very friendly to ducktyping, so you just need to provide some semblance of the response object's properties to mock it.
For example:
class MockResponse(object):
def __init__(self, resp_data, code=200, msg='OK'):
self.resp_data = resp_data
self.code = code
self.msg = msg
self.headers = {'content-type': 'text/xml; charset=utf-8'}
def read(self):
return self.resp_data
def getcode(self):
return self.code
# Define other members and properties you want
def mock_urlopen(request):
return MockResponse(r'<xml document>')
Granted, some of these are difficult to mock, because for example I believe the normal "headers" is an HTTPMessage which implements fun stuff like case-insensitive header names. But, you might be able to simply construct an HTTPMessage with your response data.
Build a separate class or module responsible for communicating with your external feeds.
Make this class able to be a test double. You're using python, so you're pretty golden there; if you were using C#, I'd suggest either in interface or virtual methods.
In your unit test, insert a test double of the external feed class. Test that your code uses the class correctly, assuming that the class does the work of communicating with your external resources correctly. Have your test double return fake data rather than live data; test various combinations of the data and of course the possible exceptions urllib2 could throw.
Aand... that's it.
You can't effectively automate unit tests that rely on external sources, so you're best off not doing it. Run an occasional integration test on your communication module, but don't include those tests as part of your automated tests.
Edit:
Just a note on the difference between my answer and #Crast's answer. Both are essentially correct, but they involve different approaches. In Crast's approach, you use a test double on the library itself. In my approach, you abstract the use of the library away into a separate module and test double that module.
Which approach you use is entirely subjective; there's no "correct" answer there. I prefer my approach because it allows me to build more modular, flexible code, something I value. But it comes at a cost in terms of additional code to write, something that may not be valued in many agile situations.
You can use pymox to mock the behavior of anything and everything in the urllib2 (or any other) package. It's 2010, you shouldn't be writing your own mock classes.
I think the easiest thing to do is to actually create a simple web server in your unit test. When you start the test, create a new thread that listens on some arbitrary port and when a client connects just returns a known set of headers and XML, then terminates.
I can elaborate if you need more info.
Here's some code:
import threading, SocketServer, time
# a request handler
class SimpleRequestHandler(SocketServer.BaseRequestHandler):
def handle(self):
data = self.request.recv(102400) # token receive
senddata = file(self.server.datafile).read() # read data from unit test file
self.request.send(senddata)
time.sleep(0.1) # make sure it finishes receiving request before closing
self.request.close()
def serve_data(datafile):
server = SocketServer.TCPServer(('127.0.0.1', 12345), SimpleRequestHandler)
server.datafile = datafile
http_server_thread = threading.Thread(target=server.handle_request())
To run your unit test, call serve_data() then call your code that requests a URL that looks like http://localhost:12345/anythingyouwant.
Why not just mock a website that returns the response you expect? then start the server in a thread in setup and kill it in the teardown. I ended up doing this for testing code that would send email by mocking an smtp server and it works great. Surely something more trivial could be done for http...
from smtpd import SMTPServer
from time import sleep
import asyncore
SMTP_PORT = 6544
class MockSMTPServer(SMTPServer):
def __init__(self, localaddr, remoteaddr, cb = None):
self.cb = cb
SMTPServer.__init__(self, localaddr, remoteaddr)
def process_message(self, peer, mailfrom, rcpttos, data):
print (peer, mailfrom, rcpttos, data)
if self.cb:
self.cb(peer, mailfrom, rcpttos, data)
self.close()
def start_smtp(cb, port=SMTP_PORT):
def smtp_thread():
_smtp = MockSMTPServer(("127.0.0.1", port), (None, 0), cb)
asyncore.loop()
return Thread(None, smtp_thread)
def test_stuff():
#.......snip noise
email_result = None
def email_back(*args):
email_result = args
t = start_smtp(email_back)
t.start()
sleep(1)
res.form["email"]= self.admin_email
res = res.form.submit()
assert res.status_int == 302,"should've redirected"
sleep(1)
assert email_result is not None, "didn't get an email"
Trying to improve a bit on #john-la-rooy answer, I've made a small class allowing simple mocking for unit tests
Should work with python 2 and 3
try:
import urllib.request as urllib
except ImportError:
import urllib2 as urllib
from io import BytesIO
class MockHTTPHandler(urllib.HTTPHandler):
def mock_response(self, req):
url = req.get_full_url()
print("incomming request:", url)
if url.endswith('.json'):
resdata = b'[{"hello": "world"}]'
headers = {'Content-Type': 'application/json'}
resp = urllib.addinfourl(BytesIO(resdata), header, url, 200)
resp.msg = "OK"
return resp
raise RuntimeError('Unhandled URL', url)
http_open = mock_response
#classmethod
def install(cls):
previous = urllib._opener
urllib.install_opener(urllib.build_opener(cls))
return previous
#classmethod
def remove(cls, previous=None):
urllib.install_opener(previous)
Used like this:
class TestOther(unittest.TestCase):
def setUp(self):
previous = MockHTTPHandler.install()
self.addCleanup(MockHTTPHandler.remove, previous)
Given that webtest doesn't seem to have a 3.x version (or any plans to develop one), are there any solutions for automated system testing of a WSGI application? I know unittest for unit testing - I'm more interested in the moment in whole systems tests.
I'm not looking for tools to help develop an application - just test it.
In case anyone else comes upon this, I ended up writing a solution myself. Here's a very simple class I use - I just inherit from WSGIBaseTest instead of TestCase, and get a method self.request() that I can pass requests into. It stores cookies, and will automatically send them into the application on later requests (until self.new_session() is called).
import unittest
from wsgiref import util
import io
class WSGIBaseTest(unittest.TestCase):
'''Base class for unit-tests. Provides up a simple interface to make requests
as though they came through a wsgi interface from a user.'''
def setUp(self):
'''Set up a fresh testing environment before each test.'''
self.cookies = []
def request(self, application, url, post_data = None):
'''Hand a request to the application as if sent by a client.
#param application: The callable wsgi application to test.
#param url: The URL to make the request against.
#param post_data: A string.'''
self.response_started = False
temp = io.StringIO(post)
environ = {
'PATH_INFO': url,
'REQUEST_METHOD': 'POST' if post_data else 'GET',
'CONTENT_LENGTH': len(post),
'wsgi.input': temp,
}
util.setup_testing_defaults(environ)
if self.cookies:
environ['HTTP_COOKIE'] = ';'.join(self.cookies)
self.response = ''
for ret in application(environ, self._start_response):
assert self.response_started
self.response += str(ret)
temp.close()
return response
def _start_response(self, status, headers):
'''A callback passed into the application, to simulate a wsgi
environment.
#param status: The response status of the application ("200", "404", etc)
#param headers: Any headers to begin the response with.
'''
assert not self.response_started
self.response_started = True
self.status = status
self.headers = headers
for header in headers:
# Parse out any cookies and save them to send with later requests.
if header[0] == 'Set-Cookie':
var = header[1].split(';', 1)
if len(var) > 1 and var[1][0:9] == ' Max-Age=':
if int(var[1][9:]) > 0:
# An approximation, since our cookies never expire unless
# explicitly deleted (by setting Max-Age=0).
self.cookies.append(var[0])
else:
index = self.cookies.index(var[0])
self.cookies.pop(index)
def new_session(self):
'''Start a new session (or pretend to be a different user) by deleting
all current cookies.'''
self.cookies = []