We Keep Coding

Python unit test coverage using pytest

I am new to pytest and wanted to add the below 3 methods for unit test coverage without actually using a real mongo db instance but rather mock it.
Could try using a real db instance but it isn't recommended.
Request for an example on how to mock mongodb client and get a document
import os
import logging
import urllib.parse
from dotenv import load_dotenv
from pymongo import MongoClient
from logger import *
load_dotenv()
def getMongoConnection():
userName = urllib.parse.quote_plus(os.getenv("USER_NAME"))
password = urllib.parse.quote_plus(os.getenv("PASSWORD"))
hostName1_port = os.getenv("HOST_NAME1")
hostName2_port = os.getenv("HOST_NAME2")
hostName3_port = os.getenv("HOST_NAME3")
authSourceDatabase = os.getenv("AUTH_SOURCE_DATABASE")
replicaSet = os.getenv("REPLICA_SET")
connectTimeoutMS = "1000"
socketTimeoutMS = "30000"
maxPoolSize = "100"
try:
client = MongoClient('mongodb://'+userName+':'+password+'#'+hostName1_port+','+hostName2_port+','+hostName3_port+'/'+authSourceDatabase+'?ssl=true&replicaSet='+replicaSet +
'&authSource='+authSourceDatabase+'&retryWrites=true&w=majority&connectTimeoutMS='+connectTimeoutMS+'&socketTimeoutMS='+socketTimeoutMS+'&maxPoolSize='+maxPoolSize)
return client
except Exception as e:
logging.error("Error while connecting to mongoDB.")
return False
def connectToDBCollection(client, databaseName, collectionName):
db = client[databaseName]
collection = db[collectionName]
return collection
def getDoc(bucketName, databaseName, collectionName):
try:
client = getMongoConnection()
if client != False:
collection = connectToDBCollection(
client, databaseName, collectionName)
return collection.find_one({'bucket': bucketName})
except Exception as e:
logging.error("An exception occurred while fetching doc, error is ", e)
Edit : (Tried using below code and was able to cover most of the cases but seeing an error)
def test_mongo():
db_conn = mongomock.MongoClient()
assert isinstance(getMongoConnection(), MongoClient)
def test_connect_mongo():
return connectToDBCollection(mongomock.MongoClient(), "sampleDB", "sampleCollection")
//trying to cover exception block for getMongoConnection()
def test_exception():
with pytest.raises(Exception) as excinfo:
getMongoConnection()
assert str(excinfo.value) == False
def test_getDoc():
collection = mongomock.MongoClient().db.collection
stored_obj = collection.find_one({'_id': 1})
assert stored_obj == getDoc("bucket", "db", "collection")
def test_createDoc():
collection = mongomock.MongoClient().db.collection
stored_obj = collection.insert_one({'_id': 1})
assert stored_obj == createDoc("bucket", "db", "collection")
def test_updateDoc():
collection = mongomock.MongoClient().db.collection
stored_obj = collection.replace_one({'_id': 1}, {'_id': 2})
assert stored_obj == updateDoc(
{'_id': 1}, {'$set': {'_id': 2}}, "db", "collection")
Errors :
test_exception - Failed: DID NOT RAISE <class 'Exception'>
test_createDoc - TypeError: not all arguments converted during string formatting
AssertionError: assert <pymongo.results.UpdateResult object at 0x7fc0e835a400> == <pymongo.results.UpdateResult object at 0x7fc0e8211900>

Looks like MongoClient is a nested dict with databaseName and collectionName or implemented with a key accessor.
You could mock the client first with
import unittest
mocked_collection = unittest.mock.MagicMock()
# mock the find_one method
mocked_collection.find_one.return_value = {'data': 'collection_find_one_result'}
mocked_client = unittest.mock.patch('pymongo.MongoClient').start()
mocked_client.return_value = {
'databaseName': {'collectionname': mocked_collection}
}

Maybe try a specialized mocking library like MongoMock?
In particular the last example using #mongomock.patch looks like it can be relevant for your code.

Unable to import module - Lambda handler Error

When I am trying to run my python code in lambda passing the handler to the function.module getting the below error, any suggestions how i could resolve this?
the below file test_client_visitor is triggered to call the client_visitor and send an email to the clients accordingly, when i run thd python file test_client_visitor in my local i get the email triggered successfully but in lambda facing the issue.
file_name: test_client_visitor
function = __import__('client_visitor')
handler = function.scan_clients
class TestFunction(unittest.TestCase):
def test_function(self):
file = open('event.json', 'rb')
try:
ba = bytearray(file.read())
event = jsonpickle.decode(ba)
print('## EVENT')
print(jsonpickle.encode(event))
context = {'requestid': '1234'}
result = handler(event, context)
print(result)
self.assertTrue(result, 'Emails could not be sent!')
finally:
file.close()
file.close()
if __name__ == '__main__':
unittest.main()
file_name: client_visitor.py
import datetime
import boto3
from aws_ses import send_bulk_templated_email
# boto3.set_stream_logger('botocore', level='DEBUG')
from mongodb import get_mongo_db
def process_clients(clients, developers, clients_to_be_notified, days):
if not clients:
pass
check_date = datetime.datetime.now() + datetime.timedelta(days)
for client in clients:
client_id_ = client['client_id']
if 'developer_id' in client:
developers[client_id_] = client['developer_id']
else:
if 'secrets' in client:
secrets = client['secrets']
for secret in secrets:
if 'not_on_or_after' in secret and secret['not_on_or_after'] < check_date.timestamp():
clients_to_be_notified.append({'client_id': client_id_,
'expiration_date': datetime.datetime.fromtimestamp(
secret['not_on_or_after']).strftime('%m/%d/%Y')})
print("adding client to notify List", client_id_, ":", client['sort'])
def notify_clients(clients_to_be_notified, developers):
developer_id_list = []
for client_secret in clients_to_be_notified:
developer_id_list.append(developers[client_secret['client_id']])
if developer_id_list:
db = get_mongo_db()
if db:
users = list(db.users.find({'guid': {'$in': developer_id_list}}, {'email', 'guid'}))
need_to_send_email = False
for user in users:
for client_secret in clients_to_be_notified:
if developers[client_secret['client_id']] == user['guid']:
client_secret['email'] = user['email']
need_to_send_email = True
break
if need_to_send_email:
return send_bulk_templated_email(clients_to_be_notified)
else:
return False
return True
def scan_clients(event, context):
local = False
if 'local' in event:
local = event['local'] == 'True'
if local:
dynamodb = boto3.resource('dynamodb', endpoint_url="http://localhost:8000")
else:
dynamodb = boto3.resource('dynamodb')
days = 30
if 'days' in event:
days = int(event['days'])
print(f"Scanning Clients with {days} or less to secret expiration")
table = dynamodb.Table('****')
scan_kwargs = {
'ProjectionExpression': 'client_id, sort, developer_id, secrets, approved'
}
test = False
if 'test' in event:
test = event['test'] == 'True'
done = False
start_key = None
developers = {}
clients_to_be_notified = []
if test:
developers['idm-portal1'] = '***'
clients_to_be_notified = [{'client_id': 'idm-portal1', 'expiration_date': '04/17/2021'}]
while not done:
if start_key:
scan_kwargs['ExclusiveStartKey'] = start_key
response = table.scan(**scan_kwargs)
process_clients(response.get('Items', []), developers, clients_to_be_notified, days)
start_key = response.get('LastEvaluatedKey', None)
done = start_key is None
print("total developers ", len(developers), " total clients_to_be_notified ", len(clients_to_be_notified))
return notify_clients(clients_to_be_notified, developers)
if __name__ == '__main__':
scan_clients(event={'days': 30, 'local': False, 'test': True}, context=None)
Response
{
"errorMessage": "Unable to import module 'test_client_visitor': No module named 'test_client_visitor'",
"errorType": "Runtime.ImportModuleError",
"stackTrace": []
}

Your file must be named test_client_visitor.py. The way lambda runs the code is by trying to import the main file and call the handler function. See the AWS docs to set up a handler for Python.
The reason you didn't run into this issue locally is because I assume you are calling python directly on the command line — python test_client_visitor. When you import a module in Python, the file has to end in the .py extension.

Able to fix this issue with right packaging of the contents to zip, avoided the creation of extra folder with the below command.
Command:
cd folder; zip -r ../filename.zip *
Thankyou everyone for your inputs.

Python access the Login directly without using With Function as Key :

Here is the complete code where i am trying to use ForexConnect().get_history(.... instead of fx.get_history( and i do not want this line of code "with ForexConnect() as fx:" how to achieve this ,the last section of code given produces excpetion issues .
Why i do not want to use "with ForexConnect() as fx:" The reason is once the session "with ForexConnect() as fx:" the function is logged out .My idea is to be in the session after once logged in .So i do not want to try this with "with ForexConnect() as fx:"
import argparse
import pandas as pd
from forexconnect import ForexConnect, fxcorepy
import common_samples
parser = False
def parse_args():
parser = argparse.ArgumentParser(description='Process command parameters.')
common_samples.add_main_arguments(parser)
common_samples.add_instrument_timeframe_arguments(parser)
common_samples.add_date_arguments(parser)
common_samples.add_max_bars_arguments(parser)
args = parser.parse_args()
return args
def main():
if parser == False :
#args = parse_args()
str_user_id = 'Dxxxx'
str_password = 'xxxxx'
str_url = "http://www.fxcorporate.com/Hosts.jsp"
str_connection = 'Demo'
str_session_id = 'None'
str_pin = 'None'
str_instrument = 'USOil'
str_timeframe = 'W1'
quotes_count = 5
date_from = None
date_to = None
else :
args = parse_args()
str_user_id = args.l
str_password = args.p
str_url = args.u
str_connection = args.c
str_session_id = args.session
str_pin = args.pin
str_instrument = args.i
str_timeframe = args.timeframe
quotes_count = args.quotescount
date_from = args.datefrom
date_to = args.dateto
with ForexConnect() as fx:
try:
fx.login(str_user_id, str_password, str_url,
str_connection, str_session_id, str_pin,
common_samples.session_status_changed)
print("")
print("Requesting a price history...")
print(str_instrument,str_timeframe,date_from,date_to,quotes_count)
history = fx.get_history(str_instrument, str_timeframe, date_from, date_to, quotes_count)
current_unit, _ = ForexConnect.parse_timeframe(str_timeframe)
date_format = '%d.%m.%Y %H:%M:%S'
print("print history ",history)
df = pd.DataFrame(history, columns=['Date', 'BidOpen', 'BidHigh','BidLow', 'BidClose', 'AskOpen', 'AskHigh', 'AskLow', 'AskClose', 'Volume'])
print(df)
if current_unit == fxcorepy.O2GTimeFrameUnit.TICK:
print("Date, Bid, Ask")
print(history.dtype.names)
for row in history:
print("{0:s}, {1:,.5f}, {2:,.5f}".format(
pd.to_datetime(str(row['Date'])).strftime(date_format), row['Bid'], row['Ask']))
else:
print("Date, BidOpen, BidHigh, BidLow, BidClose, Volume")
for row in history:
print("{0:s}, {1:,.5f}, {2:,.5f}, {3:,.5f}, {4:,.5f}, {5:d}".format(
pd.to_datetime(str(row['Date'])).strftime(date_format), row['BidOpen'], row['BidHigh'],
row['BidLow'], row['BidClose'], row['Volume']))
except Exception as e:
common_samples.print_exception(e)
try:
fx.logout()
except Exception as e:
common_samples.print_exception(e)
if __name__ == "__main__":
main()
print("")
input("Done! Press enter key to exit\n")
Here i want to login once and be in the logged in session forever.
With the below function this is working fine .But here the problem is once the With section is over the session is disconnected.
with ForexConnect() as fx:
try:
fx.login(str_user_id, str_password, str_url,
str_connection, str_session_id, str_pin,
common_samples.session_status_changed)
history = fx.get_history(str_instrument, str_timeframe, date_from, date_to, quotes_count)
current_unit, _ = ForexConnect.parse_timeframe(str_timeframe)
To stay in the session tried the below code without using the "With" and as :
Here the login is successful but could not get the data in history = ForexConnect().get_history
Error Code :
ForexConnect().login(str_user_id, str_password, str_url,
str_connection, str_session_id, str_pin,
common_samples.session_status_changed)
**history = ForexConnect().get_history(str_instrument, str_timeframe, date_from, date_to, quotes_count)**
How to make it the ** ** code work without any exception error without using With --- as : keyworkds.
what is the alternative for this with and as : why when i try to access as history = ForexConnect().get_history ( ... it is giving error how to overcome this issue.

requests.session() object not recognized in another class

I am trying to pass my session object from one class to another. But I am not sure whats happening.
class CreateSession:
def __init__(self, user, pwd, url="http://url_to_hit"):
self.url = url
self.user = user
self.pwd = pwd
def get_session(self):
sess = requests.Session()
r = sess.get(self.url + "/", auth=(self.user, self.pwd))
print(r.content)
return sess
class TestGet(CreateSession):
def get_response(self):
s = self.get_session()
print(s)
data = s.get(self.url + '/some-get')
print(data.status_code)
print(data)
if __name__ == "__main__":
TestGet(user='user', pwd='pwd').get_response()
I am getting 401 for get_response(). Not able to understand this.

What's a 401?
The response you're getting means that you're unauthorised to access the resource.
A session is used in order to persist headers and other prerequisites throughout requests, why are you creating the session every time rather than storing it in a variable?
As is, the session should work the only issue is that you're trying to call a resource that you don't have access to. - You're not passing the url parameter either in the initialisation.
Example of how you can effectively use Session:
from requests import Session
from requests.exceptions import HTTPError
class TestGet:
__session = None
__username = None
__password = None
def __init__(self, username, password):
self.__username = username
self.__password = password
#property
def session(self):
if self.__session is None:
self.__session = Session()
self.__session.auth = (self.__user, self.__pwd)
return self.__session
#session.setter
def session(self, value):
raise AttributeError('Setting \'session\' attribute is prohibited.')
def get_response(self, url):
try:
response = self.session.get(url)
# raises if the status code is an error - 4xx, 5xx
response.raise_for_status()
return response
except HTTPError as e:
# you received an http error .. handle it here (e contains the request and response)
pass
test_get = TestGet('my_user', 'my_pass')
first_response = test_get.get_response('http://your-website-with-basic-auth.com')
second_response = test_get.get_response('http://another-url.com')
my_session = test_get.session
my_session.get('http://url.com')

Function not getting executed in python

I have 2 functions(recharge_list and sms_list) in my below Server() class
import os
import json
import requests
import cherrypy
import ConfigParser
from bs4 import BeautifulSoup
class Server():
#cherrypy.expose
def index(self):
return "Seems Like You're Lost :D"
#cherrypy.expose
def recharge_list(self,carrier, state):
details_array=[]
small_details_array=[]
price_cell_array=[]
lst = []
url = "link{}/{}".format(carrier,state)
try:
if self.t_arr.get(url) is not None:
return json.dumps({'data': self.t_arr[url]})
except AttributeError:
self.t_arr = {}
r = requests.get(url)
data = r.text
soup = BeautifulSoup(data,"html.parser")
table = soup.find('table',{'class':'table'})
s=""
detailtext = table.findAll('div',{'class':'detailtext'})
for det in detailtext:
details_array.append(det.text)
smalldetails = table.findAll('div',{'style':'padding-top:5px'})
for smallDet in smalldetails:
small_details_array.append(smallDet.text);
price_cells = table.findAll('td', {'class': 'pricecell'})
for price_cell in price_cells:
price_cell_array.append(price_cell.text)
for i in range(len(details_array)):
d_arr = {}
d_arr['detail']=details_array[i]
temp = small_details_array[i].split('\n')
d_arr['talktime'] = temp[1]
d_arr['keyword']=temp[3]
tempnew = price_cell_array[i].split('\n')
d_arr['price'] = tempnew[1]
d_arr['validity'] = tempnew[3]
# global list
lst.append(d_arr)
self.t_arr[url] = lst
return json.dumps({'data': self.t_arr[url]})
#cherrypy.expose
def sms_list(self,carrier, state):
details_array=[]
price_cell_array=[]
lst = []
url = "link/{}/{}".format(carrier,state)
try:
if self.t_arr.get(url) is not None:
return json.dumps({'data': self.t_arr[url]})
except AttributeError:
self.t_arr = {}
r = requests.get(url)
data = r.text
soup = BeautifulSoup(data,"html.parser")
table = soup.find('div',{'id':'SMS'})
table2 = table.find('table',{'class':'table'})
print(table2)
s=""
detailtext = table2.findAll('div',{'class':'detailtext'})
for det in detailtext:
details_array.append(det.text)
smalldetails = table2.findAll('div',{'style':'padding-top:5px'})
price_cells = table.findAll('td', {'class': 'pricecell'})
for price_cell in price_cells:
price_cell_array.append(price_cell.text)
for i in range(len(details_array)):
d_arr = {}
d_arr['detail']=details_array[i]
tempnew = price_cell_array[i].split('\n')
d_arr['price'] = tempnew[1]
d_arr['validity'] = tempnew[3]
# global list
lst.append(d_arr)
self.t_arr[url] = lst
return json.dumps({'data': self.t_arr[url]})
if __name__ == '__main__':
''' Setting up the Server with Specified Configuration'''
cherrypy.config.update({'server.socket_host': '0.0.0.0',})
cherrypy.config.update({'server.socket_port': int(os.environ.get('PORT', '5000')),})
cherrypy.quickstart(Server())
The problem is, when I run my server with recharge_list it works, but then I have to terminate my server from terminal and re-start the server to execute the sms_list function.
By my understanding the object once created by Server class is able to execute only the first called function.
What should I edit in my code such that I can execute the functions without terminating the server.

By my understanding the object once created by Server class is able to execute only the first called function.
This is not so. Each time an HTTP request is provided, the web server calls the function associated to the URL of that request.
What should I edit in my code such that I can execute the functions without terminating the server.
In sms_list (and not in recharge_list), replace every instance of t_arr with t_sms_arr.