I am new to pytest and wanted to add the below 3 methods for unit test coverage without actually using a real mongo db instance but rather mock it.
Could try using a real db instance but it isn't recommended.
Request for an example on how to mock mongodb client and get a document
import os
import logging
import urllib.parse
from dotenv import load_dotenv
from pymongo import MongoClient
from logger import *
load_dotenv()
def getMongoConnection():
userName = urllib.parse.quote_plus(os.getenv("USER_NAME"))
password = urllib.parse.quote_plus(os.getenv("PASSWORD"))
hostName1_port = os.getenv("HOST_NAME1")
hostName2_port = os.getenv("HOST_NAME2")
hostName3_port = os.getenv("HOST_NAME3")
authSourceDatabase = os.getenv("AUTH_SOURCE_DATABASE")
replicaSet = os.getenv("REPLICA_SET")
connectTimeoutMS = "1000"
socketTimeoutMS = "30000"
maxPoolSize = "100"
try:
client = MongoClient('mongodb://'+userName+':'+password+'#'+hostName1_port+','+hostName2_port+','+hostName3_port+'/'+authSourceDatabase+'?ssl=true&replicaSet='+replicaSet +
'&authSource='+authSourceDatabase+'&retryWrites=true&w=majority&connectTimeoutMS='+connectTimeoutMS+'&socketTimeoutMS='+socketTimeoutMS+'&maxPoolSize='+maxPoolSize)
return client
except Exception as e:
logging.error("Error while connecting to mongoDB.")
return False
def connectToDBCollection(client, databaseName, collectionName):
db = client[databaseName]
collection = db[collectionName]
return collection
def getDoc(bucketName, databaseName, collectionName):
try:
client = getMongoConnection()
if client != False:
collection = connectToDBCollection(
client, databaseName, collectionName)
return collection.find_one({'bucket': bucketName})
except Exception as e:
logging.error("An exception occurred while fetching doc, error is ", e)
Edit : (Tried using below code and was able to cover most of the cases but seeing an error)
def test_mongo():
db_conn = mongomock.MongoClient()
assert isinstance(getMongoConnection(), MongoClient)
def test_connect_mongo():
return connectToDBCollection(mongomock.MongoClient(), "sampleDB", "sampleCollection")
//trying to cover exception block for getMongoConnection()
def test_exception():
with pytest.raises(Exception) as excinfo:
getMongoConnection()
assert str(excinfo.value) == False
def test_getDoc():
collection = mongomock.MongoClient().db.collection
stored_obj = collection.find_one({'_id': 1})
assert stored_obj == getDoc("bucket", "db", "collection")
def test_createDoc():
collection = mongomock.MongoClient().db.collection
stored_obj = collection.insert_one({'_id': 1})
assert stored_obj == createDoc("bucket", "db", "collection")
def test_updateDoc():
collection = mongomock.MongoClient().db.collection
stored_obj = collection.replace_one({'_id': 1}, {'_id': 2})
assert stored_obj == updateDoc(
{'_id': 1}, {'$set': {'_id': 2}}, "db", "collection")
Errors :
test_exception - Failed: DID NOT RAISE <class 'Exception'>
test_createDoc - TypeError: not all arguments converted during string formatting
AssertionError: assert <pymongo.results.UpdateResult object at 0x7fc0e835a400> == <pymongo.results.UpdateResult object at 0x7fc0e8211900>
Looks like MongoClient is a nested dict with databaseName and collectionName or implemented with a key accessor.
You could mock the client first with
import unittest
mocked_collection = unittest.mock.MagicMock()
# mock the find_one method
mocked_collection.find_one.return_value = {'data': 'collection_find_one_result'}
mocked_client = unittest.mock.patch('pymongo.MongoClient').start()
mocked_client.return_value = {
'databaseName': {'collectionname': mocked_collection}
}
Maybe try a specialized mocking library like MongoMock?
In particular the last example using #mongomock.patch looks like it can be relevant for your code.
I have a column in my dataframe that is of type suds.sax.text.Text and I want to convert it to a string. I can't find much on how to do this, except for this site. Using pandas.DataFrame.astype does not work. I'm sure there is an easy way to do this. The documentation is just going over my head. I am using a web service to use some of its functions to return some metadata on weather stations. This metadata gets returned back as a suds object. Link to the web service is here.
from suds.client import Client
from suds.transport.https import HttpAuthenticated
from urllib.error import URLError
from urllib.request import HTTPSHandler
import ssl
import pandas as pd
ssl._create_default_https_context = ssl._create_unverified_context
_URL_AWDB_WSDL = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
def _execute_awdb_call(a_func, ntries_max=3, sleep_sec=5, **kwargs):
ntries = 0
while 1:
try:
a_result = a_func(**kwargs)
break
except Exception as e:
ntries += 1
if ntries == ntries_max:
raise
else:
print(("WARNING: Received error executing AWDB function %s:"
" %s. Sleeping %d seconds and trying again." %
(str(a_func.method.name), str(e), sleep_sec)))
sleep(sleep_sec)
return a_result
def _stationMetadata_to_tuple(a_meta):
list_meta = [None] * len(_stnmeta_attrs)
for i, a_attr in enumerate(_stnmeta_attrs):
try:
list_meta[i] = a_meta[a_attr]
except AttributeError:
# Doesn't have attribute
continue
return tuple(list_meta)
try:
_client = Client(_URL_AWDB_WSDL)
_stnmeta_attrs = (_client.factory.
create('stationMetaData').__keylist__)
except URLError as e:
if type(e.reason) == ssl.SSLError:
print("Warning: SSL Error connecting to AWDB web service. Skipping verification...")
_client = Client(_URL_AWDB_WSDL, transport=_CustomTransport())
_stnmeta_attrs = (_client.factory.
create('stationMetaData').__keylist__)
else:
raise
stn_triplets = ['878:WY:SNTL', '1033:CO:SNTL']
stn_metas = _execute_awdb_call(_client.service.
getStationMetadataMultiple,
stationTriplets=stn_triplets)
stn_tups = [_stationMetadata_to_tuple(a) for a in stn_metas]
df_stns = pd.DataFrame(stn_tups, columns=_stnmeta_attrs)
stns = df_stns.rename(columns={'actonId': 'station_id',
'name': 'station_name'})
stns['station_id'] = stns.station_id.fillna(stns.stationTriplet)
stns = stns[~stns.station_id.isnull()]
print(type(stns.beginDate[0]))
I'm using openapi to define my API. I just added this security schema:
securitySchemes:
api_key:
type: apiKey
name: X-Auth
in: header
x-apikeyInfoFunc: apikey_auth
where apikey_auth is defined like this
def apikey_auth(token, required_scopes):
decrypted_token = None
try:
decrypted_token = mydecrypter.decrypt(token)
except InvalidToken:
raise OAuthProblem('Invalid token')
return {'decrypted_token': decrypted_token}
Now i'd like to use this authentication for my actual endpoints which are defined in openapi like this:
/myendpoint
get:
operationId: operation
#more stuff
security:
- api_key: []
When now calling myendpoint the authentication is being done and works as expected. What I would like to have now is the return value of apikey_auth being passed into the call of operation so i can access decrypted_token in operation like this:
def operation(decrypted_token):
data = get_data_for_token(decrypted_token)
return data
Does anyone have an idea if this is possible somehow without having an extra parameter in the endpoint defintion?
Solved. For whatever reasons with these changes it works:
def apikey_auth(token, required_scopes):
decrypted_token = None
try:
decrypted_token = mydecrypter.decrypt(token)
except InvalidToken:
raise OAuthProblem('Invalid token')
return {'sub': decrypted_token}
def operation(user):
data = get_data_for_token(user)
return data
I'm having difficulty in writing test cases for the ZabbixAPILayer class mentioned below. I'm not sure how I can mock the 'zabbix_conn_obj' there. Any help will be appreciated. Thanks!
file: externalapi/apilayer.py
from zabbix.api import ZabbixAPI
import json
import time
class ZabbixAPILayer(object):
def uptime(self,arg,zabbix_conn_obj):
try:
getUpdateItem = zabbix_conn_obj.do_request("item.get", {"host":arg})
lastclock=getUpdateItem['result'][37].get('lastclock')
lastclock=int(lastclock)
curclock=int(time.time())
check_val=curclock-lastclock
limit=60*1000
if check_val<limit:
lastval=getUpdateItem['result'][37].get('lastvalue')
return time.strftime("%H:%M:%S", time.gmtime(float(getUpdateItem['result'][37].get('lastvalue'))))
else:
return "-"
except:
return "NOT AVAILABLE"
.....
class APILayer(ZabbixAPILayer):
def __init__(self):
self.zabbix_conn_obj=ZabbixAPI(url=settings.ZABBIX_URL, user=settings.ZABBIX_USER, password=settings.ZABBIX_PWD)
def uptime(self,arg):
return super(APILayer,self).uptime(arg,self.zabbix_conn_obj)
.....
file: base/admin.py
......
from ..externalapis.apilayer import APILayer
......
gen_obj= APILayer()
gen_obj.uptime()
......
Thanks for your comments. Got this working! Here the way i'm doing this
import mock
...
def test_uptime(self):
zabbix_conn_obj = mock.Mock()
json_blob = {} # This is the json blob i'm using as return value from do request
zabbix_conn_obj.do_request = mock.Mock(return_value=json_blob)
obj = ZabbixAPILayer()
value = obj.uptime("TestHOST",zabbix_conn_obj)
desired_result = '' #whatever my desired value is
self.assertEqual(value, desired_result)
I trid the following code,
import pyodbc
try:
pyodbc.connect('DRIVER={%s};SERVER=%s;DATABASE=%s;UID=%s;PWD=%s' % (driver, server, database, uid, password))
except pyodbc.Error, err:
logging.warn(err)
The error message format i get is
('HY000', "[HY000] [MySQL][ODBC 5.1 Driver]Access denied for user 'root'#'192.168.2.27' (using password: YES) (1045) (SQLDriverConnect)")
I want to receive just the message part of the error i.e.
Access denied for user 'root'#'192.168.2.27'(using password: YES)
I dont know if I can catch errors specifically like, driver not found, host down etc..
I also tried catching errors as:
except pyodbc.OperationalError, err:
logging.warn(err)
except pyodbc.DataError, err:
logging.warn(err)
except pyodbc.IntegrityError, err:
logging.warn(err)
except pyodbc.ProgrammingError, err:
logging.warn(err)
except pyodbc.NotSupportedError, err:
logging.warn(err)
except pyodbc.DatabaseError, err:
logging.warn(err)
except pyodbc.Error, err:
logging.warn(err)
but the last one always catches the error.
Fruthermore i saw the pyodbc.Error.message is always empty.
How can i get just the message in the error.
Thanks
This worked for me.
try:
cnxn = pyodbc.connect(...)
except pyodbc.Error as ex:
sqlstate = ex.args[0]
if sqlstate == '28000':
print("LDAP Connection failed: check password")
There are different SQLSTATES and you can have if-else statements to print out the cause.
Similarly,
try:
cnxn = pyodbc.connect(...)
except pyodbc.Error as ex:
sqlstate = ex.args[1]
print(sqlstate)
will give you the second part of the error with description.
For exampleex.args[0] give you 28000 and ex.args[1] gives [28000] LDAP authentication failed for user 'user' (24) (SQLDriverConnect)
You can then use String manipulation techniques there to just print out what you want. Hope this helps.
pyodbc seems to just wrap the errors/exceptions from the underlying ODBC implementation, so it's unlikely that you will be able to do this.
It's been very long since op asked this question, but here goes a snippet of code to parse out pyodbc error messages into nice Python exceptions that can be used. This is also meant to be extended, I didn't handle every possible sqlserver error code.
import re
from enum import Enum, IntEnum, unique
class PyODBCError(Exception):
"""
Handle errors for PyODBC. Offers a error message parser
to apply specific logic depending on the error raise
ODBC error identifier: 23000
pyodbc_error_message (str) -- message raised by PyODBC
Example:
[23000] [Microsoft][ODBC Driver 17 for SQL Server][SQL Server] \
Cannot insert explicit value for identity column in table \
'building' when IDENTITY_INSERT is set to OFF.
(544) (SQLExecDirectW) \
"""
error_pattern = re.compile(
r"\[(?P<error_id>.*?)\] \[(?P<operator>.*?)\]\[(?P<driver>.*?)\]\[(?P<database_type>.*?)\](?P<error_message>.+?(?= \()) \((?P<sql_server_error_id>\d*?)\) \(SQLExecDirectW\)"
)
sql_error_code_pattern = re.compile(r"\((?P<sql_server_error_code>\d*?)\) \(SQLExecDirectW\)")
column_pattern = re.compile(r"column \'(?P<column_name>.+?)\'")
table_pattern = re.compile(r"table \'(?P<table_name>.+?)\'")
pyodbc_error_code = 'HY000'
def __init__(self, pyodbc_error_message: str) -> None:
self._parse_error_message(pyodbc_error_message)
def __str__(self) -> str:
return self.error_message
def _parse_error_message(self, pyodbc_error_message: str) -> None:
m = re.match(self.error_pattern, pyodbc_error_message)
self.operator = m.group('operator')
self.error_id = m.group('error_id')
self.driver = m.group('driver')
self.database_type = m.group('database_type')
self.error_message = m.group('error_message')
self.sql_server_error_id = m.group('sql_server_error_id')
#classmethod
def get_message(cls, pyodbc_exception: Exception) -> str:
if pyodbc_exception.args[1] == cls.pyodbc_error_code:
return pyodbc_exception.args[0]
else:
return pyodbc_exception.args[1]
#classmethod
def get_pyodbc_code(cls, pyodbc_exception: Exception) -> str:
if pyodbc_exception.args[1] == cls.pyodbc_error_code:
return pyodbc_exception.args[1]
else:
return pyodbc_exception.args[0]
#staticmethod
def get_exception(error_code: int):
return {
515: IdentityInsertNull,
544: IdentityInsertSetToOff,
2627: PrimaryKeyViolation,
8114: FailedTypeConversion,
102: IncorrectSyntax,
32: InvalidNumberParametersSupplied
}.get(error_code, DefaultException)
#classmethod
def get_sql_server_error_code(cls, pyodbc_code: str, message: str) -> int:
"""
Parses error message raised by PyODBC and return SQL Server Error Code
Looks for the following pattern:
(544) (SQLExecDirectW) -> 544
Args:
pyodbc_error_message (str): Error string raised by PyODBC
Returns:
(int) - SQL Server Error Code
"""
if pyodbc_code == cls.pyodbc_error_code:
return 32
else:
m = re.search(cls.sql_error_code_pattern, message)
if m:
return int(m.group('sql_server_error_code'))
else:
raise ValueError(f"Error raised is not from SQL Server: {message}")
#classmethod
def build_pyodbc_exception(cls, pyodbc_exception: Exception):
pyodbc_code = cls.get_pyodbc_code(pyodbc_exception)
error_message = cls.get_message(pyodbc_exception)
error_code = cls.get_sql_server_error_code(pyodbc_code, error_message)
exception = cls.get_exception(error_code)
raise exception(error_message)
class IdentityInsertNull(PyODBCError):
"""
Handle specific PyODBC error related to Null Value Inserted on Identity Column
"""
def __init__(self, pyodbc_error_message):
super().__init__(pyodbc_error_message)
m = re.search(self.table_pattern, self.error_message)
self.table_name = m.group('table_name')
m = re.search(self.column_pattern, self.error_message)
self.column_name = m.group('column_name')
class IdentityInsertSetToOff(PyODBCError):
"""
Handle specific PyODBC error related to Identity Not Set to On/Off
"""
def __init__(self, pyodbc_error_message):
super().__init__(pyodbc_error_message)
m = re.search(self.table_pattern, self.error_message)
self.table_name = m.group('table_name')
class FailedTypeConversion(PyODBCError):
"""
Handle specific PyODBC error related to data type conversion
"""
def __init__(self, pyodbc_error_message):
super().__init__(pyodbc_error_message)
class PrimaryKeyViolation(PyODBCError):
"""
Handle specific PyODBC error related to Primary Key Violation
"""
def __init__(self, pyodbc_error_message):
super().__init__(pyodbc_error_message)
class IncorrectSyntax(PyODBCError):
"""
Handle specific PyODBC error related to incorrect syntax in query
"""
def __init__(self, pyodbc_error_message):
super().__init__(pyodbc_error_message)
class DefaultException(PyODBCError):
"""
Handle default PyODBC errors
"""
def __init__(self, pyodbc_error_message):
super().__init__(pyodbc_error_message)
def __str__(self) -> str:
return f"{self.sql_server_error_id} - {self.error_message}"
class InvalidNumberParametersSupplied(Exception):
def __init__(self, error_message) -> None:
self.message = error_message
def __str__(self) -> str:
return self.message
In pyodbc 3.0.7, it works fine to catch pyodbc.ProgrammingError (and presumably the other error types, although I haven't tried). The contents of the error are still sort of cryptic, though, so it may be hard to do finer-grained handling of errors.
this will give you more clear and readable error message when connecting to mssql using myodbc:
try:
cnxn = pyodbc.connect(...)
except pyodbc.Error as ex:
sqlstate = ex.args[1]
sqlstate = sqlstate.split(".")
print(sqlstate[-3])
Be carreful it looks like an pyodbc exceptions but it may not be:
In my case I read pyodbc.IntegrityError in "(pyodbc.IntegrityError) ('23000', '[23000]..."
In the python console I can read this: "sqlalchemy.exc.IntegrityError: (pyodbc.IntegrityError) ('23000', '[23000]..."
It is actually a sqlalchemy.exc.IntegrityError exception, which wraps an pyodbc.IntegrityError