I have the following block of Python code talking to DynamoDB on AWS:
try:
response = conn.batch_write_item(batch_list)
except Exception ,e:
try:
mess = e.message
except:
mess = "NOMESS"
try:
earg0 = e.args[0]
except:
earg0 = "NOEARG0"
try:
stre = str(e)
except:
stre = "NOSTRE"
print "mess = '%s'" % mess
print "earg0 = '%s'" % earg0
print "stre = '%s'" % stre
What I get is this:
mess = ''
earg0 = 'NOEARG0'
stre = 'DynamoDBValidationError: 400 Bad Request {'message': 'Item size has exceeded the maximum allowed size', '__type': 'com.amazon.coral.validate#ValidationException'}'
What I need to somehow reliably extract the message string such as 'Item size has exceeded the maximum allowed size' from e. How can I do it?
I'm assuming you're using boto to access DynamoDB.
Here is the JSONResponseError (supersuperclass of DynamoDBValidationError) __init__ method:
self.status = status
self.reason = reason
self.body = body
if self.body:
self.error_message = self.body.get('message', None)
self.error_code = self.body.get('__type', None)
if self.error_code:
self.error_code = self.error_code.split('#')[-1]
Wild guess: I would go with e.error_message to get 'Item size has exceeded ...'.
You can also print all attributes (and their values) of e:
for attr in dir(e):
print "e[%r] = '''%s'''" % (attr, getattr(e, attr))
Take e.body, You will get the error as a dictionary.
example:
{u'message': u'The conditional request failed', u'__type': u'com.amazonaws.dynamodb.v20120810#ConditionalCheckFailedException'}
From this easily you will get message.
Related
I have modified this code python-paged-ldap-snippet.py from https://gist.github.com/mattfahrner/c228ead9c516fc322d3a
My problem is that when I change my SEARCHFILTER from '(&(objectCategory=person)(objectClass=user))' to '(&(objectCategory=person)(objectClass=user)(memberOf=CN=Users0,OU=Groups,DC=ad,DC=company,DC=com))'
it runs just fine.
If it is on SEARCHFILTER='(&(objectCategory=person)(objectClass=user))', I notice that the code is not entering the writeToFile function.
The objective of the code is to dump all the user information and parse the info into a file.
I tried running LDAPSEARCH against '(&(objectCategory=person)(objectClass=user))' and I manage to get the output .
Not sure what is wrong. Suggestions are greatly appreciated.
Thank you.
#!/usr/bin/python
import sys
import ldap
import os
LDAPSERVER='ldap://xxx.xxx.xxx.xxx:389'
BASEDN='dc=ad,dc=company,dc=com'
LDAPUSER = "CN=LDAPuser,OU=XXX,OU=Users,DC=ad,DC=company,DC=com"
LDAPPASSWORD = 'LDAPpassword'
PAGESIZE = 20000
ATTRLIST = ['sAMAccountName','uid']
SEARCHFILTER='(&(objectCategory=person)(objectClass=user))'
#SEARCHFILTER='(&(objectCategory=person)(objectClass=user)(memberOf=CN=Users0,OU=Groups,DC=ad,DC=company,DC=com))'
data = []
ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_ALLOW)
ldap.set_option(ldap.OPT_REFERRALS, 0)
l = ldap.initialize(LDAPSERVER)
l.protocol_version = 3 # Paged results only apply to LDAP v3
try:
l.simple_bind_s(LDAPUSER, LDAPPASSWORD)
print ' Login Done, Searching data'
except ldap.LDAPError as e:
exit('LDAP bind failed: %s' % e)
lc = ldap.controls.SimplePagedResultsControl(True,size=PAGESIZE,cookie='')
def writeToFile(data):
print ' Writing data to file'
#code to print all output into CVS file
while True:
try:
msgid = l.search_ext(BASEDN, ldap.SCOPE_SUBTREE, SEARCHFILTER, ATTRLIST, serverctrls=[lc])
except ldap.LDAPError as e:
sys.exit('LDAP search failed: %s' % e)
try:
rtype, rdata, rmsgid, serverctrls = l.result3(msgid)
except ldap.LDAPError as e:
sys.exit('Could not pull LDAP results: %s' % e)
for dn, attrs in rdata:
data.append(attrs)
pctrls = [
c for c in serverctrls if c.controlType == ldap.controls.SimplePagedResultsControl.controlType ]
if not pctrls:
print >> sys.stderr, 'Warning: Server ignores RFC 2696 control.'
break
cookie = pctrls[0].cookie
if not cookie:
writeToFile(data)
print 'Task Complete'
break
lc.controlValue = (PAGESIZE, cookie)
PAGESIZE = 20000
Lower your page size to a value <= 1000, since that's the max AD will give you at a time anyway. It's possible that it's waiting for 20000 records before requesting the next page and never getting it.
I have a column in my dataframe that is of type suds.sax.text.Text and I want to convert it to a string. I can't find much on how to do this, except for this site. Using pandas.DataFrame.astype does not work. I'm sure there is an easy way to do this. The documentation is just going over my head. I am using a web service to use some of its functions to return some metadata on weather stations. This metadata gets returned back as a suds object. Link to the web service is here.
from suds.client import Client
from suds.transport.https import HttpAuthenticated
from urllib.error import URLError
from urllib.request import HTTPSHandler
import ssl
import pandas as pd
ssl._create_default_https_context = ssl._create_unverified_context
_URL_AWDB_WSDL = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
def _execute_awdb_call(a_func, ntries_max=3, sleep_sec=5, **kwargs):
ntries = 0
while 1:
try:
a_result = a_func(**kwargs)
break
except Exception as e:
ntries += 1
if ntries == ntries_max:
raise
else:
print(("WARNING: Received error executing AWDB function %s:"
" %s. Sleeping %d seconds and trying again." %
(str(a_func.method.name), str(e), sleep_sec)))
sleep(sleep_sec)
return a_result
def _stationMetadata_to_tuple(a_meta):
list_meta = [None] * len(_stnmeta_attrs)
for i, a_attr in enumerate(_stnmeta_attrs):
try:
list_meta[i] = a_meta[a_attr]
except AttributeError:
# Doesn't have attribute
continue
return tuple(list_meta)
try:
_client = Client(_URL_AWDB_WSDL)
_stnmeta_attrs = (_client.factory.
create('stationMetaData').__keylist__)
except URLError as e:
if type(e.reason) == ssl.SSLError:
print("Warning: SSL Error connecting to AWDB web service. Skipping verification...")
_client = Client(_URL_AWDB_WSDL, transport=_CustomTransport())
_stnmeta_attrs = (_client.factory.
create('stationMetaData').__keylist__)
else:
raise
stn_triplets = ['878:WY:SNTL', '1033:CO:SNTL']
stn_metas = _execute_awdb_call(_client.service.
getStationMetadataMultiple,
stationTriplets=stn_triplets)
stn_tups = [_stationMetadata_to_tuple(a) for a in stn_metas]
df_stns = pd.DataFrame(stn_tups, columns=_stnmeta_attrs)
stns = df_stns.rename(columns={'actonId': 'station_id',
'name': 'station_name'})
stns['station_id'] = stns.station_id.fillna(stns.stationTriplet)
stns = stns[~stns.station_id.isnull()]
print(type(stns.beginDate[0]))
This is my test so far:
test_500(self):
client = ClientConfiguration(token=token, url=url)
client.url = 'https://localhost:1234/v1/' + bucket
keys = None
try:
get_bucket = json.loads(str(client.get_bucket(bucket)))
result = get_bucket['result']
except Exception as e:
expected_status_code = 500
failure_message = "Expected status code %s but got status code %s" % (expected_status_code, e)
self.assertEquals(e, expected_status_code, failure_message)
I need to write a mock that will return a 500 response when the 'https://localhost:1234/v1/' + bucket url is used. Can this be done with unittest and if so, how or where can I find some documentation on this? I've been through this site, the unittest documentation and Youtube and can't find anythingspecific to what I want to do.
I ended up using this to create my test.
The end result is:
#responses.activate
test_500(self):
responses.add(responses.GET, 'https://localhost:1234/v1/' + bucket,
json={'error': 'server error'}, status=500)
client = ClientConfiguration(token=token, url=url)
client.url = 'https://localhost:1234/v1/'
keys = None
try:
get_bucket = json.loads(str(client.get_bucket(bucket)))
result = get_bucket['result']
except Exception as e:
expected_status_code = 500
failure_message = "Expected status code %s but got status code %s" % (expected_status_code, e)
self.assertEquals(e, expected_status_code, failure_message)
def get_email_body(self, email):
user = self.email_usr
password = self.app_email_pwd
connection = imaplib.IMAP4_SSL('imap.gmail.com')
connection.login(user, password)
connection.list()
connection.select('"INBOX"')
time.sleep(5)
result_search, data_search = connection.search(None, 'TO', email, 'SUBJECT', '"some subject"')
required_email = data_search[0]
result_fetch, data_fetch = connection.fetch(required_email, '(RFC822)')
email_body_string = data_fetch[0][1].decode('utf-8')
confirmation_link = self.parse_confirmation_link(email_body_string)
return confirmation_link
This function works like 2 times of 4 runs. Usually it fails with:
self = <imaplib.IMAP4_SSL object at 0x7fa853614b00>, name = 'FETCH'
tag = b'JAAL5'
def _command_complete(self, name, tag):
# BYE is expected after LOGOUT
if name != 'LOGOUT':
self._check_bye()
try:
typ, data = self._get_tagged_response(tag)
except self.abort as val:
raise self.abort('command: %s => %s' % (name, val))
except self.error as val:
raise self.error('command: %s => %s' % (name, val))
if name != 'LOGOUT':
self._check_bye()
if typ == 'BAD':
raise self.error('%s command error: %s %s' % (name, typ, data))
E imaplib.error: FETCH command error: BAD [b'Could not parse command']
/usr/lib/python3.4/imaplib.py:964: error
My suggestion was that sometimes email isn't delivered at the moment of .search that's why I added time.sleep (I'm searching for the email immediately after it was sent).
Else I did try search while result_fetch is not 'OK' but is also didn't help.
Any other suggestions?
oooops, my suggestion was correct, but time.sleep was in the incorrect place. Moved sleep before connection and all go smooth
I have some a Python function written this way:
def verify_secondary_hostnames(self, publishedName, hostnames):
cname = self.get_cname(publishedName)
edge_ip = requests.get('http://{}'.format(cname)).headers['X-Edge-IP']
for hostname in hostnames:
headers = {"host": hostname}
data = requests.head("http://{}".format(edge_ip), headers=headers)
data.raise_for_status()
The unit test that I have tried to write for it looks like this:
#patch("requests.get")
#patch("requests.head")
#patch("FrontEnd.FrontEnd.get_cname")
def test_verify_secondary_hostnames(self, mock_get_cname, mock_head, mock_get):
mock_get_cname.return_value = "hulahoop"
mock_get.return_value.headers = {"X-Edge-IP": "hulahoop"}
self.fe.verify_secondary_hostnames('publishedName', ['hostname'])
self.assertEqual(mock_head.call_args,
call('http://hulahoop', headers={'host': 'hostname'}))
In this test the last part raise_for_status which actually does this:
def raise_for_status(self):
"""Raises stored :class:`HTTPError`, if one occurred."""
http_error_msg = ''
if 400 <= self.status_code < 500:
http_error_msg = '%s Client Error: %s' % (self.status_code, self.reason)
elif 500 <= self.status_code < 600:
http_error_msg = '%s Server Error: %s' % (self.status_code, self.reason)
if http_error_msg:
raise HTTPError(http_error_msg, response=self)
I can't do it here since I have already mocked the requests.head. How can I write my test to cover for that as well ? What I mean by "cover" here is that since the object is a mock object, How can I make it raise_for_status ? Do I have to create a mock method to do that or emulate a raise in my test ?
Edit: This is the latest test I have with the problems outlined:
#patch("requests.get")
#patch("requests.head")
#patch("FrontEnd.FrontEnd.get_cname")
#patch("requests.models.Response.raise_for_status")
def test_verify_secondary_hostnames(self, mock_raise,
mock_get_cname, mock_head, mock_get):
mock_get_cname.return_value = "hulahoop"
mock_get.return_value.headers = {"X-Edge-IP": "hulahoop"}
mock_head.return_value.status_code = 200
self.fe.verify_secondary_hostnames('publishedName', ['hostname'])
self.assertEqual(mock_head.call_args,
call('http://hulahoop', headers={'host': 'hostname'}))
mock_head.raise_for_status.called <=== False here
mock_head.reset_mock()
mock_head.return_value.status_code = 400
self.fe.verify_secondary_hostnames('publishedName', ['hostname'])
mock_head.raise_for_status.called <=== Still false here.
You can try something like this in your unit test:
mock_data = mock.MagicMock()
mock_data.status_code = 200 # TODO (Low Kian Seong): Test raise_for_status
mock_head.return_value = mock_data
...
mock_data.raise_for_status.assert_called_once_with()