We Keep Coding

Set attribue in custom class derived from uuid.UUID: AttributeError

I'm porting some code from IronPython to CPython (3.8) + Python.NET and I have a custom class that's broken in an odd way: it gives me an AttributeError even though the member is present in __dict__. This class derives from uuid.UUID to add support to BLE short UUIDs:
import uuid
class UUID(uuid.UUID):
"""Represents a more general version of the uuid.UUID, so we can have both
regular and short UUIDs.
"""
def __init__(self, hex=None, bytes=None, is_short=False, **kwargs):
try:
super(UUID, self).__init__(hex=hex, bytes=bytes, **kwargs)
self.__dict__['is_short'] = False
except ValueError as val_ex:
if hex is None or len(hex) != 4:
raise val_ex
# Remove braces GUIDs
hex_digits = hex.strip('{}').replace('-', '')
# remove RFC 4122 URN's 'urn:uuid:deadbeef-1234-fedc-5678-deadbeefaaaa'
hex_digits = hex_digits.replace('uuid:', '').replace('urn:', '')
if len(hex_digits) != 4:
raise ValueError('badly formed hexadecimal UUID string')
self.__dict__['int'] = int(hex, 16)
self.__dict__['is_short'] = True
if is_short is True:
self.__dict__['is_short'] = True
def __str__(self):
if self.is_short:
hex = '%04x' % self.int
return '%s' % (hex[:4])
else:
return super(UUID, self).__str__()
And I'm using it like so:
class Test_UUID(unittest.TestCase):
def test_create(self):
x = pybletestlibrary.UUID("1800")
pprint(x.__dict__)
print(x)
The above code yields:
{'int': 6144, 'is_short': True}
E
======================================================================
ERROR: test_create (__main__.Test_UUID)
----------------------------------------------------------------------
Traceback (most recent call last):
File "test\test_pybletestlibrary.py", line 27, in test_create
print(x)
File "my_uuid.py", line 66, in __str__
hex = '%04x' % self.int
AttributeError: int
----------------------------------------------------------------------
Ran 1 test in 0.003s
FAILED (errors=1)
I can't use setattr() because uuid.UUID is immutable. I don't understand why the AttributeError as int is present in __dict__. Any suggestions?
Thanks!!

Here's what I mean:
import uuid
class UUID(uuid.UUID):
"""Represents a more general version of the uuid.UUID, so we can have both
regular and short UUIDs.
"""
def __init__(self, hex=None, bytes=None, is_short=False, **kwargs):
self.is_short = False
if len(hex) == 4:
self.is_short = True
hex = "0000"+hex+"-0000-1000-8000-00805f9b34fb"
super(UUID, self).__init__(hex=hex, bytes=bytes, **kwargs)
def __str__(self):
val = super(UUID, self).__str__()
if self.is_short:
return val[4:8]
else:
return val
You might have to override some of the other attributes, too.

Identify Linux passwd file

I need help in writing a function (python preferably) to identify if a file is /etc/passwd or etc/shadow. So far I have tried using print(pw.getpwall()) but this reads the file from the os env. I need a library that takes input and can tell if a file is passwd/shadow file or not

passwd and shadow file format differs.
You can write a short function or class. First iteration would be:
Find root user, almost 100% true that root is the first entry
Check 2nd, 6th and 7th column (separator is : sign)
If 2nd is x and 6th is /root and 7th is /bin/*sh then it is a password file almost in 100%
If 2nd is a salt and hash (format: $salt$hash) and 6th is a number and 7th is empy then it is a shadow file almost in 100%
Naturally there could be problems:
Linux is configured not to use shadow file. In this case pasword file 2nd column contains the password
Linux is configured not to use salt (I guess is it possible or not)
Please check manuals: man 5 passwd and man 5 shadow
EDIT, 2020-04-24:
Here is the my corrected pwd.py:
#!/usr/bin/env python3
import os
import sys
passwd_file=('./passwd')
# path conversion handlers
def __nullpathconv(path):
return path
def __unixpathconv(path):
return path
# decide what field separator we can try to use - Unix standard, with
# the platform's path separator as an option. No special field conversion
# handler is required when using the platform's path separator as field
# separator, but are required for the home directory and shell fields when
# using the standard Unix (":") field separator.
__field_sep = {':': __unixpathconv}
if os.pathsep:
if os.pathsep != ':':
__field_sep[os.pathsep] = __nullpathconv
# helper routine to identify which separator character is in use
def __get_field_sep(record):
fs = None
for c in list(__field_sep.keys()):
# there should be 6 delimiter characters (for 7 fields)
if record.count(c) == 6:
fs = c
break
if fs:
return fs
else:
raise KeyError
# class to match the new record field name accessors.
# the resulting object is intended to behave like a read-only tuple,
# with each member also accessible by a field name.
class Passwd:
def __init__(self, name, passwd, uid, gid, gecos, dir, shell):
self.__dict__['pw_name'] = name
self.__dict__['pw_passwd'] = passwd
self.__dict__['pw_uid'] = uid
self.__dict__['pw_gid'] = gid
self.__dict__['pw_gecos'] = gecos
self.__dict__['pw_dir'] = dir
self.__dict__['pw_shell'] = shell
self.__dict__['_record'] = (self.pw_name, self.pw_passwd,
self.pw_uid, self.pw_gid,
self.pw_gecos, self.pw_dir,
self.pw_shell)
def __len__(self):
return 7
def __getitem__(self, key):
return self._record[key]
def __setattr__(self, name, value):
raise AttributeError('attribute read-only: %s' % name)
def __repr__(self):
return str(self._record)
def __cmp__(self, other):
this = str(self._record)
if this == other:
return 0
elif this < other:
return -1
else:
return 1
# read the whole file, parsing each entry into tuple form
# with dictionaries to speed recall by UID or passwd name
def __read_passwd_file():
if passwd_file:
passwd = open(passwd_file, 'r')
else:
raise KeyError
uidx = {}
namx = {}
sep = None
while 1:
entry = passwd.readline().strip()
if len(entry) > 6:
if sep is None:
sep = __get_field_sep(entry)
fields = entry.split(sep)
for i in (2, 3):
fields[i] = int(fields[i])
for i in (5, 6):
fields[i] = __field_sep[sep](fields[i])
record = Passwd(*fields)
if fields[2] not in uidx:
uidx[fields[2]] = record
if fields[0] not in namx:
namx[fields[0]] = record
elif len(entry) > 0:
pass # skip empty or malformed records
else:
break
passwd.close()
if len(uidx) == 0:
raise KeyError
return (uidx, namx)
# return the passwd database entry by UID
def getpwuid(uid):
u, n = __read_passwd_file()
return u[uid]
# return the passwd database entry by passwd name
def getpwnam(name):
u, n = __read_passwd_file()
return n[name]
# return all the passwd database entries
def getpwall():
u, n = __read_passwd_file()
return list(n.values())
# test harness
if __name__ == '__main__':
print(getpwall())

functools stops working when switch to Python 3

We have migrated our Django project code base from Python 2.7 to 3.6 and suddenly what used to work stopped. Specifically, this:
map(functools.partial(self._assocUser, user=user), persistedGroupIds)
needed to be replaced with:
for group_id in persistedGroupIds:
self._assocUser(group_id, user)
and this:
persistedGroupIds = map(functools.partial(self._persistGroup, grp_mappings=attrAll.entitlements), saml_authorization_attributes)
needed to go to:
persistedGroupIds = []
for idp_group_name in saml_authorization_attributes:
persistedGroupIds.append(self._persistGroup(idp_group_name, attrAll.entitlements))
before the old functionality reappeared. Python 3's functools don't seem to work.
Here's the full listing of the code that works fine under Python 2:
from django.contrib.auth.models import User
from django.contrib.auth.models import Group
import functools
from mappings import SAMLAttributesConfig
from django.conf import settings
import logging
log = logging.getLogger(__name__)
class SAMLServiceProviderBackend(object):
empty_entitlements_message="IdP supplied incorrect authorization entitlements. Please contact their support."
def _assocUser(self, group_id, user):
group = Group.objects.get(id=group_id)
group.user_set.add(user)
return None
def _persistGroup(self,idp_group_name, grp_mappings):
group_name = grp_mappings[idp_group_name]
try:
group = Group.objects.get(name=group_name)
except Group.DoesNotExist:
group = Group(name=group_name)
group.save()
return group.id
def _extract_grp_entitlements(self,saml_authentication_attributes,groups):
result = []
input_length = len(saml_authentication_attributes[groups])
if input_length == 0:
log.error(self.empty_entitlements_message)
raise RuntimeError(self.empty_entitlements_message)
if input_length == 1:
result = [t.strip() for t in saml_authentication_attributes[groups][0].split(',')]
elif input_length:
result = saml_authentication_attributes[groups]
return result
# return [t.strip() for t in saml_authentication_attributes[groups][0].split(',')] \
# if len(saml_authentication_attributes[groups]) == 1\
# else saml_authentication_attributes[groups]
def authenticate(self, saml_authentication=None):
if not saml_authentication: # Using another authentication method
return None
attrAll = SAMLAttributesConfig(mappings_file_name=settings.AUTH_MAPPINGS_FILE).get_config()
groups = attrAll.entitlements.containerName
if saml_authentication.is_authenticated():
saml_authentication_attributes = saml_authentication.get_attributes()
saml_authorization_attributes = self._extract_grp_entitlements(saml_authentication_attributes,groups)
persistedGroupIds = map(functools.partial(self._persistGroup, grp_mappings=attrAll.entitlements), saml_authorization_attributes)
try:
user = User.objects.get(username=saml_authentication.get_nameid())
except User.DoesNotExist:
user = User(username=saml_authentication.get_nameid())
user.set_unusable_password()
try:
user.first_name = saml_authentication_attributes['samlNameId'][0]
except KeyError:
pass
try:
setattr(user, "first_name", saml_authentication_attributes[attrAll.subject.first_name][0])
except KeyError:
pass
#user.last_name = attributes['Last name'][0]
user.save()
map(functools.partial(self._assocUser, user=user), persistedGroupIds)
user.save()
return user
return None
def get_user(self, user_id):
try:
return User.objects.get(pk=user_id)
except User.DoesNotExist:
return None
The above code no longer works under Python 3 environment and only starts working to something like this, with a functools.partial() calls spelled out in a for loop:
from django.contrib.auth.models import User
from django.contrib.auth.models import Group
import functools
from .mappings import SAMLAttributesConfig
from django.conf import settings
import logging
log = logging.getLogger(__name__)
class SAMLServiceProviderBackend(object):
empty_entitlements_message="IdP supplied incorrect authorization entitlements. Please contact their support."
def _assocUser(self, group_id, user):
group = Group.objects.get(id=group_id)
group.user_set.add(user)
return None
def _persistGroup(self,idp_group_name, grp_mappings):
group_name = grp_mappings[idp_group_name]
try:
group = Group.objects.get(name=group_name)
except Group.DoesNotExist:
group = Group(name=group_name)
group.save()
return group.id
def _extract_grp_entitlements(self,saml_authentication_attributes,groups):
result = []
input_length = len(saml_authentication_attributes[groups])
if input_length == 0:
log.error(self.empty_entitlements_message)
raise RuntimeError(self.empty_entitlements_message)
if input_length == 1:
result = [t.strip() for t in saml_authentication_attributes[groups][0].split(',')]
elif input_length:
result = saml_authentication_attributes[groups]
return result
# return [t.strip() for t in saml_authentication_attributes[groups][0].split(',')] \
# if len(saml_authentication_attributes[groups]) == 1\
# else saml_authentication_attributes[groups]
def authenticate(self, saml_authentication=None):
if not saml_authentication: # Using another authentication method
return None
attrAll = SAMLAttributesConfig(mappings_file_name=settings.AUTH_MAPPINGS_FILE).get_config()
groups = attrAll.entitlements.containerName
if saml_authentication.is_authenticated():
saml_authentication_attributes = saml_authentication.get_attributes()
saml_authorization_attributes = self._extract_grp_entitlements(saml_authentication_attributes,groups)
persistedGroupIds = map(functools.partial(self._persistGroup, grp_mappings=attrAll.entitlements), saml_authorization_attributes)
try:
user = User.objects.get(username=saml_authentication.get_nameid())
except User.DoesNotExist:
user = User(username=saml_authentication.get_nameid())
user.set_unusable_password()
try:
user.first_name = saml_authentication_attributes['samlNameId'][0]
except KeyError:
pass
try:
setattr(user, "first_name", saml_authentication_attributes[attrAll.subject.first_name][0])
except KeyError:
pass
#user.last_name = attributes['Last name'][0]
user.save()
for group_id in persistedGroupIds:
self._assocUser(user = user, group_id = group_id)
# map(functools.partial(self._assocUser, user=user), persistedGroupIds)
user.save()
return user
return None
def get_user(self, user_id):
try:
return User.objects.get(pk=user_id)
except User.DoesNotExist:
return None
What can be wrong?
I'm using PyDev plugin in Eclipse. Here's how my Python interpreter is configured there:
Here's Eclipse's .pydevproject file:
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?eclipse-pydev version="1.0"?><pydev_project>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">venv3.6</pydev_property>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python interpreter</pydev_property>
<pydev_variables_property name="org.python.pydev.PROJECT_VARIABLE_SUBSTITUTION">
<key>DJANGO_SETTINGS_MODULE</key>
<value>reporting.settings</value>
<key>DJANGO_MANAGE_LOCATION</key>
<value>./manage.py</value>
<key>SAML_PLUGIN</key>
<value>/Users/sl/abc/venv3.6/lib/python3.6/site-packages/onelogin/saml2</value>
<key>PY</key>
<value>36</value>
</pydev_variables_property>
<pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
<path>/${PROJECT_DIR_NAME}</path>
</pydev_pathproperty>
<pydev_pathproperty name="org.python.pydev.PROJECT_EXTERNAL_SOURCE_PATH">
<path>${SAML_PLUGIN}</path>
</pydev_pathproperty>
</pydev_project>

In Python 3, the map function returns an iterator instead of a list.
This means that if you call map on a collection, the effects of the call are not materialised until you iterate over the resulting iterator.
Consider this class:
>>> class C:
... def __init__(self, x):
... self.x = x
... def double(self):
... self.x *= 2
... def __repr__(self):
... return '<C:{}>'.format(self.x)
...
Let's make a list of instances:
>>> cs = [C(x) for x in range(1, 4)]
>>> cs
[<C:1>, <C:2>, <C:3>]
Now use map to call each instance's double method:
>>> res = map(C.double, cs)
Note the result is not a list:
>>> res
<map object at 0x7ff276350470>
And the instances have not changed:
>>> cs
[<C:1>, <C:2>, <C:3>]
if we call next on the iterator, the instances are updated in turn.
>>> next(res)
>>> cs
[<C:2>, <C:2>, <C:3>]
>>> next(res)
>>> cs
[<C:2>, <C:4>, <C:3>]
>>> next(res)
>>> cs
[<C:2>, <C:4>, <C:6>]
In the code samples that you have provided, the result of calling map is not assigned to a variable, so map is being used for its side effects rather than its output. In Python 3 the correct way to do this is to loop over the iterable and call the function on each element:
>>> for c in cs:
c.double()
As the linked doc puts it:
Particularly tricky is map() invoked for the side effects of the function; the correct transformation is to use a regular for loop (since creating a list would just be wasteful).

Plus sign breaks regex validation

I have numerous regular expressions, such as:
quantity_validation = re.compile("""^-?[0-9]+$""", re.I)
I use these regex in functions like:
def validate_quantity(self, value):
if value != self.context.quantity:
# while not self.quantity_validation.match(value):
# return 0, "Value is not a number"
if not self.quantity_validation.match(value):
return 0, "Value is not a number"
return 1, ""
and the validation method:
#view_config(name="validate", renderer='json')
def validate(self, full={}):
def do_validation(field, value):
message = ""
valid = 1
if getattr(self, 'validate_%s' % field, False):
valid, message = getattr(self, 'validate_%s' % field)(value)
out = dict(message=message, valid=valid, value=value)
return out
if not full:
field = self.request.params.get('field')
if not field:
return "INVALID"
return do_validation(field, self.request.params.get('value'))
else:
return dict(((field, do_validation(field, value)) for field, value in full.items()))
On the HTML form I have a modal with a quantity field on which I do ajax validation(I use jqBootstrapValidation to validate my forms)
The regex validation works fine until I type a + then the validation stop working until I refresh the page.
Why does the validation stop working?
Is it perhaps a bug in jqBootstrapValidation?
or is it a bug in the re module?

You are not encoding your URLs/inputs correctly. The '+' gets parsed into a ' ' when you pass it via the parameter string. Try '%2B' instead which should be decoded into a '+' through the form library.

Python serializable objects json [duplicate]

This question already has answers here:
How to make a class JSON serializable
(41 answers)
Closed 6 months ago.
class gpagelet:
"""
Holds 1) the pagelet xpath, which is a string
2) the list of pagelet shingles, list
"""
def __init__(self, parent):
if not isinstance( parent, gwebpage):
raise Exception("Parent must be an instance of gwebpage")
self.parent = parent # This must be a gwebpage instance
self.xpath = None # String
self.visibleShingles = [] # list of tuples
self.invisibleShingles = [] # list of tuples
self.urls = [] # list of string
class gwebpage:
"""
Holds all the datastructure after the results have been parsed
holds: 1) lists of gpagelets
2) loc, string, location of the file that represents it
"""
def __init__(self, url):
self.url = url # Str
self.netloc = False # Str
self.gpagelets = [] # gpagelets instance
self.page_key = "" # str
Is there a way for me to make my class json serializable? The thing that I am worried is the recursive reference.

Write your own encoder and decoder, which can be very simple like return __dict__
e.g. here is a encoder to dump totally recursive tree structure, you can enhance it or use as it is for your own purpose
import json
class Tree(object):
def __init__(self, name, childTrees=None):
self.name = name
if childTrees is None:
childTrees = []
self.childTrees = childTrees
class MyEncoder(json.JSONEncoder):
def default(self, obj):
if not isinstance(obj, Tree):
return super(MyEncoder, self).default(obj)
return obj.__dict__
c1 = Tree("c1")
c2 = Tree("c2")
t = Tree("t",[c1,c2])
print json.dumps(t, cls=MyEncoder)
it prints
{"childTrees": [{"childTrees": [], "name": "c1"}, {"childTrees": [], "name": "c2"}], "name": "t"}
you can similarly write a decoder but there you will somehow need to identify is it is your object or not, so may be you can put a type too if needed.

Indirect answer: instead of using JSON, you could use YAML, which has no problem doing what you want. (JSON is essentially a subset of YAML.)
Example:
import yaml
o1 = gwebpage("url")
o2 = gpagelet(o1)
o1.gpagelets = [o2]
print yaml.dump(o1)
In fact, YAML nicely handles cyclic references for you.

I implemented a very simple todict method with the help of https://stackoverflow.com/a/11637457/1766716
Iterate over properties that is not starts with __
Eliminate methods
Eliminate some properties manually which is not necessary (for my case, coming from sqlalcemy)
And used getattr to build dictionary.
class User(Base):
id = Column(Integer, primary_key=True)
firstname = Column(String(50))
lastname = Column(String(50))
password = Column(String(20))
def props(self):
return filter(
lambda a:
not a.startswith('__')
and a not in ['_decl_class_registry', '_sa_instance_state', '_sa_class_manager', 'metadata']
and not callable(getattr(self, a)),
dir(self))
def todict(self):
return {k: self.__getattribute__(k) for k in self.props()}

My solution for this was to extend the 'dict' class and perform checks around required/allowed attributes by overriding init, update, and set class methods.
class StrictDict(dict):
required=set()
at_least_one_required=set()
cannot_coexist=set()
allowed=set()
def __init__(self, iterable={}, **kwargs):
super(StrictDict, self).__init__({})
keys = set(iterable.keys()).union(set(kwargs.keys()))
if not keys.issuperset(self.required):
msg = str(self.__class__.__name__) + " requires: " + str([str(key) for key in self.required])
raise AttributeError(msg)
if len(list(self.at_least_one_required)) and len(list(keys.intersection(self.at_least_one_required))) < 1:
msg = str(self.__class__.__name__) + " requires at least one: " + str([str(key) for key in self.at_least_one_required])
raise AttributeError(msg)
for key, val in iterable.iteritems():
self.__setitem__(key, val)
for key, val in kwargs.iteritems():
self.__setitem__(key, val)
def update(self, E=None, **F):
for key, val in E.iteritems():
self.__setitem__(key, val)
for key, val in F.iteritems():
self.__setitem__(key, val)
super(StrictDict, self).update({})
def __setitem__(self, key, value):
all_allowed = self.allowed.union(self.required).union(self.at_least_one_required).union(self.cannot_coexist)
if key not in list(all_allowed):
msg = str(self.__class__.__name__) + " does not allow member '" + key + "'"
raise AttributeError(msg)
if key in list(self.cannot_coexist):
for item in list(self.cannot_coexist):
if key != item and item in self.keys():
msg = str(self.__class__.__name__) + "does not allow members '" + key + "' and '" + item + "' to coexist'"
raise AttributeError(msg)
super(StrictDict, self).__setitem__(key, value)
Example usage:
class JSONDoc(StrictDict):
"""
Class corresponding to JSON API top-level document structure
http://jsonapi.org/format/#document-top-level
"""
at_least_one_required={'data', 'errors', 'meta'}
allowed={"jsonapi", "links", "included"}
cannot_coexist={"data", "errors"}
def __setitem__(self, key, value):
if key == "included" and "data" not in self.keys():
msg = str(self.__class__.__name__) + " does not allow 'included' member if 'data' member is not present"
raise AttributeError(msg)
super(JSONDoc, self).__setitem__(key, value)
json_doc = JSONDoc(
data={
"id": 5,
"type": "movies"
},
links={
"self": "http://url.com"
}
)