We Keep Coding

Python dataclass attributes monkeypatching

I have problem with my class Config, that is works as proxy between user and ini file. It can load parameters from ini files and set them to its name equivalent in dataclass. I've realized, that it I want to get some attribute with dot like Config()._BASE_DIR, it returns str value, because ConfigParser can get values as a str. My idea is to create some method, which will patch all my attributes with property and property.setter to make possible to access dataclass attributes using dot, but wrap them with annotation classes, so, for example, Config()._minAR will return not 4.0 as string but as float.
Is my idea acceptable, or do I need to do it differently?
Config code parts:
import configparser
import pathlib
from dataclasses import dataclass
from itertools import zip_longest
#dataclass
class Config:
_IGNORE_FIELDS = {'_IGNORE_FIELDS' ,'_CONF_PARSER'}
_CONF_PARSER: configparser.ConfigParser = configparser.ConfigParser()
_BASE_TABLE_FILE_SUFFIX: str = '.csv'
_BASE_DIR: pathlib.Path = pathlib.Path().absolute()
_CONF_PATH: pathlib.Path = _BASE_DIR / 'conf'
_CONF_FILE_PATH: pathlib.Path = _CONF_PATH / 'settings.ini'
_DATA_TABLE_PATH: pathlib.Path = _CONF_PATH / ('_data_table' + _BASE_TABLE_FILE_SUFFIX)
_minAR: float = 4.0
_maxAR: float = 5.0
CATCH_TIME: int = 6
def __init__(self) -> None:
self.prepare()
def check_synchronized(self) -> tuple[bool, str]:
if not self.CONF_PARSER.has_section('settings'):
return False, 'ini'
parser_config = self.CONF_PARSER['settings'].items()
python_config = {
k: v
for k, v in self.__dataclass_fields__.items()
if k not in self._IGNORE_FIELDS
}.items()
for pair_1, pair_2 in zip_longest(python_config, parser_config, fillvalue=(None, None)):
key_1, val_1 = pair_1
if key_1 is None:
return False, 'script'
key_2, val_2 = pair_2
if key_2 is None:
return False, 'ini'
if key_2 in self._IGNORE_FIELDS:
continue
if key_1.lower() != key_2.lower() or (default := str(val_1.default)) != val_2:
mode = 'ini' if default != str(getattr(self, key_1)) else 'script'
return False, mode
return True, 'both'
def updateFromIni(self):
for key, value in self.CONF_PARSER['settings'].items():
upper_key = key.upper()
if str(getattr(self, upper_key)) == value:
continue
setattr(self, upper_key, value)
def prepare(self):
self._createConfDir()
is_sync, mode = self.check_synchronized()
if is_sync:
return
if mode == 'ini' or mode == 'both':
self._writeAll()
elif mode == 'script':
self.updateFromIni()
def _writeAll(self):
if not self.CONF_PARSER.has_section('settings'):
self.CONF_PARSER.add_section('settings')
for key, field in self.__dataclass_fields__.items():
if key in self._IGNORE_FIELDS:
continue
self.CONF_PARSER.set('settings', key, str(field.default))
self._writeInFile()
def _writeInFile(self):
with open(self.CONF_FILE_PATH, 'w') as file:
self.CONF_PARSER.write(file)
def _createConfDir(self) -> None:
if not self.CONF_PATH.exists():
self.CONF_PATH.mkdir(parents=True, exist_ok=True)
def setValue(self, field, value):
if not hasattr(self, field) or field in self._IGNORE_FIELDS:
return
setattr(self, field, value)
if not isinstance(value, str):
value = str(value)
self.CONF_PARSER.set('settings', field, value)
self._writeInFile()
More context: I use dataclass with configParser to make my Config class able to do the following things:
Sync attributes with ini file (if no ini file, create it from Config structure with default values; if Config not syncronized with ini file, load from ini, and write to ini, it ini-file has wrong structure, or some values are incorrect) to avoid the situation, when user accidentally delete ini file;
Set and Get all existing values in config from any part of my program (it is PyQt6 application);
Save it state from one session (application run) to another.
So, I had no idea, what other structure of config class I should have used, except for this. If you have better idea for synchronizable config, tell me.

I've discovered, that only one change, that I need to make my Config class make custom dot access to attributes, is to write custom magic method __getattribute__ in my class.
result:
import configparser
import pathlib
from dataclasses import dataclass
from itertools import zip_longest
from typing import Any
ACCESS_FIELDS = {
'BASE_TABLE_FILE_SUFFIX', 'BASE_DIR', 'CONF_PATH', 'CONF_FILE_PATH',
'DATA_TABLE_PATH', 'minAR', 'CATCH_TIME'
}
class Config:
# some code ...
def __getattribute__(self, __name: str) -> Any:
if __name == 'ACCESS_FIELDS':
return ACCESS_FIELDS
attr = super().__getattribute__(__name)
if __name in ACCESS_FIELDS:
_type = self.__annotations__[__name]
return _type(attr)
return attr
# other code ...
I created variable with accessed fields not in class body, because in other cases, if I get ACCESS_FIELDS by using Config.ACCESS_FIELDS or self.ACCESS_FIELDS, it will call __getattrubute__ method again and cause recursion error.
Basically, I got all what I need by using this solution, but I still has problem with setValue method. I've discovered, that __setattr__ overriden method works not so good with __getattribute__ overriden method in my class (it cause recursion error too). Probably, I'll restructure my Config class, but not now.

Look up items in list of dataclasses by value

I'm using python to filter data in elasticsearch based on request params provided. I've got a working example, but know it can be improved and am trying to think of a better way. The current code is like this:
#dataclass(frozen=True)
class Filter:
param: str
custom_es_field: Optional[str] = None
is_bool_query: bool = False
is_date_query: bool = False
is_range_query: bool = False
def es_field(self) -> str:
if self.custom_es_field:
field = self.custom_es_field
elif "." in self.param:
field = self.param.replace(".", "__")
else:
field = self.param
return field
filters = [
Filter(param="publication_year", is_range_query=True),
Filter(param="publication_date", is_date_query=True),
Filter(param="venue.issn").
...
]
def filter_records(filter_params, s):
for filter in filters:
# range query
if filter.param in filter_params and filter.is_range_query:
param = filter_params[filter.param]
if "<" in param:
param = param[1:]
validate_range_param(filter, param)
kwargs = {filter.es_field(): {"lte": int(param)}}
s = s.filter("range", **kwargs)
elif filter.param in filter_params and filter.is_bool_query:
....
The thing I think is slow is I am looping through all of the filters in order to use the one that came in as a request variable. I'm tempted to convert this to a dictionary so I can do filter["publication_year"], but I like having the extra methods available via the dataclass. Would love to hear any thoughts.

Using Boolean in Django Custom Query Set

Lately I had came to one strange thing when I was working with Django custom querysets. So, I am using request.GET for passing boolean value to my django view. Then I pass this value to my custom queryset function. It seems weird because it works in case of basic query but not with '__isnull'. Are there any differencies?
views.py
if request.GET.get('available'):
items = Item.objects.is_available(request.GET.get('available')) # this works
...
if request.GET.get('order'):
transfers = Transfer.objects.not_working_is_order(request.GET.get('order')) # not working
managers.py
class ItemQuerySet(QuerySet):
def is_available(self, is_available):
return self.filter(is_available=is_available) # this works
class TransferQuerySet(QuerySet):
def is_order(self, bool):
if bool == 'False':
return self.filter(order__isnull=False)
elif bool == 'True':
return self.filter(order__isnull=True) # this works
def not_working_is_order(self, bool):
return self.filter(order__isnull=bool) # not working
mytemplate.html
Not available
myothertemplate.html
Not order

That's because bool is not a boolean, it's a string, you can do something like self.filter(order__isnull=bool == 'True')

Static classes being initialised on import. How does python 2 initialise static classes on import

I am trying to introduce python 3 support for the package mime and the code is doing something I have never seen before.
There is a class Types() that is used in the package as a static class.
class Types(with_metaclass(ItemMeta, object)): # I changed this for 2-3 compatibility
type_variants = defaultdict(list)
extension_index = defaultdict(list)
# __metaclass__ = ItemMeta # unnessecary now
def __init__(self, data_version=None):
self.data_version = data_version
The type_variants defaultdict is what is getting filled in python 2 but not in 3.
It very much seems to be getting filled by this class when is in a different file called mime_types.py.
class MIMETypes(object):
_types = Types(VERSION)
def __repr__(self):
return '<MIMETypes version:%s>' % VERSION
#classmethod
def load_from_file(cls, type_file):
data = open(type_file).read()
data = data.split('\n')
mime_types = Types()
for index, line in enumerate(data):
item = line.strip()
if not item:
continue
try:
ret = TEXT_FORMAT_RE.match(item).groups()
except Exception as e:
__parsing_error(type_file, index, line, e)
(unregistered, obsolete, platform, mediatype, subtype, extensions,
encoding, urls, docs, comment) = ret
if mediatype is None:
if comment is None:
__parsing_error(type_file, index, line, RuntimeError)
continue
extensions = extensions and extensions.split(',') or []
urls = urls and urls.split(',') or []
mime_type = Type('%s/%s' % (mediatype, subtype))
mime_type.extensions = extensions
...
mime_type.url = urls
mime_types.add(mime_type) # instance of Type() is being filled?
return mime_types
The function startup() is being run whenever mime_types.py is imported and it does this.
def startup():
global STARTUP
if STARTUP:
type_files = glob(join(DIR, 'types', '*'))
type_files.sort()
for type_file in type_files:
MIMETypes.load_from_file(type_file) # class method is filling Types?
STARTUP = False
This all seems pretty weird to me. The MIMETypes class first creates an instance of Types() on the first line. _types = Types(VERSION). It then seems to do nothing with this instance and only use the mime_types instance created in the load_from_file() class method. mime_types = Types().
This sort of thing vaguely reminds me of javascript class construction. How is the instance mime_types filling Types.type_variants so that when it is imported like this.
from mime import Type, Types
The class's type_variants defaultdict can be used. And why isn't this working in python 3?
EDIT:
Adding extra code to show how type_variants is filled
(In "Types" Class)
#classmethod
def add_type_variant(cls, mime_type):
cls.type_veriants[mime_type.simplified].append(mime_type)
#classmethod
def add(cls, *types):
for mime_type in types:
if isinstance(mime_type, Types):
cls.add(*mime_type.defined_types())
else:
mts = cls.type_veriants.get(mime_type.simplified)
if mts and mime_type in mts:
Warning('Type %s already registered as a variant of %s.',
mime_type, mime_type.simplified)
cls.add_type_variant(mime_type)
cls.index_extensions(mime_type)
You can see that MIMETypes uses the add() classmethod.

Without posting more of your code, it's hard to say. I will say that I was able to get that package ported to Python 3 with only a few changes (print statement -> function, basestring -> str, adding a dot before same-package imports, and a really ugly hack to compensate for their love of cmp:
def cmp(x,y):
if isinstance(x, Type): return x.__cmp__(y)
if isinstance(y, Type): return y.__cmp__(x) * -1
return 0 if x == y else (1 if x > y else -1)
Note, I'm not even sure this is correct.
Then
import mime
print(mime.Types.type_veriants) # sic
printed out a 1590 entry defaultdict.
Regarding your question about MIMETypes._types not being used, I agree, it's not.
Regarding your question about how the dictionary is being populated, it's quite simple, and you've identified most of it.
import mime
Imports the package's __init__.py which contains the line:
from .mime_types import MIMETypes, VERSION
And mime_types.py includes the lines:
def startup():
global STARTUP
if STARTUP:
type_files = glob(join(DIR, 'types', '*'))
type_files.sort()
for type_file in type_files:
MIMETypes.load_from_file(type_file)
STARTUP = False
startup()
And MIMETypes.load_from_file() has the lines:
mime_types = Types()
#...
for ... in ...:
mime_types.add(mime_type)
And Types.add(): has the line:
cls.add_type_variant(mime_type)
And that classmethod contains:
cls.type_veriants[mime_type.simplified].append(mime_type)

GAE converting dictionary to NDB datastore entity

I would like to ask some guidelines on a small task that I am trying to solve.
I am experimenting with a small app that uses JSON data to save entities.
I know that you can easily convert a dict to an entity by just creating the model but, I am trying to build a more generic approach that would convert any dict to an entity.
My steps are:
Get the dict.
Validate that the dict keys correspond to an entitys model definitions by reading the class.dict of the model.
Try to unpack the validated properties in the model class contructor (create the model instance)
return it.
So far I am ok but lack of my python knowledge, is either constraining me, or confusing me.
Maybe I am as well forgetting or unaware of more simple way to do it.
So here is it:
#classmethod
def entity_from_dict(cls, parent_key, dict):
valid_properties = {}
logging.info(cls.__dict__)
for property,value in dict.iteritems():
if property in cls.__dict__: # should not iterate over functions, classmethods, and #property
logging.info(cls.__dict__[property]) # this outputs eg: StringProperty('title', required=True)
logging.info(type(cls.__dict__[property])) #this is more interesting <class 'google.appengine.ext.ndb.model.StringProperty'>
valid_properties.update({property: value})
# Update the id from the dict
if 'id' in dict: # if not creating a new entity
valid_properties['id'] = dict['id']
# Add the parent
valid_properties['parent'] = parent_key
#logging.info(valid_properties)
try:
entity = cls(**valid_properties)
except Exception as e:
logging.exception('Could not create entity \n' + repr(e))
return False
return entity
My problem is that I want only to validate ndb. Properties and not #classmethods, #property as well because this causes a conflict.
I am also using expando classes, so any property in the dict that is extra gets stored.
How can I check against these specific types?

Solved it as #Tim Hoffman proposed using the ._properties of the Ndb model.
A thing I didn't know is that via the ._properties I could get the model definition properties and I thought that it would only return the instance properties :-).
Also I did not use populate because I find that it does the same as passing the valid dict unpacked in the model's contructor ;-)
So here it is:
#classmethod
def entity_from_dict(cls, parent_key, data_dict):
valid_properties = {}
for cls_property in cls._properties:
if cls_property in data_dict:
valid_properties.update({cls_property: data_dict[cls_property]})
#logging.info(valid_properties)
# Update the id from the data_dict
if 'id' in data_dict: # if creating a new entity
valid_properties['id'] = data_dict['id']
# Add the parent
valid_properties['parent'] = parent_key
try:
entity = cls(**valid_properties)
except Exception as e:
logging.exception('Could not create entity \n' + repr(e))
return False
return entity

The JSON dump method in python which we using during the converting models to JSON for export converts non-strings into strings. Therefore Jimmy Kane methods throw the error due to model incompatibility. To avoid this problem I updated his method and added a method named prop_literal just for converting non-string characters which capsuled in the string into their literal type.
I also added the entity.put() to add the entity to datastore because the aim was that :)
def prop_literal(prop_type,prop_val):
"""
Convert non-string encapsulated in the string into literal type
"""
if "Integer" in prop_type:
return int(prop_val)
elif "Float" in prop_type:
return float(prop_val)
elif "DateTime" in prop_type:
# bos gecsin neticede locale
return None
elif ("String" in prop_type) or ("Text" in prop_type):
return prop_val
elif "Bool" in prop_type:
return True if prop_val == True else False
else:
return prop_val
def entity_from_dict(cls, parent_key, data_dict):
valid_properties = {}
for cls_property in cls._properties:
if cls_property in data_dict:
prop_type = str(cls._properties[cls_property])
# logging.info(prop_type)
real_val = prop_literal(prop_type,data_dict[cls_property])
try:
valid_properties.update({cls_property: real_val})
except Exception as ex:
# logging.info("Veri aktariminda hata:"+str(ex))
else:
# logging.info("prop skipped")
#logging.info(valid_properties)
# Update the id from the data_dict
if 'id' in data_dict: # if creating a new entity
valid_properties['id'] = data_dict['id']
# Add the parent
valid_properties['parent'] = parent_key
try:
entity = cls(**valid_properties)
logging.info(entity)
entity.put()
except Exception as e:
logging.exception('Could not create entity \n' + repr(e))
return False
return entity