I was recently hired as a junior dev as my first job for a bigger company which uses NetSuite. An old dev wrote a python script which handles pictures made by designers, that uploads pictures to NetSuite when they are uploaded to a specific folder.
Since the Script uses SHA1 I need to change the TBA to SHA256 because NetSuite does not support SHA1 anymore.
I have a hard time understanding the old dev's code, and find documentation on how to change the TBA from SHA1 to SHA256..
These are snippets from the code.
import datetime
import requests
import os
import oauth2 as oauth
import json
import time
import base64
import sys
import hashlib
import hmac
url = "https://xxxxx=1"
token = oauth.Token(key="xxxxxxxxxxx",secret="xxxxxxxxxx")
consumer = oauth.Consumer(key="xxxxxxxxxxxxxxxx",secret="xxxxxxxxxxxxxxxx")
realm="xxxxxxxxxxxxxx"
signature_method = oauth.SignatureMethod_HMAC_SHA1()
In this part I understand he initialises the method oauth.SignatureMethod_HMAC_SHA1().
Then when I go to the oauth file I find this
class SignatureMethod_HMAC_SHA1(SignatureMethod):
name = 'HMAC-SHA1'
def signing_base(self, request, consumer, token):
if (not hasattr(request, 'normalized_url') or request.normalized_url is None):
raise ValueError("Base URL for request is not set.")
sig = (
escape(request.method),
escape(request.normalized_url),
escape(request.get_normalized_parameters()),
)
key = '%s&' % escape(consumer.secret)
if token:
key += escape(token.secret)
raw = '&'.join(sig)
return key.encode('ascii'), raw.encode('ascii')
def sign(self, request, consumer, token):
"""Builds the base signature string."""
key, raw = self.signing_base(request, consumer, token)
hashed = hmac.new(key, raw, sha1)
# Calculate the digest base 64.
return binascii.b2a_base64(hashed.digest())[:-1]
I looked this file through, and it does not contain any methods containing SHA256.. Only SHA1 and PLAINTEXT.
I tried to change the values to SHA256 but that did not work of course.
I tried to look up documentation on oAuth2 but I only found very small amounts of information, and it seems like it only contains SHA1 and PLAINTEXT..
So how do I change the script to function with SHA256 instead of SHA1?
EDIT to answer comment
Hashlib contains this:
class _Hash(object):
digest_size: int
block_size: int
# [Python documentation note] Changed in version 3.4: The name attribute has
# been present in CPython since its inception, but until Python 3.4 was not
# formally specified, so may not exist on some platforms
name: str
def __init__(self, data: _DataType = ...) -> None: ...
def copy(self) -> _Hash: ...
def digest(self) -> bytes: ...
def hexdigest(self) -> str: ...
def update(self, arg: _DataType) -> None: ...
def md5(arg: _DataType = ...) -> _Hash: ...
def sha1(arg: _DataType = ...) -> _Hash: ...
def sha224(arg: _DataType = ...) -> _Hash: ...
def sha256(arg: _DataType = ...) -> _Hash: ...
def sha384(arg: _DataType = ...) -> _Hash: ...
def sha512(arg: _DataType = ...) -> _Hash: ...
def new(name: str, data: _DataType = ...) -> _Hash: ...
algorithms_guaranteed: AbstractSet[str]
algorithms_available: AbstractSet[str]
def pbkdf2_hmac(hash_name: str, password: _DataType, salt: _DataType, iterations: int, dklen: Optional[int] = ...) -> bytes: ...
if sys.version_info >= (3, 6):
class _VarLenHash(object):
digest_size: int
block_size: int
name: str
def __init__(self, data: _DataType = ...) -> None: ...
def copy(self) -> _VarLenHash: ...
def digest(self, length: int) -> bytes: ...
def hexdigest(self, length: int) -> str: ...
def update(self, arg: _DataType) -> None: ...
sha3_224 = _Hash
sha3_256 = _Hash
sha3_384 = _Hash
sha3_512 = _Hash
shake_128 = _VarLenHash
shake_256 = _VarLenHash
def scrypt(password: _DataType, *, salt: _DataType, n: int, r: int, p: int, maxmem: int = ..., dklen: int = ...) -> bytes: ...
class _BlakeHash(_Hash):
MAX_DIGEST_SIZE: int
MAX_KEY_SIZE: int
PERSON_SIZE: int
SALT_SIZE: int
def __init__(self, data: _DataType = ..., digest_size: int = ..., key: _DataType = ..., salt: _DataType = ..., person: _DataType = ..., fanout: int = ..., depth: int = ..., leaf_size: int = ..., node_offset: int = ..., node_depth: int = ..., inner_size: int = ..., last_node: bool = ...) -> None: ...
blake2b = _BlakeHash
blake2s = _BlakeHash
There is already sha256() function in Haslib file,
so you can try to add a new class SignatureMethod_HMAC_SHA256 into the oauth file which can be similar to that SHA1.
Just change parameters of hmac.new() function like this:
hashed = hmac.new(key, raw, sha256)
Whole class can look like this:
class SignatureMethod_HMAC_SHA256(SignatureMethod):
name = 'HMAC-SHA256'
def signing_base(self, request, consumer, token):
if (not hasattr(request, 'normalized_url') or request.normalized_url is None):
raise ValueError("Base URL for request is not set.")
sig = (
escape(request.method),
escape(request.normalized_url),
escape(request.get_normalized_parameters()),
)
key = '%s&' % escape(consumer.secret)
if token:
key += escape(token.secret)
raw = '&'.join(sig)
return key.encode('ascii'), raw.encode('ascii')
def sign(self, request, consumer, token):
"""Builds the base signature string."""
key, raw = self.signing_base(request, consumer, token)
hashed = hmac.new(key, raw, sha256)
# Calculate the digest base 64.
return binascii.b2a_base64(hashed.digest())[:-1]
Then you can simply call in your script new SHA256 method instead of that deprecated SHA1 method:
signature_method = oauth.SignatureMethod_HMAC_SHA256()
Related
Any suggestion to not make repetitive code?
I want a better edit method with less complexity.
Generating the payload dict for each kwarg and then setting the value back to the self also uploading the new data in the database seems complex.
from __future__ import annotations
from typing import Union, TYPE_CHECKING, Dict
import attr
from utils.util import MISSING
if TYPE_CHECKING:
from .profile import Profile
#attr.s(auto_attribs=True, slots=True)
class Stats(object):
attack: int = 1
defence: int = 1
healing: Union[int, float] = 1
health: int = 100
health_cap: int = 100
def to_dict(self):
return attr.asdict(self)
async def edit(
self,
profile: Profile,
*,
attack: int = MISSING,
defence: int = MISSING,
healing: Union[int, float] = MISSING,
health: int = MISSING,
health_cap: int = MISSING
):
payload: Dict[str, int] = {}
for kwag, field in (
(attack, "attack"),
(defence, "defence"),
(health, "health"),
(healing, "healing_speed"),
(health_cap, "health_cap"),
):
if kwag is not MISSING:
payload[field] = kwag
setattr(self, field, getattr(self, field) + kwag)
await profile._http.increment(profile.id, {"stats": payload})
I'm having a trouble while mocking Google Cloud Storage Client in Python.
I've tried a couple solutions, but none of them seems to be working in my case. This is what I've tried:
Unit testing Mock GCS
Python unittest.mock google storage - how to achieve exceptions.NotFound as side effect
How to Mock a Google API Library with Python 3.7 for Unit Testing
So basically I have a class for downloading the decoded GCS object.
This is the class that I want to test:
class Extract:
def __init__(self, storage_client: storage.Client) -> None:
self.client = storage_client
#retry.Retry(initial=3.0, maximum=60.0, deadline=240.0)
def _get_storage_object(self, bucket: str, blob: str) -> bytes:
bucket_obj = self.client.get_bucket(bucket)
blob_obj = bucket_obj.blob(blob)
data = blob_obj.download_as_bytes(timeout=60)
return data
def get_decoded_blob(self, bucket: str, blob: str) -> Union[dict, list]:
raw_blob = self._get_storage_object(bucket=bucket, blob=blob)
decoded_blob = decode_data(raw_blob)
return decoded_blob
And this last version of code for unit testing of the class.
def decode_data(data: bytes) -> List[Dict[str, Any]]:
decoded_blob = data.decode('utf-8')
decoded_data = ast.literal_eval(decoded_blob)
return decoded_data
class TestExtract:
#pytest.fixture()
#mock.patch('src.pipeline.extract.storage', spec=True)
def storage_client(self, mock_storage):
mock_storage_client = mock_storage.Client.return_value
mock_bucket = mock.Mock()
mock_bucket.blob.return_value.download_as_bytes.return_value = 'data'.encode('utf-8')
mock_storage_client.bucket.return_value = mock_bucket
return mock_storage_client
def test_storage_client(self, storage_client):
gcs = extract.Extract(storage_client)
output = gcs._get_storage_object('bucket', 'blob')
assert output == b'data'
The error for the above code is that mock does not return return_value which was assigned in the fixture. Instead, it returns the MagicMock object as below:
AssertionError: assert <MagicMock na...806039010368'> == b'data'
Anyone can help me in this case?
My prof gave us this code to play with, but I have troubles calling an async function from a class
import asyncio
import aiohttp
import time
from pathlib import Path
from typing import List, Any, Dict, Union, Awaitable, Optional
import json
import toml
from mypy_extensions import TypedDict
# apikey = …
_coin_list: Dict[str, Any] = {}
async def fetch(url: str) -> Dict[str, Any]:
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp:
text = await resp.text()
return json.loads(text)
async def fetch_price_data(from_currencies: List[str], to_currencies: List[str], full: bool=False) -> Dict[str, Any]:
if full:
endpoint = 'pricemultifull'
from_parameter = 'fsyms'
if 'USD' not in to_currencies:
to_currencies.append('USD')
else:
endpoint = 'pricemulti'
from_parameter = 'fsyms'
price_url = f'https://min-api.cryptocompare.com/data/{endpoint}?' \
f'{from_parameter}={",".join(from_currencies)}&' \
f'tsyms={",".join(to_currencies)}'
resp = await fetch(price_url)
if full:
resp = resp['RAW']
return resp
async def fetch_coin_list() -> Dict[str, Any]:
global _coin_list
async with asyncio.Lock():
if not _coin_list:
coin_list_url = 'https://min-api.cryptocompare.com/data/all/coinlist'
_coin_list = await fetch(coin_list_url)
return _coin_list['Data']
# warnings
class CryptoPriceException(Exception):
pass
class CurrencyNotFound(CryptoPriceException):
pass
class UnfetchedInformation(CryptoPriceException):
pass
CurrencyKwargs = TypedDict('CurrencyKwargs', {
'cache': int,
'target_currencies': List[str],
'full': bool,
'historical': Optional[str],
'human': bool
})
class Prices:
"""Prices object"""
def __init__(self, currency: 'Currency') -> None:
self._prices: Dict[str, float] = {}
self._currency = currency
async def get(self, target_currency: str, default: float=0.0) -> float:
"""
Gets the price for a specified currency, if currency is not in target currencies,
it's added there for the specific currency
:param target_currency: Currency to get converted price for
:param default: What to return if the desired currency is not found in fetched prices
"""
target_currency = target_currency.upper()
if target_currency not in self._currency.target_currencies:
self._currency.target_currencies.append(target_currency)
await self._currency.load()
# TODO float should be in the dict from when it's put there -> stop using .update() with data from api
return float(self._prices.get(target_currency, default))
def __getitem__(self, item: str) -> float:
try:
return float(self._prices[item.upper()])
except KeyError:
raise CurrencyNotFound("Desired target currency not found, make sure it's in desired_currencies "
"and that cryptocompare.com supports it.")
def __setitem__(self, key: str, value: float) -> None:
self._prices[key.upper()] = value
def __getattr__(self, item: str) -> float:
return self[item]
class Currency:
"""
Currency object
"""
def __init__(self, symbol: str, cache: int=60, target_currencies: List[str] = None,
full: bool = False, historical: Optional[str] = None, human: bool = False) -> None:
"""
:param symbol: Symbol of the currency (e.g. ZEC) - will be converted to uppercase automatically
:param cache: Seconds to keep prices in cache
:param target_currencies: Which currencies to convert prices to
:param full: Whether to fetch full data, like change, market cap, volume etc.
:param historical: Whether to fetch movement data, either None, or 'minute', 'hour', 'day'
:param human: Whether to fetch information that concern humans (logo, full name)
"""
self.symbol = symbol.upper()
self.cache = cache
self.target_currencies = target_currencies or ['USD', 'BTC']
self.last_loaded: Union[bool, float] = False
self.prices = Prices(self)
self.full = full
self.historical = historical
self.human = human
self.human_data: Dict[str, Any] = {}
self.full_data: Dict[str, Any] = {}
# #property
# def image_url(self) -> str:
# """Available only if human is True - url to a image of currency's logo"""
# if not self.human:
# raise UnfetchedInformation('human must be True to fetch image_url')
# return f'https://www.cryptocompare.com{self.human_data.get("ImageUrl")}'
# #property
# def name(self) -> str:
# """Available only if human is True - name of the currency (e.g. Bitcoin from BTC)"""
# if not self.human:
# raise UnfetchedInformation('human must be True to fetch name')
# return self.human_data.get('CoinName', '')
#property
def supply(self) -> float:
if not self.full:
raise UnfetchedInformation('full must be True to fetch supply')
return float(self.full_data['USD']['SUPPLY'])
#property
def market_cap(self) -> float:
# TODO should be in self.prices
if not self.full:
raise UnfetchedInformation('full must be True to fetch market_cap')
return float(self.full_data['USD']['MKTCAP'])
def volume(self):
raise NotImplementedError
async def load(self) -> None:
"""Loads the data if they are not cached"""
tasks: List[Awaitable[Any]] = []
if not self.last_loaded:
if self.human:
tasks.append(fetch_coin_list())
if not self.last_loaded or time.time() < self.last_loaded + self.cache:
tasks.append(self.__load())
await asyncio.gather(*tasks)
if self.human and not self.human_data:
extra_data = await fetch_coin_list()
self.human_data = extra_data.get(self.symbol, {})
self.last_loaded = time.time()
async def __load(self) -> None:
try:
json_data = await fetch_price_data([self.symbol], self.target_currencies, full=self.full)
except Exception as _:
fallback = self.__load_fallback()
for tsym, price in self.prices._prices.items():
self.prices._prices[tsym] = fallback[tsym]
else:
if self.full:
self.full_data = json_data.get(self.symbol, {})
for tsym, price in self.prices._prices.items():
if self.full_data.get(tsym):
self.prices._prices[tsym] = self.full_data.get(tsym, {}).get('PRICE')
else:
self.prices._prices.update(json_data.get(self.symbol, {}))
def __load_fallback(self):
fallback_toml = (Path(__file__).resolve().parent / 'fallbacks.tml')
with fallback_toml.open(mode='r') as f:
return toml.load(f)
class Currencies:
"""
Wrapper around currencies.
Paramaters will propagate to all currencies gotten through this wrapper.
If you want to share state across modules, you should import currencies with lowercase
and set their parameters manually.
"""
def __init__(self, cache: int=60, target_currencies: List[str]=None,
full: bool=False, historical: Optional[str]=None, human: bool=False) -> None:
"""
:param cache: Seconds to keep prices in cache
:param target_currencies: Which currencies to convert prices to
:param full: Whether to fetch full data, like change, market cap, volume etc.
:param historical: Whether to fetch movement data, either None, or 'minute', 'hour', 'day'
TODO https://min-api.cryptocompare.com/data/histominute?fsym=BTC&tsym=USD&limit=60&aggregate=3&e=CCCAGG
TODO aggregate -> po kolika minutach, cas je v timestampech
Bonusove argumenty v metode a vracet jen metodou?
:param human: Whether to fetch information that concern humans (logo, full name)
Will not work with classic currencies like USD or EUR
"""
self.currencies: Dict[str, Currency] = dict()
self.cache = cache
self.target_currencies = target_currencies or ['USD', 'BTC', 'ETH']
self.full = full
self.historical = historical
self.human = human
async def load_all(self) -> None:
"""Loads data for all currencies"""
symbols = []
for _, currency in self.currencies.items():
symbols.append(currency.symbol)
if self.human:
# This is done just once, as the contents don't change
await fetch_coin_list()
# TODO fetch only if at least one isn't cached
price_data = await fetch_price_data(symbols, self.target_currencies, full=self.full)
for symbol, currency in self.currencies.items():
if self.full:
currency.full_data = price_data.get(symbol, {})
currency.prices._prices.update(price_data.get(symbol, {}))
currency.last_loaded = time.time()
if self.human:
# Update the currency with already fetched extra information
await currency.load()
def add(self, *symbols: str) -> None:
"""Add to the list of symbols for which to load prices"""
for symbol in symbols:
if symbol not in self.currencies:
self.currencies[symbol] = Currency(symbol, **self.__currency_kwargs)
#property
def __currency_kwargs(self) -> CurrencyKwargs:
"""All kwargs that are propagated to individual currencies"""
return {
'cache': self.cache,
'target_currencies': self.target_currencies,
'full': self.full,
'historical': self.historical,
'human': self.human
}
def __getitem__(self, item: str) -> Currency:
"""Gets a currency, if not present, will create one"""
item = item.upper()
if item not in self.currencies:
self.currencies[item] = Currency(item, **self.__currency_kwargs)
return self.currencies[item]
def __getattr__(self, item: str) -> Currency:
"""Same as getitem, but accessible with dots"""
return self[item]
currencies = Currency()
For example, I'm trying to call fetch_coin_list function but it gives an error. And any method I'm trying to call gives an error.
I'm pretty sure I call it the wrong way but I have no idea how to fix it. Sorry I'm really stupid and it's my first time working with async functions, please help. I'll be incredibly thankful
A simple usage example is provided on the documentation page.
import asyncio
async def main():
print('Hello ...')
await asyncio.sleep(1)
print('... World!')
# Python 3.7+
asyncio.run(main())
If you want to run multiple asynchronous tasks, then you can refer to the method load of the class Currency in the code you provided, which uses the asyncio.gather method.
What is a good design pattern to implement templated object generation (not sure that's the name) in python?
By that, I mean having a function such as:
from typing import TypeVar
T = TypeVar('T')
def mk_templated_obj_factory(template: T) -> Callable[..., T]:
"""Returns a f(**kwargs) function that returns an object of type T created by a template of the same type."""
Python has templated strings. Something like `"this {is} a {template}".format' would be how one could achieve the above. If we want to get a "proper" function that has a signature (useful for a user so they know what arguments they need to provide!), we could do this:
from inspect import signature, Signature, Parameter
from operator import itemgetter
from typing import Callable
f = "hello {name} how are you {verb}?".format
def templated_string_func(template: str) -> Callable:
"""A function making templated strings. Like template.format, but with a signature"""
f = partial(str.format, template)
names = filter(None, map(itemgetter(1), string.Formatter().parse(template)))
params = [Parameter(name=name, kind=Parameter.KEYWORD_ONLY) for name in names]
f.__signature__ = Signature(params)
return f
f = templated_string_func("hello {name} how are you {verb}?")
assert f(name='Christian', verb='doing') == 'hello Christian how are you doing?'
assert str(signature(f)) == '(*, name, verb)'
But would if we want to make dict factories? Something having this behavior:
g = templated_dict_func(template={'hello': '$name', 'how are you': ['$verb', 2]})
assert g(name='Christian', verb='doing') == {'hello': '$name', 'how are you': ['doing', 2]}
What about other types of objects?
It seems like something that would have a solid design pattern...
I would recommend using decorators to register your template function generating functions in a dictionary that maps from types to the functions that handle them. The dictionary is needed in order to be able to template objects of any type in an extensible way, without writing all the templating logic in a single big function, but instead adding handling logic for new types as needed.
The core code is in the Templater class, just grouped here for organisation:
class Templater:
templater_registry: dict[type, Callable[[Any], TemplateFunc]] = {}
#classmethod
def register(cls, handles_type: type):
def decorator(f):
cls.templater_registry[handles_type] = f
return f
return decorator
...
Where TemplateFunc is defined as Generator[str, None, Callable[..., T]], a generator that yields strs and returns a function that returns some type T. This is chosen so that the template handlers can yield the names of their keyword arguments and then return their template function. The Templater.template_func method uses a something of type TemplateFunc to generate a function with the correct signature.
The register decorator presented above is written such that:
#Templater.register(dict)
def templated_dict_func(template: dict[K, V]):
pass
is equivalent to:
def templated_dict_func(template: dict[K, V]):
pass
Templater.templater_registry[dict] = templated_dict_func
The code for templating any type is fairly self-explainatory:
class Templater:
...
#classmethod
def template_func_generator(cls, template: T) -> TemplateFunc[T]:
# if it is a type that can be a template
if type(template) in cls.templater_registry:
# then return the template handler
template_factory = cls.templater_registry[type(template)]
return template_factory(template)
else:
# else: an empty generator that returns a function that returns the template unchanged,
# since we don't know how to handle it
def just_return():
return lambda: template
yield # this yield is needed to tell python that this is a generator
return just_return()
The code for templating strings is fairly unchanged, except that the argument names are yielded instead of put in the function signature:
#Templater.register(str)
def templated_string_func(template: str) -> TemplateFunc[str]:
"""A function making templated strings. Like template.format, but with a signature"""
f = partial(str.format, template)
yield from filter(None, map(itemgetter(1), string.Formatter().parse(template)))
return f
The list template function could look like this:
#Templater.register(list)
def templated_list_func(template: list[T]) -> TemplateFunc[list[T]]:
entries = []
for item in template:
item_template_func = yield from Templater.template_func_generator(item)
entries.append(item_template_func)
def template_func(**kwargs):
return [
item_template_func(**kwargs)
for item_template_func in entries
]
return template_func
Although, if you cannot guarantee that every template function can handle extra arguments, you need to track which arguments belong to which elements and only pass the necessary ones. I use the get_generator_return utility function (defined later on) to capture both the yielded values and the return value of the recursive calls.
#Templater.register(list)
def templated_list_func(template: list[T]) -> TemplateFunc[list[T]]:
entries = []
for item in template:
params, item_template_func = get_generator_return(Templater.template_func_generator(item))
params = tuple(params)
yield from params
entries.append((item_template_func, params))
def template_func(**kwargs):
return [
item_template_func(**{arg: kwargs[arg] for arg in args})
for item_template_func, args in entries
]
return template_func
The dict handler is implemented similarly. This system could be extended to support all kinds of different objects, including arbitrary dataclasses and more, but I leave that as an exercise for the reader!
Here is the entire working example:
import string
from functools import partial
from inspect import Signature, Parameter
from operator import itemgetter
from typing import Callable, Any, TypeVar, Generator, Tuple, Dict, List
from collections import namedtuple
T = TypeVar('T')
U = TypeVar('U')
def get_generator_return(gen: Generator[T, Any, U]) -> Tuple[Generator[T, Any, U], U]:
return_value = None
def inner():
nonlocal return_value
return_value = yield from gen
gen_items = list(inner())
def new_gen():
yield from gen_items
return return_value
return new_gen(), return_value
# TemplateFunc: TypeAlias = Generator[str, None, Callable[..., T]]
TemplateFunc = Generator[str, None, Callable[..., T]]
class Templater:
templater_registry: Dict[type, Callable[[Any], TemplateFunc]] = {}
#classmethod
def register(cls, handles_type: type):
def decorator(f):
cls.templater_registry[handles_type] = f
return f
return decorator
#classmethod
def template_func_generator(cls, template: T) -> TemplateFunc[T]:
if type(template) in cls.templater_registry:
template_factory = cls.templater_registry[type(template)]
return template_factory(template)
else:
# an empty generator that returns a function that returns the template unchanged,
# since we don't know how to handle it
def just_return():
return lambda: template
yield # this yield is needed to tell python that this is a generator
return just_return()
#classmethod
def template_func(cls, template: T) -> Callable[..., T]:
gen = cls.template_func_generator(template)
params, f = get_generator_return(gen)
f.__signature__ = Signature(Parameter(name=param, kind=Parameter.KEYWORD_ONLY) for param in params)
return f
#Templater.register(str)
def templated_string_func(template: str) -> TemplateFunc[str]:
"""A function making templated strings. Like template.format, but with a signature"""
f = partial(str.format, template)
yield from filter(None, map(itemgetter(1), string.Formatter().parse(template)))
return f
K = TypeVar('K')
V = TypeVar('V')
#Templater.register(dict)
def templated_dict_func(template: Dict[K, V]) -> TemplateFunc[Dict[K, V]]:
DictEntryInfo = namedtuple('DictEntryInfo', ['key_func', 'value_func', 'key_args', 'value_args'])
entries: list[DictEntryInfo] = []
for key, value in template.items():
key_params, key_template_func = get_generator_return(Templater.template_func_generator(key))
value_params, value_template_func = get_generator_return(Templater.template_func_generator(value))
key_params = tuple(key_params)
value_params = tuple(value_params)
yield from key_params
yield from value_params
entries.append(DictEntryInfo(key_template_func, value_template_func, key_params, value_params))
def template_func(**kwargs):
return {
entry_info.key_func(**{arg: kwargs[arg] for arg in entry_info.key_args}):
entry_info.value_func(**{arg: kwargs[arg] for arg in entry_info.value_args})
for entry_info in entries
}
return template_func
#Templater.register(list)
def templated_list_func(template: List[T]) -> TemplateFunc[List[T]]:
entries = []
for item in template:
params, item_template_func = get_generator_return(Templater.template_func_generator(item))
params = tuple(params)
yield from params
entries.append((item_template_func, params))
def template_func(**kwargs):
return [
item_template_func(**{arg: kwargs[arg] for arg in args})
for item_template_func, args in entries
]
return template_func
g = Templater.template_func(template={'hello': '{name}', 'how are you': ['{verb}', 2]})
assert g(name='Christian', verb='doing') == {'hello': 'Christian', 'how are you': ['doing', 2]}
print(g.__signature__)
i was trying to find a solution similar to java Jackson ObjectMapper that can do serialization/deserialization of python object to json. and find that
cattrs is closest to what i need. but it cannot do attribute mapping like use firstName in json but first_name in the deserialized object.
attrs-serde can do the attribute mapping but cannot do recursive deserialization.
the problem can be illustrated in this example,
import attr
import cattr
from attrs_serde import serde
name_path = ["contact", "personal", "Name"]
phone_path = ["contact", "Phone"]
#serde
#attr.s(auto_attribs=True, frozen=True)
class Name:
first: str
last: str
#serde
#attr.s(auto_attribs=True, frozen=True)
class Person:
name: Name = attr.ib(metadata={"to": name_path, "from": name_path})
phone: str = attr.ib(metadata={"to": phone_path, "from": phone_path})
person_json = {"contact": {"personal": {"Name": {"first": "John", "last": "Smith"}}, "Phone": "555-112233"}}
# XXX: to/from only works on serde
p = Person(name=Name(first="John", last="Smith"), phone="555-112233")
print(p.to_dict())
# {'contact': {'personal': {'Name': {'first': 'John', 'last': 'Smith'}}, 'Phone': '555-112233'}}
p1 = Person.from_dict(person_json)
print(f"p1={p1}")
# p1=Person(name={'first': 'John', 'last': 'Smith'}, phone='555-112233')
# XXX: nested only works on cttrs
person = {"Name": {"First": "John", "Last": "Smith"}, "Phone": "555-112233"}
converter = cattr.Converter()
converter.register_structure_hook(
Person, lambda d, _: Person(name=converter.structure(d["Name"], Name), phone=d.get("Phone"))
)
converter.register_structure_hook(Name, lambda d, _: Name(first=d["First"], last=d.get("Last")))
p2 = converter.structure(person, Person)
print(p2)
assert p == p2
print(converter.unstructure(p2))
# {'name': {'first': 'John', 'last': 'Smith'}, 'phone': '555-112233'}
# {"contact": {"personal": {"name": "John"}, "phone": "555-112233"}}
any more elegant solution using cattr?
you can use humps to do case convert
import humps
import cattr
class CAttrConverter:
converter = cattr.Converter()
def __init__(self):
"""
structure hook for load
unstructure hook for dump
"""
def load(self, params, data_cls, camel_to_snake=True):
"""
:param params: params, mostly from front end
:param data_cls:
:param camel_to_snake: need to convert from camel style to snake style
"""
if camel_to_snake:
params = humps.depascalize(params)
return self.converter.structure(params, data_cls)
def dump(self, data, snake_to_camel=False):
"""
:param data:
:param snake_to_camel: dump as camel case
"""
result: dict = self.converter.unstructure(data)
if snake_to_camel:
result = humps.camelize(result)
return result
Posting this for those in the future.
Yeah you can achieve this by overloading the convert classes methods:
def unstructure_attrs_asdict(self, obj) -> Dict[str, Any]:
def structure_attrs_fromdict(
self, obj: Mapping[str, Any], cl: Type[T]
) -> T:
Or if you want tuples
def unstructure_attrs_astuple(self, obj) -> Tuple[Any, ...]:
def structure_attrs_fromtuple(
self, obj: Tuple[Any, ...], cl: Type[T]
) -> T:
Simple Converter classes that uses the to and from fields from metadata. I will leave handling nested fields to your imagination.
from typing import TypeVar, Dict, Any, Mapping, Type
from cattr import Converter
from cattr._compat import fields
T = TypeVar("T")
class ConverterWithMetaDataOverrides(Converter):
# Classes to Python primitives.
def unstructure_attrs_asdict(self, obj) -> Dict[str, Any]:
"""Our version of `attrs.asdict`, so we can call back to us."""
attrs = fields(obj.__class__)
dispatch = self._unstructure_func.dispatch
rv = self._dict_factory()
for a in attrs:
name = a.name
serialize_as = name
if 'to' in a.metadata:
serialize_as = a.metadata['to']
v = getattr(obj, name)
rv[serialize_as] = dispatch(a.type or v.__class__)(v)
return rv
def structure_attrs_fromdict(
self, obj: Mapping[str, Any], cl: Type[T]
) -> T:
"""Instantiate an attrs class from a mapping (dict)."""
# For public use.
conv_obj = {} # Start with a fresh dict, to ignore extra keys.
dispatch = self._structure_func.dispatch
for a in fields(cl): # type: ignore
# We detect the type by metadata.
type_ = a.type
name = a.name
serialize_from = name
if 'from' in a.metadata:
serialize_from = a.metadata['from']
try:
val = obj[serialize_from]
except KeyError:
continue
if name[0] == "_":
name = name[1:]
conv_obj[name] = (
dispatch(type_)(val, type_) if type_ is not None else val
)
return cl(**conv_obj) # type: ignore
converter = ConverterWithMetaDataOverrides()
Usage:
#attrs(slots=True, frozen=True, auto_attribs=True)
class LevelTwo(object):
a: str = ib(metadata={'from': 'haha_not_a', 'to': 'haha_not_a'})
b: str
c: int
#attrs(slots=True, frozen=True, auto_attribs=True)
class LevelOne(object):
leveltwo: LevelTwo = ib(metadata={'from': 'level_two', 'to': 'level_two'})
#attrs(slots=True, frozen=True, auto_attribs=True)
class Root(object):
levelone: LevelOne = ib(metadata={'from': 'levelOne', 'to': 'levelOne'})
converter.structure(converter.unstructure(Root(levelone=LevelOne(leveltwo=LevelTwo(a='here', b='here_again', c=42)))),
Root)
>>> converter.unstructure(Root(levelone=LevelOne(leveltwo=LevelTwo(a='here', b='here_again', c=42)))
>>> {'levelOne': {'level_two': {'haha_not_a': 'here', 'b': 'here_again', 'c': 42}}}