Consider the following:
import typing
import pickle
import os
import redis
T = typing.TypeVar("T")
class Cache(typing.Protocol[T]):
def set(self, key: str, value: T) -> None:
pass
def get(self, key: str) -> T | None:
pass
class LocalCache(typing.Generic[T]):
def __init__(self) -> None:
self.cache: dict[str, T] = {}
def set(self, key: str, value: T) -> None:
self.cache[key] = value
def get(self, key: str) -> T | None:
return self.cache.get(key, None)
class RedisCache(typing.Generic[T]):
def __init__(self, host: str = "localhost", port: int = 6379) -> None:
redis_host = os.environ.get("REDISHOST", host)
redis_port = int(os.environ.get("REDISPORT", str(port)))
self.redis_client = redis.Redis(host=redis_host, port=redis_port)
def set(self, key: str, value: T) -> None:
self.redis_client.set(key, pickle.dumps(value))
def get(self, key: str) -> T | None:
value = self.redis_client.get(key)
if value is None:
return value
return typing.cast(T, pickle.loads(value))
def create_cache(t: typing.Type[T], use_redis: bool = False) -> Cache[T]:
if use_redis:
return RedisCache[t]()
return LocalCache[t]()
cache = create_cache(int, use_redis=False)
I'm getting the following error from mypy:
Variable "t" is not valid as a type [valid-type]mypy(error) See
https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliasesmypy(note)
I can't seem to understand how to satisfy mypy with create_cache. Any suggestions? Thanks
Related
How do I check if a module implements specific functions in Python?
Say I have two modules, both implement two functions: f and g; with equal arity. However, in mod0 g returns a float and in mod1 g returns an int.
mod0/__init__.py
def f(a: int) -> int: return a
def g(a: int) -> float: return float(a)
mod1/__init__.py
def f(a: int) -> int: return a
def g(a: int) -> int: return a
# Or even `g = f`?
Attempt:
type_checker.py
from typing import Protocol
class ProtoFg(Protocol):
#staticmethod
def f(a: int) -> int: pass
#staticmethod
def g(a: int) -> float: pass
def check_conformance(mod: ProtoFg): pass
import mod0, mod1
check_conformance(mod0)
check_conformance(mod1) # I want an error here
What I am looking for is an error similar to: "mod1 doesn't conform to ProtoFg as mod1.f has return type of float not int"
Related: equivalent in TypeScript
A static typing way of solving the problem of "ensuring that a signature matches <X>" is normally a job for a series of no-op decorators:
from __future__ import annotations
import collections.abc as cx
import typing as t
P = t.ParamSpec("P")
R = t.TypeVar("R")
class SignatureDispenser(t.Protocol[P, R]):
def __call__(self, *args: P.args, **kwargs: P.kwargs) -> R:
...
def dispense_signature(
self: cx.Callable[P, R], f: cx.Callable[P, R], /
) -> cx.Callable[P, R]:
...
def dispense_signature(f: cx.Callable[P, R], /) -> cx.Callable[P, R]:
return f
def as_signature_dispenser(f: cx.Callable[P, R], /) -> SignatureDispenser[P, R]:
if not t.TYPE_CHECKING:
f.dispense_signature = dispense_signature
return t.cast(SignatureDispenser[P, R], f)
#as_signature_dispenser
def f(a: int) -> int:
return a
#f.dispense_signature # mypy: Argument 1 to "dispense_signature" of "SignatureDispenser" has incompatible type "Callable[[int], float]"; expected "Callable[[int], int]" [arg-type]
def g(a: int) -> float:
return float(a)
If you wanted to check this at runtime, I would opt for an ast.NodeVisitor which inspects your f and g. The following is a prototype implementation (only works for Python 3.9+):
import ast
import re
from inspect import getsource
from types import ModuleType
typing_Any_pattern: re.Pattern[str] = re.compile(r"^(?:[_typing]+\.)?Any$")
class ConformanceChecker(ast.NodeVisitor):
__f1_name: str
__f2_name: str
__f1_signature_str: str | None
__f2_signature_str: str | None
#classmethod
def check_conformance(
cls, module_: ModuleType, /, *, f1_name: str, f2_name: str
) -> bool:
"""
Returns `True` if the signature of the function named by `f1_name` matches the
signature named by `f2_name`, otherwise `False`.
"""
f1_tree: ast.Module = ast.parse(getsource(getattr(module_, f1_name)))
f2_tree: ast.Module = ast.parse(getsource(getattr(module_, f2_name)))
checker: ConformanceChecker = cls(f1_name=f1_name, f2_name=f2_name)
checker.visit(f1_tree)
checker.visit(f2_tree)
return checker.__f1_signature_str == checker.__f2_signature_str
def __init__(self, *, f1_name: str, f2_name: str) -> None:
self.__f1_name = f1_name
self.__f2_name = f2_name
self.__f1_signature_str = None
self.__f2_signature_str = None
def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
func_name: str = node.name
if func_name not in (self.__f1_name, self.__f2_name):
return
# Wipe the function body
node.body[:] = [ast.Expr(value=ast.Ellipsis())]
# Uniformly name positional-only arguments
arg_name_template: str = "__arg%d"
arg_node: ast.arg
for i, arg_node in enumerate(node.args.posonlyargs):
arg_node.arg = arg_name_template % i
self.generic_visit(node)
if _is_typing_Any_annotation(node.returns):
node.returns = None
node.name = "f" # Uniformly set function name for string comparison
if func_name == self.__f1_name:
self.__f1_signature_str = ast.unparse(node)
else:
self.__f2_signature_str = ast.unparse(node)
def visit_arg(self, node: ast.arg) -> None:
"""
Wipe any annotations in the signature which look like it refers to `typing.Any`
"""
if _is_typing_Any_annotation(node.annotation):
node.annotation = None
def _is_typing_Any_annotation(expr: ast.expr | None, /) -> bool:
return (expr is not None) and (
typing_Any_pattern.fullmatch(ast.unparse(expr)) is not None
)
>>> import mod0, mod1
>>> ConformanceChecker.check_conformance(mod0, f1_name="f", f2_name="g")
False
>>> ConformanceChecker.check_conformance(mod1, f1_name="f", f2_name="g")
True
I have a custom class collection with custom classes, but typehints wont work for me this way
foo = CustomClass("abc")
bar = CustomClass("cba")
collection = CustomClassCollection([foo, bar])
for item in collection:
item.<typehints here won't show up>
how can i workaround here to show type hints for iterated item?
here's CustomClassCollection iter method:
def __iter__(self) -> CustomClass:
return next(self)
implementation of CustomClassCollection:
class CustomClassCollection(KeyIndexedCollection, JsonSerializable):
def __init__(self, items: Optional[List[CustomClass]] = None):
super().__init__(items)
self._collection = {}
self._add_items_impl(self._collection, take_with_default(items, []))
def _add_items_impl(self, dst_collection, items):
for item in items:
self._add_impl(dst_collection, item)
def add(self, item: CustomClass) -> CustomClassCollection:
return self.clone(items=[*self.items(), item])
def add_items(self, items: List[CustomClass]) -> CustomClassCollection:
return self.clone(items=[*self.items(), *items])
def get(self, key: str, default: Optional[Any] = None) -> ObjClass:
return self._collection.get(key, default)
def __next__(self) -> CustomClass:
yield from self._collection.values()
def __iter__(self) -> CustomClass:
return next(self)
def items(self) -> List[CustomClass]:
return list(self._collection.values())
Implementation of CustomClass:
class CustomClass(KeyObject, JsonSerializable):
def __init__(self, name: str):
self._name = name
def name(self) -> str:
return self._name
Changing __iter__ to:
def __iter__(self) -> Iterator[CustomClass]:
return next(self)
Did the job
I've recently written my own OrderedSet implementation since I had problems with the publicly available ordered/sorted set implementations. The class uses a dict proxy object in the background and mostly forwards operations. I implemented all (in my opinion) relevant methods, incl. __iter__, and calls like list(myset) work as expected.
However, the call set(myset) always returns empty sets.
Here's the full code of OrderedSet:
from typing import TypeVar, Generic, Optional, Iterable, Set, AbstractSet, Union, Iterator, Any, Dict
T = TypeVar("T")
S = TypeVar("S")
class OrderedSet(Generic[T], Set[T]):
def __init__(self, base: Optional[Union[Dict[T, None], Iterable[T]]] = None):
super().__init__()
self.the_dict: Dict[T, None]
if not base:
self.the_dict = {}
elif isinstance(base, dict):
self.the_dict = base
else:
self.the_dict = dict.fromkeys(base)
def __eq__(self, o: object) -> bool:
return isinstance(o, OrderedSet) and list(self.the_dict) == list(o.the_dict)
def __ne__(self, o: object) -> bool:
return not self.__eq__(o)
def __str__(self) -> str:
return "{" + ", ".join(list(map(str, self.the_dict))) + "}"
def __repr__(self) -> str:
return f"OrderedSet({repr(self.the_dict)})"
def add(self, element: T) -> None:
self.the_dict = {**self.the_dict, **{element: None}}
def clear(self) -> None:
self.the_dict.clear()
def copy(self) -> 'OrderedSet[T]':
return OrderedSet(self.the_dict.copy())
def difference(self, s: Iterable[Any]) -> 'OrderedSet[T]':
return OrderedSet({e: None for e in self.the_dict if e not in s})
def difference_update(self, s: Iterable[Any]) -> None:
self.the_dict = {e: None for e in self.the_dict if e not in s}
def discard(self, element: T) -> None:
del self.the_dict[element]
def intersection(self, s: Iterable[Any]) -> 'OrderedSet[T]':
return OrderedSet({e: None for e in self.the_dict if e in s})
def intersection_update(self, s: Iterable[Any]) -> None:
self.the_dict = {e: None for e in self.the_dict if e in s}
def isdisjoint(self, s: Iterable[Any]) -> bool:
return self.the_dict.keys().isdisjoint(s)
def issubset(self, s: Iterable[Any]) -> bool:
return set(iter(self)).issubset(iter(s))
def issuperset(self, s: Iterable[Any]) -> bool:
return set(iter(self)).issuperset(iter(s))
def pop(self) -> T:
items = list(self.the_dict)
result = items.pop()
self.the_dict = dict.fromkeys(items)
return result
def remove(self, element: T) -> None:
del self.the_dict[element]
def symmetric_difference(self, s: Iterable[T]) -> 'OrderedSet[T]':
return OrderedSet(
dict.fromkeys([e for e in self.the_dict if e not in s] +
[e for e in s if e not in self.the_dict]))
def symmetric_difference_update(self, s: Iterable[T]) -> None:
self.the_dict = self.symmetric_difference(s).the_dict
def union(self, s: Iterable[T]) -> 'OrderedSet[T]':
return OrderedSet({**self.the_dict, **dict.fromkeys(s)})
def update(self, s: Iterable[T]) -> None:
self.the_dict = self.union(s).the_dict
def __len__(self) -> int:
return len(self.the_dict)
def __contains__(self, o: object) -> bool:
return o in self.the_dict
def __iter__(self) -> Iterator[T]:
return iter(self.the_dict)
def __and__(self, s: AbstractSet[object]) -> 'OrderedSet[T]':
return self.intersection(s)
def __iand__(self, s: AbstractSet[object]) -> 'OrderedSet[T]':
result = self.intersection(s)
self.the_dict = result.the_dict
return result
def __or__(self, s: AbstractSet[S]) -> 'OrderedSet[Union[T, S]]':
return self.union(s)
def __ior__(self, s: AbstractSet[S]) -> 'OrderedSet[Union[T, S]]':
result = self.union(s)
self.the_dict = result.the_dict
return result
def __sub__(self, s: AbstractSet[Optional[T]]) -> 'OrderedSet[T]':
return self.difference(s)
def __isub__(self, s: AbstractSet[Optional[T]]) -> 'OrderedSet[T]':
result = self.difference(s)
self.the_dict = result.the_dict
return result
def __xor__(self, s: AbstractSet[S]) -> 'OrderedSet[Union[T, S]]':
return self.symmetric_difference(s)
def __ixor__(self, s: AbstractSet[S]) -> 'OrderedSet[Union[T, S]]':
result = self.symmetric_difference(s)
self.the_dict = result.the_dict
return result
def __le__(self, s: AbstractSet[object]) -> bool:
return self.issubset(s)
def __lt__(self, s: AbstractSet[object]) -> bool:
return self.issubset(s) and len(self) < len(s)
def __ge__(self, s: AbstractSet[object]) -> bool:
return set(iter(self)) >= set(iter(s))
def __gt__(self, s: AbstractSet[object]) -> bool:
return set(iter(self)) > set(iter(s))
And here some example calls:
>>> from orderedset import OrderedSet
>>> s = OrderedSet([3, 1, 2])
>>> s
OrderedSet({3: None, 1: None, 2: None})
>>> list(s)
[3, 1, 2]
>>> set(s)
set()
Does anybody have an idea what's going on here? The set() function seems to be implemented in lengthy C code, which I don't understand well enough to deduce anything. It seems, though, that __iter__ is called for conversion to list, but not to set...
Any ideas?
Thanks a lot!
Python 3.9 or above
Your class should be inheriting from collections.abc.Set instead of typing.Set(which has been deprecated anyway)
from collections.abc import Set
...
class OrderedSet(Generic[T], Set[T]):
...
obj = OrderedSet()
obj.add(1)
obj.add(2)
obj.add(3)
print(set(obj)) # {1,2,3}
Python 3.8 or lower
Before 3.9, using generics with collections.abc.Set was not possible. Inheriting from AbstractSet solves the problem.
from typing import Generic, AbstractSet
...
class OrderedSet(Generic[T], AbstractSet[T]):
...
You need to initialize super or otherwise it would be an empty set
def __init__(self, base: Optional[Union[Dict[T, None], Iterable[T]]] = None):
self.the_dict: Dict[T, None]
if not base:
self.the_dict = {}
elif isinstance(base, dict):
self.the_dict = base
else:
self.the_dict = dict.fromkeys(base)
super().__init__(self.the_dict.keys())
I'm trying to get the following behavior with pydantic.BaseModel:
class MyClass:
def __init__(self, value: T) -> None:
self._value = value
# Maybe:
#property
def value(self) -> T:
return self._value
# Maybe:
#value.setter
def value(self, value: T) -> None:
# ...
self._value = value
If T is also a pydantic model, then recursive initialization using dictionaries should work:
# Initialize `x._value` with `T(foo="bar", spam="ham")`:
x = MyClass(value={"foo": "bar", "spam": "ham"})
Note that _value is initialized using the kwargs value. Validation must also be available for private fields.
The pydantic docs (PrivateAttr, etc.) seem to imply that pydantic will never expose private attributes. I'm sure there is some hack for this. But is there an idiomatic way to achieve the behavior in pydantic? Or should I just use a custom class?
Not sure it this solution is advisable, based on: https://github.com/samuelcolvin/pydantic/issues/1577
https://github.com/samuelcolvin/pydantic/issues/655
import inspect
from typing import Dict
from pydantic import BaseModel, PrivateAttr
from pydantic.main import no_type_check
class PatchedModel(BaseModel):
#no_type_check
def __setattr__(self, name, value):
"""
To be able to use properties with setters
"""
try:
super().__setattr__(name, value)
except ValueError as e:
setters = inspect.getmembers(
self.__class__,
predicate=lambda x: isinstance(x, property) and x.fset is not None
)
for setter_name, func in setters:
if setter_name == name:
object.__setattr__(self, name, value)
break
else:
raise e
class T(BaseModel):
value1: str
value2: int
class MyClassPydantic(PatchedModel):
_value: T = PrivateAttr()
def __init__(self, value: Dict, **kwargs):
super().__init__(**kwargs)
object.__setattr__(self, "_value", T(**value))
#property
def value(self) -> T:
return self._value
#value.setter
def value(self, value: T) -> None:
self._value: T = value
# To avoid the PatchedModel(BaseModel) use instead
# def set_value(self, value: T) -> None:
# self._value: T = value
if __name__ == "__main__":
my_pydantic_class = MyClassPydantic({"value1": "test1", "value2": 1})
print(my_pydantic_class.value)
my_pydantic_class.value = T(value1="test2", value2=2)
# my_pydantic_class.set_value(T(value1="test2", value2=2))
print(my_pydantic_class.value)
I ended up with something like this, it acts like a private field, but i can change it by public methods:
import inspect
from typing import Optional
from uuid import UUID
from pydantic import BaseModel, Field
class Entity(BaseModel):
"""Base entity class."""
def __setattr__(self, name, value):
if "self" not in inspect.currentframe().f_back.f_locals:
raise Exception("set attr is protected")
super().__setattr__(name, value)
class PostId(UUID):
"""Post unique id."""
class Post(Entity):
"""Post."""
post_id: PostId = Field(description='unique post id')
title: Optional[str] = Field(None, description='title')
def change_title(self, new_title: str) -> None:
"""Changes title."""
self.title = new_title
I just looking at inspect.currentframe().f_back.f_locals and looking for self key.
Ispired by accessify
Tested with this little test:
from uuid import uuid4
import pytest
import post_pydantic
def test_pydantic():
"""Test pydantic varriant."""
post_id = uuid4()
post = post_pydantic.Post(post_id=post_id)
with pytest.raises(Exception) as e:
post.post_id = uuid4()
assert post.post_id == post_id
assert e.value.args[0] == "set attr is protected"
new_title = "New title"
post.change_title(new_title)
assert post.title == new_title
I have following function signature:
def get(key, default=None):
pass
I want to specify return type to be as default type, but if default is None then it should be str:
T = TypeVar('T')
def get(key: str, default: Optional[T]=None) -> T:
pass
^ this solves first problem, but I can't figure out how to tell linter that alternative return type should be str and not Any.
I also tried -> Union[str, T] and no luck.
As #jonrsharpe suggested I tried to use #overload decorator:
from typing import TypeVar, overload
T = TypeVar('T')
#overload
def get(key: str, default=None) -> str: ...
#overload
def get(key: str, default: T) -> T: ...
def get(key: str, default: T=None) -> T:
if default is None:
return ''
print(key)
return 2
def main():
a = get('key')
a = get('key', None)
a = get('key', 1)
a = get('key', 1.23)
a = get('key', b'fd')
a = get('key', 'fds')
if __name__ == '__main__':
main()
But because of default is None (by default) detected return type is now Union[str, T].
Final edit
Should be def get(key: str, default: Optional[T]=None) -> T + overloads. Now everything is fine.