Add type hints #576

getzze · 2024-03-26T16:10:25Z

This is a huge PR, but hopefully it is not too hard to review.

To type check the code I had to setup mypy and I found it was simpler to also add a pyproject.toml file and bump the minimal python version to 3.8 (3.7 is not supported anymore anyway).
I also set up a ruff workflow in the pyproject.toml file, I find it very useful to lint and format the code.

All tests are passing, mypy and ruff give no errors.

The problem to review this PR is that I modified the python code sometimes, to improve type checking. Maybe I can add comments where the code was modified to make it easier to review. Some examples of things that I modified:

make no_default an Enum sentinel: using the string "no_default" do not work anymore
same for no_pad.
split def some_func(*args, ...) in def some_func(func, /, *args, ...) when args needed to be a function followed by its arguments.
use an explicit list of keyword arguments instead of **kwargs in function definitions to be able to type check the arguments.
add missing returns
transform assert to if blocks.

Also, I added a dependency to typing_extensions, that is a backport of typing improvements to older python versions. It can be removed, losing a bit of type checking for functoolz.excepts and functoolz.juxt.

Related PR: #516 #502
Issue: #496

getzze · 2024-03-26T16:11:34Z

tlz/_build_tlz.py

        if package == 'tlz':
            return ModuleSpec(fullname, self)
+        return None


Missing return

getzze · 2024-03-26T16:12:10Z

tlz/_build_tlz.py

        if package == 'tlz':
            return self
+        return None


missing return

getzze · 2024-03-26T16:12:33Z

tlz/_build_tlz.py


-class TlzLoader:
+
+class TlzLoader(Loader):


Make it an explicit subclass of Loader

getzze · 2024-03-26T16:13:19Z

toolz/_signatures.py

+    if sigspec is None:
+        return -1


account for sigspec=None

getzze · 2024-03-26T16:13:43Z

toolz/_signatures.py

    """ Return the names of position-only arguments if func has **kwargs"""
-    if num_pos_only == 0:
+    if num_pos_only == 0 or sigspec is None:


account for sigspec=None

getzze · 2024-03-26T16:14:19Z

toolz/_signatures.py

@@ -641,7 +673,7 @@ def expand_sig(sig):
    if isinstance(sig, tuple):
        if len(sig) == 3:
            num_pos_only, func, keyword_only = sig
-            assert isinstance(sig[-1], tuple)
+            # assert isinstance(sig[-1], tuple)


remove assert, replace with nothing here

getzze · 2024-03-26T16:14:59Z

toolz/curried/__init__.py

@@ -36,6 +35,7 @@
    count,
    curry,
    diff,
+    excepts,


For the tests to pass

getzze · 2024-03-26T16:15:22Z

toolz/curried/__init__.py

 from .exceptions import merge, merge_with

 accumulate = toolz.curry(toolz.accumulate)
+apply = toolz.curry(toolz.apply)


For the tests to pass, because the signature of apply changed.

getzze · 2024-03-26T16:16:11Z

toolz/dicttoolz.py

-def merge(*dicts, **kwargs):
+def merge(
+    *dicts: Mapping[_S, _T],
+    factory: type[_DictType] = dict,


explicitly add a factory keyword argument

eriknw · 2024-03-26T16:16:33Z

wow, nice! Thanks @getzze! I'm in the process of moving so it may take me some time to properly review this, but know that this is appreciated and likely to go in :)

getzze · 2024-03-26T16:16:43Z

toolz/dicttoolz.py

+def merge_with(
+    func: Callable[[Sequence[_T]], _U],
+    *dicts: Mapping[_S, _T],
+    factory: type[_DictType] = dict,


explicitly add a factory keyword argument

getzze · 2024-03-26T16:17:25Z

toolz/dicttoolz.py


-    values = collections.defaultdict(lambda: [].append)
+    values = defaultdict(list)


use defaultdict(list) instead of a lambda.

getzze · 2024-03-26T16:18:08Z

toolz/dicttoolz.py

+def dissoc(
+    d: Mapping[_S, _T],
+    *keys: _S,
+    factory: type[_DictType] = dict,


explicitly add a factory keyword argument

getzze · 2024-03-26T16:19:34Z

toolz/sandbox/core.py

-        if self.key == self._default_hashkey:
-            val = self.key
+    def __hash__(self) -> int:
+        if not callable(self.key):


check if the key is callable instead of a specific string.

getzze · 2024-03-26T16:22:04Z

toolz/sandbox/parallel.py

@@ -50,7 +70,8 @@ def fold(binop, seq, default=no_default, map=map, chunksize=128, combine=None):
    >>> fold(add, [1, 2, 3, 4], chunksize=2, map=map)
    10
    """
-    assert chunksize > 1
+    # assert chunksize > 1
+    chunksize = max(chunksize, 1)


update the variable instead of using an assert

getzze · 2024-03-26T16:23:33Z

toolz/utils.py

+class NoDefaultType(Enum):
+    no_default = '__no_default__'
+
+
+no_default = NoDefaultType.no_default
+
+# no_default = '__no__default__'
+# NoDefaultType = Literal['__no__default__']
+
+
+class NoPadType(Enum):
+    no_pad = '__no_pad__'
+
+
+no_pad = NoPadType.no_pad
+
+# no_pad = '__no__pad__'
+# NoPadType = Literal['__no__pad__']


use an Enum for no_default and no_pad instead of a string

getzze · 2024-03-26T16:26:49Z

toolz/itertoolz.py

+    d: dict[_S, list[_T]] = collections.defaultdict(list)
    for item in seq:
-        d[key(item)](item)
-    rv = {}
-    for k, v in d.items():
-        rv[k] = v.__self__
-    return rv
+        vals = d[key(item)]
+        vals.append(item)
+    return d


use defaultdict(list) instead of a lambda and loop only once over items

getzze · 2024-03-26T16:27:03Z

toolz/itertoolz.py

-def merge_sorted(*seqs, **kwargs):
+def merge_sorted(
+    *seqs: Iterable[_CT],
+    key: UnaryOp[_CT] | None = None,


explicitly add a key keyword argument

getzze · 2024-03-26T16:27:52Z

toolz/itertoolz.py

@@ -259,21 +305,20 @@ def unique(seq, key=None):
    ('cat', 'mouse')
    """
    seen = set()
-    seen_add = seen.add


do not cache the set.add function

getzze · 2024-03-26T16:28:39Z

toolz/itertoolz.py

-            seen_add(item)
-        return True
-    else:
+    if isinstance(seq, Sequence):


use isinstance(seq, Sequence) instead of not (iter(seq) is seq)

getzze · 2024-03-26T16:34:48Z

toolz/itertoolz.py

+    default: _T | NoDefaultType = no_default,
+    key: TransformOp[_T, Any] | None = None,


explicitly add a default and key keyword arguments

getzze · 2024-03-26T16:55:23Z

toolz/functoolz.py

@@ -26,7 +68,7 @@ def identity(x):
    return x


-def apply(*func_and_args, **kwargs):
+def apply(func: Callable[..., _T], /, *args: Any, **kwargs: Any) -> _T:


explicitly add func first positional only argument.

getzze · 2024-03-26T16:57:33Z

toolz/functoolz.py

-            args = (val,) + args
-            return func(*args)
-    return reduce(evalform_front, forms, val)
+        # if isinstance(form, tuple):


according to the types, no need to check for tuple type, but it would be good to check anyway.

getzze · 2024-03-26T16:57:43Z

toolz/functoolz.py

+    ) -> _T:
+        if callable(form):
+            return form(val)
+        # if isinstance(form, tuple):


according to the types, no need to check for tuple type, but it would be good to check anyway.

getzze · 2024-03-26T16:58:47Z

toolz/functoolz.py

+    def __init__(
+        self,
+        # TODO: type hint that the returned value of a `partial` is _T
+        func: curry[_T] | partial | Callable[..., _T],


add an explicit func first position only argument.

getzze · 2024-03-26T17:00:36Z

toolz/functoolz.py

        func = obj.func
-    obj = cls(func, *args, **(kwargs or {}))
+    # TODO: check that func is callable


TODO: check that func is really callable

add pyproject.toml, type_checking and ruff linting

00ab8f2