first commit

This commit is contained in:
Ayxan
2022-05-23 00:16:32 +04:00
commit d660f2a4ca
24786 changed files with 4428337 additions and 0 deletions

View File

@@ -0,0 +1,26 @@
from .itertoolz import *
from .functoolz import *
from .dicttoolz import *
from .recipes import *
from functools import partial, reduce
sorted = sorted
map = map
filter = filter
# Aliases
comp = compose
from . import curried, sandbox
functoolz._sigs.create_signature_registry()
from ._version import get_versions
__version__ = get_versions()['version']
del get_versions

View File

@@ -0,0 +1,785 @@
"""Internal module for better introspection of builtins.
The main functions are ``is_builtin_valid_args``, ``is_builtin_partial_args``,
and ``has_unknown_args``. Other functions in this module support these three.
Notably, we create a ``signatures`` registry to enable introspection of
builtin functions in any Python version. This includes builtins that
have more than one valid signature. Currently, the registry includes
builtins from ``builtins``, ``functools``, ``itertools``, and ``operator``
modules. More can be added as requested. We don't guarantee full coverage.
Everything in this module should be regarded as implementation details.
Users should try to not use this module directly.
"""
import functools
import inspect
import itertools
import operator
from importlib import import_module
from .functoolz import (is_partial_args, is_arity, has_varargs,
has_keywords, num_required_args)
import builtins
# We mock builtin callables using lists of tuples with lambda functions.
#
# The tuple spec is (num_position_args, lambda_func, keyword_only_args).
#
# num_position_args:
# - The number of positional-only arguments. If not specified,
# all positional arguments are considered positional-only.
#
# lambda_func:
# - lambda function that matches a signature of a builtin, but does
# not include keyword-only arguments.
#
# keyword_only_args: (optional)
# - Tuple of keyword-only argumemts.
module_info = {}
module_info[builtins] = dict(
abs=[
lambda x: None],
all=[
lambda iterable: None],
anext=[
lambda aiterator: None,
lambda aiterator, default: None],
any=[
lambda iterable: None],
apply=[
lambda object: None,
lambda object, args: None,
lambda object, args, kwargs: None],
ascii=[
lambda obj: None],
bin=[
lambda number: None],
bool=[
lambda x=False: None],
buffer=[
lambda object: None,
lambda object, offset: None,
lambda object, offset, size: None],
bytearray=[
lambda: None,
lambda int: None,
lambda string, encoding='utf8', errors='strict': None],
callable=[
lambda obj: None],
chr=[
lambda i: None],
classmethod=[
lambda function: None],
cmp=[
lambda x, y: None],
coerce=[
lambda x, y: None],
complex=[
lambda real=0, imag=0: None],
delattr=[
lambda obj, name: None],
dict=[
lambda **kwargs: None,
lambda mapping, **kwargs: None],
dir=[
lambda: None,
lambda object: None],
divmod=[
lambda x, y: None],
enumerate=[
(0, lambda iterable, start=0: None)],
eval=[
lambda source: None,
lambda source, globals: None,
lambda source, globals, locals: None],
execfile=[
lambda filename: None,
lambda filename, globals: None,
lambda filename, globals, locals: None],
file=[
(0, lambda name, mode='r', buffering=-1: None)],
filter=[
lambda function, iterable: None],
float=[
lambda x=0.0: None],
format=[
lambda value: None,
lambda value, format_spec: None],
frozenset=[
lambda: None,
lambda iterable: None],
getattr=[
lambda object, name: None,
lambda object, name, default: None],
globals=[
lambda: None],
hasattr=[
lambda obj, name: None],
hash=[
lambda obj: None],
hex=[
lambda number: None],
id=[
lambda obj: None],
input=[
lambda: None,
lambda prompt: None],
int=[
lambda x=0: None,
(0, lambda x, base=10: None)],
intern=[
lambda string: None],
isinstance=[
lambda obj, class_or_tuple: None],
issubclass=[
lambda cls, class_or_tuple: None],
iter=[
lambda iterable: None,
lambda callable, sentinel: None],
len=[
lambda obj: None],
list=[
lambda: None,
lambda iterable: None],
locals=[
lambda: None],
long=[
lambda x=0: None,
(0, lambda x, base=10: None)],
map=[
lambda func, sequence, *iterables: None],
memoryview=[
(0, lambda object: None)],
next=[
lambda iterator: None,
lambda iterator, default: None],
object=[
lambda: None],
oct=[
lambda number: None],
ord=[
lambda c: None],
pow=[
lambda x, y: None,
lambda x, y, z: None],
property=[
lambda fget=None, fset=None, fdel=None, doc=None: None],
range=[
lambda stop: None,
lambda start, stop: None,
lambda start, stop, step: None],
raw_input=[
lambda: None,
lambda prompt: None],
reduce=[
lambda function, sequence: None,
lambda function, sequence, initial: None],
reload=[
lambda module: None],
repr=[
lambda obj: None],
reversed=[
lambda sequence: None],
round=[
(0, lambda number, ndigits=0: None)],
set=[
lambda: None,
lambda iterable: None],
setattr=[
lambda obj, name, value: None],
slice=[
lambda stop: None,
lambda start, stop: None,
lambda start, stop, step: None],
staticmethod=[
lambda function: None],
sum=[
lambda iterable: None,
lambda iterable, start: None],
super=[
lambda type: None,
lambda type, obj: None],
tuple=[
lambda: None,
lambda iterable: None],
type=[
lambda object: None,
lambda name, bases, dict: None],
unichr=[
lambda i: None],
unicode=[
lambda object: None,
lambda string='', encoding='utf8', errors='strict': None],
vars=[
lambda: None,
lambda object: None],
xrange=[
lambda stop: None,
lambda start, stop: None,
lambda start, stop, step: None],
zip=[
lambda *iterables: None],
__build_class__=[
(2, lambda func, name, *bases, **kwds: None, ('metaclass',))],
__import__=[
(0, lambda name, globals=None, locals=None, fromlist=None,
level=None: None)],
)
module_info[builtins]['exec'] = [
lambda source: None,
lambda source, globals: None,
lambda source, globals, locals: None]
module_info[builtins].update(
breakpoint=[
lambda *args, **kws: None],
bytes=[
lambda: None,
lambda int: None,
lambda string, encoding='utf8', errors='strict': None],
compile=[
(0, lambda source, filename, mode, flags=0,
dont_inherit=False, optimize=-1: None)],
max=[
(1, lambda iterable: None, ('default', 'key',)),
(1, lambda arg1, arg2, *args: None, ('key',))],
min=[
(1, lambda iterable: None, ('default', 'key',)),
(1, lambda arg1, arg2, *args: None, ('key',))],
open=[
(0, lambda file, mode='r', buffering=-1, encoding=None,
errors=None, newline=None, closefd=True, opener=None: None)],
sorted=[
(1, lambda iterable: None, ('key', 'reverse'))],
str=[
lambda object='', encoding='utf', errors='strict': None],
)
module_info[builtins]['print'] = [
(0, lambda *args: None, ('sep', 'end', 'file', 'flush',))]
module_info[functools] = dict(
cmp_to_key=[
(0, lambda mycmp: None)],
partial=[
lambda func, *args, **kwargs: None],
partialmethod=[
lambda func, *args, **kwargs: None],
reduce=[
lambda function, sequence: None,
lambda function, sequence, initial: None],
)
module_info[itertools] = dict(
accumulate=[
(0, lambda iterable, func=None: None)],
chain=[
lambda *iterables: None],
combinations=[
(0, lambda iterable, r: None)],
combinations_with_replacement=[
(0, lambda iterable, r: None)],
compress=[
(0, lambda data, selectors: None)],
count=[
lambda start=0, step=1: None],
cycle=[
lambda iterable: None],
dropwhile=[
lambda predicate, iterable: None],
filterfalse=[
lambda function, sequence: None],
groupby=[
(0, lambda iterable, key=None: None)],
ifilter=[
lambda function, sequence: None],
ifilterfalse=[
lambda function, sequence: None],
imap=[
lambda func, sequence, *iterables: None],
islice=[
lambda iterable, stop: None,
lambda iterable, start, stop: None,
lambda iterable, start, stop, step: None],
izip=[
lambda *iterables: None],
izip_longest=[
(0, lambda *iterables: None, ('fillvalue',))],
permutations=[
(0, lambda iterable, r=0: None)],
repeat=[
(0, lambda object, times=0: None)],
starmap=[
lambda function, sequence: None],
takewhile=[
lambda predicate, iterable: None],
tee=[
lambda iterable: None,
lambda iterable, n: None],
zip_longest=[
(0, lambda *iterables: None, ('fillvalue',))],
)
module_info[itertools].update(
product=[
(0, lambda *iterables: None, ('repeat',))],
)
module_info[operator] = dict(
__abs__=[
lambda a: None],
__add__=[
lambda a, b: None],
__and__=[
lambda a, b: None],
__concat__=[
lambda a, b: None],
__contains__=[
lambda a, b: None],
__delitem__=[
lambda a, b: None],
__delslice__=[
lambda a, b, c: None],
__div__=[
lambda a, b: None],
__eq__=[
lambda a, b: None],
__floordiv__=[
lambda a, b: None],
__ge__=[
lambda a, b: None],
__getitem__=[
lambda a, b: None],
__getslice__=[
lambda a, b, c: None],
__gt__=[
lambda a, b: None],
__iadd__=[
lambda a, b: None],
__iand__=[
lambda a, b: None],
__iconcat__=[
lambda a, b: None],
__idiv__=[
lambda a, b: None],
__ifloordiv__=[
lambda a, b: None],
__ilshift__=[
lambda a, b: None],
__imatmul__=[
lambda a, b: None],
__imod__=[
lambda a, b: None],
__imul__=[
lambda a, b: None],
__index__=[
lambda a: None],
__inv__=[
lambda a: None],
__invert__=[
lambda a: None],
__ior__=[
lambda a, b: None],
__ipow__=[
lambda a, b: None],
__irepeat__=[
lambda a, b: None],
__irshift__=[
lambda a, b: None],
__isub__=[
lambda a, b: None],
__itruediv__=[
lambda a, b: None],
__ixor__=[
lambda a, b: None],
__le__=[
lambda a, b: None],
__lshift__=[
lambda a, b: None],
__lt__=[
lambda a, b: None],
__matmul__=[
lambda a, b: None],
__mod__=[
lambda a, b: None],
__mul__=[
lambda a, b: None],
__ne__=[
lambda a, b: None],
__neg__=[
lambda a: None],
__not__=[
lambda a: None],
__or__=[
lambda a, b: None],
__pos__=[
lambda a: None],
__pow__=[
lambda a, b: None],
__repeat__=[
lambda a, b: None],
__rshift__=[
lambda a, b: None],
__setitem__=[
lambda a, b, c: None],
__setslice__=[
lambda a, b, c, d: None],
__sub__=[
lambda a, b: None],
__truediv__=[
lambda a, b: None],
__xor__=[
lambda a, b: None],
_abs=[
lambda x: None],
_compare_digest=[
lambda a, b: None],
abs=[
lambda a: None],
add=[
lambda a, b: None],
and_=[
lambda a, b: None],
attrgetter=[
lambda attr, *args: None],
concat=[
lambda a, b: None],
contains=[
lambda a, b: None],
countOf=[
lambda a, b: None],
delitem=[
lambda a, b: None],
delslice=[
lambda a, b, c: None],
div=[
lambda a, b: None],
eq=[
lambda a, b: None],
floordiv=[
lambda a, b: None],
ge=[
lambda a, b: None],
getitem=[
lambda a, b: None],
getslice=[
lambda a, b, c: None],
gt=[
lambda a, b: None],
iadd=[
lambda a, b: None],
iand=[
lambda a, b: None],
iconcat=[
lambda a, b: None],
idiv=[
lambda a, b: None],
ifloordiv=[
lambda a, b: None],
ilshift=[
lambda a, b: None],
imatmul=[
lambda a, b: None],
imod=[
lambda a, b: None],
imul=[
lambda a, b: None],
index=[
lambda a: None],
indexOf=[
lambda a, b: None],
inv=[
lambda a: None],
invert=[
lambda a: None],
ior=[
lambda a, b: None],
ipow=[
lambda a, b: None],
irepeat=[
lambda a, b: None],
irshift=[
lambda a, b: None],
is_=[
lambda a, b: None],
is_not=[
lambda a, b: None],
isCallable=[
lambda a: None],
isMappingType=[
lambda a: None],
isNumberType=[
lambda a: None],
isSequenceType=[
lambda a: None],
isub=[
lambda a, b: None],
itemgetter=[
lambda item, *args: None],
itruediv=[
lambda a, b: None],
ixor=[
lambda a, b: None],
le=[
lambda a, b: None],
length_hint=[
lambda obj: None,
lambda obj, default: None],
lshift=[
lambda a, b: None],
lt=[
lambda a, b: None],
matmul=[
lambda a, b: None],
methodcaller=[
lambda name, *args, **kwargs: None],
mod=[
lambda a, b: None],
mul=[
lambda a, b: None],
ne=[
lambda a, b: None],
neg=[
lambda a: None],
not_=[
lambda a: None],
or_=[
lambda a, b: None],
pos=[
lambda a: None],
pow=[
lambda a, b: None],
repeat=[
lambda a, b: None],
rshift=[
lambda a, b: None],
sequenceIncludes=[
lambda a, b: None],
setitem=[
lambda a, b, c: None],
setslice=[
lambda a, b, c, d: None],
sub=[
lambda a, b: None],
truediv=[
lambda a, b: None],
truth=[
lambda a: None],
xor=[
lambda a, b: None],
)
module_info['toolz'] = dict(
curry=[
(0, lambda *args, **kwargs: None)],
excepts=[
(0, lambda exc, func, handler=None: None)],
flip=[
(0, lambda func=None, a=None, b=None: None)],
juxt=[
(0, lambda *funcs: None)],
memoize=[
(0, lambda func=None, cache=None, key=None: None)],
)
module_info['toolz.functoolz'] = dict(
Compose=[
(0, lambda funcs: None)],
InstanceProperty=[
(0, lambda fget=None, fset=None, fdel=None, doc=None,
classval=None: None)],
)
def num_pos_args(sigspec):
""" Return the number of positional arguments. ``f(x, y=1)`` has 1"""
return sum(1 for x in sigspec.parameters.values()
if x.kind == x.POSITIONAL_OR_KEYWORD
and x.default is x.empty)
def get_exclude_keywords(num_pos_only, sigspec):
""" Return the names of position-only arguments if func has **kwargs"""
if num_pos_only == 0:
return ()
has_kwargs = any(x.kind == x.VAR_KEYWORD
for x in sigspec.parameters.values())
if not has_kwargs:
return ()
pos_args = list(sigspec.parameters.values())[:num_pos_only]
return tuple(x.name for x in pos_args)
def signature_or_spec(func):
try:
return inspect.signature(func)
except (ValueError, TypeError):
return None
def expand_sig(sig):
""" Convert the signature spec in ``module_info`` to add to ``signatures``
The input signature spec is one of:
- ``lambda_func``
- ``(num_position_args, lambda_func)``
- ``(num_position_args, lambda_func, keyword_only_args)``
The output signature spec is:
``(num_position_args, lambda_func, keyword_exclude, sigspec)``
where ``keyword_exclude`` includes keyword only arguments and, if variadic
keywords is present, the names of position-only argument. The latter is
included to support builtins such as ``partial(func, *args, **kwargs)``,
which allows ``func=`` to be used as a keyword even though it's the name
of a positional argument.
"""
if isinstance(sig, tuple):
if len(sig) == 3:
num_pos_only, func, keyword_only = sig
assert isinstance(sig[-1], tuple)
else:
num_pos_only, func = sig
keyword_only = ()
sigspec = signature_or_spec(func)
else:
func = sig
sigspec = signature_or_spec(func)
num_pos_only = num_pos_args(sigspec)
keyword_only = ()
keyword_exclude = get_exclude_keywords(num_pos_only, sigspec)
return num_pos_only, func, keyword_only + keyword_exclude, sigspec
signatures = {}
def create_signature_registry(module_info=module_info, signatures=signatures):
for module, info in module_info.items():
if isinstance(module, str):
module = import_module(module)
for name, sigs in info.items():
if hasattr(module, name):
new_sigs = tuple(expand_sig(sig) for sig in sigs)
signatures[getattr(module, name)] = new_sigs
def check_valid(sig, args, kwargs):
""" Like ``is_valid_args`` for the given signature spec"""
num_pos_only, func, keyword_exclude, sigspec = sig
if len(args) < num_pos_only:
return False
if keyword_exclude:
kwargs = dict(kwargs)
for item in keyword_exclude:
kwargs.pop(item, None)
try:
func(*args, **kwargs)
return True
except TypeError:
return False
def _is_valid_args(func, args, kwargs):
""" Like ``is_valid_args`` for builtins in our ``signatures`` registry"""
if func not in signatures:
return None
sigs = signatures[func]
return any(check_valid(sig, args, kwargs) for sig in sigs)
def check_partial(sig, args, kwargs):
""" Like ``is_partial_args`` for the given signature spec"""
num_pos_only, func, keyword_exclude, sigspec = sig
if len(args) < num_pos_only:
pad = (None,) * (num_pos_only - len(args))
args = args + pad
if keyword_exclude:
kwargs = dict(kwargs)
for item in keyword_exclude:
kwargs.pop(item, None)
return is_partial_args(func, args, kwargs, sigspec)
def _is_partial_args(func, args, kwargs):
""" Like ``is_partial_args`` for builtins in our ``signatures`` registry"""
if func not in signatures:
return None
sigs = signatures[func]
return any(check_partial(sig, args, kwargs) for sig in sigs)
def check_arity(n, sig):
num_pos_only, func, keyword_exclude, sigspec = sig
if keyword_exclude or num_pos_only > n:
return False
return is_arity(n, func, sigspec)
def _is_arity(n, func):
if func not in signatures:
return None
sigs = signatures[func]
checks = [check_arity(n, sig) for sig in sigs]
if all(checks):
return True
elif any(checks):
return None
return False
def check_varargs(sig):
num_pos_only, func, keyword_exclude, sigspec = sig
return has_varargs(func, sigspec)
def _has_varargs(func):
if func not in signatures:
return None
sigs = signatures[func]
checks = [check_varargs(sig) for sig in sigs]
if all(checks):
return True
elif any(checks):
return None
return False
def check_keywords(sig):
num_pos_only, func, keyword_exclude, sigspec = sig
if keyword_exclude:
return True
return has_keywords(func, sigspec)
def _has_keywords(func):
if func not in signatures:
return None
sigs = signatures[func]
checks = [check_keywords(sig) for sig in sigs]
if all(checks):
return True
elif any(checks):
return None
return False
def check_required_args(sig):
num_pos_only, func, keyword_exclude, sigspec = sig
return num_required_args(func, sigspec)
def _num_required_args(func):
if func not in signatures:
return None
sigs = signatures[func]
vals = [check_required_args(sig) for sig in sigs]
val = vals[0]
if all(x == val for x in vals):
return val
return None

View File

@@ -0,0 +1,21 @@
# This file was generated by 'versioneer.py' (0.18) from
# revision-control system data, or from the parent directory name of an
# unpacked source archive. Distribution tarballs contain a pre-generated copy
# of this file.
import json
version_json = '''
{
"date": "2021-11-06T00:07:13-0500",
"dirty": false,
"error": null,
"full-revisionid": "294e981edad035a7ac6f0e2b48f1738368fa4b34",
"version": "0.11.2"
}
''' # END VERSION_JSON
def get_versions():
return json.loads(version_json)

View File

@@ -0,0 +1,30 @@
import warnings
warnings.warn("The toolz.compatibility module is no longer "
"needed in Python 3 and has been deprecated. Please "
"import these utilities directly from the standard library. "
"This module will be removed in a future release.",
category=DeprecationWarning, stacklevel=2)
import operator
import sys
PY3 = sys.version_info[0] > 2
PY34 = sys.version_info[0] == 3 and sys.version_info[1] == 4
PYPY = hasattr(sys, 'pypy_version_info') and PY3
__all__ = ('map', 'filter', 'range', 'zip', 'reduce', 'zip_longest',
'iteritems', 'iterkeys', 'itervalues', 'filterfalse',
'PY3', 'PY34', 'PYPY')
map = map
filter = filter
range = range
zip = zip
from functools import reduce
from itertools import zip_longest
from itertools import filterfalse
iteritems = operator.methodcaller('items')
iterkeys = operator.methodcaller('keys')
itervalues = operator.methodcaller('values')
from collections.abc import Sequence

View File

@@ -0,0 +1,103 @@
"""
Alternate namespace for toolz such that all functions are curried
Currying provides implicit partial evaluation of all functions
Example:
Get usually requires two arguments, an index and a collection
>>> from toolz.curried import get
>>> get(0, ('a', 'b'))
'a'
When we use it in higher order functions we often want to pass a partially
evaluated form
>>> data = [(1, 2), (11, 22), (111, 222)]
>>> list(map(lambda seq: get(0, seq), data))
[1, 11, 111]
The curried version allows simple expression of partial evaluation
>>> list(map(get(0), data))
[1, 11, 111]
See Also:
toolz.functoolz.curry
"""
import toolz
from . import operator
from toolz import (
apply,
comp,
complement,
compose,
compose_left,
concat,
concatv,
count,
curry,
diff,
first,
flip,
frequencies,
identity,
interleave,
isdistinct,
isiterable,
juxt,
last,
memoize,
merge_sorted,
peek,
pipe,
second,
thread_first,
thread_last,
)
from .exceptions import merge, merge_with
accumulate = toolz.curry(toolz.accumulate)
assoc = toolz.curry(toolz.assoc)
assoc_in = toolz.curry(toolz.assoc_in)
cons = toolz.curry(toolz.cons)
countby = toolz.curry(toolz.countby)
dissoc = toolz.curry(toolz.dissoc)
do = toolz.curry(toolz.do)
drop = toolz.curry(toolz.drop)
excepts = toolz.curry(toolz.excepts)
filter = toolz.curry(toolz.filter)
get = toolz.curry(toolz.get)
get_in = toolz.curry(toolz.get_in)
groupby = toolz.curry(toolz.groupby)
interpose = toolz.curry(toolz.interpose)
itemfilter = toolz.curry(toolz.itemfilter)
itemmap = toolz.curry(toolz.itemmap)
iterate = toolz.curry(toolz.iterate)
join = toolz.curry(toolz.join)
keyfilter = toolz.curry(toolz.keyfilter)
keymap = toolz.curry(toolz.keymap)
map = toolz.curry(toolz.map)
mapcat = toolz.curry(toolz.mapcat)
nth = toolz.curry(toolz.nth)
partial = toolz.curry(toolz.partial)
partition = toolz.curry(toolz.partition)
partition_all = toolz.curry(toolz.partition_all)
partitionby = toolz.curry(toolz.partitionby)
peekn = toolz.curry(toolz.peekn)
pluck = toolz.curry(toolz.pluck)
random_sample = toolz.curry(toolz.random_sample)
reduce = toolz.curry(toolz.reduce)
reduceby = toolz.curry(toolz.reduceby)
remove = toolz.curry(toolz.remove)
sliding_window = toolz.curry(toolz.sliding_window)
sorted = toolz.curry(toolz.sorted)
tail = toolz.curry(toolz.tail)
take = toolz.curry(toolz.take)
take_nth = toolz.curry(toolz.take_nth)
topk = toolz.curry(toolz.topk)
unique = toolz.curry(toolz.unique)
update_in = toolz.curry(toolz.update_in)
valfilter = toolz.curry(toolz.valfilter)
valmap = toolz.curry(toolz.valmap)
del exceptions
del toolz

View File

@@ -0,0 +1,18 @@
import toolz
__all__ = ['merge_with', 'merge']
@toolz.curry
def merge_with(func, d, *dicts, **kwargs):
return toolz.merge_with(func, d, *dicts, **kwargs)
@toolz.curry
def merge(d, *dicts, **kwargs):
return toolz.merge(d, *dicts, **kwargs)
merge_with.__doc__ = toolz.merge_with.__doc__
merge.__doc__ = toolz.merge.__doc__

View File

@@ -0,0 +1,23 @@
from __future__ import absolute_import
import operator
from toolz.functoolz import curry, num_required_args, has_keywords
def should_curry(f):
num = num_required_args(f)
return num is None or num > 1 or num == 1 and has_keywords(f) is not False
locals().update(
{name: curry(f) if should_curry(f) else f
for name, f in vars(operator).items() if callable(f)},
)
# Clean up the namespace.
del curry
del num_required_args
del has_keywords
del operator
del should_curry

View File

@@ -0,0 +1,337 @@
import operator
from functools import reduce
from collections.abc import Mapping
__all__ = ('merge', 'merge_with', 'valmap', 'keymap', 'itemmap',
'valfilter', 'keyfilter', 'itemfilter',
'assoc', 'dissoc', 'assoc_in', 'update_in', 'get_in')
def _get_factory(f, kwargs):
factory = kwargs.pop('factory', dict)
if kwargs:
raise TypeError("{}() got an unexpected keyword argument "
"'{}'".format(f.__name__, kwargs.popitem()[0]))
return factory
def merge(*dicts, **kwargs):
""" Merge a collection of dictionaries
>>> merge({1: 'one'}, {2: 'two'})
{1: 'one', 2: 'two'}
Later dictionaries have precedence
>>> merge({1: 2, 3: 4}, {3: 3, 4: 4})
{1: 2, 3: 3, 4: 4}
See Also:
merge_with
"""
if len(dicts) == 1 and not isinstance(dicts[0], Mapping):
dicts = dicts[0]
factory = _get_factory(merge, kwargs)
rv = factory()
for d in dicts:
rv.update(d)
return rv
def merge_with(func, *dicts, **kwargs):
""" Merge dictionaries and apply function to combined values
A key may occur in more than one dict, and all values mapped from the key
will be passed to the function as a list, such as func([val1, val2, ...]).
>>> merge_with(sum, {1: 1, 2: 2}, {1: 10, 2: 20})
{1: 11, 2: 22}
>>> merge_with(first, {1: 1, 2: 2}, {2: 20, 3: 30}) # doctest: +SKIP
{1: 1, 2: 2, 3: 30}
See Also:
merge
"""
if len(dicts) == 1 and not isinstance(dicts[0], Mapping):
dicts = dicts[0]
factory = _get_factory(merge_with, kwargs)
result = factory()
for d in dicts:
for k, v in d.items():
if k not in result:
result[k] = [v]
else:
result[k].append(v)
return valmap(func, result, factory)
def valmap(func, d, factory=dict):
""" Apply function to values of dictionary
>>> bills = {"Alice": [20, 15, 30], "Bob": [10, 35]}
>>> valmap(sum, bills) # doctest: +SKIP
{'Alice': 65, 'Bob': 45}
See Also:
keymap
itemmap
"""
rv = factory()
rv.update(zip(d.keys(), map(func, d.values())))
return rv
def keymap(func, d, factory=dict):
""" Apply function to keys of dictionary
>>> bills = {"Alice": [20, 15, 30], "Bob": [10, 35]}
>>> keymap(str.lower, bills) # doctest: +SKIP
{'alice': [20, 15, 30], 'bob': [10, 35]}
See Also:
valmap
itemmap
"""
rv = factory()
rv.update(zip(map(func, d.keys()), d.values()))
return rv
def itemmap(func, d, factory=dict):
""" Apply function to items of dictionary
>>> accountids = {"Alice": 10, "Bob": 20}
>>> itemmap(reversed, accountids) # doctest: +SKIP
{10: "Alice", 20: "Bob"}
See Also:
keymap
valmap
"""
rv = factory()
rv.update(map(func, d.items()))
return rv
def valfilter(predicate, d, factory=dict):
""" Filter items in dictionary by value
>>> iseven = lambda x: x % 2 == 0
>>> d = {1: 2, 2: 3, 3: 4, 4: 5}
>>> valfilter(iseven, d)
{1: 2, 3: 4}
See Also:
keyfilter
itemfilter
valmap
"""
rv = factory()
for k, v in d.items():
if predicate(v):
rv[k] = v
return rv
def keyfilter(predicate, d, factory=dict):
""" Filter items in dictionary by key
>>> iseven = lambda x: x % 2 == 0
>>> d = {1: 2, 2: 3, 3: 4, 4: 5}
>>> keyfilter(iseven, d)
{2: 3, 4: 5}
See Also:
valfilter
itemfilter
keymap
"""
rv = factory()
for k, v in d.items():
if predicate(k):
rv[k] = v
return rv
def itemfilter(predicate, d, factory=dict):
""" Filter items in dictionary by item
>>> def isvalid(item):
... k, v = item
... return k % 2 == 0 and v < 4
>>> d = {1: 2, 2: 3, 3: 4, 4: 5}
>>> itemfilter(isvalid, d)
{2: 3}
See Also:
keyfilter
valfilter
itemmap
"""
rv = factory()
for item in d.items():
if predicate(item):
k, v = item
rv[k] = v
return rv
def assoc(d, key, value, factory=dict):
""" Return a new dict with new key value pair
New dict has d[key] set to value. Does not modify the initial dictionary.
>>> assoc({'x': 1}, 'x', 2)
{'x': 2}
>>> assoc({'x': 1}, 'y', 3) # doctest: +SKIP
{'x': 1, 'y': 3}
"""
d2 = factory()
d2.update(d)
d2[key] = value
return d2
def dissoc(d, *keys, **kwargs):
""" Return a new dict with the given key(s) removed.
New dict has d[key] deleted for each supplied key.
Does not modify the initial dictionary.
>>> dissoc({'x': 1, 'y': 2}, 'y')
{'x': 1}
>>> dissoc({'x': 1, 'y': 2}, 'y', 'x')
{}
>>> dissoc({'x': 1}, 'y') # Ignores missing keys
{'x': 1}
"""
factory = _get_factory(dissoc, kwargs)
d2 = factory()
if len(keys) < len(d) * .6:
d2.update(d)
for key in keys:
if key in d2:
del d2[key]
else:
remaining = set(d)
remaining.difference_update(keys)
for k in remaining:
d2[k] = d[k]
return d2
def assoc_in(d, keys, value, factory=dict):
""" Return a new dict with new, potentially nested, key value pair
>>> purchase = {'name': 'Alice',
... 'order': {'items': ['Apple', 'Orange'],
... 'costs': [0.50, 1.25]},
... 'credit card': '5555-1234-1234-1234'}
>>> assoc_in(purchase, ['order', 'costs'], [0.25, 1.00]) # doctest: +SKIP
{'credit card': '5555-1234-1234-1234',
'name': 'Alice',
'order': {'costs': [0.25, 1.00], 'items': ['Apple', 'Orange']}}
"""
return update_in(d, keys, lambda x: value, value, factory)
def update_in(d, keys, func, default=None, factory=dict):
""" Update value in a (potentially) nested dictionary
inputs:
d - dictionary on which to operate
keys - list or tuple giving the location of the value to be changed in d
func - function to operate on that value
If keys == [k0,..,kX] and d[k0]..[kX] == v, update_in returns a copy of the
original dictionary with v replaced by func(v), but does not mutate the
original dictionary.
If k0 is not a key in d, update_in creates nested dictionaries to the depth
specified by the keys, with the innermost value set to func(default).
>>> inc = lambda x: x + 1
>>> update_in({'a': 0}, ['a'], inc)
{'a': 1}
>>> transaction = {'name': 'Alice',
... 'purchase': {'items': ['Apple', 'Orange'],
... 'costs': [0.50, 1.25]},
... 'credit card': '5555-1234-1234-1234'}
>>> update_in(transaction, ['purchase', 'costs'], sum) # doctest: +SKIP
{'credit card': '5555-1234-1234-1234',
'name': 'Alice',
'purchase': {'costs': 1.75, 'items': ['Apple', 'Orange']}}
>>> # updating a value when k0 is not in d
>>> update_in({}, [1, 2, 3], str, default="bar")
{1: {2: {3: 'bar'}}}
>>> update_in({1: 'foo'}, [2, 3, 4], inc, 0)
{1: 'foo', 2: {3: {4: 1}}}
"""
ks = iter(keys)
k = next(ks)
rv = inner = factory()
rv.update(d)
for key in ks:
if k in d:
d = d[k]
dtemp = factory()
dtemp.update(d)
else:
d = dtemp = factory()
inner[k] = inner = dtemp
k = key
if k in d:
inner[k] = func(d[k])
else:
inner[k] = func(default)
return rv
def get_in(keys, coll, default=None, no_default=False):
""" Returns coll[i0][i1]...[iX] where [i0, i1, ..., iX]==keys.
If coll[i0][i1]...[iX] cannot be found, returns ``default``, unless
``no_default`` is specified, then it raises KeyError or IndexError.
``get_in`` is a generalization of ``operator.getitem`` for nested data
structures such as dictionaries and lists.
>>> transaction = {'name': 'Alice',
... 'purchase': {'items': ['Apple', 'Orange'],
... 'costs': [0.50, 1.25]},
... 'credit card': '5555-1234-1234-1234'}
>>> get_in(['purchase', 'items', 0], transaction)
'Apple'
>>> get_in(['name'], transaction)
'Alice'
>>> get_in(['purchase', 'total'], transaction)
>>> get_in(['purchase', 'items', 'apple'], transaction)
>>> get_in(['purchase', 'items', 10], transaction)
>>> get_in(['purchase', 'total'], transaction, 0)
0
>>> get_in(['y'], {}, no_default=True)
Traceback (most recent call last):
...
KeyError: 'y'
See Also:
itertoolz.get
operator.getitem
"""
try:
return reduce(operator.getitem, keys, coll)
except (KeyError, IndexError, TypeError):
if no_default:
raise
return default

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,46 @@
import itertools
from .itertoolz import frequencies, pluck, getter
__all__ = ('countby', 'partitionby')
def countby(key, seq):
""" Count elements of a collection by a key function
>>> countby(len, ['cat', 'mouse', 'dog'])
{3: 2, 5: 1}
>>> def iseven(x): return x % 2 == 0
>>> countby(iseven, [1, 2, 3]) # doctest:+SKIP
{True: 1, False: 2}
See Also:
groupby
"""
if not callable(key):
key = getter(key)
return frequencies(map(key, seq))
def partitionby(func, seq):
""" Partition a sequence according to a function
Partition `s` into a sequence of lists such that, when traversing
`s`, every time the output of `func` changes a new list is started
and that and subsequent items are collected into that list.
>>> is_space = lambda c: c == " "
>>> list(partitionby(is_space, "I have space"))
[('I',), (' ',), ('h', 'a', 'v', 'e'), (' ',), ('s', 'p', 'a', 'c', 'e')]
>>> is_large = lambda x: x > 10
>>> list(partitionby(is_large, [1, 2, 1, 99, 88, 33, 99, -1, 5]))
[(1, 2, 1), (99, 88, 33, 99), (-1, 5)]
See also:
partition
groupby
itertools.groupby
"""
return map(tuple, pluck(1, itertools.groupby(seq, key=func)))

View File

@@ -0,0 +1,2 @@
from .core import EqualityHashKey, unzip
from .parallel import fold

View File

@@ -0,0 +1,133 @@
from toolz.itertoolz import getter, cons, pluck
from itertools import tee, starmap
# See #166: https://github.com/pytoolz/toolz/issues/166
# See #173: https://github.com/pytoolz/toolz/pull/173
class EqualityHashKey(object):
""" Create a hash key that uses equality comparisons between items.
This may be used to create hash keys for otherwise unhashable types:
>>> from toolz import curry
>>> EqualityHashDefault = curry(EqualityHashKey, None)
>>> set(map(EqualityHashDefault, [[], (), [1], [1]])) # doctest: +SKIP
{=[]=, =()=, =[1]=}
**Caution:** adding N ``EqualityHashKey`` items to a hash container
may require O(N**2) operations, not O(N) as for typical hashable types.
Therefore, a suitable key function such as ``tuple`` or ``frozenset``
is usually preferred over using ``EqualityHashKey`` if possible.
The ``key`` argument to ``EqualityHashKey`` should be a function or
index that returns a hashable object that effectively distinguishes
unequal items. This helps avoid the poor scaling that occurs when
using the default key. For example, the above example can be improved
by using a key function that distinguishes items by length or type:
>>> EqualityHashLen = curry(EqualityHashKey, len)
>>> EqualityHashType = curry(EqualityHashKey, type) # this works too
>>> set(map(EqualityHashLen, [[], (), [1], [1]])) # doctest: +SKIP
{=[]=, =()=, =[1]=}
``EqualityHashKey`` is convenient to use when a suitable key function
is complicated or unavailable. For example, the following returns all
unique values based on equality:
>>> from toolz import unique
>>> vals = [[], [], (), [1], [1], [2], {}, {}, {}]
>>> list(unique(vals, key=EqualityHashDefault))
[[], (), [1], [2], {}]
**Warning:** don't change the equality value of an item already in a hash
containter. Unhashable types are unhashable for a reason. For example:
>>> L1 = [1] ; L2 = [2]
>>> s = set(map(EqualityHashDefault, [L1, L2]))
>>> s # doctest: +SKIP
{=[1]=, =[2]=}
>>> L1[0] = 2 # Don't do this! ``s`` now has duplicate items!
>>> s # doctest: +SKIP
{=[2]=, =[2]=}
Although this may appear problematic, immutable data types is a common
idiom in functional programming, and``EqualityHashKey`` easily allows
the same idiom to be used by convention rather than strict requirement.
See Also:
identity
"""
__slots__ = ['item', 'key']
_default_hashkey = '__default__hashkey__'
def __init__(self, key, item):
if key is None:
self.key = self._default_hashkey
elif not callable(key):
self.key = getter(key)
else:
self.key = key
self.item = item
def __hash__(self):
if self.key == self._default_hashkey:
val = self.key
else:
val = self.key(self.item)
return hash(val)
def __eq__(self, other):
try:
return (self._default_hashkey == other._default_hashkey and
self.item == other.item)
except AttributeError:
return False
def __ne__(self, other):
return not self.__eq__(other)
def __str__(self):
return '=%s=' % str(self.item)
def __repr__(self):
return '=%s=' % repr(self.item)
# See issue #293: https://github.com/pytoolz/toolz/issues/239
def unzip(seq):
"""Inverse of ``zip``
>>> a, b = unzip([('a', 1), ('b', 2)])
>>> list(a)
['a', 'b']
>>> list(b)
[1, 2]
Unlike the naive implementation ``def unzip(seq): zip(*seq)`` this
implementation can handle an infinite sequence ``seq``.
Caveats:
* The implementation uses ``tee``, and so can use a significant amount
of auxiliary storage if the resulting iterators are consumed at
different times.
* The inner sequence cannot be infinite. In Python 3 ``zip(*seq)`` can be
used if ``seq`` is a finite sequence of infinite sequences.
"""
seq = iter(seq)
# Check how many iterators we need
try:
first = tuple(next(seq))
except StopIteration:
return tuple()
# and create them
niters = len(first)
seqs = tee(cons(first, seq), niters)
return tuple(starmap(pluck, enumerate(seqs)))

View File

@@ -0,0 +1,75 @@
import functools
from toolz.itertoolz import partition_all
from toolz.utils import no_default
def _reduce(func, seq, initial=None):
if initial is None:
return functools.reduce(func, seq)
else:
return functools.reduce(func, seq, initial)
def fold(binop, seq, default=no_default, map=map, chunksize=128, combine=None):
"""
Reduce without guarantee of ordered reduction.
inputs:
``binop`` - associative operator. The associative property allows us to
leverage a parallel map to perform reductions in parallel.
``seq`` - a sequence to be aggregated
``default`` - an identity element like 0 for ``add`` or 1 for mul
``map`` - an implementation of ``map``. This may be parallel and
determines how work is distributed.
``chunksize`` - Number of elements of ``seq`` that should be handled
within a single function call
``combine`` - Binary operator to combine two intermediate results.
If ``binop`` is of type (total, item) -> total
then ``combine`` is of type (total, total) -> total
Defaults to ``binop`` for common case of operators like add
Fold chunks up the collection into blocks of size ``chunksize`` and then
feeds each of these to calls to ``reduce``. This work is distributed
with a call to ``map``, gathered back and then refolded to finish the
computation. In this way ``fold`` specifies only how to chunk up data but
leaves the distribution of this work to an externally provided ``map``
function. This function can be sequential or rely on multithreading,
multiprocessing, or even distributed solutions.
If ``map`` intends to serialize functions it should be prepared to accept
and serialize lambdas. Note that the standard ``pickle`` module fails
here.
Example
-------
>>> # Provide a parallel map to accomplish a parallel sum
>>> from operator import add
>>> fold(add, [1, 2, 3, 4], chunksize=2, map=map)
10
"""
assert chunksize > 1
if combine is None:
combine = binop
chunks = partition_all(chunksize, seq)
# Evaluate sequence in chunks via map
if default == no_default:
results = map(
functools.partial(_reduce, binop),
chunks)
else:
results = map(
functools.partial(_reduce, binop, initial=default),
chunks)
results = list(results) # TODO: Support complete laziness
if len(results) == 1: # Return completed result
return results[0]
else: # Recurse to reaggregate intermediate results
return fold(combine, results, map=map, chunksize=chunksize)

View File

@@ -0,0 +1,10 @@
import pytest
import importlib
def test_compat_warn():
with pytest.warns(DeprecationWarning):
# something else is importing this,
import toolz.compatibility
# reload to be sure we warn
importlib.reload(toolz.compatibility)

View File

@@ -0,0 +1,113 @@
import toolz
import toolz.curried
from toolz.curried import (take, first, second, sorted, merge_with, reduce,
merge, operator as cop)
from collections import defaultdict
from importlib import import_module
from operator import add
def test_take():
assert list(take(2)([1, 2, 3])) == [1, 2]
def test_first():
assert first is toolz.itertoolz.first
def test_merge():
assert merge(factory=lambda: defaultdict(int))({1: 1}) == {1: 1}
assert merge({1: 1}) == {1: 1}
assert merge({1: 1}, factory=lambda: defaultdict(int)) == {1: 1}
def test_merge_with():
assert merge_with(sum)({1: 1}, {1: 2}) == {1: 3}
def test_merge_with_list():
assert merge_with(sum, [{'a': 1}, {'a': 2}]) == {'a': 3}
def test_sorted():
assert sorted(key=second)([(1, 2), (2, 1)]) == [(2, 1), (1, 2)]
def test_reduce():
assert reduce(add)((1, 2, 3)) == 6
def test_module_name():
assert toolz.curried.__name__ == 'toolz.curried'
def test_curried_operator():
for k, v in vars(cop).items():
if not callable(v):
continue
if not isinstance(v, toolz.curry):
try:
# Make sure it is unary
v(1)
except TypeError:
try:
v('x')
except TypeError:
pass
else:
continue
raise AssertionError(
'toolz.curried.operator.%s is not curried!' % k,
)
# Make sure this isn't totally empty.
assert len(set(vars(cop)) & {'add', 'sub', 'mul'}) == 3
def test_curried_namespace():
exceptions = import_module('toolz.curried.exceptions')
namespace = {}
def should_curry(func):
if not callable(func) or isinstance(func, toolz.curry):
return False
nargs = toolz.functoolz.num_required_args(func)
if nargs is None or nargs > 1:
return True
return nargs == 1 and toolz.functoolz.has_keywords(func)
def curry_namespace(ns):
return {
name: toolz.curry(f) if should_curry(f) else f
for name, f in ns.items() if '__' not in name
}
from_toolz = curry_namespace(vars(toolz))
from_exceptions = curry_namespace(vars(exceptions))
namespace.update(toolz.merge(from_toolz, from_exceptions))
namespace = toolz.valfilter(callable, namespace)
curried_namespace = toolz.valfilter(callable, toolz.curried.__dict__)
if namespace != curried_namespace:
missing = set(namespace) - set(curried_namespace)
if missing:
raise AssertionError('There are missing functions in toolz.curried:\n %s'
% ' \n'.join(sorted(missing)))
extra = set(curried_namespace) - set(namespace)
if extra:
raise AssertionError('There are extra functions in toolz.curried:\n %s'
% ' \n'.join(sorted(extra)))
unequal = toolz.merge_with(list, namespace, curried_namespace)
unequal = toolz.valfilter(lambda x: x[0] != x[1], unequal)
messages = []
for name, (orig_func, auto_func) in sorted(unequal.items()):
if name in from_exceptions:
messages.append('%s should come from toolz.curried.exceptions' % name)
elif should_curry(getattr(toolz, name)):
messages.append('%s should be curried from toolz' % name)
else:
messages.append('%s should come from toolz and NOT be curried' % name)
raise AssertionError('\n'.join(messages))

View File

@@ -0,0 +1,11 @@
import doctest
import toolz
def test_doctests():
toolz.__test__ = {}
for name, func in vars(toolz).items():
if isinstance(func, toolz.curry):
toolz.__test__[name] = func.func
assert doctest.testmod(toolz).failed == 0
del toolz.__test__

View File

@@ -0,0 +1,270 @@
from collections import defaultdict as _defaultdict
from collections.abc import Mapping
import os
from toolz.dicttoolz import (merge, merge_with, valmap, keymap, update_in,
assoc, dissoc, keyfilter, valfilter, itemmap,
itemfilter, assoc_in)
from toolz.functoolz import identity
from toolz.utils import raises
def inc(x):
return x + 1
def iseven(i):
return i % 2 == 0
class TestDict(object):
"""Test typical usage: dict inputs, no factory keyword.
Class attributes:
D: callable that inputs a dict and creates or returns a MutableMapping
kw: kwargs dict to specify "factory" keyword (if applicable)
"""
D = dict
kw = {}
def test_merge(self):
D, kw = self.D, self.kw
assert merge(D({1: 1, 2: 2}), D({3: 4}), **kw) == D({1: 1, 2: 2, 3: 4})
def test_merge_iterable_arg(self):
D, kw = self.D, self.kw
assert merge([D({1: 1, 2: 2}), D({3: 4})], **kw) == D({1: 1, 2: 2, 3: 4})
def test_merge_with(self):
D, kw = self.D, self.kw
dicts = D({1: 1, 2: 2}), D({1: 10, 2: 20})
assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22})
assert merge_with(tuple, *dicts, **kw) == D({1: (1, 10), 2: (2, 20)})
dicts = D({1: 1, 2: 2, 3: 3}), D({1: 10, 2: 20})
assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22, 3: 3})
assert merge_with(tuple, *dicts, **kw) == D({1: (1, 10), 2: (2, 20), 3: (3,)})
assert not merge_with(sum)
def test_merge_with_iterable_arg(self):
D, kw = self.D, self.kw
dicts = D({1: 1, 2: 2}), D({1: 10, 2: 20})
assert merge_with(sum, *dicts, **kw) == D({1: 11, 2: 22})
assert merge_with(sum, dicts, **kw) == D({1: 11, 2: 22})
assert merge_with(sum, iter(dicts), **kw) == D({1: 11, 2: 22})
def test_valmap(self):
D, kw = self.D, self.kw
assert valmap(inc, D({1: 1, 2: 2}), **kw) == D({1: 2, 2: 3})
def test_keymap(self):
D, kw = self.D, self.kw
assert keymap(inc, D({1: 1, 2: 2}), **kw) == D({2: 1, 3: 2})
def test_itemmap(self):
D, kw = self.D, self.kw
assert itemmap(reversed, D({1: 2, 2: 4}), **kw) == D({2: 1, 4: 2})
def test_valfilter(self):
D, kw = self.D, self.kw
assert valfilter(iseven, D({1: 2, 2: 3}), **kw) == D({1: 2})
def test_keyfilter(self):
D, kw = self.D, self.kw
assert keyfilter(iseven, D({1: 2, 2: 3}), **kw) == D({2: 3})
def test_itemfilter(self):
D, kw = self.D, self.kw
assert itemfilter(lambda item: iseven(item[0]), D({1: 2, 2: 3}), **kw) == D({2: 3})
assert itemfilter(lambda item: iseven(item[1]), D({1: 2, 2: 3}), **kw) == D({1: 2})
def test_assoc(self):
D, kw = self.D, self.kw
assert assoc(D({}), "a", 1, **kw) == D({"a": 1})
assert assoc(D({"a": 1}), "a", 3, **kw) == D({"a": 3})
assert assoc(D({"a": 1}), "b", 3, **kw) == D({"a": 1, "b": 3})
# Verify immutability:
d = D({'x': 1})
oldd = d
assoc(d, 'x', 2, **kw)
assert d is oldd
def test_dissoc(self):
D, kw = self.D, self.kw
assert dissoc(D({"a": 1}), "a", **kw) == D({})
assert dissoc(D({"a": 1, "b": 2}), "a", **kw) == D({"b": 2})
assert dissoc(D({"a": 1, "b": 2}), "b", **kw) == D({"a": 1})
assert dissoc(D({"a": 1, "b": 2}), "a", "b", **kw) == D({})
assert dissoc(D({"a": 1}), "a", **kw) == dissoc(dissoc(D({"a": 1}), "a", **kw), "a", **kw)
# Verify immutability:
d = D({'x': 1})
oldd = d
d2 = dissoc(d, 'x', **kw)
assert d is oldd
assert d2 is not oldd
def test_assoc_in(self):
D, kw = self.D, self.kw
assert assoc_in(D({"a": 1}), ["a"], 2, **kw) == D({"a": 2})
assert (assoc_in(D({"a": D({"b": 1})}), ["a", "b"], 2, **kw) ==
D({"a": D({"b": 2})}))
assert assoc_in(D({}), ["a", "b"], 1, **kw) == D({"a": D({"b": 1})})
# Verify immutability:
d = D({'x': 1})
oldd = d
d2 = assoc_in(d, ['x'], 2, **kw)
assert d is oldd
assert d2 is not oldd
def test_update_in(self):
D, kw = self.D, self.kw
assert update_in(D({"a": 0}), ["a"], inc, **kw) == D({"a": 1})
assert update_in(D({"a": 0, "b": 1}), ["b"], str, **kw) == D({"a": 0, "b": "1"})
assert (update_in(D({"t": 1, "v": D({"a": 0})}), ["v", "a"], inc, **kw) ==
D({"t": 1, "v": D({"a": 1})}))
# Handle one missing key.
assert update_in(D({}), ["z"], str, None, **kw) == D({"z": "None"})
assert update_in(D({}), ["z"], inc, 0, **kw) == D({"z": 1})
assert update_in(D({}), ["z"], lambda x: x+"ar", default="b", **kw) == D({"z": "bar"})
# Same semantics as Clojure for multiple missing keys, ie. recursively
# create nested empty dictionaries to the depth specified by the
# keys with the innermost value set to f(default).
assert update_in(D({}), [0, 1], inc, default=-1, **kw) == D({0: D({1: 0})})
assert update_in(D({}), [0, 1], str, default=100, **kw) == D({0: D({1: "100"})})
assert (update_in(D({"foo": "bar", 1: 50}), ["d", 1, 0], str, 20, **kw) ==
D({"foo": "bar", 1: 50, "d": D({1: D({0: "20"})})}))
# Verify immutability:
d = D({'x': 1})
oldd = d
update_in(d, ['x'], inc, **kw)
assert d is oldd
def test_factory(self):
D, kw = self.D, self.kw
assert merge(defaultdict(int, D({1: 2})), D({2: 3})) == {1: 2, 2: 3}
assert (merge(defaultdict(int, D({1: 2})), D({2: 3}),
factory=lambda: defaultdict(int)) ==
defaultdict(int, D({1: 2, 2: 3})))
assert not (merge(defaultdict(int, D({1: 2})), D({2: 3}),
factory=lambda: defaultdict(int)) == {1: 2, 2: 3})
assert raises(TypeError, lambda: merge(D({1: 2}), D({2: 3}), factoryy=dict))
class defaultdict(_defaultdict):
def __eq__(self, other):
return (super(defaultdict, self).__eq__(other) and
isinstance(other, _defaultdict) and
self.default_factory == other.default_factory)
class TestDefaultDict(TestDict):
"""Test defaultdict as input and factory
Class attributes:
D: callable that inputs a dict and creates or returns a MutableMapping
kw: kwargs dict to specify "factory" keyword (if applicable)
"""
@staticmethod
def D(dict_):
return defaultdict(int, dict_)
kw = {'factory': lambda: defaultdict(int)}
class CustomMapping(object):
"""Define methods of the MutableMapping protocol required by dicttoolz"""
def __init__(self, *args, **kwargs):
self._d = dict(*args, **kwargs)
def __getitem__(self, key):
return self._d[key]
def __setitem__(self, key, val):
self._d[key] = val
def __delitem__(self, key):
del self._d[key]
def __iter__(self):
return iter(self._d)
def __len__(self):
return len(self._d)
def __contains__(self, key):
return key in self._d
def __eq__(self, other):
return isinstance(other, CustomMapping) and self._d == other._d
def __ne__(self, other):
return not isinstance(other, CustomMapping) or self._d != other._d
def keys(self):
return self._d.keys()
def values(self):
return self._d.values()
def items(self):
return self._d.items()
def update(self, *args, **kwargs):
self._d.update(*args, **kwargs)
# Unused methods that are part of the MutableMapping protocol
#def get(self, key, *args):
# return self._d.get(key, *args)
#def pop(self, key, *args):
# return self._d.pop(key, *args)
#def popitem(self, key):
# return self._d.popitem()
#def clear(self):
# self._d.clear()
#def setdefault(self, key, *args):
# return self._d.setdefault(self, key, *args)
class TestCustomMapping(TestDict):
"""Test CustomMapping as input and factory
Class attributes:
D: callable that inputs a dict and creates or returns a MutableMapping
kw: kwargs dict to specify "factory" keyword (if applicable)
"""
D = CustomMapping
kw = {'factory': lambda: CustomMapping()}
def test_environ():
# See: https://github.com/pytoolz/cytoolz/issues/127
assert keymap(identity, os.environ) == os.environ
assert valmap(identity, os.environ) == os.environ
assert itemmap(identity, os.environ) == os.environ
def test_merge_with_non_dict_mappings():
class Foo(Mapping):
def __init__(self, d):
self.d = d
def __iter__(self):
return iter(self.d)
def __getitem__(self, key):
return self.d[key]
def __len__(self):
return len(self.d)
d = Foo({1: 1})
assert merge(d) is d or merge(d) == {1: 1}
assert merge_with(sum, d) == {1: 1}

View File

@@ -0,0 +1,796 @@
import inspect
import toolz
from toolz.functoolz import (thread_first, thread_last, memoize, curry,
compose, compose_left, pipe, complement, do, juxt,
flip, excepts, apply)
from operator import add, mul, itemgetter
from toolz.utils import raises
from functools import partial
def iseven(x):
return x % 2 == 0
def isodd(x):
return x % 2 == 1
def inc(x):
return x + 1
def double(x):
return 2 * x
class AlwaysEquals(object):
"""useful to test correct __eq__ implementation of other objects"""
def __eq__(self, other):
return True
def __ne__(self, other):
return False
class NeverEquals(object):
"""useful to test correct __eq__ implementation of other objects"""
def __eq__(self, other):
return False
def __ne__(self, other):
return True
def test_apply():
assert apply(double, 5) == 10
assert tuple(map(apply, [double, inc, double], [10, 500, 8000])) == (20, 501, 16000)
assert raises(TypeError, apply)
def test_thread_first():
assert thread_first(2) == 2
assert thread_first(2, inc) == 3
assert thread_first(2, inc, inc) == 4
assert thread_first(2, double, inc) == 5
assert thread_first(2, (add, 5), double) == 14
def test_thread_last():
assert list(thread_last([1, 2, 3], (map, inc), (filter, iseven))) == [2, 4]
assert list(thread_last([1, 2, 3], (map, inc), (filter, isodd))) == [3]
assert thread_last(2, (add, 5), double) == 14
def test_memoize():
fn_calls = [0] # Storage for side effects
def f(x, y):
""" A docstring """
fn_calls[0] += 1
return x + y
mf = memoize(f)
assert mf(2, 3) is mf(2, 3)
assert fn_calls == [1] # function was only called once
assert mf.__doc__ == f.__doc__
assert raises(TypeError, lambda: mf(1, {}))
def test_memoize_kwargs():
fn_calls = [0] # Storage for side effects
def f(x, y=0):
return x + y
mf = memoize(f)
assert mf(1) == f(1)
assert mf(1, 2) == f(1, 2)
assert mf(1, y=2) == f(1, y=2)
assert mf(1, y=3) == f(1, y=3)
def test_memoize_curried():
@curry
def f(x, y=0):
return x + y
f2 = f(y=1)
fm2 = memoize(f2)
assert fm2(3) == f2(3)
assert fm2(3) == f2(3)
def test_memoize_partial():
def f(x, y=0):
return x + y
f2 = partial(f, y=1)
fm2 = memoize(f2)
assert fm2(3) == f2(3)
assert fm2(3) == f2(3)
def test_memoize_key_signature():
# Single argument should not be tupled as a key. No keywords.
mf = memoize(lambda x: False, cache={1: True})
assert mf(1) is True
assert mf(2) is False
# Single argument must be tupled if signature has varargs. No keywords.
mf = memoize(lambda x, *args: False, cache={(1,): True, (1, 2): 2})
assert mf(1) is True
assert mf(2) is False
assert mf(1, 1) is False
assert mf(1, 2) == 2
assert mf((1, 2)) is False
# More than one argument is always tupled. No keywords.
mf = memoize(lambda x, y: False, cache={(1, 2): True})
assert mf(1, 2) is True
assert mf(1, 3) is False
assert raises(TypeError, lambda: mf((1, 2)))
# Nullary function (no inputs) uses empty tuple as the key
mf = memoize(lambda: False, cache={(): True})
assert mf() is True
# Single argument must be tupled if there are keyword arguments, because
# keyword arguments may be passed as unnamed args.
mf = memoize(lambda x, y=0: False,
cache={((1,), frozenset((('y', 2),))): 2,
((1, 2), None): 3})
assert mf(1, y=2) == 2
assert mf(1, 2) == 3
assert mf(2, y=2) is False
assert mf(2, 2) is False
assert mf(1) is False
assert mf((1, 2)) is False
# Keyword-only signatures must still have an "args" tuple.
mf = memoize(lambda x=0: False, cache={(None, frozenset((('x', 1),))): 1,
((1,), None): 2})
assert mf() is False
assert mf(x=1) == 1
assert mf(1) == 2
def test_memoize_curry_cache():
@memoize(cache={1: True})
def f(x):
return False
assert f(1) is True
assert f(2) is False
def test_memoize_key():
@memoize(key=lambda args, kwargs: args[0])
def f(x, y, *args, **kwargs):
return x + y
assert f(1, 2) == 3
assert f(1, 3) == 3
def test_memoize_wrapped():
def foo():
"""
Docstring
"""
pass
memoized_foo = memoize(foo)
assert memoized_foo.__wrapped__ is foo
def test_curry_simple():
cmul = curry(mul)
double = cmul(2)
assert callable(double)
assert double(10) == 20
assert repr(cmul) == repr(mul)
cmap = curry(map)
assert list(cmap(inc)([1, 2, 3])) == [2, 3, 4]
assert raises(TypeError, lambda: curry())
assert raises(TypeError, lambda: curry({1: 2}))
def test_curry_kwargs():
def f(a, b, c=10):
return (a + b) * c
f = curry(f)
assert f(1, 2, 3) == 9
assert f(1)(2, 3) == 9
assert f(1, 2) == 30
assert f(1, c=3)(2) == 9
assert f(c=3)(1, 2) == 9
def g(a=1, b=10, c=0):
return a + b + c
cg = curry(g, b=2)
assert cg() == 3
assert cg(b=3) == 4
assert cg(a=0) == 2
assert cg(a=0, b=1) == 1
assert cg(0) == 2 # pass "a" as arg, not kwarg
assert raises(TypeError, lambda: cg(1, 2)) # pass "b" as arg AND kwarg
def h(x, func=int):
return func(x)
# __init__ must not pick func as positional arg
assert curry(h)(0.0) == 0
assert curry(h)(func=str)(0.0) == '0.0'
assert curry(h, func=str)(0.0) == '0.0'
def test_curry_passes_errors():
@curry
def f(a, b):
if not isinstance(a, int):
raise TypeError()
return a + b
assert f(1, 2) == 3
assert raises(TypeError, lambda: f('1', 2))
assert raises(TypeError, lambda: f('1')(2))
assert raises(TypeError, lambda: f(1, 2, 3))
def test_curry_docstring():
def f(x, y):
""" A docstring """
return x
g = curry(f)
assert g.__doc__ == f.__doc__
assert str(g) == str(f)
assert f(1, 2) == g(1, 2)
def test_curry_is_like_partial():
def foo(a, b, c=1):
return a + b + c
p, c = partial(foo, 1, c=2), curry(foo)(1, c=2)
assert p.keywords == c.keywords
assert p.args == c.args
assert p(3) == c(3)
p, c = partial(foo, 1), curry(foo)(1)
assert p.keywords == c.keywords
assert p.args == c.args
assert p(3) == c(3)
assert p(3, c=2) == c(3, c=2)
p, c = partial(foo, c=1), curry(foo)(c=1)
assert p.keywords == c.keywords
assert p.args == c.args
assert p(1, 2) == c(1, 2)
def test_curry_is_idempotent():
def foo(a, b, c=1):
return a + b + c
f = curry(foo, 1, c=2)
g = curry(f)
assert isinstance(f, curry)
assert isinstance(g, curry)
assert not isinstance(g.func, curry)
assert not hasattr(g.func, 'func')
assert f.func == g.func
assert f.args == g.args
assert f.keywords == g.keywords
def test_curry_attributes_readonly():
def foo(a, b, c=1):
return a + b + c
f = curry(foo, 1, c=2)
assert raises(AttributeError, lambda: setattr(f, 'args', (2,)))
assert raises(AttributeError, lambda: setattr(f, 'keywords', {'c': 3}))
assert raises(AttributeError, lambda: setattr(f, 'func', f))
assert raises(AttributeError, lambda: delattr(f, 'args'))
assert raises(AttributeError, lambda: delattr(f, 'keywords'))
assert raises(AttributeError, lambda: delattr(f, 'func'))
def test_curry_attributes_writable():
def foo(a, b, c=1):
return a + b + c
foo.__qualname__ = 'this.is.foo'
f = curry(foo, 1, c=2)
assert f.__qualname__ == 'this.is.foo'
f.__name__ = 'newname'
f.__doc__ = 'newdoc'
f.__module__ = 'newmodule'
f.__qualname__ = 'newqualname'
assert f.__name__ == 'newname'
assert f.__doc__ == 'newdoc'
assert f.__module__ == 'newmodule'
assert f.__qualname__ == 'newqualname'
if hasattr(f, 'func_name'):
assert f.__name__ == f.func_name
def test_curry_module():
from toolz.curried.exceptions import merge
assert merge.__module__ == 'toolz.curried.exceptions'
def test_curry_comparable():
def foo(a, b, c=1):
return a + b + c
f1 = curry(foo, 1, c=2)
f2 = curry(foo, 1, c=2)
g1 = curry(foo, 1, c=3)
h1 = curry(foo, c=2)
h2 = h1(c=2)
h3 = h1()
assert f1 == f2
assert not (f1 != f2)
assert f1 != g1
assert not (f1 == g1)
assert f1 != h1
assert h1 == h2
assert h1 == h3
# test function comparison works
def bar(a, b, c=1):
return a + b + c
b1 = curry(bar, 1, c=2)
assert b1 != f1
assert {f1, f2, g1, h1, h2, h3, b1, b1()} == {f1, g1, h1, b1}
# test unhashable input
unhash1 = curry(foo, [])
assert raises(TypeError, lambda: hash(unhash1))
unhash2 = curry(foo, c=[])
assert raises(TypeError, lambda: hash(unhash2))
def test_curry_doesnot_transmogrify():
# Early versions of `curry` transmogrified to `partial` objects if
# only one positional argument remained even if keyword arguments
# were present. Now, `curry` should always remain `curry`.
def f(x, y=0):
return x + y
cf = curry(f)
assert cf(y=1)(y=2)(y=3)(1) == f(1, 3)
def test_curry_on_classmethods():
class A(object):
BASE = 10
def __init__(self, base):
self.BASE = base
@curry
def addmethod(self, x, y):
return self.BASE + x + y
@classmethod
@curry
def addclass(cls, x, y):
return cls.BASE + x + y
@staticmethod
@curry
def addstatic(x, y):
return x + y
a = A(100)
assert a.addmethod(3, 4) == 107
assert a.addmethod(3)(4) == 107
assert A.addmethod(a, 3, 4) == 107
assert A.addmethod(a)(3)(4) == 107
assert a.addclass(3, 4) == 17
assert a.addclass(3)(4) == 17
assert A.addclass(3, 4) == 17
assert A.addclass(3)(4) == 17
assert a.addstatic(3, 4) == 7
assert a.addstatic(3)(4) == 7
assert A.addstatic(3, 4) == 7
assert A.addstatic(3)(4) == 7
# we want this to be of type curry
assert isinstance(a.addmethod, curry)
assert isinstance(A.addmethod, curry)
def test_memoize_on_classmethods():
class A(object):
BASE = 10
HASH = 10
def __init__(self, base):
self.BASE = base
@memoize
def addmethod(self, x, y):
return self.BASE + x + y
@classmethod
@memoize
def addclass(cls, x, y):
return cls.BASE + x + y
@staticmethod
@memoize
def addstatic(x, y):
return x + y
def __hash__(self):
return self.HASH
a = A(100)
assert a.addmethod(3, 4) == 107
assert A.addmethod(a, 3, 4) == 107
a.BASE = 200
assert a.addmethod(3, 4) == 107
a.HASH = 200
assert a.addmethod(3, 4) == 207
assert a.addclass(3, 4) == 17
assert A.addclass(3, 4) == 17
A.BASE = 20
assert A.addclass(3, 4) == 17
A.HASH = 20 # hashing of class is handled by metaclass
assert A.addclass(3, 4) == 17 # hence, != 27
assert a.addstatic(3, 4) == 7
assert A.addstatic(3, 4) == 7
def test_curry_call():
@curry
def add(x, y):
return x + y
assert raises(TypeError, lambda: add.call(1))
assert add(1)(2) == add.call(1, 2)
assert add(1)(2) == add(1).call(2)
def test_curry_bind():
@curry
def add(x=1, y=2):
return x + y
assert add() == add(1, 2)
assert add.bind(10)(20) == add(10, 20)
assert add.bind(10).bind(20)() == add(10, 20)
assert add.bind(x=10)(y=20) == add(10, 20)
assert add.bind(x=10).bind(y=20)() == add(10, 20)
def test_curry_unknown_args():
def add3(x, y, z):
return x + y + z
@curry
def f(*args):
return add3(*args)
assert f()(1)(2)(3) == 6
assert f(1)(2)(3) == 6
assert f(1, 2)(3) == 6
assert f(1, 2, 3) == 6
assert f(1, 2)(3, 4) == f(1, 2, 3, 4)
def test_curry_bad_types():
assert raises(TypeError, lambda: curry(1))
def test_curry_subclassable():
class mycurry(curry):
pass
add = mycurry(lambda x, y: x+y)
assert isinstance(add, curry)
assert isinstance(add, mycurry)
assert isinstance(add(1), mycurry)
assert isinstance(add()(1), mycurry)
assert add(1)(2) == 3
# Should we make `_should_curry` public?
"""
class curry2(curry):
def _should_curry(self, args, kwargs, exc=None):
return len(self.args) + len(args) < 2
add = curry2(lambda x, y: x+y)
assert isinstance(add(1), curry2)
assert add(1)(2) == 3
assert isinstance(add(1)(x=2), curry2)
assert raises(TypeError, lambda: add(1)(x=2)(3))
"""
def generate_compose_test_cases():
"""
Generate test cases for parametrized tests of the compose function.
"""
def add_then_multiply(a, b, c=10):
return (a + b) * c
return (
(
(), # arguments to compose()
(0,), {}, # positional and keyword args to the Composed object
0 # expected result
),
(
(inc,),
(0,), {},
1
),
(
(double, inc),
(0,), {},
2
),
(
(str, iseven, inc, double),
(3,), {},
"False"
),
(
(str, add),
(1, 2), {},
'3'
),
(
(str, inc, add_then_multiply),
(1, 2), {"c": 3},
'10'
),
)
def test_compose():
for (compose_args, args, kw, expected) in generate_compose_test_cases():
assert compose(*compose_args)(*args, **kw) == expected
def test_compose_metadata():
# Define two functions with different names
def f(a):
return a
def g(a):
return a
composed = compose(f, g)
assert composed.__name__ == 'f_of_g'
assert composed.__doc__ == 'lambda *args, **kwargs: f(g(*args, **kwargs))'
# Create an object with no __name__.
h = object()
composed = compose(f, h)
assert composed.__name__ == 'Compose'
assert composed.__doc__ == 'A composition of functions'
assert repr(composed) == 'Compose({!r}, {!r})'.format(f, h)
assert composed == compose(f, h)
assert composed == AlwaysEquals()
assert not composed == compose(h, f)
assert not composed == object()
assert not composed == NeverEquals()
assert composed != compose(h, f)
assert composed != NeverEquals()
assert composed != object()
assert not composed != compose(f, h)
assert not composed != AlwaysEquals()
assert hash(composed) == hash(compose(f, h))
assert hash(composed) != hash(compose(h, f))
bindable = compose(str, lambda x: x*2, lambda x, y=0: int(x) + y)
class MyClass:
def __int__(self):
return 8
my_method = bindable
my_static_method = staticmethod(bindable)
assert MyClass.my_method(3) == '6'
assert MyClass.my_method(3, y=2) == '10'
assert MyClass.my_static_method(2) == '4'
assert MyClass().my_method() == '16'
assert MyClass().my_method(y=3) == '22'
assert MyClass().my_static_method(0) == '0'
assert MyClass().my_static_method(0, 1) == '2'
assert compose(f, h).__wrapped__ is h
if hasattr(toolz, 'sandbox'): # only test this with Python version (i.e., not Cython)
assert compose(f, h).__class__.__wrapped__ is None
# __signature__ is python3 only
def myfunc(a, b, c, *d, **e):
return 4
def otherfunc(f):
return 'result: {}'.format(f)
# set annotations compatibly with python2 syntax
myfunc.__annotations__ = {
'a': int,
'b': str,
'c': float,
'd': int,
'e': bool,
'return': int,
}
otherfunc.__annotations__ = {'f': int, 'return': str}
composed = compose(otherfunc, myfunc)
sig = inspect.signature(composed)
assert sig.parameters == inspect.signature(myfunc).parameters
assert sig.return_annotation == str
class MyClass:
method = composed
assert len(inspect.signature(MyClass().method).parameters) == 4
def generate_compose_left_test_cases():
"""
Generate test cases for parametrized tests of the compose function.
These are based on, and equivalent to, those produced by
enerate_compose_test_cases().
"""
return tuple(
(tuple(reversed(compose_args)), args, kwargs, expected)
for (compose_args, args, kwargs, expected)
in generate_compose_test_cases()
)
def test_compose_left():
for (compose_left_args, args, kw, expected) in generate_compose_left_test_cases():
assert compose_left(*compose_left_args)(*args, **kw) == expected
def test_pipe():
assert pipe(1, inc) == 2
assert pipe(1, inc, inc) == 3
assert pipe(1, double, inc, iseven) is False
def test_complement():
# No args:
assert complement(lambda: False)()
assert not complement(lambda: True)()
# Single arity:
assert complement(iseven)(1)
assert not complement(iseven)(2)
assert complement(complement(iseven))(2)
assert not complement(complement(isodd))(2)
# Multiple arities:
both_even = lambda a, b: iseven(a) and iseven(b)
assert complement(both_even)(1, 2)
assert not complement(both_even)(2, 2)
# Generic truthiness:
assert complement(lambda: "")()
assert complement(lambda: 0)()
assert complement(lambda: None)()
assert complement(lambda: [])()
assert not complement(lambda: "x")()
assert not complement(lambda: 1)()
assert not complement(lambda: [1])()
def test_do():
inc = lambda x: x + 1
assert do(inc, 1) == 1
log = []
assert do(log.append, 1) == 1
assert log == [1]
def test_juxt_generator_input():
data = list(range(10))
juxtfunc = juxt(itemgetter(2*i) for i in range(5))
assert juxtfunc(data) == (0, 2, 4, 6, 8)
assert juxtfunc(data) == (0, 2, 4, 6, 8)
def test_flip():
def f(a, b):
return a, b
assert flip(f, 'a', 'b') == ('b', 'a')
def test_excepts():
# These are descriptors, make sure this works correctly.
assert excepts.__name__ == 'excepts'
assert (
'A wrapper around a function to catch exceptions and\n'
' dispatch to a handler.\n'
) in excepts.__doc__
def idx(a):
"""idx docstring
"""
return [1, 2].index(a)
def handler(e):
"""handler docstring
"""
assert isinstance(e, ValueError)
return -1
excepting = excepts(ValueError, idx, handler)
assert excepting(1) == 0
assert excepting(2) == 1
assert excepting(3) == -1
assert excepting.__name__ == 'idx_excepting_ValueError'
assert 'idx docstring' in excepting.__doc__
assert 'ValueError' in excepting.__doc__
assert 'handler docstring' in excepting.__doc__
def getzero(a):
"""getzero docstring
"""
return a[0]
excepting = excepts((IndexError, KeyError), getzero)
assert excepting([]) is None
assert excepting([1]) == 1
assert excepting({}) is None
assert excepting({0: 1}) == 1
assert excepting.__name__ == 'getzero_excepting_IndexError_or_KeyError'
assert 'getzero docstring' in excepting.__doc__
assert 'return_none' in excepting.__doc__
assert 'Returns None' in excepting.__doc__
def raise_(a):
"""A function that raises an instance of the exception type given.
"""
raise a()
excepting = excepts((ValueError, KeyError), raise_)
assert excepting(ValueError) is None
assert excepting(KeyError) is None
assert raises(TypeError, lambda: excepting(TypeError))
assert raises(NotImplementedError, lambda: excepting(NotImplementedError))
excepting = excepts(object(), object(), object())
assert excepting.__name__ == 'excepting'
assert excepting.__doc__ == excepts.__doc__

View File

@@ -0,0 +1,487 @@
import functools
import inspect
import itertools
import operator
import toolz
from toolz.functoolz import (curry, is_valid_args, is_partial_args, is_arity,
num_required_args, has_varargs, has_keywords)
from toolz._signatures import builtins
import toolz._signatures as _sigs
from toolz.utils import raises
def make_func(param_string, raise_if_called=True):
if not param_string.startswith('('):
param_string = '(%s)' % param_string
if raise_if_called:
body = 'raise ValueError("function should not be called")'
else:
body = 'return True'
d = {}
exec('def func%s:\n %s' % (param_string, body), globals(), d)
return d['func']
def test_make_func():
f = make_func('')
assert raises(ValueError, lambda: f())
assert raises(TypeError, lambda: f(1))
f = make_func('', raise_if_called=False)
assert f()
assert raises(TypeError, lambda: f(1))
f = make_func('x, y=1', raise_if_called=False)
assert f(1)
assert f(x=1)
assert f(1, 2)
assert f(x=1, y=2)
assert raises(TypeError, lambda: f(1, 2, 3))
f = make_func('(x, y=1)', raise_if_called=False)
assert f(1)
assert f(x=1)
assert f(1, 2)
assert f(x=1, y=2)
assert raises(TypeError, lambda: f(1, 2, 3))
def test_is_valid(check_valid=is_valid_args, incomplete=False):
orig_check_valid = check_valid
check_valid = lambda func, *args, **kwargs: orig_check_valid(func, args, kwargs)
f = make_func('')
assert check_valid(f)
assert check_valid(f, 1) is False
assert check_valid(f, x=1) is False
f = make_func('x')
assert check_valid(f) is incomplete
assert check_valid(f, 1)
assert check_valid(f, x=1)
assert check_valid(f, 1, x=2) is False
assert check_valid(f, 1, y=2) is False
assert check_valid(f, 1, 2) is False
assert check_valid(f, x=1, y=2) is False
f = make_func('x=1')
assert check_valid(f)
assert check_valid(f, 1)
assert check_valid(f, x=1)
assert check_valid(f, 1, x=2) is False
assert check_valid(f, 1, y=2) is False
assert check_valid(f, 1, 2) is False
assert check_valid(f, x=1, y=2) is False
f = make_func('*args')
assert check_valid(f)
assert check_valid(f, 1)
assert check_valid(f, 1, 2)
assert check_valid(f, x=1) is False
f = make_func('**kwargs')
assert check_valid(f)
assert check_valid(f, x=1)
assert check_valid(f, x=1, y=2)
assert check_valid(f, 1) is False
f = make_func('x, *args')
assert check_valid(f) is incomplete
assert check_valid(f, 1)
assert check_valid(f, 1, 2)
assert check_valid(f, x=1)
assert check_valid(f, 1, x=1) is False
assert check_valid(f, 1, y=1) is False
f = make_func('x, y=1, **kwargs')
assert check_valid(f) is incomplete
assert check_valid(f, 1)
assert check_valid(f, x=1)
assert check_valid(f, 1, 2)
assert check_valid(f, x=1, y=2, z=3)
assert check_valid(f, 1, 2, y=3) is False
f = make_func('a, b, c=3, d=4')
assert check_valid(f) is incomplete
assert check_valid(f, 1) is incomplete
assert check_valid(f, 1, 2)
assert check_valid(f, 1, c=3) is incomplete
assert check_valid(f, 1, e=3) is False
assert check_valid(f, 1, 2, e=3) is False
assert check_valid(f, 1, 2, b=3) is False
assert check_valid(1) is False
def test_is_valid_py3(check_valid=is_valid_args, incomplete=False):
orig_check_valid = check_valid
check_valid = lambda func, *args, **kwargs: orig_check_valid(func, args, kwargs)
f = make_func('x, *, y=1')
assert check_valid(f) is incomplete
assert check_valid(f, 1)
assert check_valid(f, x=1)
assert check_valid(f, 1, y=2)
assert check_valid(f, 1, 2) is False
assert check_valid(f, 1, z=2) is False
f = make_func('x, *args, y=1')
assert check_valid(f) is incomplete
assert check_valid(f, 1)
assert check_valid(f, x=1)
assert check_valid(f, 1, y=2)
assert check_valid(f, 1, 2, y=2)
assert check_valid(f, 1, 2)
assert check_valid(f, 1, z=2) is False
f = make_func('*, y=1')
assert check_valid(f)
assert check_valid(f, 1) is False
assert check_valid(f, y=1)
assert check_valid(f, z=1) is False
f = make_func('x, *, y')
assert check_valid(f) is incomplete
assert check_valid(f, 1) is incomplete
assert check_valid(f, x=1) is incomplete
assert check_valid(f, 1, y=2)
assert check_valid(f, x=1, y=2)
assert check_valid(f, 1, 2) is False
assert check_valid(f, 1, z=2) is False
assert check_valid(f, 1, y=1, z=2) is False
f = make_func('x=1, *, y, z=3')
assert check_valid(f) is incomplete
assert check_valid(f, 1, z=3) is incomplete
assert check_valid(f, y=2)
assert check_valid(f, 1, y=2)
assert check_valid(f, x=1, y=2)
assert check_valid(f, x=1, y=2, z=3)
assert check_valid(f, 1, x=1, y=2) is False
assert check_valid(f, 1, 3, y=2) is False
f = make_func('w, x=2, *args, y, z=4')
assert check_valid(f) is incomplete
assert check_valid(f, 1) is incomplete
assert check_valid(f, 1, y=3)
f = make_func('a, b, c=3, d=4, *args, e=5, f=6, g, h')
assert check_valid(f) is incomplete
assert check_valid(f, 1) is incomplete
assert check_valid(f, 1, 2) is incomplete
assert check_valid(f, 1, 2, g=7) is incomplete
assert check_valid(f, 1, 2, g=7, h=8)
assert check_valid(f, 1, 2, 3, 4, 5, 6, 7, 8, 9) is incomplete
f = make_func('a: int, b: float')
assert check_valid(f) is incomplete
assert check_valid(f, 1) is incomplete
assert check_valid(f, b=1) is incomplete
assert check_valid(f, 1, 2)
f = make_func('(a: int, b: float) -> float')
assert check_valid(f) is incomplete
assert check_valid(f, 1) is incomplete
assert check_valid(f, b=1) is incomplete
assert check_valid(f, 1, 2)
f.__signature__ = 34
assert check_valid(f) is False
class RaisesValueError(object):
def __call__(self):
pass
@property
def __signature__(self):
raise ValueError('Testing Python 3.4')
f = RaisesValueError()
assert check_valid(f) is None
def test_is_partial():
test_is_valid(check_valid=is_partial_args, incomplete=True)
test_is_valid_py3(check_valid=is_partial_args, incomplete=True)
def test_is_valid_curry():
def check_curry(func, args, kwargs, incomplete=True):
try:
curry(func)(*args, **kwargs)
curry(func, *args)(**kwargs)
curry(func, **kwargs)(*args)
curry(func, *args, **kwargs)()
if not isinstance(func, type(lambda: None)):
return None
return incomplete
except ValueError:
return True
except TypeError:
return False
check_valid = functools.partial(check_curry, incomplete=True)
test_is_valid(check_valid=check_valid, incomplete=True)
test_is_valid_py3(check_valid=check_valid, incomplete=True)
check_valid = functools.partial(check_curry, incomplete=False)
test_is_valid(check_valid=check_valid, incomplete=False)
test_is_valid_py3(check_valid=check_valid, incomplete=False)
def test_func_keyword():
def f(func=None):
pass
assert is_valid_args(f, (), {})
assert is_valid_args(f, (None,), {})
assert is_valid_args(f, (), {'func': None})
assert is_valid_args(f, (None,), {'func': None}) is False
assert is_partial_args(f, (), {})
assert is_partial_args(f, (None,), {})
assert is_partial_args(f, (), {'func': None})
assert is_partial_args(f, (None,), {'func': None}) is False
def test_has_unknown_args():
assert has_varargs(1) is False
assert has_varargs(map)
assert has_varargs(make_func('')) is False
assert has_varargs(make_func('x, y, z')) is False
assert has_varargs(make_func('*args'))
assert has_varargs(make_func('**kwargs')) is False
assert has_varargs(make_func('x, y, *args, **kwargs'))
assert has_varargs(make_func('x, y, z=1')) is False
assert has_varargs(make_func('x, y, z=1, **kwargs')) is False
f = make_func('*args')
f.__signature__ = 34
assert has_varargs(f) is False
class RaisesValueError(object):
def __call__(self):
pass
@property
def __signature__(self):
raise ValueError('Testing Python 3.4')
f = RaisesValueError()
assert has_varargs(f) is None
def test_num_required_args():
assert num_required_args(lambda: None) == 0
assert num_required_args(lambda x: None) == 1
assert num_required_args(lambda x, *args: None) == 1
assert num_required_args(lambda x, **kwargs: None) == 1
assert num_required_args(lambda x, y, *args, **kwargs: None) == 2
assert num_required_args(map) == 2
assert num_required_args(dict) is None
def test_has_keywords():
assert has_keywords(lambda: None) is False
assert has_keywords(lambda x: None) is False
assert has_keywords(lambda x=1: None)
assert has_keywords(lambda **kwargs: None)
assert has_keywords(int)
assert has_keywords(sorted)
assert has_keywords(max)
assert has_keywords(map) is False
assert has_keywords(bytearray) is None
def test_has_varargs():
assert has_varargs(lambda: None) is False
assert has_varargs(lambda *args: None)
assert has_varargs(lambda **kwargs: None) is False
assert has_varargs(map)
assert has_varargs(max) is None
def test_is_arity():
assert is_arity(0, lambda: None)
assert is_arity(1, lambda: None) is False
assert is_arity(1, lambda x: None)
assert is_arity(3, lambda x, y, z: None)
assert is_arity(1, lambda x, *args: None) is False
assert is_arity(1, lambda x, **kwargs: None) is False
assert is_arity(1, all)
assert is_arity(2, map) is False
assert is_arity(2, range) is None
def test_introspect_curry_valid_py3(check_valid=is_valid_args, incomplete=False):
orig_check_valid = check_valid
check_valid = lambda _func, *args, **kwargs: orig_check_valid(_func, args, kwargs)
f = toolz.curry(make_func('x, y, z=0'))
assert check_valid(f)
assert check_valid(f, 1)
assert check_valid(f, 1, 2)
assert check_valid(f, 1, 2, 3)
assert check_valid(f, 1, 2, 3, 4) is False
assert check_valid(f, invalid_keyword=True) is False
assert check_valid(f(1))
assert check_valid(f(1), 2)
assert check_valid(f(1), 2, 3)
assert check_valid(f(1), 2, 3, 4) is False
assert check_valid(f(1), x=2) is False
assert check_valid(f(1), y=2)
assert check_valid(f(x=1), 2) is False
assert check_valid(f(x=1), y=2)
assert check_valid(f(y=2), 1)
assert check_valid(f(y=2), 1, z=3)
assert check_valid(f(y=2), 1, 3) is False
f = toolz.curry(make_func('x, y, z=0'), 1, x=1)
assert check_valid(f) is False
assert check_valid(f, z=3) is False
f = toolz.curry(make_func('x, y, *args, z'))
assert check_valid(f)
assert check_valid(f, 0)
assert check_valid(f(1), 0)
assert check_valid(f(1, 2), 0)
assert check_valid(f(1, 2, 3), 0)
assert check_valid(f(1, 2, 3, 4), 0)
assert check_valid(f(1, 2, 3, 4), z=4)
assert check_valid(f(x=1))
assert check_valid(f(x=1), 1) is False
assert check_valid(f(x=1), y=2)
def test_introspect_curry_partial_py3():
test_introspect_curry_valid_py3(check_valid=is_partial_args, incomplete=True)
def test_introspect_curry_py3():
f = toolz.curry(make_func(''))
assert num_required_args(f) == 0
assert is_arity(0, f)
assert has_varargs(f) is False
assert has_keywords(f) is False
f = toolz.curry(make_func('x'))
assert num_required_args(f) == 0
assert is_arity(0, f) is False
assert is_arity(1, f) is False
assert has_varargs(f) is False
assert has_keywords(f) # A side-effect of being curried
f = toolz.curry(make_func('x, y, z=0'))
assert num_required_args(f) == 0
assert is_arity(0, f) is False
assert is_arity(1, f) is False
assert is_arity(2, f) is False
assert is_arity(3, f) is False
assert has_varargs(f) is False
assert has_keywords(f)
f = toolz.curry(make_func('*args, **kwargs'))
assert num_required_args(f) == 0
assert has_varargs(f)
assert has_keywords(f)
def test_introspect_builtin_modules():
mods = [builtins, functools, itertools, operator, toolz,
toolz.functoolz, toolz.itertoolz, toolz.dicttoolz, toolz.recipes]
denylist = set()
def add_denylist(mod, attr):
if hasattr(mod, attr):
denylist.add(getattr(mod, attr))
add_denylist(builtins, 'basestring')
add_denylist(builtins, 'NoneType')
add_denylist(builtins, '__metaclass__')
add_denylist(builtins, 'sequenceiterator')
def is_missing(modname, name, func):
if name.startswith('_') and not name.startswith('__'):
return False
if name.startswith('__pyx_unpickle_') or name.endswith('_cython__'):
return False
try:
if issubclass(func, BaseException):
return False
except TypeError:
pass
try:
return (callable(func)
and func.__module__ is not None
and modname in func.__module__
and is_partial_args(func, (), {}) is not True
and func not in denylist)
except AttributeError:
return False
missing = {}
for mod in mods:
modname = mod.__name__
for name, func in vars(mod).items():
if is_missing(modname, name, func):
if modname not in missing:
missing[modname] = []
missing[modname].append(name)
if missing:
messages = []
for modname, names in sorted(missing.items()):
msg = '{}:\n {}'.format(modname, '\n '.join(sorted(names)))
messages.append(msg)
message = 'Missing introspection for the following callables:\n\n'
raise AssertionError(message + '\n\n'.join(messages))
def test_inspect_signature_property():
# By adding AddX to our signature registry, we can inspect the class
# itself and objects of the class. `inspect.signature` doesn't like
# it when `obj.__signature__` is a property.
class AddX(object):
def __init__(self, func):
self.func = func
def __call__(self, addx, *args, **kwargs):
return addx + self.func(*args, **kwargs)
@property
def __signature__(self):
sig = inspect.signature(self.func)
params = list(sig.parameters.values())
kind = inspect.Parameter.POSITIONAL_OR_KEYWORD
newparam = inspect.Parameter('addx', kind)
params = [newparam] + params
return sig.replace(parameters=params)
addx = AddX(lambda x: x)
sig = inspect.signature(addx)
assert sig == inspect.Signature(parameters=[
inspect.Parameter('addx', inspect.Parameter.POSITIONAL_OR_KEYWORD),
inspect.Parameter('x', inspect.Parameter.POSITIONAL_OR_KEYWORD)])
assert num_required_args(AddX) is False
_sigs.signatures[AddX] = (_sigs.expand_sig((0, lambda func: None)),)
assert num_required_args(AddX) == 1
del _sigs.signatures[AddX]
def test_inspect_wrapped_property():
class Wrapped(object):
def __init__(self, func):
self.func = func
def __call__(self, *args, **kwargs):
return self.func(*args, **kwargs)
@property
def __wrapped__(self):
return self.func
func = lambda x: x
wrapped = Wrapped(func)
assert inspect.signature(func) == inspect.signature(wrapped)
assert num_required_args(Wrapped) is None
_sigs.signatures[Wrapped] = (_sigs.expand_sig((0, lambda func: None)),)
assert num_required_args(Wrapped) == 1

View File

@@ -0,0 +1,549 @@
import itertools
from itertools import starmap
from toolz.utils import raises
from functools import partial
from random import Random
from pickle import dumps, loads
from toolz.itertoolz import (remove, groupby, merge_sorted,
concat, concatv, interleave, unique,
isiterable, getter,
mapcat, isdistinct, first, second,
nth, take, tail, drop, interpose, get,
rest, last, cons, frequencies,
reduceby, iterate, accumulate,
sliding_window, count, partition,
partition_all, take_nth, pluck, join,
diff, topk, peek, peekn, random_sample)
from operator import add, mul
# is comparison will fail between this and no_default
no_default2 = loads(dumps('__no__default__'))
def identity(x):
return x
def iseven(x):
return x % 2 == 0
def isodd(x):
return x % 2 == 1
def inc(x):
return x + 1
def double(x):
return 2 * x
def test_remove():
r = remove(iseven, range(5))
assert type(r) is not list
assert list(r) == list(filter(isodd, range(5)))
def test_groupby():
assert groupby(iseven, [1, 2, 3, 4]) == {True: [2, 4], False: [1, 3]}
def test_groupby_non_callable():
assert groupby(0, [(1, 2), (1, 3), (2, 2), (2, 4)]) == \
{1: [(1, 2), (1, 3)],
2: [(2, 2), (2, 4)]}
assert groupby([0], [(1, 2), (1, 3), (2, 2), (2, 4)]) == \
{(1,): [(1, 2), (1, 3)],
(2,): [(2, 2), (2, 4)]}
assert groupby([0, 0], [(1, 2), (1, 3), (2, 2), (2, 4)]) == \
{(1, 1): [(1, 2), (1, 3)],
(2, 2): [(2, 2), (2, 4)]}
def test_merge_sorted():
assert list(merge_sorted([1, 2, 3], [1, 2, 3])) == [1, 1, 2, 2, 3, 3]
assert list(merge_sorted([1, 3, 5], [2, 4, 6])) == [1, 2, 3, 4, 5, 6]
assert list(merge_sorted([1], [2, 4], [3], [])) == [1, 2, 3, 4]
assert list(merge_sorted([5, 3, 1], [6, 4, 3], [],
key=lambda x: -x)) == [6, 5, 4, 3, 3, 1]
assert list(merge_sorted([2, 1, 3], [1, 2, 3],
key=lambda x: x // 3)) == [2, 1, 1, 2, 3, 3]
assert list(merge_sorted([2, 3], [1, 3],
key=lambda x: x // 3)) == [2, 1, 3, 3]
assert ''.join(merge_sorted('abc', 'abc', 'abc')) == 'aaabbbccc'
assert ''.join(merge_sorted('abc', 'abc', 'abc', key=ord)) == 'aaabbbccc'
assert ''.join(merge_sorted('cba', 'cba', 'cba',
key=lambda x: -ord(x))) == 'cccbbbaaa'
assert list(merge_sorted([1], [2, 3, 4], key=identity)) == [1, 2, 3, 4]
data = [[(1, 2), (0, 4), (3, 6)], [(5, 3), (6, 5), (8, 8)],
[(9, 1), (9, 8), (9, 9)]]
assert list(merge_sorted(*data, key=lambda x: x[1])) == [
(9, 1), (1, 2), (5, 3), (0, 4), (6, 5), (3, 6), (8, 8), (9, 8), (9, 9)]
assert list(merge_sorted()) == []
assert list(merge_sorted([1, 2, 3])) == [1, 2, 3]
assert list(merge_sorted([1, 4, 5], [2, 3])) == [1, 2, 3, 4, 5]
assert list(merge_sorted([1, 4, 5], [2, 3], key=identity)) == [
1, 2, 3, 4, 5]
assert list(merge_sorted([1, 5], [2], [4, 7], [3, 6], key=identity)) == [
1, 2, 3, 4, 5, 6, 7]
def test_interleave():
assert ''.join(interleave(('ABC', '123'))) == 'A1B2C3'
assert ''.join(interleave(('ABC', '1'))) == 'A1BC'
def test_unique():
assert tuple(unique((1, 2, 3))) == (1, 2, 3)
assert tuple(unique((1, 2, 1, 3))) == (1, 2, 3)
assert tuple(unique((1, 2, 3), key=iseven)) == (1, 2)
def test_isiterable():
assert isiterable([1, 2, 3]) is True
assert isiterable('abc') is True
assert isiterable(5) is False
def test_isdistinct():
assert isdistinct([1, 2, 3]) is True
assert isdistinct([1, 2, 1]) is False
assert isdistinct("Hello") is False
assert isdistinct("World") is True
assert isdistinct(iter([1, 2, 3])) is True
assert isdistinct(iter([1, 2, 1])) is False
def test_nth():
assert nth(2, 'ABCDE') == 'C'
assert nth(2, iter('ABCDE')) == 'C'
assert nth(1, (3, 2, 1)) == 2
assert nth(0, {'foo': 'bar'}) == 'foo'
assert raises(StopIteration, lambda: nth(10, {10: 'foo'}))
assert nth(-2, 'ABCDE') == 'D'
assert raises(ValueError, lambda: nth(-2, iter('ABCDE')))
def test_first():
assert first('ABCDE') == 'A'
assert first((3, 2, 1)) == 3
assert isinstance(first({0: 'zero', 1: 'one'}), int)
def test_second():
assert second('ABCDE') == 'B'
assert second((3, 2, 1)) == 2
assert isinstance(second({0: 'zero', 1: 'one'}), int)
def test_last():
assert last('ABCDE') == 'E'
assert last((3, 2, 1)) == 1
assert isinstance(last({0: 'zero', 1: 'one'}), int)
def test_rest():
assert list(rest('ABCDE')) == list('BCDE')
assert list(rest((3, 2, 1))) == list((2, 1))
def test_take():
assert list(take(3, 'ABCDE')) == list('ABC')
assert list(take(2, (3, 2, 1))) == list((3, 2))
def test_tail():
assert list(tail(3, 'ABCDE')) == list('CDE')
assert list(tail(3, iter('ABCDE'))) == list('CDE')
assert list(tail(2, (3, 2, 1))) == list((2, 1))
def test_drop():
assert list(drop(3, 'ABCDE')) == list('DE')
assert list(drop(1, (3, 2, 1))) == list((2, 1))
def test_take_nth():
assert list(take_nth(2, 'ABCDE')) == list('ACE')
def test_get():
assert get(1, 'ABCDE') == 'B'
assert list(get([1, 3], 'ABCDE')) == list('BD')
assert get('a', {'a': 1, 'b': 2, 'c': 3}) == 1
assert get(['a', 'b'], {'a': 1, 'b': 2, 'c': 3}) == (1, 2)
assert get('foo', {}, default='bar') == 'bar'
assert get({}, [1, 2, 3], default='bar') == 'bar'
assert get([0, 2], 'AB', 'C') == ('A', 'C')
assert get([0], 'AB') == ('A',)
assert get([], 'AB') == ()
assert raises(IndexError, lambda: get(10, 'ABC'))
assert raises(KeyError, lambda: get(10, {'a': 1}))
assert raises(TypeError, lambda: get({}, [1, 2, 3]))
assert raises(TypeError, lambda: get([1, 2, 3], 1, None))
assert raises(KeyError, lambda: get('foo', {}, default=no_default2))
def test_mapcat():
assert (list(mapcat(identity, [[1, 2, 3], [4, 5, 6]])) ==
[1, 2, 3, 4, 5, 6])
assert (list(mapcat(reversed, [[3, 2, 1, 0], [6, 5, 4], [9, 8, 7]])) ==
list(range(10)))
inc = lambda i: i + 1
assert ([4, 5, 6, 7, 8, 9] ==
list(mapcat(partial(map, inc), [[3, 4, 5], [6, 7, 8]])))
def test_cons():
assert list(cons(1, [2, 3])) == [1, 2, 3]
def test_concat():
assert list(concat([[], [], []])) == []
assert (list(take(5, concat([['a', 'b'], range(1000000000)]))) ==
['a', 'b', 0, 1, 2])
def test_concatv():
assert list(concatv([], [], [])) == []
assert (list(take(5, concatv(['a', 'b'], range(1000000000)))) ==
['a', 'b', 0, 1, 2])
def test_interpose():
assert "a" == first(rest(interpose("a", range(1000000000))))
assert "tXaXrXzXaXn" == "".join(interpose("X", "tarzan"))
assert list(interpose(0, itertools.repeat(1, 4))) == [1, 0, 1, 0, 1, 0, 1]
assert list(interpose('.', ['a', 'b', 'c'])) == ['a', '.', 'b', '.', 'c']
def test_frequencies():
assert (frequencies(["cat", "pig", "cat", "eel",
"pig", "dog", "dog", "dog"]) ==
{"cat": 2, "eel": 1, "pig": 2, "dog": 3})
assert frequencies([]) == {}
assert frequencies("onomatopoeia") == {"a": 2, "e": 1, "i": 1, "m": 1,
"o": 4, "n": 1, "p": 1, "t": 1}
def test_reduceby():
data = [1, 2, 3, 4, 5]
iseven = lambda x: x % 2 == 0
assert reduceby(iseven, add, data, 0) == {False: 9, True: 6}
assert reduceby(iseven, mul, data, 1) == {False: 15, True: 8}
projects = [{'name': 'build roads', 'state': 'CA', 'cost': 1000000},
{'name': 'fight crime', 'state': 'IL', 'cost': 100000},
{'name': 'help farmers', 'state': 'IL', 'cost': 2000000},
{'name': 'help farmers', 'state': 'CA', 'cost': 200000}]
assert reduceby(lambda x: x['state'],
lambda acc, x: acc + x['cost'],
projects, 0) == {'CA': 1200000, 'IL': 2100000}
assert reduceby('state',
lambda acc, x: acc + x['cost'],
projects, 0) == {'CA': 1200000, 'IL': 2100000}
def test_reduce_by_init():
assert reduceby(iseven, add, [1, 2, 3, 4]) == {True: 2 + 4, False: 1 + 3}
assert reduceby(iseven, add, [1, 2, 3, 4], no_default2) == {True: 2 + 4,
False: 1 + 3}
def test_reduce_by_callable_default():
def set_add(s, i):
s.add(i)
return s
assert reduceby(iseven, set_add, [1, 2, 3, 4, 1, 2], set) == \
{True: {2, 4}, False: {1, 3}}
def test_iterate():
assert list(itertools.islice(iterate(inc, 0), 0, 5)) == [0, 1, 2, 3, 4]
assert list(take(4, iterate(double, 1))) == [1, 2, 4, 8]
def test_accumulate():
assert list(accumulate(add, [1, 2, 3, 4, 5])) == [1, 3, 6, 10, 15]
assert list(accumulate(mul, [1, 2, 3, 4, 5])) == [1, 2, 6, 24, 120]
assert list(accumulate(add, [1, 2, 3, 4, 5], -1)) == [-1, 0, 2, 5, 9, 14]
def binop(a, b):
raise AssertionError('binop should not be called')
start = object()
assert list(accumulate(binop, [], start)) == [start]
assert list(accumulate(binop, [])) == []
assert list(accumulate(add, [1, 2, 3], no_default2)) == [1, 3, 6]
def test_accumulate_works_on_consumable_iterables():
assert list(accumulate(add, iter((1, 2, 3)))) == [1, 3, 6]
def test_sliding_window():
assert list(sliding_window(2, [1, 2, 3, 4])) == [(1, 2), (2, 3), (3, 4)]
assert list(sliding_window(3, [1, 2, 3, 4])) == [(1, 2, 3), (2, 3, 4)]
def test_sliding_window_of_short_iterator():
assert list(sliding_window(3, [1, 2])) == []
assert list(sliding_window(7, [1, 2])) == []
def test_partition():
assert list(partition(2, [1, 2, 3, 4])) == [(1, 2), (3, 4)]
assert list(partition(3, range(7))) == [(0, 1, 2), (3, 4, 5)]
assert list(partition(3, range(4), pad=-1)) == [(0, 1, 2),
(3, -1, -1)]
assert list(partition(2, [])) == []
def test_partition_all():
assert list(partition_all(2, [1, 2, 3, 4])) == [(1, 2), (3, 4)]
assert list(partition_all(3, range(5))) == [(0, 1, 2), (3, 4)]
assert list(partition_all(2, [])) == []
# Regression test: https://github.com/pytoolz/toolz/issues/387
class NoCompare(object):
def __eq__(self, other):
if self.__class__ == other.__class__:
return True
raise ValueError()
obj = NoCompare()
result = [(obj, obj, obj, obj), (obj, obj, obj)]
assert list(partition_all(4, [obj]*7)) == result
assert list(partition_all(4, iter([obj]*7))) == result
def test_count():
assert count((1, 2, 3)) == 3
assert count([]) == 0
assert count(iter((1, 2, 3, 4))) == 4
assert count('hello') == 5
assert count(iter('hello')) == 5
def test_pluck():
assert list(pluck(0, [[0, 1], [2, 3], [4, 5]])) == [0, 2, 4]
assert list(pluck([0, 1], [[0, 1, 2], [3, 4, 5]])) == [(0, 1), (3, 4)]
assert list(pluck(1, [[0], [0, 1]], None)) == [None, 1]
data = [{'id': 1, 'name': 'cheese'}, {'id': 2, 'name': 'pies', 'price': 1}]
assert list(pluck('id', data)) == [1, 2]
assert list(pluck('price', data, 0)) == [0, 1]
assert list(pluck(['id', 'name'], data)) == [(1, 'cheese'), (2, 'pies')]
assert list(pluck(['name'], data)) == [('cheese',), ('pies',)]
assert list(pluck(['price', 'other'], data, 0)) == [(0, 0), (1, 0)]
assert raises(IndexError, lambda: list(pluck(1, [[0]])))
assert raises(KeyError, lambda: list(pluck('name', [{'id': 1}])))
assert list(pluck(0, [[0, 1], [2, 3], [4, 5]], no_default2)) == [0, 2, 4]
assert raises(IndexError, lambda: list(pluck(1, [[0]], no_default2)))
def test_join():
names = [(1, 'one'), (2, 'two'), (3, 'three')]
fruit = [('apple', 1), ('orange', 1), ('banana', 2), ('coconut', 2)]
def addpair(pair):
return pair[0] + pair[1]
result = set(starmap(add, join(first, names, second, fruit)))
expected = {(1, 'one', 'apple', 1),
(1, 'one', 'orange', 1),
(2, 'two', 'banana', 2),
(2, 'two', 'coconut', 2)}
assert result == expected
result = set(starmap(add, join(first, names, second, fruit,
left_default=no_default2,
right_default=no_default2)))
assert result == expected
def test_getter():
assert getter(0)('Alice') == 'A'
assert getter([0])('Alice') == ('A',)
assert getter([])('Alice') == ()
def test_key_as_getter():
squares = [(i, i**2) for i in range(5)]
pows = [(i, i**2, i**3) for i in range(5)]
assert set(join(0, squares, 0, pows)) == set(join(lambda x: x[0], squares,
lambda x: x[0], pows))
get = lambda x: (x[0], x[1])
assert set(join([0, 1], squares, [0, 1], pows)) == set(join(get, squares,
get, pows))
get = lambda x: (x[0],)
assert set(join([0], squares, [0], pows)) == set(join(get, squares,
get, pows))
def test_join_double_repeats():
names = [(1, 'one'), (2, 'two'), (3, 'three'), (1, 'uno'), (2, 'dos')]
fruit = [('apple', 1), ('orange', 1), ('banana', 2), ('coconut', 2)]
result = set(starmap(add, join(first, names, second, fruit)))
expected = {(1, 'one', 'apple', 1),
(1, 'one', 'orange', 1),
(2, 'two', 'banana', 2),
(2, 'two', 'coconut', 2),
(1, 'uno', 'apple', 1),
(1, 'uno', 'orange', 1),
(2, 'dos', 'banana', 2),
(2, 'dos', 'coconut', 2)}
assert result == expected
def test_join_missing_element():
names = [(1, 'one'), (2, 'two'), (3, 'three')]
fruit = [('apple', 5), ('orange', 1)]
result = set(starmap(add, join(first, names, second, fruit)))
expected = {(1, 'one', 'orange', 1)}
assert result == expected
def test_left_outer_join():
result = set(join(identity, [1, 2], identity, [2, 3], left_default=None))
expected = {(2, 2), (None, 3)}
assert result == expected
def test_right_outer_join():
result = set(join(identity, [1, 2], identity, [2, 3], right_default=None))
expected = {(2, 2), (1, None)}
assert result == expected
def test_outer_join():
result = set(join(identity, [1, 2], identity, [2, 3],
left_default=None, right_default=None))
expected = {(2, 2), (1, None), (None, 3)}
assert result == expected
def test_diff():
assert raises(TypeError, lambda: list(diff()))
assert raises(TypeError, lambda: list(diff([1, 2])))
assert raises(TypeError, lambda: list(diff([1, 2], 3)))
assert list(diff([1, 2], (1, 2), iter([1, 2]))) == []
assert list(diff([1, 2, 3], (1, 10, 3), iter([1, 2, 10]))) == [
(2, 10, 2), (3, 3, 10)]
assert list(diff([1, 2], [10])) == [(1, 10)]
assert list(diff([1, 2], [10], default=None)) == [(1, 10), (2, None)]
# non-variadic usage
assert raises(TypeError, lambda: list(diff([])))
assert raises(TypeError, lambda: list(diff([[]])))
assert raises(TypeError, lambda: list(diff([[1, 2]])))
assert raises(TypeError, lambda: list(diff([[1, 2], 3])))
assert list(diff([(1, 2), (1, 3)])) == [(2, 3)]
data1 = [{'cost': 1, 'currency': 'dollar'},
{'cost': 2, 'currency': 'dollar'}]
data2 = [{'cost': 100, 'currency': 'yen'},
{'cost': 300, 'currency': 'yen'}]
conversions = {'dollar': 1, 'yen': 0.01}
def indollars(item):
return conversions[item['currency']] * item['cost']
list(diff(data1, data2, key=indollars)) == [
({'cost': 2, 'currency': 'dollar'}, {'cost': 300, 'currency': 'yen'})]
def test_topk():
assert topk(2, [4, 1, 5, 2]) == (5, 4)
assert topk(2, [4, 1, 5, 2], key=lambda x: -x) == (1, 2)
assert topk(2, iter([5, 1, 4, 2]), key=lambda x: -x) == (1, 2)
assert topk(2, [{'a': 1, 'b': 10}, {'a': 2, 'b': 9},
{'a': 10, 'b': 1}, {'a': 9, 'b': 2}], key='a') == \
({'a': 10, 'b': 1}, {'a': 9, 'b': 2})
assert topk(2, [{'a': 1, 'b': 10}, {'a': 2, 'b': 9},
{'a': 10, 'b': 1}, {'a': 9, 'b': 2}], key='b') == \
({'a': 1, 'b': 10}, {'a': 2, 'b': 9})
assert topk(2, [(0, 4), (1, 3), (2, 2), (3, 1), (4, 0)], 0) == \
((4, 0), (3, 1))
def test_topk_is_stable():
assert topk(4, [5, 9, 2, 1, 5, 3], key=lambda x: 1) == (5, 9, 2, 1)
def test_peek():
alist = ["Alice", "Bob", "Carol"]
element, blist = peek(alist)
assert element == alist[0]
assert list(blist) == alist
assert raises(StopIteration, lambda: peek([]))
def test_peekn():
alist = ("Alice", "Bob", "Carol")
elements, blist = peekn(2, alist)
assert elements == alist[:2]
assert tuple(blist) == alist
elements, blist = peekn(len(alist) * 4, alist)
assert elements == alist
assert tuple(blist) == alist
def test_random_sample():
alist = list(range(100))
assert list(random_sample(prob=1, seq=alist, random_state=2016)) == alist
mk_rsample = lambda rs=1: list(random_sample(prob=0.1,
seq=alist,
random_state=rs))
rsample1 = mk_rsample()
assert rsample1 == mk_rsample()
rsample2 = mk_rsample(1984)
randobj = Random(1984)
assert rsample2 == mk_rsample(randobj)
assert rsample1 != rsample2
assert mk_rsample(hash(object)) == mk_rsample(hash(object))
assert mk_rsample(hash(object)) != mk_rsample(hash(object()))
assert mk_rsample(b"a") == mk_rsample(u"a")
assert raises(TypeError, lambda: mk_rsample([]))

View File

@@ -0,0 +1,27 @@
from toolz import first, identity, countby, partitionby
def iseven(x):
return x % 2 == 0
def test_countby():
assert countby(iseven, [1, 2, 3]) == {True: 1, False: 2}
assert countby(len, ['cat', 'dog', 'mouse']) == {3: 2, 5: 1}
assert countby(0, ('ab', 'ac', 'bc')) == {'a': 2, 'b': 1}
def test_partitionby():
assert list(partitionby(identity, [])) == []
vowels = "aeiou"
assert (list(partitionby(vowels.__contains__, "abcdefghi")) ==
[("a",), ("b", "c", "d"), ("e",), ("f", "g", "h"), ("i",)])
assert (list(map(first,
partitionby(identity,
[1, 1, 1, 2, 3, 3, 2, 2, 3]))) ==
[1, 2, 3, 2, 3])
assert ''.join(map(first,
partitionby(identity, "Khhhaaaaannnnn!!!!"))) == 'Khan!'

View File

@@ -0,0 +1,192 @@
from toolz import *
import toolz
import toolz.curried
import pickle
from toolz.utils import raises
def test_compose():
f = compose(str, sum)
g = pickle.loads(pickle.dumps(f))
assert f((1, 2)) == g((1, 2))
def test_curry():
f = curry(map)(str)
g = pickle.loads(pickle.dumps(f))
assert list(f((1, 2, 3))) == list(g((1, 2, 3)))
def test_juxt():
f = juxt(str, int, bool)
g = pickle.loads(pickle.dumps(f))
assert f(1) == g(1)
assert f.funcs == g.funcs
def test_complement():
f = complement(bool)
assert f(True) is False
assert f(False) is True
g = pickle.loads(pickle.dumps(f))
assert f(True) == g(True)
assert f(False) == g(False)
def test_instanceproperty():
p = toolz.functoolz.InstanceProperty(bool)
assert p.__get__(None) is None
assert p.__get__(0) is False
assert p.__get__(1) is True
p2 = pickle.loads(pickle.dumps(p))
assert p2.__get__(None) is None
assert p2.__get__(0) is False
assert p2.__get__(1) is True
def f(x, y):
return x, y
def test_flip():
flip = pickle.loads(pickle.dumps(toolz.functoolz.flip))
assert flip is toolz.functoolz.flip
g1 = flip(f)
g2 = pickle.loads(pickle.dumps(g1))
assert g1(1, 2) == g2(1, 2) == f(2, 1)
g1 = flip(f)(1)
g2 = pickle.loads(pickle.dumps(g1))
assert g1(2) == g2(2) == f(2, 1)
def test_curried_exceptions():
# This tests a global curried object that isn't defined in toolz.functoolz
merge = pickle.loads(pickle.dumps(toolz.curried.merge))
assert merge is toolz.curried.merge
@toolz.curry
class GlobalCurried(object):
def __init__(self, x, y):
self.x = x
self.y = y
@toolz.curry
def f1(self, a, b):
return self.x + self.y + a + b
def g1(self):
pass
def __reduce__(self):
"""Allow us to serialize instances of GlobalCurried"""
return GlobalCurried, (self.x, self.y)
@toolz.curry
class NestedCurried(object):
def __init__(self, x, y):
self.x = x
self.y = y
@toolz.curry
def f2(self, a, b):
return self.x + self.y + a + b
def g2(self):
pass
def __reduce__(self):
"""Allow us to serialize instances of NestedCurried"""
return GlobalCurried.NestedCurried, (self.x, self.y)
class Nested(object):
def __init__(self, x, y):
self.x = x
self.y = y
@toolz.curry
def f3(self, a, b):
return self.x + self.y + a + b
def g3(self):
pass
def test_curried_qualname():
def preserves_identity(obj):
return pickle.loads(pickle.dumps(obj)) is obj
assert preserves_identity(GlobalCurried)
assert preserves_identity(GlobalCurried.func.f1)
assert preserves_identity(GlobalCurried.func.NestedCurried)
assert preserves_identity(GlobalCurried.func.NestedCurried.func.f2)
assert preserves_identity(GlobalCurried.func.Nested.f3)
global_curried1 = GlobalCurried(1)
global_curried2 = pickle.loads(pickle.dumps(global_curried1))
assert global_curried1 is not global_curried2
assert global_curried1(2).f1(3, 4) == global_curried2(2).f1(3, 4) == 10
global_curried3 = global_curried1(2)
global_curried4 = pickle.loads(pickle.dumps(global_curried3))
assert global_curried3 is not global_curried4
assert global_curried3.f1(3, 4) == global_curried4.f1(3, 4) == 10
func1 = global_curried1(2).f1(3)
func2 = pickle.loads(pickle.dumps(func1))
assert func1 is not func2
assert func1(4) == func2(4) == 10
nested_curried1 = GlobalCurried.func.NestedCurried(1)
nested_curried2 = pickle.loads(pickle.dumps(nested_curried1))
assert nested_curried1 is not nested_curried2
assert nested_curried1(2).f2(3, 4) == nested_curried2(2).f2(3, 4) == 10
# If we add `curry.__getattr__` forwarding, the following tests will pass
# if not PY34:
# assert preserves_identity(GlobalCurried.func.g1)
# assert preserves_identity(GlobalCurried.func.NestedCurried.func.g2)
# assert preserves_identity(GlobalCurried.func.Nested)
# assert preserves_identity(GlobalCurried.func.Nested.g3)
#
# # Rely on curry.__getattr__
# assert preserves_identity(GlobalCurried.f1)
# assert preserves_identity(GlobalCurried.NestedCurried)
# assert preserves_identity(GlobalCurried.NestedCurried.f2)
# assert preserves_identity(GlobalCurried.Nested.f3)
# if not PY34:
# assert preserves_identity(GlobalCurried.g1)
# assert preserves_identity(GlobalCurried.NestedCurried.g2)
# assert preserves_identity(GlobalCurried.Nested)
# assert preserves_identity(GlobalCurried.Nested.g3)
#
# nested_curried3 = nested_curried1(2)
# nested_curried4 = pickle.loads(pickle.dumps(nested_curried3))
# assert nested_curried3 is not nested_curried4
# assert nested_curried3.f2(3, 4) == nested_curried4.f2(3, 4) == 10
#
# func1 = nested_curried1(2).f2(3)
# func2 = pickle.loads(pickle.dumps(func1))
# assert func1 is not func2
# assert func1(4) == func2(4) == 10
#
# if not PY34:
# nested3 = GlobalCurried.func.Nested(1, 2)
# nested4 = pickle.loads(pickle.dumps(nested3))
# assert nested3 is not nested4
# assert nested3.f3(3, 4) == nested4.f3(3, 4) == 10
#
# func1 = nested3.f3(3)
# func2 = pickle.loads(pickle.dumps(func1))
# assert func1 is not func2
# assert func1(4) == func2(4) == 10
def test_curried_bad_qualname():
@toolz.curry
class Bad(object):
__qualname__ = 'toolz.functoolz.not.a.valid.path'
assert raises(pickle.PicklingError, lambda: pickle.dumps(Bad))

View File

@@ -0,0 +1,85 @@
import functools
import toolz._signatures as _sigs
from toolz._signatures import builtins, _is_valid_args, _is_partial_args
def test_is_valid(check_valid=_is_valid_args, incomplete=False):
orig_check_valid = check_valid
check_valid = lambda func, *args, **kwargs: orig_check_valid(func, args, kwargs)
assert check_valid(lambda x: None) is None
f = builtins.abs
assert check_valid(f) is incomplete
assert check_valid(f, 1)
assert check_valid(f, x=1) is False
assert check_valid(f, 1, 2) is False
f = builtins.complex
assert check_valid(f)
assert check_valid(f, 1)
assert check_valid(f, real=1)
assert check_valid(f, 1, 2)
assert check_valid(f, 1, imag=2)
assert check_valid(f, 1, real=2) is False
assert check_valid(f, 1, 2, 3) is False
assert check_valid(f, 1, 2, imag=3) is False
f = builtins.int
assert check_valid(f)
assert check_valid(f, 1)
assert check_valid(f, x=1)
assert check_valid(f, 1, 2)
assert check_valid(f, 1, base=2)
assert check_valid(f, x=1, base=2)
assert check_valid(f, base=2) is incomplete
assert check_valid(f, 1, 2, 3) is False
f = builtins.map
assert check_valid(f) is incomplete
assert check_valid(f, 1) is incomplete
assert check_valid(f, 1, 2)
assert check_valid(f, 1, 2, 3)
assert check_valid(f, 1, 2, 3, 4)
f = builtins.min
assert check_valid(f) is incomplete
assert check_valid(f, 1)
assert check_valid(f, iterable=1) is False
assert check_valid(f, 1, 2)
assert check_valid(f, 1, 2, 3)
assert check_valid(f, key=None) is incomplete
assert check_valid(f, 1, key=None)
assert check_valid(f, 1, 2, key=None)
assert check_valid(f, 1, 2, 3, key=None)
assert check_valid(f, key=None, default=None) is incomplete
assert check_valid(f, 1, key=None, default=None)
assert check_valid(f, 1, 2, key=None, default=None) is False
assert check_valid(f, 1, 2, 3, key=None, default=None) is False
f = builtins.range
assert check_valid(f) is incomplete
assert check_valid(f, 1)
assert check_valid(f, 1, 2)
assert check_valid(f, 1, 2, 3)
assert check_valid(f, 1, 2, step=3) is False
assert check_valid(f, 1, 2, 3, 4) is False
f = functools.partial
assert orig_check_valid(f, (), {}) is incomplete
assert orig_check_valid(f, (), {'func': 1}) is incomplete
assert orig_check_valid(f, (1,), {})
assert orig_check_valid(f, (1,), {'func': 1})
assert orig_check_valid(f, (1, 2), {})
def test_is_partial():
test_is_valid(check_valid=_is_partial_args, incomplete=True)
def test_for_coverage(): # :)
assert _sigs._is_arity(1, 1) is None
assert _sigs._is_arity(1, all)
assert _sigs._has_varargs(None) is None
assert _sigs._has_keywords(None) is None
assert _sigs._num_required_args(None) is None

View File

@@ -0,0 +1,57 @@
import toolz
def test_tlz():
import tlz
tlz.curry
tlz.functoolz.curry
assert tlz.__package__ == 'tlz'
assert tlz.__name__ == 'tlz'
import tlz.curried
assert tlz.curried.__package__ == 'tlz.curried'
assert tlz.curried.__name__ == 'tlz.curried'
tlz.curried.curry
import tlz.curried.operator
assert tlz.curried.operator.__package__ in (None, 'tlz.curried')
assert tlz.curried.operator.__name__ == 'tlz.curried.operator'
assert tlz.functoolz.__name__ == 'tlz.functoolz'
m1 = tlz.functoolz
import tlz.functoolz as m2
assert m1 is m2
import tlz.sandbox
try:
import tlzthisisabadname.curried
1/0
except ImportError:
pass
try:
import tlz.curry
1/0
except ImportError:
pass
try:
import tlz.badsubmodulename
1/0
except ImportError:
pass
assert toolz.__package__ == 'toolz'
assert toolz.curried.__package__ == 'toolz.curried'
assert toolz.functoolz.__name__ == 'toolz.functoolz'
try:
import cytoolz
assert cytoolz.__package__ == 'cytoolz'
assert cytoolz.curried.__package__ == 'cytoolz.curried'
assert cytoolz.functoolz.__name__ == 'cytoolz.functoolz'
except ImportError:
pass
if hasattr(tlz, '__file__'):
assert tlz.__file__ == toolz.__file__
if hasattr(tlz.functoolz, '__file__'):
assert tlz.functoolz.__file__ == toolz.functoolz.__file__
assert tlz.pipe is toolz.pipe
assert 'tlz' in tlz.__doc__
assert tlz.curried.__doc__ is not None

View File

@@ -0,0 +1,6 @@
from toolz.utils import raises
def test_raises():
assert raises(ZeroDivisionError, lambda: 1 / 0)
assert not raises(ZeroDivisionError, lambda: 1)

View File

@@ -0,0 +1,9 @@
def raises(err, lamda):
try:
lamda()
return False
except err:
return True
no_default = '__no__default__'