From 2960244f578115f4f6ae0d892a392582ca43e3eb Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 23 Aug 2026 22:11:35 +0300 Subject: [PATCH 1/4] gh-64862: Add the stop_exception parameter in iter() and aiter() The created iterator stops when the callable raises the specified exception. The second parameter of iter() is now named stop_value and can be passed as a keyword argument. aiter() now accepts the same stop_value and stop_exception parameters, calling an asynchronous callable and awaiting the result. Co-Authored-By: Claude Opus 5 (1M context) --- Doc/library/functions.rst | 65 ++- Doc/whatsnew/3.16.rst | 7 + Include/internal/pycore_genobject.h | 3 + .../pycore_global_objects_fini_generated.h | 2 + Include/internal/pycore_global_strings.h | 2 + Include/internal/pycore_interp_structs.h | 2 +- Include/internal/pycore_iterobject.h | 41 ++ .../internal/pycore_runtime_init_generated.h | 2 + .../internal/pycore_unicodeobject_generated.h | 8 + Lib/test/test_asyncgen.py | 142 ++++++ Lib/test/test_inspect/test_inspect.py | 4 +- Lib/test/test_iter.py | 95 ++++ Lib/test/test_sys.py | 2 +- Makefile.pre.in | 1 + ...6-08-23-21-30-00.gh-issue-64862.Kx7Qm2.rst | 5 + Objects/genobject.c | 8 +- Objects/iterobject.c | 467 +++++++++++++++++- Objects/object.c | 4 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + Python/bltinmodule.c | 84 +++- Python/clinic/bltinmodule.c.h | 161 +++++- 22 files changed, 1046 insertions(+), 63 deletions(-) create mode 100644 Include/internal/pycore_iterobject.h create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-08-23-21-30-00.gh-issue-64862.Kx7Qm2.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index f45ab397e936938..0e8ebf1b5254632 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -65,14 +65,46 @@ are always available. They are listed here in alphabetical order. .. function:: aiter(async_iterable, /) + aiter(callable, /, stop_value, *, stop_exception=StopAsyncIteration) + aiter(callable, /, *, stop_exception) Return an :term:`asynchronous iterator` for an :term:`asynchronous iterable`. Equivalent to calling ``x.__aiter__()``. - Note: Unlike :func:`iter`, :func:`aiter` has no 2-argument variant. + If *stop_value* or *stop_exception* is given, + then the first argument must be a callable object. + The asynchronous iterator created in this case + calls *callable* with no arguments and awaits the result + for each call to its :meth:`~object.__anext__` method; + if the awaited value is equal to *stop_value*, + or if the call raises :exc:`StopAsyncIteration` or an exception + matching *stop_exception*, :exc:`StopAsyncIteration` will be raised, + otherwise the value will be returned. + The callable is only called when the result of :meth:`~object.__anext__` + is awaited. + + *stop_exception* is an exception class or a tuple of exception classes. + If *stop_value* is not specified, + the iteration stops only when the callable raises an exception. + + For example, reading fixed-size chunks from an asynchronous stream + until the end of file is reached:: + + from functools import partial + async for chunk in aiter(partial(reader.read, 1024), b''): + process_chunk(chunk) + + Or consuming an :class:`asyncio.Queue` until it is shut down:: + + from asyncio import QueueShutDown + async for item in aiter(queue.get, stop_exception=QueueShutDown): + process_item(item) .. versionadded:: 3.10 + .. versionchanged:: next + Added the *stop_value* and *stop_exception* parameters. + .. function:: all(iterable, /) Return ``True`` if all elements of the *iterable* are true (or if the iterable @@ -1143,21 +1175,29 @@ are always available. They are listed here in alphabetical order. .. function:: iter(iterable, /) - iter(callable, sentinel, /) + iter(callable, /, stop_value, *, stop_exception=StopIteration) + iter(callable, /, *, stop_exception) Return an :term:`iterator` object. The first argument is interpreted very - differently depending on the presence of the second argument. Without a - second argument, the single argument must be a collection object which supports the + differently depending on the presence of the other arguments. Without other + arguments, the single argument must be a collection object which supports the :term:`iterable` protocol (the :meth:`~object.__iter__` method), or it must support the sequence protocol (the :meth:`~object.__getitem__` method with integer arguments starting at ``0``). If it does not support either of those protocols, - :exc:`TypeError` is raised. If the second argument, *sentinel*, is given, + :exc:`TypeError` is raised. + + If *stop_value* or *stop_exception* is given, then the first argument must be a callable object. The iterator created in this case will call *callable* with no arguments for each call to its :meth:`~iterator.__next__` method; if the value returned is equal to - *sentinel*, :exc:`StopIteration` will be raised, otherwise the value will - be returned. + *stop_value*, or if the call raises :exc:`StopIteration` or an exception + matching *stop_exception*, :exc:`StopIteration` will be raised, otherwise the + value will be returned. + + *stop_exception* is an exception class or a tuple of exception classes. + If *stop_value* is not specified, + the iteration stops only when the callable raises an exception. See also :ref:`typeiter`. @@ -1170,6 +1210,17 @@ are always available. They are listed here in alphabetical order. for block in iter(partial(f.read, 64), b''): process_block(block) + *stop_exception* is useful for callables which report exhaustion by raising an + exception instead of returning a special value. + For example, draining a queue:: + + from queue import Empty + for item in iter(queue.get_nowait, stop_exception=Empty): + process_item(item) + + .. versionchanged:: next + Added the *stop_exception* parameter and allowed passing *stop_value* by keyword. + .. function:: len(object, /) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index a1a8415482b97aa..aa8b1633d11ee43 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -75,6 +75,13 @@ New features Other language changes ====================== +* The :func:`iter` function now accepts the *stop_exception* parameter. + The created iterator stops when the callable raises the specified exception. + The second parameter is now named *stop_value* and can be passed by keyword. + :func:`aiter` now accepts the same *stop_value* and *stop_exception* + parameters, calling an asynchronous callable and awaiting the result. + (Contributed by Serhiy Storchaka in :gh:`64862`.) + * :meth:`memoryview.cast` now allows casting a multidimensional F-contiguous view to a one-dimensional view. (Contributed by Jaemin Park in :gh:`91484`.) diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index c86ae242feac1ed..266add0fb7a9c34 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -29,6 +29,9 @@ PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); +// Set the exception passed to throw(typ[, val[, tb]]). +// Return 0 on success, -1 on failure. +extern int _PyGen_SetException(PyObject *typ, PyObject *val, PyObject *tb); PyAPI_FUNC(PyObject *)_PyCoro_GetAwaitableIter(PyObject *o); PyAPI_FUNC(PyObject *)_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index c982f76431c52d1..7b869d16b81ac2f 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -2104,6 +2104,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stdout)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(step)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(steps)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stop_exception)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stop_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(store_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(strategy)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(strftime)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index c430d0fb99bc41d..6e7835f2aa7a0b3 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -827,6 +827,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(stdout) STRUCT_FOR_ID(step) STRUCT_FOR_ID(steps) + STRUCT_FOR_ID(stop_exception) + STRUCT_FOR_ID(stop_value) STRUCT_FOR_ID(store_name) STRUCT_FOR_ID(strategy) STRUCT_FOR_ID(strftime) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 0623adce693d465..6c907e0cf79894d 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -538,7 +538,7 @@ struct _py_func_state { If you add a new static type to the standard library, you may have to update one of these numbers. */ -#define _Py_NUM_MANAGED_PREINITIALIZED_TYPES 120 +#define _Py_NUM_MANAGED_PREINITIALIZED_TYPES 122 #define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES \ (_Py_NUM_MANAGED_PREINITIALIZED_TYPES + 83) #define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10 diff --git a/Include/internal/pycore_iterobject.h b/Include/internal/pycore_iterobject.h new file mode 100644 index 000000000000000..610fe2aa5ce6736 --- /dev/null +++ b/Include/internal/pycore_iterobject.h @@ -0,0 +1,41 @@ +#ifndef Py_INTERNAL_ITEROBJECT_H +#define Py_INTERNAL_ITEROBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +extern PyTypeObject _PyACallIter_Type; +extern PyTypeObject _PyACallIterAwaitable_Type; + +// Like PyCallIter_New(), but the iteration also stops when *callable* raises +// an exception matching *stop_exc* (an exception class or a tuple of exception +// classes). Both *sentinel* and *stop_exc* can be NULL. +extern PyObject *_PyCallIter_NewEx(PyObject *callable, PyObject *sentinel, + PyObject *stop_exc); + +// The asynchronous counterpart of _PyCallIter_NewEx(): the result of +// *callable* is awaited, and StopAsyncIteration stops the iteration. +extern PyObject *_PyACallIter_New(PyObject *callable, PyObject *sentinel, + PyObject *stop_exc); + +// Return NULL if *stop_exc* has no effect: *implied_exc* stops the iteration +// in any case, and an empty tuple never matches a raised exception. +static inline PyObject * +_PyIter_NormalizeStopException(PyObject *stop_exc, PyObject *implied_exc) +{ + if (stop_exc == implied_exc || + (PyTuple_Check(stop_exc) && PyTuple_GET_SIZE(stop_exc) == 0)) + { + return NULL; + } + return stop_exc; +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_ITEROBJECT_H */ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 5a194478f562314..730c5e5ae94c766 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -2102,6 +2102,8 @@ extern "C" { INIT_ID(stdout), \ INIT_ID(step), \ INIT_ID(steps), \ + INIT_ID(stop_exception), \ + INIT_ID(stop_value), \ INIT_ID(store_name), \ INIT_ID(strategy), \ INIT_ID(strftime), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 25529c5ca713a95..13cc349358b5479 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -3088,6 +3088,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(stop_exception); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(stop_value); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(store_name); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 70a285dd91f385f..2ae6de229bb33f4 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -789,6 +789,148 @@ async def gen(): applied_twice = aiter(applied_once) self.assertIs(applied_once, applied_twice) + def make_counter(self): + state = {'n': 0} + async def counter(): + state['n'] += 1 + return state['n'] + return counter + + def collect(self, ait): + async def consume(): + return [i async for i in ait] + return self.loop.run_until_complete(consume()) + + def test_aiter_callable_stop(self): + self.assertEqual(self.collect(aiter(self.make_counter(), 4)), [1, 2, 3]) + self.assertEqual(self.collect(aiter(self.make_counter(), stop_value=4)), + [1, 2, 3]) + + def test_aiter_callable_stop_exception(self): + counter = self.make_counter() + async def spam(): + value = await counter() + if value > 3: + raise LookupError + return value + self.assertEqual(self.collect(aiter(spam, stop_exception=LookupError)), + [1, 2, 3]) + counter = self.make_counter() + self.assertEqual( + self.collect(aiter(spam, stop_exception=(ZeroDivisionError, + LookupError))), + [1, 2, 3]) + + def test_aiter_callable_stop_and_exception(self): + counter = self.make_counter() + async def spam(): + value = await counter() + if value > 5: + raise LookupError + return value + self.assertEqual( + self.collect(aiter(spam, 3, stop_exception=LookupError)), [1, 2]) + counter = self.make_counter() + self.assertEqual( + self.collect(aiter(spam, 100, stop_exception=LookupError)), + [1, 2, 3, 4, 5]) + + def test_aiter_callable_stop_exception_redundant(self): + # StopAsyncIteration and an empty tuple stop the iteration in any + # case, so they are the same as no exception argument + counter = self.make_counter() + async def spam(): + value = await counter() + if value > 3: + raise StopAsyncIteration + return value + self.assertEqual( + self.collect(aiter(spam, stop_exception=StopAsyncIteration)), + [1, 2, 3]) + counter = self.make_counter() + self.assertEqual(self.collect(aiter(spam, stop_exception=())), + [1, 2, 3]) + + def test_aiter_callable_stop_async_iteration(self): + # StopAsyncIteration stops the iteration even if other exception + # is specified + counter = self.make_counter() + async def spam(): + value = await counter() + if value > 3: + raise StopAsyncIteration + return value + self.assertEqual(self.collect(aiter(spam, stop_exception=LookupError)), + [1, 2, 3]) + + def test_aiter_callable_other_exception(self): + async def spam(): + raise ZeroDivisionError + it = aiter(spam, stop_exception=LookupError) + with self.assertRaises(ZeroDivisionError): + self.loop.run_until_complete(anext(it)) + + def test_aiter_callable_exhausted(self): + it = aiter(self.make_counter(), 3) + self.assertEqual(self.collect(it), [1, 2]) + self.assertEqual(self.loop.run_until_complete(anext(it, 'default')), + 'default') + with self.assertRaises(StopAsyncIteration): + self.loop.run_until_complete(anext(it)) + + def test_aiter_callable_lazy(self): + # The callable is only called when the awaitable is awaited + calls = [] + async def spam(): + calls.append(1) + return len(calls) + it = aiter(spam, 10) + awaitable = it.__anext__() + self.assertEqual(calls, []) + self.assertEqual(self.loop.run_until_complete(awaitable), 1) + self.assertEqual(calls, [1]) + + def test_aiter_callable_awaitable(self): + it = aiter(self.make_counter(), 10) + awaitable = it.__anext__() + self.assertIsNone(awaitable.close()) + with self.assertRaises(RuntimeError): + self.loop.run_until_complete(awaitable) + awaitable = it.__anext__() + with self.assertRaises(KeyError): + awaitable.throw(KeyError('injected')) + + def test_aiter_callable_cancel(self): + # Cancellation is delivered to the awaited callable result + cancelled = [] + async def spam(): + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + cancelled.append(1) + raise + async def consume(): + async for _ in aiter(spam, None): + pass + async def main(): + task = asyncio.ensure_future(consume()) + await asyncio.sleep(0) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + self.loop.run_until_complete(main()) + self.assertEqual(cancelled, [1]) + + def test_aiter_callable_errors(self): + async def gen(): + yield 1 + self.assertRaises(TypeError, aiter, gen(), 1) + self.assertRaises(TypeError, aiter, [1, 2], stop_exception=LookupError) + self.assertRaises(TypeError, aiter, len, stop_exception=42) + self.assertRaises(TypeError, aiter, len, + stop_exception=(LookupError, 42)) + self.assertRaises(TypeError, aiter, len, stop_exception=LookupError()) + def test_anext_bad_args(self): async def gen(): yield 1 diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index c68c643cb97fb45..9e42f883c1f3f42 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -6171,10 +6171,12 @@ def test_builtins_have_signatures(self): 'dict', 'frozendict', 'int', 'str'} # These need PEP 457 groups needs_groups = {"range", "slice", "dir", "getattr", - "next", "iter", "vars"} + "next", "vars"} no_signature |= needs_groups # These have unrepresentable parameter default values of NULL unsupported_signature = {"anext"} + # These have text signatures with PEP 457 groups + unsupported_signature |= {"aiter", "iter"} # These need *args support in Argument Clinic needs_varargs = {"min", "max", "__build_class__"} no_signature |= needs_varargs diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 18e4b676c532368..7f5dfde201cba64 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -350,6 +350,101 @@ def spam(state=[0]): return i self.check_iterator(iter(spam, 20), list(range(10)), pickle=False) + # Test iter() with the stop value passed by keyword + def test_iter_keyword_stop(self): + self.check_iterator(iter(CallableIterClass(), stop_value=10), list(range(10))) + + # Test iter() with the exception argument + def test_iter_exception(self): + self.check_iterator(iter(CallableIterClass(), stop_exception=IndexError), + list(range(101))) + + def test_iter_exception_tuple(self): + self.check_iterator( + iter(CallableIterClass(), stop_exception=(ZeroDivisionError, IndexError)), + list(range(101))) + + # Test iter() with both the stop value and the exception argument + def test_iter_exception_and_stop(self): + self.check_iterator(iter(CallableIterClass(), 10, stop_exception=IndexError), + list(range(10))) + self.check_iterator(iter(CallableIterClass(), 200, stop_exception=IndexError), + list(range(101))) + + # StopIteration stops the iteration even if other exception is specified + def test_iter_exception_stop_iteration(self): + def spam(state=[0]): + i = state[0] + if i == 10: + raise StopIteration + state[0] = i+1 + return i + self.check_iterator(iter(spam, stop_exception=IndexError), list(range(10)), + pickle=False) + + # Other exceptions are propagated + def test_iter_exception_not_matching(self): + def spam(): + raise ZeroDivisionError + it = iter(spam, stop_exception=IndexError) + self.assertRaises(ZeroDivisionError, next, it) + + def test_iter_exception_errors(self): + self.assertRaises(TypeError, iter, [1, 2], stop_exception=IndexError) + self.assertRaises(TypeError, iter, len, stop_exception=42) + self.assertRaises(TypeError, iter, len, stop_exception=(IndexError, 42)) + self.assertRaises(TypeError, iter, len, stop_exception=IndexError()) + + # StopIteration and an empty tuple stop the iteration in any case, + # so they are the same as no exception argument + def test_iter_exception_redundant(self): + def make_spam(): + state = [0] + def spam(): + if state[0] == 10: + raise StopIteration + state[0] += 1 + return state[0] - 1 + return spam + for stop_exception in StopIteration, (): + with self.subTest(stop_exception=stop_exception): + self.check_iterator( + iter(make_spam(), stop_exception=stop_exception), + list(range(10)), pickle=False) + + def test_calliter_reduce(self): + c = CallableIterClass() + # The form without the stop exception is pickled as iter(c, stop) + self.assertEqual(iter(c, 10).__reduce__(), (iter, (c, 10))) + self.assertEqual(iter(c, 10, stop_exception=StopIteration).__reduce__(), + (iter, (c, 10))) + self.assertEqual(iter(c, 10, stop_exception=()).__reduce__(), + (iter, (c, 10))) + self.assertEqual(iter(c, stop_exception=StopIteration).__reduce__(), + (iter, (c, None), ((), ()))) + self.assertEqual(iter(c, stop_exception=IndexError).__reduce__(), + (iter, (c, None), ((), IndexError))) + self.assertEqual(iter(c, 10, stop_exception=IndexError).__reduce__(), + (iter, (c, None), ((10,), IndexError))) + + def test_calliter_setstate(self): + c = CallableIterClass() + it = iter(c, stop_exception=IndexError) + self.assertRaises(TypeError, it.__setstate__, 42) + self.assertRaises(TypeError, it.__setstate__, ((), IndexError, ())) + self.assertRaises(TypeError, it.__setstate__, ([], IndexError)) + self.assertRaises(TypeError, it.__setstate__, ((1, 2), IndexError)) + self.assertRaises(TypeError, it.__setstate__, ((), 42)) + self.assertRaises(TypeError, it.__setstate__, ((), None)) + it.__setstate__(((10,), StopIteration)) + self.assertEqual(it.__reduce__(), (iter, (c, 10))) + it.__setstate__(((10,), ())) + self.assertEqual(it.__reduce__(), (iter, (c, 10))) + it.__setstate__(((), IndexError)) + self.assertEqual(it.__reduce__(), (iter, (c, None), ((), IndexError))) + it.__setstate__(((10,), ())) + self.assertEqual(list(it), list(range(10))) + def test_iter_function_concealing_reentrant_exhaustion(self): # gh-101892: Test two-argument iter() with a function that # exhausts its associated iterator but forgets to either return diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index f2adce532595e70..4308de227a46ce4 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1726,7 +1726,7 @@ def get_gen(): yield 1 check(iter('abc'), size('lP')) # callable-iterator import re - check(re.finditer('',''), size('2P')) + check(re.finditer('',''), size('3P')) # list check(list([]), vsize('Pn')) check(list([1]), vsize('Pn') + 2*self.P) diff --git a/Makefile.pre.in b/Makefile.pre.in index d3d24a13898d992..6a50ef51394f560 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1357,6 +1357,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_interpframe_structs.h \ $(srcdir)/Include/internal/pycore_interpolation.h \ $(srcdir)/Include/internal/pycore_intrinsics.h \ + $(srcdir)/Include/internal/pycore_iterobject.h \ $(srcdir)/Include/internal/pycore_jit.h \ $(srcdir)/Include/internal/pycore_lazyimportobject.h \ $(srcdir)/Include/internal/pycore_list.h \ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-23-21-30-00.gh-issue-64862.Kx7Qm2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-23-21-30-00.gh-issue-64862.Kx7Qm2.rst new file mode 100644 index 000000000000000..61a42300994aa80 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-23-21-30-00.gh-issue-64862.Kx7Qm2.rst @@ -0,0 +1,5 @@ +The :func:`iter` function now accepts the *stop_exception* parameter. +The created iterator stops when the callable raises the specified exception. +The second parameter is now named *stop_value* and can be passed by keyword. +:func:`aiter` now accepts the same *stop_value* and *stop_exception* +parameters, calling an asynchronous callable and awaiting the result. diff --git a/Objects/genobject.c b/Objects/genobject.c index 6529a66fc35a6b4..6a96bc27d9a950d 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -542,8 +542,8 @@ gen_close(PyObject *self, PyObject *args) // Set an exception for a gen.throw() call. // Return 0 on success, -1 on failure. -static int -gen_set_exception(PyObject *typ, PyObject *val, PyObject *tb) +int +_PyGen_SetException(PyObject *typ, PyObject *val, PyObject *tb) { /* First, check the traceback argument, replacing None with NULL. */ @@ -640,7 +640,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, "cannot reuse already awaited coroutine"); return NULL; } - gen_set_exception(typ, val, tb); + _PyGen_SetException(typ, val, tb); return NULL; } @@ -718,7 +718,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, throw_here: assert(FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state) == FRAME_EXECUTING); - if (gen_set_exception(typ, val, tb) < 0) { + if (_PyGen_SetException(typ, val, tb) < 0) { FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, frame_state); return NULL; } diff --git a/Objects/iterobject.c b/Objects/iterobject.c index e323987601d5d4c..7b4d5931e10cbcc 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -5,7 +5,9 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_genobject.h" // _PyCoro_GetAwaitableIter() +#include "pycore_iterobject.h" // _PyCallIter_NewEx() #include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_pystate.h" // _PyThreadState_GET() typedef struct { @@ -185,22 +187,37 @@ PyTypeObject PySeqIter_Type = { typedef struct { PyObject_HEAD - PyObject *it_callable; /* Set to NULL when iterator is exhausted */ - PyObject *it_sentinel; /* Set to NULL when iterator is exhausted */ + /* All are set to NULL when the iterator is exhausted */ + PyObject *it_callable; + PyObject *it_sentinel; /* can be NULL */ + PyObject *it_stop_exc; /* can be NULL */ } calliterobject; PyObject * -PyCallIter_New(PyObject *callable, PyObject *sentinel) +_PyCallIter_NewEx(PyObject *callable, PyObject *sentinel, PyObject *stop_exc) { calliterobject *it; + if (stop_exc != NULL && + _PyEval_CheckExceptTypeValid(_PyThreadState_GET(), stop_exc) < 0) + { + return NULL; + } it = PyObject_GC_New(calliterobject, &PyCallIter_Type); if (it == NULL) return NULL; it->it_callable = Py_NewRef(callable); - it->it_sentinel = Py_NewRef(sentinel); + it->it_sentinel = Py_XNewRef(sentinel); + it->it_stop_exc = Py_XNewRef(stop_exc); _PyObject_GC_TRACK(it); return (PyObject *)it; } + +PyObject * +PyCallIter_New(PyObject *callable, PyObject *sentinel) +{ + return _PyCallIter_NewEx(callable, sentinel, NULL); +} + static void calliter_dealloc(PyObject *op) { @@ -208,6 +225,7 @@ calliter_dealloc(PyObject *op) _PyObject_GC_UNTRACK(it); Py_XDECREF(it->it_callable); Py_XDECREF(it->it_sentinel); + Py_XDECREF(it->it_stop_exc); PyObject_GC_Del(it); } @@ -217,6 +235,7 @@ calliter_traverse(PyObject *op, visitproc visit, void *arg) calliterobject *it = (calliterobject*)op; Py_VISIT(it->it_callable); Py_VISIT(it->it_sentinel); + Py_VISIT(it->it_stop_exc); return 0; } @@ -231,10 +250,12 @@ calliter_iternext(PyObject *op) } result = _PyObject_CallNoArgs(it->it_callable); - if (result != NULL && it->it_sentinel != NULL){ - int ok; - - ok = PyObject_RichCompareBool(it->it_sentinel, result, Py_EQ); + /* The call can exhaust the iterator re-entrantly. */ + if (result != NULL && it->it_callable != NULL) { + if (it->it_sentinel == NULL) { + return result; /* Common case, fast path */ + } + int ok = PyObject_RichCompareBool(it->it_sentinel, result, Py_EQ); if (ok == 0) { return result; /* Common case, fast path */ } @@ -242,12 +263,17 @@ calliter_iternext(PyObject *op) if (ok > 0) { Py_CLEAR(it->it_callable); Py_CLEAR(it->it_sentinel); + Py_CLEAR(it->it_stop_exc); } } - else if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + else if ((it->it_stop_exc != NULL && + PyErr_ExceptionMatches(it->it_stop_exc)) || + PyErr_ExceptionMatches(PyExc_StopIteration)) + { PyErr_Clear(); Py_CLEAR(it->it_callable); Py_CLEAR(it->it_sentinel); + Py_CLEAR(it->it_stop_exc); } Py_XDECREF(result); return NULL; @@ -263,14 +289,66 @@ calliter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) * call must be before access of iterator pointers. * see issue #101765 */ - if (it->it_callable != NULL && it->it_sentinel != NULL) - return Py_BuildValue("N(OO)", iter, it->it_callable, it->it_sentinel); - else + if (it->it_callable == NULL) { return Py_BuildValue("N(())", iter); + } + /* Only the sentinel can be passed as an argument of iter(), so other + attributes are restored from the state (see calliter_setstate()). */ + if (it->it_sentinel == NULL) { + if (it->it_stop_exc == NULL) { + return Py_BuildValue("N(OO)(()())", iter, it->it_callable, Py_None); + } + else { + return Py_BuildValue("N(OO)(()O)", iter, it->it_callable, Py_None, + it->it_stop_exc); + } + } + else { + if (it->it_stop_exc == NULL) { + return Py_BuildValue("N(OO)", iter, it->it_callable, + it->it_sentinel); + } + else { + return Py_BuildValue("N(OO)((O)O)", iter, it->it_callable, Py_None, + it->it_sentinel, it->it_stop_exc); + } + } +} + +static PyObject * +calliter_setstate(PyObject *op, PyObject *state) +{ + calliterobject *it = (calliterobject*)op; + PyObject *sentinel, *stop_exc; + + if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != 2) { + goto error; + } + sentinel = PyTuple_GET_ITEM(state, 0); + stop_exc = PyTuple_GET_ITEM(state, 1); + if (!PyTuple_Check(sentinel) || PyTuple_GET_SIZE(sentinel) > 1) { + goto error; + } + if (_PyEval_CheckExceptTypeValid(_PyThreadState_GET(), stop_exc) < 0) { + return NULL; + } + stop_exc = _PyIter_NormalizeStopException(stop_exc, PyExc_StopIteration); + if (it->it_callable != NULL) { + Py_XSETREF(it->it_sentinel, + PyTuple_GET_SIZE(sentinel) ? + Py_NewRef(PyTuple_GET_ITEM(sentinel, 0)) : NULL); + Py_XSETREF(it->it_stop_exc, Py_XNewRef(stop_exc)); + } + Py_RETURN_NONE; + +error: + PyErr_SetString(PyExc_TypeError, "invalid state for callable_iterator"); + return NULL; } static PyMethodDef calliter_methods[] = { {"__reduce__", calliter_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", calliter_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -337,10 +415,10 @@ anextawaitable_traverse(PyObject *op, visitproc visit, void *arg) } static PyObject * -anextawaitable_getiter(anextawaitableobject *obj) +awaitable_getiter(PyObject *owner, PyObject *wrapped) { - assert(obj->wrapped != NULL); - PyObject *awaitable = _PyCoro_GetAwaitableIter(obj->wrapped); + assert(wrapped != NULL); + PyObject *awaitable = _PyCoro_GetAwaitableIter(wrapped); if (awaitable == NULL) { return NULL; } @@ -359,7 +437,7 @@ anextawaitable_getiter(anextawaitableobject *obj) if (!PyIter_Check(awaitable)) { PyErr_Format(PyExc_TypeError, "%T.__await__() must return an iterable, not %T", - obj, awaitable); + owner, awaitable); Py_DECREF(awaitable); return NULL; } @@ -391,7 +469,7 @@ anextawaitable_iternext(PyObject *op) * gen.__anext__().__next__() */ anextawaitableobject *obj = anextawaitableobject_CAST(op); - PyObject *awaitable = anextawaitable_getiter(obj); + PyObject *awaitable = awaitable_getiter(op, obj->wrapped); if (awaitable == NULL) { return NULL; } @@ -411,7 +489,7 @@ anextawaitable_iternext(PyObject *op) static PyObject * anextawaitable_proxy(anextawaitableobject *obj, char *meth, PyObject *arg) { - PyObject *awaitable = anextawaitable_getiter(obj); + PyObject *awaitable = awaitable_getiter((PyObject *)obj, obj->wrapped); if (awaitable == NULL) { return NULL; } @@ -540,3 +618,356 @@ PyAnextAwaitable_New(PyObject *awaitable, PyObject *default_value) _PyObject_GC_TRACK(anext); return (PyObject *)anext; } + + +/* -------------------------------------- */ + +/* The asynchronous counterpart of calliterobject: the callable is called + and its result is awaited for every __anext__(). */ + +typedef struct { + PyObject_HEAD + /* All are set to NULL when the iterator is exhausted */ + PyObject *it_callable; + PyObject *it_sentinel; /* can be NULL */ + PyObject *it_stop_exc; /* can be NULL */ +} acalliterobject; + +#define acalliterobject_CAST(op) ((acalliterobject *)(op)) + +/* The awaitable returned by acalliter_anext(). The callable is only + called when this object is awaited. */ +typedef struct { + PyObject_HEAD + PyObject *aw_iterator; /* the iterator which created this object */ + PyObject *aw_wrapped; /* the awaitable returned by the callable */ + char aw_closed; +} acallawaitableobject; + +#define acallawaitableobject_CAST(op) ((acallawaitableobject *)(op)) + +PyObject * +_PyACallIter_New(PyObject *callable, PyObject *sentinel, PyObject *stop_exc) +{ + if (stop_exc != NULL && + _PyEval_CheckExceptTypeValid(_PyThreadState_GET(), stop_exc) < 0) + { + return NULL; + } + acalliterobject *it = PyObject_GC_New(acalliterobject, &_PyACallIter_Type); + if (it == NULL) { + return NULL; + } + it->it_callable = Py_NewRef(callable); + it->it_sentinel = Py_XNewRef(sentinel); + it->it_stop_exc = Py_XNewRef(stop_exc); + _PyObject_GC_TRACK(it); + return (PyObject *)it; +} + +static void +acalliter_exhaust(acalliterobject *it) +{ + Py_CLEAR(it->it_callable); + Py_CLEAR(it->it_sentinel); + Py_CLEAR(it->it_stop_exc); +} + +/* Return 1 if the raised exception ends the iteration. */ +static int +acalliter_stop_matches(acalliterobject *it) +{ + return ((it->it_stop_exc != NULL && + PyErr_ExceptionMatches(it->it_stop_exc)) || + PyErr_ExceptionMatches(PyExc_StopAsyncIteration)); +} + +static void +acalliter_dealloc(PyObject *op) +{ + acalliterobject *it = acalliterobject_CAST(op); + _PyObject_GC_UNTRACK(it); + Py_XDECREF(it->it_callable); + Py_XDECREF(it->it_sentinel); + Py_XDECREF(it->it_stop_exc); + PyObject_GC_Del(it); +} + +static int +acalliter_traverse(PyObject *op, visitproc visit, void *arg) +{ + acalliterobject *it = acalliterobject_CAST(op); + Py_VISIT(it->it_callable); + Py_VISIT(it->it_sentinel); + Py_VISIT(it->it_stop_exc); + return 0; +} + +static PyObject *acallawaitable_new(PyObject *iterator); + +static PyObject * +acalliter_anext(PyObject *op) +{ + return acallawaitable_new(op); +} + +static PyAsyncMethods acalliter_as_async = { + 0, /* am_await */ + PyObject_SelfIter, /* am_aiter */ + acalliter_anext, /* am_anext */ + 0, /* am_send */ +}; + +PyTypeObject _PyACallIter_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "async_callable_iterator", /* tp_name */ + sizeof(acalliterobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + acalliter_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + &acalliter_as_async, /* tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + acalliter_traverse, /* tp_traverse */ +}; + +/* -------------------------------------- */ + +static PyObject * +acallawaitable_new(PyObject *iterator) +{ + acallawaitableobject *aw = PyObject_GC_New( + acallawaitableobject, &_PyACallIterAwaitable_Type); + if (aw == NULL) { + return NULL; + } + aw->aw_iterator = Py_NewRef(iterator); + aw->aw_wrapped = NULL; + aw->aw_closed = 0; + _PyObject_GC_TRACK(aw); + return (PyObject *)aw; +} + +static void +acallawaitable_dealloc(PyObject *op) +{ + acallawaitableobject *aw = acallawaitableobject_CAST(op); + _PyObject_GC_UNTRACK(aw); + Py_XDECREF(aw->aw_iterator); + Py_XDECREF(aw->aw_wrapped); + PyObject_GC_Del(aw); +} + +static int +acallawaitable_traverse(PyObject *op, visitproc visit, void *arg) +{ + acallawaitableobject *aw = acallawaitableobject_CAST(op); + Py_VISIT(aw->aw_iterator); + Py_VISIT(aw->aw_wrapped); + return 0; +} + +/* Call the callable. Return 0 on success, -1 on failure. */ +static int +acallawaitable_start(acallawaitableobject *aw) +{ + acalliterobject *it = acalliterobject_CAST(aw->aw_iterator); + + if (aw->aw_closed) { + PyErr_SetString(PyExc_RuntimeError, + "cannot reuse already awaited __anext__()"); + return -1; + } + if (it->it_callable == NULL) { + PyErr_SetNone(PyExc_StopAsyncIteration); + return -1; + } + PyObject *awaitable = _PyObject_CallNoArgs(it->it_callable); + if (awaitable == NULL) { + if (acalliter_stop_matches(it)) { + PyErr_Clear(); + acalliter_exhaust(it); + PyErr_SetNone(PyExc_StopAsyncIteration); + } + return -1; + } + aw->aw_wrapped = awaitable; + return 0; +} + +/* Turn the exception raised by the wrapped awaitable into the result of + the await. Always returns NULL. */ +static PyObject * +acallawaitable_handle_error(acallawaitableobject *aw) +{ + acalliterobject *it = acalliterobject_CAST(aw->aw_iterator); + + if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + PyObject *value; + if (_PyGen_FetchStopIterationValue(&value) < 0) { + return NULL; + } + int ok = 0; + if (it->it_sentinel != NULL) { + ok = PyObject_RichCompareBool(it->it_sentinel, value, Py_EQ); + } + if (ok == 0) { + (void)_PyGen_SetStopIterationValue(value); + } + else if (ok > 0) { + acalliter_exhaust(it); + PyErr_SetNone(PyExc_StopAsyncIteration); + } + Py_DECREF(value); + return NULL; + } + if (acalliter_stop_matches(it)) { + PyErr_Clear(); + acalliter_exhaust(it); + PyErr_SetNone(PyExc_StopAsyncIteration); + } + return NULL; +} + +static PyObject * +acallawaitable_iternext(PyObject *op) +{ + acallawaitableobject *aw = acallawaitableobject_CAST(op); + + if (aw->aw_wrapped == NULL && acallawaitable_start(aw) < 0) { + return NULL; + } + PyObject *awaitable = awaitable_getiter(op, aw->aw_wrapped); + if (awaitable == NULL) { + return NULL; + } + PyObject *result = (*Py_TYPE(awaitable)->tp_iternext)(awaitable); + Py_DECREF(awaitable); + if (result != NULL) { + return result; + } + return acallawaitable_handle_error(aw); +} + +static PyObject * +acallawaitable_proxy(acallawaitableobject *aw, char *meth, PyObject *arg) +{ + PyObject *awaitable = awaitable_getiter((PyObject *)aw, aw->aw_wrapped); + if (awaitable == NULL) { + return NULL; + } + // When specified, 'arg' may be a tuple (if coming from a METH_VARARGS + // method) or a single object (if coming from a METH_O method). + PyObject *ret = arg == NULL + ? PyObject_CallMethod(awaitable, meth, NULL) + : PyObject_CallMethod(awaitable, meth, "O", arg); + Py_DECREF(awaitable); + if (ret != NULL) { + return ret; + } + return acallawaitable_handle_error(aw); +} + +static PyObject * +acallawaitable_send(PyObject *op, PyObject *arg) +{ + acallawaitableobject *aw = acallawaitableobject_CAST(op); + + if (aw->aw_wrapped == NULL && acallawaitable_start(aw) < 0) { + return NULL; + } + return acallawaitable_proxy(aw, "send", arg); +} + +static PyObject * +acallawaitable_throw(PyObject *op, PyObject *args) +{ + acallawaitableobject *aw = acallawaitableobject_CAST(op); + + if (aw->aw_wrapped == NULL) { + /* Not started, so the exception is raised at the point of the + await, as for a not started coroutine. */ + PyObject *typ, *val = NULL, *tb = NULL; + if (!PyArg_UnpackTuple(args, "throw", 1, 3, &typ, &val, &tb)) { + return NULL; + } + aw->aw_closed = 1; + (void)_PyGen_SetException(typ, val, tb); + return NULL; + } + return acallawaitable_proxy(aw, "throw", args); +} + +static PyObject * +acallawaitable_close(PyObject *op, PyObject *Py_UNUSED(dummy)) +{ + acallawaitableobject *aw = acallawaitableobject_CAST(op); + + if (aw->aw_wrapped == NULL) { + /* Not started, so there is nothing to close. */ + aw->aw_closed = 1; + Py_RETURN_NONE; + } + PyObject *result = acallawaitable_proxy(aw, "close", NULL); + aw->aw_closed = 1; + return result; +} + +static PyMethodDef acallawaitable_methods[] = { + {"send", acallawaitable_send, METH_O, send_doc}, + {"throw", acallawaitable_throw, METH_VARARGS, throw_doc}, + {"close", acallawaitable_close, METH_NOARGS, close_doc}, + {NULL, NULL} /* Sentinel */ +}; + +static PyAsyncMethods acallawaitable_as_async = { + PyObject_SelfIter, /* am_await */ + 0, /* am_aiter */ + 0, /* am_anext */ + 0, /* am_send */ +}; + +PyTypeObject _PyACallIterAwaitable_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "async_callable_iterator_awaitable", /* tp_name */ + sizeof(acallawaitableobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + acallawaitable_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + &acallawaitable_as_async, /* tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + acallawaitable_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + acallawaitable_iternext, /* tp_iternext */ + acallawaitable_methods, /* tp_methods */ +}; diff --git a/Objects/object.c b/Objects/object.c index fadd9273a36607c..c0cb0da7a0d92e5 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2519,6 +2519,8 @@ _PyObject_FiniState(PyInterpreterState *interp) } +extern PyTypeObject _PyACallIter_Type; +extern PyTypeObject _PyACallIterAwaitable_Type; extern PyTypeObject _PyAnextAwaitable_Type; extern PyTypeObject _PyLegacyEventHandler_Type; extern PyTypeObject _PyLineIterator; @@ -2612,6 +2614,8 @@ static PyTypeObject* static_types[_Py_NUM_MANAGED_PREINITIALIZED_TYPES] = { &PyWrapperDescr_Type, &PyZip_Type, &Py_GenericAliasType, + &_PyACallIter_Type, + &_PyACallIterAwaitable_Type, &_PyAnextAwaitable_Type, &_PyAsyncGenASend_Type, &_PyAsyncGenAThrow_Type, diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 7fec674c550e027..60bb98775e528bf 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -276,6 +276,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index cb0f6fce86df51a..9bf21ffbe322d10 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -735,6 +735,9 @@ Include\cpython + + Include\cpython + Include\internal diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index cbe59c8883d5a57..f2c74e9676e39a2 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -10,6 +10,7 @@ #include "pycore_floatobject.h" // _PyFloat_ExactDealloc() #include "pycore_interp.h" // _PyInterpreterState_GetConfig() #include "pycore_import.h" // _PyImport_LazyImportModuleLevelObject () +#include "pycore_iterobject.h" // _PyCallIter_NewEx() #include "pycore_long.h" // _PyLong_CompactValue #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _Py_AddToAllObjects() @@ -1893,50 +1894,79 @@ builtin_hex(PyObject *module, PyObject *integer) } -/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ +/*[clinic input] +@text_signature "($module, object, /, [stop_value], *, stop_exception=StopIteration)" +iter as builtin_iter + + object: object + / + stop_value: object = NULL + * + stop_exception: object = NULL + +Get an iterator from an object. + +In the first form, the argument must supply its own iterator, or be a +sequence. In the second form, the callable is called until it returns +the stop value or raises StopIteration or the specified exception. +[clinic start generated code]*/ + static PyObject * -builtin_iter(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +builtin_iter_impl(PyObject *module, PyObject *object, PyObject *stop_value, + PyObject *stop_exception) +/*[clinic end generated code: output=eb9c9ae8f77bf400 input=d4eb3d19c8942790]*/ { - PyObject *v; - - if (!_PyArg_CheckPositional("iter", nargs, 1, 2)) - return NULL; - v = args[0]; - if (nargs == 1) - return PyObject_GetIter(v); - if (!PyCallable_Check(v)) { + if (stop_value == NULL && stop_exception == NULL) { + return PyObject_GetIter(object); + } + if (!PyCallable_Check(object)) { PyErr_SetString(PyExc_TypeError, - "iter(v, w): v must be callable"); + "iter(): the first argument must be callable"); return NULL; } - PyObject *sentinel = args[1]; - return PyCallIter_New(v, sentinel); + if (stop_exception != NULL) { + stop_exception = _PyIter_NormalizeStopException(stop_exception, + PyExc_StopIteration); + } + return _PyCallIter_NewEx(object, stop_value, stop_exception); } -PyDoc_STRVAR(iter_doc, -"iter(iterable) -> iterator\n\ -iter(callable, sentinel) -> iterator\n\ -\n\ -Get an iterator from an object. In the first form, the argument must\n\ -supply its own iterator, or be a sequence.\n\ -In the second form, the callable is called until it returns the\n\ -sentinel."); - /*[clinic input] +@text_signature "($module, object, /, [stop_value], *, stop_exception=StopAsyncIteration)" aiter as builtin_aiter - async_iterable: object + object: object / + stop_value: object = NULL + * + stop_exception: object = NULL Return an AsyncIterator for an AsyncIterable object. + +In the second form, the callable is called and its result is awaited +until it returns the stop value or raises StopAsyncIteration or the +specified exception. [clinic start generated code]*/ static PyObject * -builtin_aiter(PyObject *module, PyObject *async_iterable) -/*[clinic end generated code: output=1bae108d86f7960e input=473993d0cacc7d23]*/ +builtin_aiter_impl(PyObject *module, PyObject *object, PyObject *stop_value, + PyObject *stop_exception) +/*[clinic end generated code: output=2865edb3fbc45693 input=3eec4f0424a7ebac]*/ { - return PyObject_GetAIter(async_iterable); + if (stop_value == NULL && stop_exception == NULL) { + return PyObject_GetAIter(object); + } + if (!PyCallable_Check(object)) { + PyErr_SetString(PyExc_TypeError, + "aiter(): the first argument must be callable"); + return NULL; + } + if (stop_exception != NULL) { + stop_exception = _PyIter_NormalizeStopException( + stop_exception, PyExc_StopAsyncIteration); + } + return _PyACallIter_New(object, stop_value, stop_exception); } PyObject *PyAnextAwaitable_New(PyObject *, PyObject *); @@ -3472,7 +3502,7 @@ static PyMethodDef builtin_methods[] = { BUILTIN_INPUT_METHODDEF BUILTIN_ISINSTANCE_METHODDEF BUILTIN_ISSUBCLASS_METHODDEF - {"iter", _PyCFunction_CAST(builtin_iter), METH_FASTCALL, iter_doc}, + BUILTIN_ITER_METHODDEF BUILTIN_AITER_METHODDEF BUILTIN_LEN_METHODDEF BUILTIN_LOCALS_METHODDEF diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 4a38e0df61708c0..30a10473eb6a407 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -850,14 +850,167 @@ PyDoc_STRVAR(builtin_hex__doc__, #define BUILTIN_HEX_METHODDEF \ {"hex", (PyCFunction)builtin_hex, METH_O, builtin_hex__doc__}, +PyDoc_STRVAR(builtin_iter__doc__, +"iter($module, object, /, [stop_value], *, stop_exception=StopIteration)\n" +"--\n" +"\n" +"Get an iterator from an object.\n" +"\n" +"In the first form, the argument must supply its own iterator, or be a\n" +"sequence. In the second form, the callable is called until it returns\n" +"the stop value or raises StopIteration or the specified exception."); + +#define BUILTIN_ITER_METHODDEF \ + {"iter", _PyCFunction_CAST(builtin_iter), METH_FASTCALL|METH_KEYWORDS, builtin_iter__doc__}, + +static PyObject * +builtin_iter_impl(PyObject *module, PyObject *object, PyObject *stop_value, + PyObject *stop_exception); + +static PyObject * +builtin_iter(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(stop_value), &_Py_ID(stop_exception), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "stop_value", "stop_exception", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "iter", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *object; + PyObject *stop_value = NULL; + PyObject *stop_exception = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + object = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + stop_value = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + stop_exception = args[2]; +skip_optional_kwonly: + return_value = builtin_iter_impl(module, object, stop_value, stop_exception); + +exit: + return return_value; +} + PyDoc_STRVAR(builtin_aiter__doc__, -"aiter($module, async_iterable, /)\n" +"aiter($module, object, /, [stop_value], *, stop_exception=StopAsyncIteration)\n" "--\n" "\n" -"Return an AsyncIterator for an AsyncIterable object."); +"Return an AsyncIterator for an AsyncIterable object.\n" +"\n" +"In the second form, the callable is called and its result is awaited\n" +"until it returns the stop value or raises StopAsyncIteration or the\n" +"specified exception."); #define BUILTIN_AITER_METHODDEF \ - {"aiter", (PyCFunction)builtin_aiter, METH_O, builtin_aiter__doc__}, + {"aiter", _PyCFunction_CAST(builtin_aiter), METH_FASTCALL|METH_KEYWORDS, builtin_aiter__doc__}, + +static PyObject * +builtin_aiter_impl(PyObject *module, PyObject *object, PyObject *stop_value, + PyObject *stop_exception); + +static PyObject * +builtin_aiter(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(stop_value), &_Py_ID(stop_exception), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "stop_value", "stop_exception", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "aiter", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *object; + PyObject *stop_value = NULL; + PyObject *stop_exception = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + object = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + stop_value = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + stop_exception = args[2]; +skip_optional_kwonly: + return_value = builtin_aiter_impl(module, object, stop_value, stop_exception); + +exit: + return return_value; +} PyDoc_STRVAR(builtin_anext__doc__, "anext($module, async_iterator, default=, /)\n" @@ -1387,4 +1540,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=84efa9c5cc737ce5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9a8a92605af72942 input=a9049054013a1b77]*/ From 25837892cbfb145321edfac0a1554d5e0550eaa9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 23 Aug 2026 23:41:15 +0300 Subject: [PATCH 2/4] Add the new static types to globals-to-fix.tsv Co-Authored-By: Claude Opus 5 (1M context) --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index db575d870be5c53..148f6e68ab806e5 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -58,6 +58,8 @@ Objects/genobject.c - _PyCoroWrapper_Type - Objects/interpolationobject.c - _PyInterpolation_Type - Objects/iterobject.c - PyCallIter_Type - Objects/iterobject.c - PySeqIter_Type - +Objects/iterobject.c - _PyACallIter_Type - +Objects/iterobject.c - _PyACallIterAwaitable_Type - Objects/iterobject.c - _PyAnextAwaitable_Type - Objects/lazyimportobject.c - PyLazyImport_Type - Objects/listobject.c - PyListIter_Type - @@ -73,6 +75,8 @@ Objects/moduleobject.c - PyModule_Type - Objects/namespaceobject.c - _PyNamespace_Type - Objects/object.c - _PyNone_Type - Objects/object.c - _PyNotImplemented_Type - +Objects/object.c - _PyACallIter_Type - +Objects/object.c - _PyACallIterAwaitable_Type - Objects/object.c - _PyAnextAwaitable_Type - Objects/odictobject.c - PyODictItems_Type - Objects/odictobject.c - PyODictIter_Type - From d85e8cef650c0a956957f85d4aa20a6ee10524c0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 24 Aug 2026 11:28:56 +0300 Subject: [PATCH 3/4] Always set it_stop_exc for a non-exhausted iterator Use StopIteration (StopAsyncIteration for aiter()) as the default instead of normalizing it to NULL, so that the check is a single PyErr_ExceptionMatches(). Co-Authored-By: Claude Opus 5 (1M context) --- Include/internal/pycore_iterobject.h | 17 +------ Lib/test/test_iter.py | 8 ++-- Objects/iterobject.c | 69 ++++++++++------------------ Python/bltinmodule.c | 8 ---- 4 files changed, 31 insertions(+), 71 deletions(-) diff --git a/Include/internal/pycore_iterobject.h b/Include/internal/pycore_iterobject.h index 610fe2aa5ce6736..90b444976e2f19c 100644 --- a/Include/internal/pycore_iterobject.h +++ b/Include/internal/pycore_iterobject.h @@ -13,28 +13,15 @@ extern PyTypeObject _PyACallIterAwaitable_Type; // Like PyCallIter_New(), but the iteration also stops when *callable* raises // an exception matching *stop_exc* (an exception class or a tuple of exception -// classes). Both *sentinel* and *stop_exc* can be NULL. +// classes). *sentinel* can be NULL; NULL *stop_exc* means StopIteration. extern PyObject *_PyCallIter_NewEx(PyObject *callable, PyObject *sentinel, PyObject *stop_exc); // The asynchronous counterpart of _PyCallIter_NewEx(): the result of -// *callable* is awaited, and StopAsyncIteration stops the iteration. +// *callable* is awaited, and NULL *stop_exc* means StopAsyncIteration. extern PyObject *_PyACallIter_New(PyObject *callable, PyObject *sentinel, PyObject *stop_exc); -// Return NULL if *stop_exc* has no effect: *implied_exc* stops the iteration -// in any case, and an empty tuple never matches a raised exception. -static inline PyObject * -_PyIter_NormalizeStopException(PyObject *stop_exc, PyObject *implied_exc) -{ - if (stop_exc == implied_exc || - (PyTuple_Check(stop_exc) && PyTuple_GET_SIZE(stop_exc) == 0)) - { - return NULL; - } - return stop_exc; -} - #ifdef __cplusplus } #endif diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 7f5dfde201cba64..65ba7376dd3d281 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -419,9 +419,9 @@ def test_calliter_reduce(self): self.assertEqual(iter(c, 10, stop_exception=StopIteration).__reduce__(), (iter, (c, 10))) self.assertEqual(iter(c, 10, stop_exception=()).__reduce__(), - (iter, (c, 10))) + (iter, (c, None), ((10,), ()))) self.assertEqual(iter(c, stop_exception=StopIteration).__reduce__(), - (iter, (c, None), ((), ()))) + (iter, (c, None), ((), StopIteration))) self.assertEqual(iter(c, stop_exception=IndexError).__reduce__(), (iter, (c, None), ((), IndexError))) self.assertEqual(iter(c, 10, stop_exception=IndexError).__reduce__(), @@ -439,10 +439,10 @@ def test_calliter_setstate(self): it.__setstate__(((10,), StopIteration)) self.assertEqual(it.__reduce__(), (iter, (c, 10))) it.__setstate__(((10,), ())) - self.assertEqual(it.__reduce__(), (iter, (c, 10))) + self.assertEqual(it.__reduce__(), (iter, (c, None), ((10,), ()))) it.__setstate__(((), IndexError)) self.assertEqual(it.__reduce__(), (iter, (c, None), ((), IndexError))) - it.__setstate__(((10,), ())) + it.__setstate__(((10,), StopIteration)) self.assertEqual(list(it), list(range(10))) def test_iter_function_concealing_reentrant_exhaustion(self): diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 7b4d5931e10cbcc..42f8dbfb35e5644 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -189,17 +189,18 @@ typedef struct { PyObject_HEAD /* All are set to NULL when the iterator is exhausted */ PyObject *it_callable; - PyObject *it_sentinel; /* can be NULL */ - PyObject *it_stop_exc; /* can be NULL */ + PyObject *it_sentinel; /* can be NULL */ + PyObject *it_stop_exc; /* not NULL if it_callable is not NULL */ } calliterobject; PyObject * _PyCallIter_NewEx(PyObject *callable, PyObject *sentinel, PyObject *stop_exc) { calliterobject *it; - if (stop_exc != NULL && - _PyEval_CheckExceptTypeValid(_PyThreadState_GET(), stop_exc) < 0) - { + if (stop_exc == NULL) { + stop_exc = PyExc_StopIteration; + } + else if (_PyEval_CheckExceptTypeValid(_PyThreadState_GET(), stop_exc) < 0) { return NULL; } it = PyObject_GC_New(calliterobject, &PyCallIter_Type); @@ -207,7 +208,7 @@ _PyCallIter_NewEx(PyObject *callable, PyObject *sentinel, PyObject *stop_exc) return NULL; it->it_callable = Py_NewRef(callable); it->it_sentinel = Py_XNewRef(sentinel); - it->it_stop_exc = Py_XNewRef(stop_exc); + it->it_stop_exc = Py_NewRef(stop_exc); _PyObject_GC_TRACK(it); return (PyObject *)it; } @@ -266,10 +267,7 @@ calliter_iternext(PyObject *op) Py_CLEAR(it->it_stop_exc); } } - else if ((it->it_stop_exc != NULL && - PyErr_ExceptionMatches(it->it_stop_exc)) || - PyErr_ExceptionMatches(PyExc_StopIteration)) - { + else if (PyErr_ExceptionMatches(it->it_stop_exc)) { PyErr_Clear(); Py_CLEAR(it->it_callable); Py_CLEAR(it->it_sentinel); @@ -295,23 +293,15 @@ calliter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) /* Only the sentinel can be passed as an argument of iter(), so other attributes are restored from the state (see calliter_setstate()). */ if (it->it_sentinel == NULL) { - if (it->it_stop_exc == NULL) { - return Py_BuildValue("N(OO)(()())", iter, it->it_callable, Py_None); - } - else { - return Py_BuildValue("N(OO)(()O)", iter, it->it_callable, Py_None, - it->it_stop_exc); - } + return Py_BuildValue("N(OO)(()O)", iter, it->it_callable, Py_None, + it->it_stop_exc); + } + else if (it->it_stop_exc == PyExc_StopIteration) { + return Py_BuildValue("N(OO)", iter, it->it_callable, it->it_sentinel); } else { - if (it->it_stop_exc == NULL) { - return Py_BuildValue("N(OO)", iter, it->it_callable, - it->it_sentinel); - } - else { - return Py_BuildValue("N(OO)((O)O)", iter, it->it_callable, Py_None, - it->it_sentinel, it->it_stop_exc); - } + return Py_BuildValue("N(OO)((O)O)", iter, it->it_callable, Py_None, + it->it_sentinel, it->it_stop_exc); } } @@ -332,12 +322,11 @@ calliter_setstate(PyObject *op, PyObject *state) if (_PyEval_CheckExceptTypeValid(_PyThreadState_GET(), stop_exc) < 0) { return NULL; } - stop_exc = _PyIter_NormalizeStopException(stop_exc, PyExc_StopIteration); if (it->it_callable != NULL) { Py_XSETREF(it->it_sentinel, PyTuple_GET_SIZE(sentinel) ? Py_NewRef(PyTuple_GET_ITEM(sentinel, 0)) : NULL); - Py_XSETREF(it->it_stop_exc, Py_XNewRef(stop_exc)); + Py_SETREF(it->it_stop_exc, Py_NewRef(stop_exc)); } Py_RETURN_NONE; @@ -629,8 +618,8 @@ typedef struct { PyObject_HEAD /* All are set to NULL when the iterator is exhausted */ PyObject *it_callable; - PyObject *it_sentinel; /* can be NULL */ - PyObject *it_stop_exc; /* can be NULL */ + PyObject *it_sentinel; /* can be NULL */ + PyObject *it_stop_exc; /* not NULL if it_callable is not NULL */ } acalliterobject; #define acalliterobject_CAST(op) ((acalliterobject *)(op)) @@ -649,9 +638,10 @@ typedef struct { PyObject * _PyACallIter_New(PyObject *callable, PyObject *sentinel, PyObject *stop_exc) { - if (stop_exc != NULL && - _PyEval_CheckExceptTypeValid(_PyThreadState_GET(), stop_exc) < 0) - { + if (stop_exc == NULL) { + stop_exc = PyExc_StopAsyncIteration; + } + else if (_PyEval_CheckExceptTypeValid(_PyThreadState_GET(), stop_exc) < 0) { return NULL; } acalliterobject *it = PyObject_GC_New(acalliterobject, &_PyACallIter_Type); @@ -660,7 +650,7 @@ _PyACallIter_New(PyObject *callable, PyObject *sentinel, PyObject *stop_exc) } it->it_callable = Py_NewRef(callable); it->it_sentinel = Py_XNewRef(sentinel); - it->it_stop_exc = Py_XNewRef(stop_exc); + it->it_stop_exc = Py_NewRef(stop_exc); _PyObject_GC_TRACK(it); return (PyObject *)it; } @@ -673,15 +663,6 @@ acalliter_exhaust(acalliterobject *it) Py_CLEAR(it->it_stop_exc); } -/* Return 1 if the raised exception ends the iteration. */ -static int -acalliter_stop_matches(acalliterobject *it) -{ - return ((it->it_stop_exc != NULL && - PyErr_ExceptionMatches(it->it_stop_exc)) || - PyErr_ExceptionMatches(PyExc_StopAsyncIteration)); -} - static void acalliter_dealloc(PyObject *op) { @@ -797,7 +778,7 @@ acallawaitable_start(acallawaitableobject *aw) } PyObject *awaitable = _PyObject_CallNoArgs(it->it_callable); if (awaitable == NULL) { - if (acalliter_stop_matches(it)) { + if (PyErr_ExceptionMatches(it->it_stop_exc)) { PyErr_Clear(); acalliter_exhaust(it); PyErr_SetNone(PyExc_StopAsyncIteration); @@ -834,7 +815,7 @@ acallawaitable_handle_error(acallawaitableobject *aw) Py_DECREF(value); return NULL; } - if (acalliter_stop_matches(it)) { + if (PyErr_ExceptionMatches(it->it_stop_exc)) { PyErr_Clear(); acalliter_exhaust(it); PyErr_SetNone(PyExc_StopAsyncIteration); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index f2c74e9676e39a2..f2744f9b3749af6 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1924,10 +1924,6 @@ builtin_iter_impl(PyObject *module, PyObject *object, PyObject *stop_value, "iter(): the first argument must be callable"); return NULL; } - if (stop_exception != NULL) { - stop_exception = _PyIter_NormalizeStopException(stop_exception, - PyExc_StopIteration); - } return _PyCallIter_NewEx(object, stop_value, stop_exception); } @@ -1962,10 +1958,6 @@ builtin_aiter_impl(PyObject *module, PyObject *object, PyObject *stop_value, "aiter(): the first argument must be callable"); return NULL; } - if (stop_exception != NULL) { - stop_exception = _PyIter_NormalizeStopException( - stop_exception, PyExc_StopAsyncIteration); - } return _PyACallIter_New(object, stop_value, stop_exception); } From 5877437a4a349a095488818b92f0d905c31f5795 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 24 Aug 2026 12:33:44 +0300 Subject: [PATCH 4/4] Replace a leaking StopIteration with RuntimeError If the callable raises StopIteration (StopAsyncIteration in aiter()) which does not match stop_exception, the consumer would mistake it for the end of the iteration, or, in the asynchronous case, for the result of the await. Replace it with RuntimeError, as PEP 479 and PEP 525 do for generators. StopIteration is therefore no longer special: it stops the iteration only because it is the default stop_exception. Co-Authored-By: Claude Opus 5 (1M context) --- Doc/library/functions.rst | 15 +++++++---- Lib/test/test_asyncgen.py | 48 +++++++++++++++++++++++------------ Lib/test/test_iter.py | 48 ++++++++++++++++------------------- Objects/iterobject.c | 32 ++++++++++++++++++----- Python/bltinmodule.c | 9 +++---- Python/clinic/bltinmodule.c.h | 7 +++-- 6 files changed, 96 insertions(+), 63 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 0e8ebf1b5254632..ddfda5445270338 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -77,8 +77,8 @@ are always available. They are listed here in alphabetical order. calls *callable* with no arguments and awaits the result for each call to its :meth:`~object.__anext__` method; if the awaited value is equal to *stop_value*, - or if the call raises :exc:`StopAsyncIteration` or an exception - matching *stop_exception*, :exc:`StopAsyncIteration` will be raised, + or if the call raises an exception matching *stop_exception*, + :exc:`StopAsyncIteration` will be raised, otherwise the value will be returned. The callable is only called when the result of :meth:`~object.__anext__` is awaited. @@ -86,6 +86,9 @@ are always available. They are listed here in alphabetical order. *stop_exception* is an exception class or a tuple of exception classes. If *stop_value* is not specified, the iteration stops only when the callable raises an exception. + If the callable raises :exc:`StopAsyncIteration` which does not match + *stop_exception*, it is replaced with a :exc:`RuntimeError`, + as for asynchronous generators (see :pep:`525`). For example, reading fixed-size chunks from an asynchronous stream until the end of file is reached:: @@ -1191,13 +1194,15 @@ are always available. They are listed here in alphabetical order. then the first argument must be a callable object. The iterator created in this case will call *callable* with no arguments for each call to its :meth:`~iterator.__next__` method; if the value returned is equal to - *stop_value*, or if the call raises :exc:`StopIteration` or an exception - matching *stop_exception*, :exc:`StopIteration` will be raised, otherwise the - value will be returned. + *stop_value*, or if the call raises an exception matching *stop_exception*, + :exc:`StopIteration` will be raised, otherwise the value will be returned. *stop_exception* is an exception class or a tuple of exception classes. If *stop_value* is not specified, the iteration stops only when the callable raises an exception. + If the callable raises :exc:`StopIteration` which does not match + *stop_exception*, it is replaced with a :exc:`RuntimeError`, + as for generators (see :pep:`479`). See also :ref:`typeiter`. diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 2ae6de229bb33f4..cdae58b3e89ae36 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -835,9 +835,8 @@ async def spam(): self.collect(aiter(spam, 100, stop_exception=LookupError)), [1, 2, 3, 4, 5]) - def test_aiter_callable_stop_exception_redundant(self): - # StopAsyncIteration and an empty tuple stop the iteration in any - # case, so they are the same as no exception argument + def test_aiter_callable_stop_async_iteration(self): + # StopAsyncIteration is the default stop exception counter = self.make_counter() async def spam(): value = await counter() @@ -847,21 +846,38 @@ async def spam(): self.assertEqual( self.collect(aiter(spam, stop_exception=StopAsyncIteration)), [1, 2, 3]) - counter = self.make_counter() - self.assertEqual(self.collect(aiter(spam, stop_exception=())), - [1, 2, 3]) - def test_aiter_callable_stop_async_iteration(self): - # StopAsyncIteration stops the iteration even if other exception - # is specified - counter = self.make_counter() + def test_aiter_callable_leak_from_await(self): + # A StopAsyncIteration leaking from the await is replaced with + # RuntimeError (see PEP 525) async def spam(): - value = await counter() - if value > 3: - raise StopAsyncIteration - return value - self.assertEqual(self.collect(aiter(spam, stop_exception=LookupError)), - [1, 2, 3]) + raise StopAsyncIteration + it = aiter(spam, 10, stop_exception=LookupError) + with self.assertRaisesRegex(RuntimeError, + 'callable raised StopAsyncIteration') as cm: + self.loop.run_until_complete(anext(it)) + self.assertIsInstance(cm.exception.__cause__, StopAsyncIteration) + # but if it matches stop_exception, it stops the iteration + it = aiter(spam, 10, stop_exception=(LookupError, StopAsyncIteration)) + with self.assertRaises(StopAsyncIteration): + self.loop.run_until_complete(anext(it)) + + def test_aiter_callable_leak_from_call(self): + # StopIteration and StopAsyncIteration leaking from the call are + # replaced with RuntimeError (see PEP 525) + for exc in StopIteration, StopAsyncIteration: + with self.subTest(exc=exc): + def spam(): + raise exc + it = aiter(spam, 10, stop_exception=LookupError) + with self.assertRaisesRegex( + RuntimeError, f'callable raised {exc.__name__}') as cm: + self.loop.run_until_complete(anext(it)) + self.assertIsInstance(cm.exception.__cause__, exc) + # but if it matches stop_exception, it stops the iteration + it = aiter(spam, 10, stop_exception=(LookupError, exc)) + with self.assertRaises(StopAsyncIteration): + self.loop.run_until_complete(anext(it)) def test_aiter_callable_other_exception(self): async def spam(): diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 65ba7376dd3d281..0de501e94917bac 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -371,16 +371,18 @@ def test_iter_exception_and_stop(self): self.check_iterator(iter(CallableIterClass(), 200, stop_exception=IndexError), list(range(101))) - # StopIteration stops the iteration even if other exception is specified - def test_iter_exception_stop_iteration(self): - def spam(state=[0]): - i = state[0] - if i == 10: - raise StopIteration - state[0] = i+1 - return i - self.check_iterator(iter(spam, stop_exception=IndexError), list(range(10)), - pickle=False) + # A leaking StopIteration is replaced with RuntimeError (see PEP 479) + def test_iter_exception_stop_iteration_leak(self): + def spam(): + raise StopIteration + it = iter(spam, stop_exception=IndexError) + with self.assertRaisesRegex(RuntimeError, + 'callable raised StopIteration') as cm: + next(it) + self.assertIsInstance(cm.exception.__cause__, StopIteration) + # but if it matches stop_exception, it stops the iteration + it = iter(spam, stop_exception=(IndexError, StopIteration)) + self.assertRaises(StopIteration, next, it) # Other exceptions are propagated def test_iter_exception_not_matching(self): @@ -395,22 +397,16 @@ def test_iter_exception_errors(self): self.assertRaises(TypeError, iter, len, stop_exception=(IndexError, 42)) self.assertRaises(TypeError, iter, len, stop_exception=IndexError()) - # StopIteration and an empty tuple stop the iteration in any case, - # so they are the same as no exception argument - def test_iter_exception_redundant(self): - def make_spam(): - state = [0] - def spam(): - if state[0] == 10: - raise StopIteration - state[0] += 1 - return state[0] - 1 - return spam - for stop_exception in StopIteration, (): - with self.subTest(stop_exception=stop_exception): - self.check_iterator( - iter(make_spam(), stop_exception=stop_exception), - list(range(10)), pickle=False) + # StopIteration is the default stop exception + def test_iter_exception_stop_iteration(self): + def spam(state=[0]): + i = state[0] + if i == 10: + raise StopIteration + state[0] = i+1 + return i + self.check_iterator(iter(spam, stop_exception=StopIteration), + list(range(10)), pickle=False) def test_calliter_reduce(self): c = CallableIterClass() diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 42f8dbfb35e5644..5a5ca8d5e1f9b8c 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -7,6 +7,7 @@ #include "pycore_genobject.h" // _PyCoro_GetAwaitableIter() #include "pycore_iterobject.h" // _PyCallIter_NewEx() #include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -187,10 +188,10 @@ PyTypeObject PySeqIter_Type = { typedef struct { PyObject_HEAD - /* All are set to NULL when the iterator is exhausted */ + /* Both are set to NULL when the iterator is exhausted */ PyObject *it_callable; PyObject *it_sentinel; /* can be NULL */ - PyObject *it_stop_exc; /* not NULL if it_callable is not NULL */ + PyObject *it_stop_exc; /* never NULL */ } calliterobject; PyObject * @@ -264,14 +265,17 @@ calliter_iternext(PyObject *op) if (ok > 0) { Py_CLEAR(it->it_callable); Py_CLEAR(it->it_sentinel); - Py_CLEAR(it->it_stop_exc); } } else if (PyErr_ExceptionMatches(it->it_stop_exc)) { PyErr_Clear(); Py_CLEAR(it->it_callable); Py_CLEAR(it->it_sentinel); - Py_CLEAR(it->it_stop_exc); + } + else if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + /* It would be mistaken for the end of the iteration (see PEP 479). */ + _PyErr_FormatFromCause(PyExc_RuntimeError, + "callable raised StopIteration"); } Py_XDECREF(result); return NULL; @@ -616,10 +620,10 @@ PyAnextAwaitable_New(PyObject *awaitable, PyObject *default_value) typedef struct { PyObject_HEAD - /* All are set to NULL when the iterator is exhausted */ + /* Both are set to NULL when the iterator is exhausted */ PyObject *it_callable; PyObject *it_sentinel; /* can be NULL */ - PyObject *it_stop_exc; /* not NULL if it_callable is not NULL */ + PyObject *it_stop_exc; /* never NULL */ } acalliterobject; #define acalliterobject_CAST(op) ((acalliterobject *)(op)) @@ -660,7 +664,6 @@ acalliter_exhaust(acalliterobject *it) { Py_CLEAR(it->it_callable); Py_CLEAR(it->it_sentinel); - Py_CLEAR(it->it_stop_exc); } static void @@ -783,6 +786,16 @@ acallawaitable_start(acallawaitableobject *aw) acalliter_exhaust(it); PyErr_SetNone(PyExc_StopAsyncIteration); } + else if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + /* It would be mistaken for the result of the await (PEP 525). */ + _PyErr_FormatFromCause(PyExc_RuntimeError, + "callable raised StopIteration"); + } + else if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) { + /* It would be mistaken for the end of the iteration (PEP 525). */ + _PyErr_FormatFromCause(PyExc_RuntimeError, + "callable raised StopAsyncIteration"); + } return -1; } aw->aw_wrapped = awaitable; @@ -820,6 +833,11 @@ acallawaitable_handle_error(acallawaitableobject *aw) acalliter_exhaust(it); PyErr_SetNone(PyExc_StopAsyncIteration); } + else if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) { + /* It would be mistaken for the end of the iteration (see PEP 525). */ + _PyErr_FormatFromCause(PyExc_RuntimeError, + "callable raised StopAsyncIteration"); + } return NULL; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index f2744f9b3749af6..d28e6fa9cd01aed 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1908,13 +1908,13 @@ Get an iterator from an object. In the first form, the argument must supply its own iterator, or be a sequence. In the second form, the callable is called until it returns -the stop value or raises StopIteration or the specified exception. +the stop value or raises the specified exception. [clinic start generated code]*/ static PyObject * builtin_iter_impl(PyObject *module, PyObject *object, PyObject *stop_value, PyObject *stop_exception) -/*[clinic end generated code: output=eb9c9ae8f77bf400 input=d4eb3d19c8942790]*/ +/*[clinic end generated code: output=eb9c9ae8f77bf400 input=d3a2f767f29d9ae6]*/ { if (stop_value == NULL && stop_exception == NULL) { return PyObject_GetIter(object); @@ -1941,14 +1941,13 @@ aiter as builtin_aiter Return an AsyncIterator for an AsyncIterable object. In the second form, the callable is called and its result is awaited -until it returns the stop value or raises StopAsyncIteration or the -specified exception. +until it returns the stop value or raises the specified exception. [clinic start generated code]*/ static PyObject * builtin_aiter_impl(PyObject *module, PyObject *object, PyObject *stop_value, PyObject *stop_exception) -/*[clinic end generated code: output=2865edb3fbc45693 input=3eec4f0424a7ebac]*/ +/*[clinic end generated code: output=2865edb3fbc45693 input=2adb37d12adafd0c]*/ { if (stop_value == NULL && stop_exception == NULL) { return PyObject_GetAIter(object); diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 30a10473eb6a407..c10bb03d8178161 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -858,7 +858,7 @@ PyDoc_STRVAR(builtin_iter__doc__, "\n" "In the first form, the argument must supply its own iterator, or be a\n" "sequence. In the second form, the callable is called until it returns\n" -"the stop value or raises StopIteration or the specified exception."); +"the stop value or raises the specified exception."); #define BUILTIN_ITER_METHODDEF \ {"iter", _PyCFunction_CAST(builtin_iter), METH_FASTCALL|METH_KEYWORDS, builtin_iter__doc__}, @@ -938,8 +938,7 @@ PyDoc_STRVAR(builtin_aiter__doc__, "Return an AsyncIterator for an AsyncIterable object.\n" "\n" "In the second form, the callable is called and its result is awaited\n" -"until it returns the stop value or raises StopAsyncIteration or the\n" -"specified exception."); +"until it returns the stop value or raises the specified exception."); #define BUILTIN_AITER_METHODDEF \ {"aiter", _PyCFunction_CAST(builtin_aiter), METH_FASTCALL|METH_KEYWORDS, builtin_aiter__doc__}, @@ -1540,4 +1539,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=9a8a92605af72942 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5fb1ac6a4253ee2f input=a9049054013a1b77]*/