Commit 6dd9b9f1 authored by Nicolas Boulenguez's avatar Nicolas Boulenguez

import upstream 1.10.

parent bc12910c
For the Trio contributing guide, see:
https://trio.readthedocs.io/en/latest/contributing.html
This software is made available under the terms of *either* of the
licenses found in LICENSE.APACHE2 or LICENSE.MIT. Contributions to
trio are made under the terms of *both* these licenses.
This diff is collapsed.
The MIT License (MIT)
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
include LICENSE.txt README.rst CODE_OF_CONDUCT.md
include LICENSE LICENSE.MIT LICENSE.APACHE2
include README.rst
include CODE_OF_CONDUCT.md CONTRIBUTING.md
include test-requirements.txt .coveragerc
recursive-include docs *
prune docs/build
Metadata-Version: 1.2
Name: async_generator
Version: 1.9
Version: 1.10
Summary: Async generators and context managers for Python 3.5+
Home-page: https://github.com/python-trio/async_generator
Author: Nathaniel J. Smith
Author-email: njs@pobox.com
License: MIT
Description-Content-Type: UNKNOWN
License: MIT -or- Apache License 2.0
Description: .. image:: https://img.shields.io/badge/chat-join%20now-blue.svg
:target: https://gitter.im/python-trio/general
:alt: Join chatroom
......@@ -15,16 +14,16 @@ Description: .. image:: https://img.shields.io/badge/chat-join%20now-blue.svg
:target: https://async-generator.readthedocs.io/en/latest/?badge=latest
:alt: Documentation Status
.. image:: https://travis-ci.org/njsmith/async_generator.svg?branch=master
:target: https://travis-ci.org/njsmith/async_generator
.. image:: https://travis-ci.org/python-trio/async_generator.svg?branch=master
:target: https://travis-ci.org/python-trio/async_generator
:alt: Automated test status
.. image:: https://ci.appveyor.com/api/projects/status/af4eyed8o8tc3t0r/branch/master?svg=true
:target: https://ci.appveyor.com/project/python-trio/trio/history
:alt: Automated test status (Windows)
.. image:: https://codecov.io/gh/njsmith/async_generator/branch/master/graph/badge.svg
:target: https://codecov.io/gh/njsmith/async_generator
.. image:: https://codecov.io/gh/python-trio/async_generator/branch/master/graph/badge.svg
:target: https://codecov.io/gh/python-trio/async_generator
:alt: Test coverage
The async_generator library
......
......@@ -6,16 +6,16 @@
:target: https://async-generator.readthedocs.io/en/latest/?badge=latest
:alt: Documentation Status
.. image:: https://travis-ci.org/njsmith/async_generator.svg?branch=master
:target: https://travis-ci.org/njsmith/async_generator
.. image:: https://travis-ci.org/python-trio/async_generator.svg?branch=master
:target: https://travis-ci.org/python-trio/async_generator
:alt: Automated test status
.. image:: https://ci.appveyor.com/api/projects/status/af4eyed8o8tc3t0r/branch/master?svg=true
:target: https://ci.appveyor.com/project/python-trio/trio/history
:alt: Automated test status (Windows)
.. image:: https://codecov.io/gh/njsmith/async_generator/branch/master/graph/badge.svg
:target: https://codecov.io/gh/njsmith/async_generator
.. image:: https://codecov.io/gh/python-trio/async_generator/branch/master/graph/badge.svg
:target: https://codecov.io/gh/python-trio/async_generator
:alt: Test coverage
The async_generator library
......
Metadata-Version: 1.2
Name: async-generator
Version: 1.9
Version: 1.10
Summary: Async generators and context managers for Python 3.5+
Home-page: https://github.com/python-trio/async_generator
Author: Nathaniel J. Smith
Author-email: njs@pobox.com
License: MIT
Description-Content-Type: UNKNOWN
License: MIT -or- Apache License 2.0
Description: .. image:: https://img.shields.io/badge/chat-join%20now-blue.svg
:target: https://gitter.im/python-trio/general
:alt: Join chatroom
......@@ -15,16 +14,16 @@ Description: .. image:: https://img.shields.io/badge/chat-join%20now-blue.svg
:target: https://async-generator.readthedocs.io/en/latest/?badge=latest
:alt: Documentation Status
.. image:: https://travis-ci.org/njsmith/async_generator.svg?branch=master
:target: https://travis-ci.org/njsmith/async_generator
.. image:: https://travis-ci.org/python-trio/async_generator.svg?branch=master
:target: https://travis-ci.org/python-trio/async_generator
:alt: Automated test status
.. image:: https://ci.appveyor.com/api/projects/status/af4eyed8o8tc3t0r/branch/master?svg=true
:target: https://ci.appveyor.com/project/python-trio/trio/history
:alt: Automated test status (Windows)
.. image:: https://codecov.io/gh/njsmith/async_generator/branch/master/graph/badge.svg
:target: https://codecov.io/gh/njsmith/async_generator
.. image:: https://codecov.io/gh/python-trio/async_generator/branch/master/graph/badge.svg
:target: https://codecov.io/gh/python-trio/async_generator
:alt: Test coverage
The async_generator library
......
.coveragerc
CODE_OF_CONDUCT.md
CONTRIBUTING.md
LICENSE
LICENSE.APACHE2
LICENSE.MIT
MANIFEST.in
README.rst
setup.py
......@@ -15,4 +19,11 @@ async_generator.egg-info/top_level.txt
async_generator/_tests/__init__.py
async_generator/_tests/conftest.py
async_generator/_tests/test_async_generator.py
async_generator/_tests/test_util.py
\ No newline at end of file
async_generator/_tests/test_util.py
docs/Makefile
docs/make.bat
docs/source/conf.py
docs/source/history.rst
docs/source/index.rst
docs/source/reference.rst
docs/source/_static/.gitkeep
\ No newline at end of file
......@@ -5,6 +5,8 @@ from ._impl import (
yield_from_,
isasyncgen,
isasyncgenfunction,
get_asyncgen_hooks,
set_asyncgen_hooks,
)
from ._util import aclosing, asynccontextmanager
......@@ -16,4 +18,6 @@ __all__ = [
"isasyncgen",
"isasyncgenfunction",
"asynccontextmanager",
"get_asyncgen_hooks",
"set_asyncgen_hooks",
]
......@@ -45,7 +45,7 @@ def _unwrap(box):
# # We used to call _PyAsyncGenValueWrapperNew to create and set up new
# # wrapper objects, but that symbol isn't available on Windows:
# #
# # https://github.com/njsmith/async_generator/issues/5
# # https://github.com/python-trio/async_generator/issues/5
# #
# # Fortunately, the type object is available, but it means we have to do
# # this the hard way.
......@@ -225,11 +225,76 @@ class ANextIter:
return result
UNSPECIFIED = object()
try:
from sys import get_asyncgen_hooks, set_asyncgen_hooks
except ImportError:
import threading
asyncgen_hooks = collections.namedtuple(
"asyncgen_hooks", ("firstiter", "finalizer")
)
class _hooks_storage(threading.local):
def __init__(self):
self.firstiter = None
self.finalizer = None
_hooks = _hooks_storage()
def get_asyncgen_hooks():
return asyncgen_hooks(
firstiter=_hooks.firstiter, finalizer=_hooks.finalizer
)
def set_asyncgen_hooks(firstiter=UNSPECIFIED, finalizer=UNSPECIFIED):
if firstiter is not UNSPECIFIED:
if firstiter is None or callable(firstiter):
_hooks.firstiter = firstiter
else:
raise TypeError(
"callable firstiter expected, got {}".format(
type(firstiter).__name__
)
)
if finalizer is not UNSPECIFIED:
if finalizer is None or callable(finalizer):
_hooks.finalizer = finalizer
else:
raise TypeError(
"callable finalizer expected, got {}".format(
type(finalizer).__name__
)
)
class AsyncGenerator:
# https://bitbucket.org/pypy/pypy/issues/2786:
# PyPy implements 'await' in a way that requires the frame object
# used to execute a coroutine to keep a weakref to that coroutine.
# During a GC pass, weakrefs to all doomed objects are broken
# before any of the doomed objects' finalizers are invoked.
# If an AsyncGenerator is unreachable, its _coroutine probably
# is too, and the weakref from ag._coroutine.cr_frame to
# ag._coroutine will be broken before ag.__del__ can do its
# one-turn close attempt or can schedule a full aclose() using
# the registered finalization hook. It doesn't look like the
# underlying issue is likely to be fully fixed anytime soon,
# so we work around it by preventing an AsyncGenerator and
# its _coroutine from being considered newly unreachable at
# the same time if the AsyncGenerator's finalizer might want
# to iterate the coroutine some more.
_pypy_issue2786_workaround = set()
def __init__(self, coroutine):
self._coroutine = coroutine
self._it = coroutine.__await__()
self.ag_running = False
self._finalizer = None
self._closed = False
self._hooks_inited = False
# On python 3.5.0 and 3.5.1, __aiter__ must be awaitable.
# Starting in 3.5.2, it should not be awaitable, and if it is, then it
......@@ -263,33 +328,49 @@ class AsyncGenerator:
# Core functionality
################################################################
# We make these async functions and use await, rather than just regular
# functions that pass back awaitables, in order to get more useful
# tracebacks when debugging.
# These need to return awaitables, rather than being async functions,
# to match the native behavior where the firstiter hook is called
# immediately on asend()/etc, even if the coroutine that asend()
# produces isn't awaited for a bit.
def __anext__(self):
return self._do_it(self._it.__next__)
async def __anext__(self):
return await self._do_it(self._it.__next__)
def asend(self, value):
return self._do_it(self._it.send, value)
async def asend(self, value):
return await self._do_it(self._it.send, value)
def athrow(self, type, value=None, traceback=None):
return self._do_it(self._it.throw, type, value, traceback)
async def athrow(self, type, value=None, traceback=None):
return await self._do_it(self._it.throw, type, value, traceback)
def _do_it(self, start_fn, *args):
if not self._hooks_inited:
self._hooks_inited = True
(firstiter, self._finalizer) = get_asyncgen_hooks()
if firstiter is not None:
firstiter(self)
if sys.implementation.name == "pypy":
self._pypy_issue2786_workaround.add(self._coroutine)
async def _do_it(self, start_fn, *args):
# On CPython 3.5.2 (but not 3.5.0), coroutines get cranky if you try
# to iterate them after they're exhausted. Generators OTOH just raise
# StopIteration. We want to convert the one into the other, so we need
# to avoid iterating stopped coroutines.
if getcoroutinestate(self._coroutine) is CORO_CLOSED:
raise StopAsyncIteration()
if self.ag_running:
raise ValueError("async generator already executing")
try:
self.ag_running = True
return await ANextIter(self._it, start_fn, *args)
finally:
self.ag_running = False
async def step():
if self.ag_running:
raise ValueError("async generator already executing")
try:
self.ag_running = True
return await ANextIter(self._it, start_fn, *args)
except StopAsyncIteration:
self._pypy_issue2786_workaround.discard(self._coroutine)
raise
finally:
self.ag_running = False
return step()
################################################################
# Cleanup
......@@ -297,32 +378,54 @@ class AsyncGenerator:
async def aclose(self):
state = getcoroutinestate(self._coroutine)
if state is CORO_CLOSED or self._closed:
return
# Make sure that even if we raise "async_generator ignored
# GeneratorExit", and thus fail to exhaust the coroutine,
# __del__ doesn't complain again.
self._closed = True
if state is CORO_CREATED:
# Make sure that aclose() on an unstarted generator returns
# successfully and prevents future iteration.
self._it.close()
return
elif state is CORO_CLOSED:
return
try:
await self.athrow(GeneratorExit)
except (GeneratorExit, StopAsyncIteration):
pass
self._pypy_issue2786_workaround.discard(self._coroutine)
else:
raise RuntimeError("async_generator ignored GeneratorExit")
def __del__(self):
self._pypy_issue2786_workaround.discard(self._coroutine)
if getcoroutinestate(self._coroutine) is CORO_CREATED:
# Never started, nothing to clean up, just suppress the "coroutine
# never awaited" message.
self._coroutine.close()
if getcoroutinestate(self._coroutine) is CORO_SUSPENDED:
# This exception will get swallowed because this is __del__, but
# it's an easy way to trigger the print-to-console logic
raise RuntimeError(
"partially-exhausted async_generator {!r} garbage collected"
.format(self._coroutine.cr_frame.f_code.co_name)
)
if getcoroutinestate(self._coroutine
) is CORO_SUSPENDED and not self._closed:
if self._finalizer is not None:
self._finalizer(self)
else:
# Mimic the behavior of native generators on GC with no finalizer:
# throw in GeneratorExit, run for one turn, and complain if it didn't
# finish.
thrower = self.athrow(GeneratorExit)
try:
thrower.send(None)
except (GeneratorExit, StopAsyncIteration):
pass
except StopIteration:
raise RuntimeError("async_generator ignored GeneratorExit")
else:
raise RuntimeError(
"async_generator {!r} awaited during finalization; install "
"a finalization hook to support this, or wrap it in "
"'async with aclosing(...):'"
.format(self.ag_code.co_name)
)
finally:
thrower.close()
if hasattr(collections.abc, "AsyncGenerator"):
......
......@@ -13,6 +13,8 @@ from .. import (
yield_from_,
isasyncgen,
isasyncgenfunction,
get_asyncgen_hooks,
set_asyncgen_hooks,
)
......@@ -190,6 +192,22 @@ async def test_reentrance_forbidden():
pass # pragma: no cover
async def test_reentrance_forbidden_simultaneous_asends():
@async_generator
async def f():
await mock_sleep()
ag = f()
sender1 = ag.asend(None)
sender2 = ag.asend(None)
assert sender1.send(None) == "mock_sleep"
with pytest.raises(ValueError):
sender2.send(None)
with pytest.raises(StopAsyncIteration):
sender1.send(None)
await ag.aclose()
# https://bugs.python.org/issue32526
async def test_reentrance_forbidden_while_suspended_in_coroutine_runner():
@async_generator
......@@ -601,31 +619,68 @@ async def test_yield_from_athrow_raises_StopAsyncIteration():
################################################################
async def test___del__():
gen = async_range(10)
async def test___del__(capfd):
completions = 0
@async_generator
async def awaits_when_unwinding():
await yield_(0)
try:
await yield_(1)
finally:
await mock_sleep()
try:
await yield_(2)
finally:
nonlocal completions
completions += 1
gen = awaits_when_unwinding()
# Hasn't started yet, so no problem
gen.__del__()
gen = async_range(10)
await collect(gen)
gen = awaits_when_unwinding()
assert await collect(gen) == [0, 1, 2]
# Exhausted, so no problem
gen.__del__()
gen = async_range(10)
await gen.aclose()
# Closed, so no problem
gen.__del__()
for stop_after_turn in (1, 2, 3):
gen = awaits_when_unwinding()
for turn in range(stop_after_turn):
assert await gen.__anext__() == turn
await gen.aclose()
# Closed, so no problem
gen.__del__()
gen = async_range(10)
await gen.__anext__()
await gen.aclose()
# Closed, so no problem
gen.__del__()
for stop_after_turn in (1, 2, 3):
gen = awaits_when_unwinding()
for turn in range(stop_after_turn):
assert await gen.__anext__() == turn
if stop_after_turn == 2:
# Stopped in the middle of a try/finally that awaits in the finally,
# so __del__ can't cleanup.
with pytest.raises(RuntimeError) as info:
gen.__del__()
assert "awaited during finalization; install a finalization hook" in str(
info.value
)
else:
# Can clean up without awaiting, so __del__ is fine
gen.__del__()
gen = async_range(10)
await gen.__anext__()
# Started, but not exhausted or closed -- big problem
with pytest.raises(RuntimeError):
assert completions == 3
@async_generator
async def yields_when_unwinding():
try:
await yield_(1)
finally:
await yield_(2)
gen = yields_when_unwinding()
assert await gen.__anext__() == 1
with pytest.raises(RuntimeError) as info:
gen.__del__()
......@@ -779,3 +834,182 @@ async def test_no_spurious_unawaited_coroutine_warning(recwarn):
for msg in recwarn: # pragma: no cover
print(msg)
assert not issubclass(msg.category, RuntimeWarning)
################################################################
#
# GC hooks
#
################################################################
@pytest.fixture
def local_asyncgen_hooks():
old_hooks = get_asyncgen_hooks()
yield
set_asyncgen_hooks(*old_hooks)
def test_gc_hooks_interface(local_asyncgen_hooks):
def one(agen): # pragma: no cover
pass
def two(agen): # pragma: no cover
pass
set_asyncgen_hooks(None, None)
assert get_asyncgen_hooks() == (None, None)
set_asyncgen_hooks(finalizer=two)
assert get_asyncgen_hooks() == (None, two)
set_asyncgen_hooks(firstiter=one)
assert get_asyncgen_hooks() == (one, two)
set_asyncgen_hooks(finalizer=None, firstiter=two)
assert get_asyncgen_hooks() == (two, None)
set_asyncgen_hooks(None, one)
assert get_asyncgen_hooks() == (None, one)
tup = (one, two)
set_asyncgen_hooks(*tup)
assert get_asyncgen_hooks() == tup
with pytest.raises(TypeError):
set_asyncgen_hooks(firstiter=42)
with pytest.raises(TypeError):
set_asyncgen_hooks(finalizer=False)
def in_thread(results):
results.append(get_asyncgen_hooks())
set_asyncgen_hooks(two, one)
results.append(get_asyncgen_hooks())
from threading import Thread
results = []
thread = Thread(target=in_thread, args=(results,))
thread.start()
thread.join()
assert results == [(None, None), (two, one)]
assert get_asyncgen_hooks() == (one, two)
async def test_gc_hooks_behavior(local_asyncgen_hooks):
events = []
to_finalize = []
def firstiter(agen):
events.append("firstiter {}".format(agen.ag_frame.f_locals["ident"]))
def finalizer(agen):
events.append("finalizer {}".format(agen.ag_frame.f_locals["ident"]))
to_finalize.append(agen)
@async_generator
async def agen(ident):
events.append("yield 1 {}".format(ident))
await yield_(1)
try:
events.append("yield 2 {}".format(ident))
await yield_(2)
events.append("after yield 2 {}".format(ident))
finally:
events.append("mock_sleep {}".format(ident))
await mock_sleep()
try:
events.append("yield 3 {}".format(ident))
await yield_(3)
finally:
events.append("unwind 3 {}".format(ident))
# this one is included to make sure we _don't_ execute it
events.append("done {}".format(ident)) # pragma: no cover
async def anext_verbosely(iter, ident):
events.append("before asend {}".format(ident))
sender = iter.asend(None)
events.append("before send {}".format(ident))
await sender
events.append("after asend {}".format(ident))
# Ensure that firstiter is called immediately on asend(),
# before the first turn of the coroutine that asend() returns,
# to match the behavior of native generators.
# Ensure that the firstiter that gets used is the one in effect
# at the time of that first call, rather than at the time of iteration.
iterA = agen("A")
iterB = agen("B")
await anext_verbosely(iterA, "A")
set_asyncgen_hooks(firstiter, finalizer)
await anext_verbosely(iterB, "B")
iterC = agen("C")
await anext_verbosely(iterC, "C")
assert events == [
"before asend A", "before send A", "yield 1 A", "after asend A",
"before asend B", "firstiter B", "before send B", "yield 1 B",
"after asend B", "before asend C", "firstiter C", "before send C",
"yield 1 C", "after asend C"
]
del events[:]
# Ensure that firstiter is only called once, even if we create
# two asend() coroutines before iterating either of them.
iterX = agen("X")
sender1 = iterX.asend(None)
sender2 = iterX.asend(None)
events.append("before close")
sender1.close()
sender2.close()
await iterX.aclose()
assert events == ["firstiter X", "before close"]
del events[:]
from weakref import ref
refA, refB, refC = map(ref, (iterA, iterB, iterC))
# iterA uses the finalizer that was in effect when it started, i.e. no finalizer
await iterA.__anext__()
await iterA.__anext__()
del iterA
# Do multiple GC passes since we're deliberately shielding the
# coroutine objects from the first pass due to PyPy issue 2786.
for _ in range(4):
gc.collect()
assert refA() is None
assert events == [
"yield 2 A", "after yield 2 A", "mock_sleep A", "yield 3 A",
"unwind 3 A"
]
assert not to_finalize
del events[:]
# iterB and iterC do use our finalizer
await iterC.__anext__()
await iterB.__anext__()
await iterC.__anext__()
idB, idC = id(iterB), id(iterC)
del iterB
for _ in range(4):
gc.collect()
del iterC
for _ in range(4):
gc.collect()
assert events == [
"yield 2 C", "yield 2 B", "after yield 2 C", "mock_sleep C",
"yield 3 C", "finalizer B", "finalizer C"
]
del events[:]
# finalizer invokes aclose() is not called again once the revived reference drops
assert list(map(id, to_finalize)) == [idB, idC]
events.append("before aclose B")
await to_finalize[0].aclose()
events.append("before aclose C")
await to_finalize[1].aclose()
events.append("after aclose both")
del to_finalize[:]
for _ in range(4):
gc.collect()
assert refB() is None and refC() is None
assert events == [
"before aclose B", "mock_sleep B", "before aclose C", "unwind 3 C",
"after aclose both"
]
__version__ = "1.9"
__version__ = "1.10"
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
SPHINXPROJ = async_generator
SOURCEDIR = source
BUILDDIR = build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
@ECHO OFF
pushd %~dp0
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=source
set BUILDDIR=build
set SPHINXPROJ=async_generator
if "%1" == "" goto help
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (