Commit d10c8653 authored by Julien Puydt's avatar Julien Puydt

New upstream version 7.0.0+ds

parent 89918492
......@@ -15,8 +15,9 @@ install:
- "mklink /d \"C:\\Program Files\\Python\" %PYTHON%"
- "SET PYTHON=\"C:\\Program Files\\Python\""
- "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
- "python -m pip install --upgrade --force-reinstall virtualenv"
- "python -m pip install tox"
- "python -m tox --notest"
- "python -m tox --notest -vv"
before_build:
......
ref-names: tag: v6.5.8
ref-names: HEAD -> master, tag: v7.0.0
......@@ -3,3 +3,5 @@
# Needed for setuptools-scm-git-archive
.git_archival.txt export-subst
CHANGES.rst merge=union
......@@ -176,3 +176,6 @@ ENV/
# mypy
.mypy_cache
# pip wheel dist info
pip-wheel-metadata/
......@@ -10,6 +10,12 @@ python:
- 3.6
- &pypy3 pypy3
env:
global:
GIT_INSTALLER_DIR_PATH: ${HOME}/.git-installers
GIT_VERSION: 2.20.1
PYTHON_INSTALLER_DIR_PATH: ${HOME}/.python-installers
_base_envs:
- &stage_lint
stage: &stage_lint_name lint
......@@ -50,8 +56,14 @@ _base_envs:
<<: *pyenv_base
<<: *stage_test_osx
os: osx
osx_image: xcode8.3
language: generic
before_install:
cache:
directories:
- $HOME/Library/Caches/Homebrew
- $PYTHON_INSTALLER_DIR_PATH
- $GIT_INSTALLER_DIR_PATH
before_install: &install-from-pyenv
- brew install zlib readline
- brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/pyenv.rb || brew upgrade pyenv
- brew install openssl@1.1
......@@ -64,6 +76,71 @@ _base_envs:
- python -m pip install tox
- env CRYPTOGRAPHY_SUPPRESS_LINK_FLAGS=1 LDFLAGS="$(brew --prefix openssl@1.1)/lib/libssl.a $(brew --prefix openssl@1.1)/lib/libcrypto.a" CFLAGS="-I$(brew --prefix openssl@1.1)/include" python -m tox --notest
- *python_ssl_openssl_version
before_install: &install-from-python_org
- |
function probe_url() {
local py_ver="$1"
[ $(curl -I --write-out '%{http_code}' --silent --output /dev/null "https://www.python.org/ftp/python/${py_ver}/python-${py_ver}-macosx10.6.pkg") == '200' ] && return 0
return 1
}
- |
function find_last_macos_py() {
for py_ver in $*
do
>&2 echo Probing $py_ver
probe_url $py_ver && >&2 echo "Found pkg: ${py_ver}" && echo $py_ver && return 0
done
>&2 echo Failed looking up macOS pkg for $*
return 1
}
- export GIT_DMG_NAME="git-${GIT_VERSION}-intel-universal-mavericks.dmg"
- export GIT_PKG_NAME="git-${GIT_VERSION}-intel-universal-mavericks.pkg"
- export GIT_DMG_PATH="${GIT_INSTALLER_DIR_PATH}/${GIT_DMG_NAME}"
- >
stat "${GIT_DMG_PATH}" &>/dev/null || wget -O "${GIT_DMG_PATH}" "https://sourceforge.net/projects/git-osx-installer/files/${GIT_DMG_NAME}/download?use_mirror=autoselect"
- stat "${GIT_DMG_PATH}" >/dev/null
- sudo hdiutil attach ${GIT_DMG_PATH}
- hdiutil info
- >
export GIT_INSTALLER_VOLUME=$(hdiutil info | tail -n1 | sed 's#^.*\(/Volumes.*\)#\1#')
- >
export GIT_INSTALLER_PATH="${GIT_INSTALLER_VOLUME}/${GIT_PKG_NAME}"
- ls -alh "${GIT_INSTALLER_VOLUME}"
- sudo installer -verboseR -dumplog -pkg "${GIT_INSTALLER_PATH}" -target /
- sudo hdiutil detach "${GIT_INSTALLER_VOLUME}"
- export PYTHON_VERSION_LONG_SUGGESTIONS=$(git ls-remote --sort -v:refname --tags git://github.com/python/cpython.git "${PYTHON_VERSION}*" "v${PYTHON_VERSION}*" | grep -v '\^{}$' | awk '{print$2}' | sed 's#^refs/tags/##;s#^v##' | grep -v '[abcepr]')
- export PYTHON_VERSION_LONG=$(find_last_macos_py $PYTHON_VERSION_LONG_SUGGESTIONS)
- export PYTHON_VERSION_SHORT=$(echo ${PYTHON_VERSION_LONG} | awk -F. '{print$1"."$2}')
- echo "Selected version vars are:"
- echo "PYTHON_VERSION=${PYTHON_VERSION}"
- echo "PYTHON_VERSION_SHORT=${PYTHON_VERSION_SHORT}"
- echo "PYTHON_VERSION_LONG=${PYTHON_VERSION_LONG}"
- export PYTHON_INSTALL_PATH="/Library/Frameworks/Python.framework/Versions/${PYTHON_VERSION_SHORT}"
- export PYTHON_INSTALL_EXE="${PYTHON_INSTALL_PATH}/bin/python${PYTHON_VERSION_SHORT}"
- export PATH="${PYTHON_INSTALL_PATH}/bin:${PATH}"
- export PYTHON_VENV_PATH="${HOME}/virtualenv/python${PYTHON_VERSION_SHORT}"
- export PYTHON_INSTALLER_PATH="${PYTHON_INSTALLER_DIR_PATH}/python-${PYTHON_VERSION_LONG}.pkg"
- echo "PYTHON_INSTALLER_PATH=${PYTHON_INSTALLER_PATH}"
- env
- >
stat "${PYTHON_INSTALLER_PATH}" &>/dev/null || wget -O "${PYTHON_INSTALLER_PATH}" "https://www.python.org/ftp/python/${PYTHON_VERSION_LONG}/python-${PYTHON_VERSION_LONG}-macosx10.6.pkg"
- stat "${PYTHON_INSTALLER_PATH}" >/dev/null
- sudo installer -verboseR -dumplog -pkg "${PYTHON_INSTALLER_PATH}" -target /
- ls "${PYTHON_INSTALL_PATH}/bin"
- ls -lh "${PYTHON_INSTALL_EXE}"
- stat "${PYTHON_INSTALL_EXE}"
- /Applications/Python\ ${PYTHON_VERSION_SHORT}/Install\ Certificates.command || echo "No need to fix certificates"
- curl https://bootstrap.pypa.io/get-pip.py | ${PYTHON_INSTALL_EXE}
- >
"${PYTHON_INSTALL_EXE}" -m pip install -U pip
- >
"${PYTHON_INSTALL_EXE}" -m pip install -U virtualenv
- >
"${PYTHON_INSTALL_EXE}" -m virtualenv "${PYTHON_VENV_PATH}"
- . "${PYTHON_VENV_PATH}/bin/activate"
- curl https://bootstrap.pypa.io/get-pip.py | python
- python --version
- pip --version
before_cache:
- brew --cache
- &python_3_7_mixture
......@@ -107,37 +184,28 @@ jobs:
<<: *manual_run_or_cron
python: 2.7
env:
- PYTHON_VERSION=2.7.13
- *env_pyenv
- *env_path
PYTHON_VERSION: 2.7
- <<: *osx_python_base
<<: *manual_run_or_cron
python: 3.4
env:
- PYTHON_VERSION=3.4.6
- *env_pyenv
- *env_path
PYTHON_VERSION: 3.4
- <<: *osx_python_base
<<: *manual_run_or_cron
python: 3.5
env:
- PYTHON_VERSION=3.5.3
- *env_pyenv
- *env_path
PYTHON_VERSION: 3.5
- <<: *osx_python_base
python: 3.6
env:
- PYTHON_VERSION=3.6.5
- *env_pyenv
- *env_path
PYTHON_VERSION: 3.6
- <<: *osx_python_base
python: *mainstream_python
env:
- PYTHON_VERSION=3.7.0
- *env_pyenv
- *env_path
PYTHON_VERSION: *mainstream_python
- <<: *osx_python_base
<<: *manual_run_or_cron
before_install: *install-from-pyenv
python: nightly
env:
- PYTHON_VERSION=3.8-dev
......@@ -146,6 +214,7 @@ jobs:
- <<: *osx_python_base
<<: *manual_run_or_cron
osx_image: xcode9.4
before_install: *install-from-pyenv
python: pypy3.6-7.1.0
env:
- PYTHON_VERSION=pypy3.6-7.1.0
......@@ -155,17 +224,10 @@ jobs:
- <<: *stage_deploy
if: tag IS present
<<: *python_3_7_mixture
install: skip
script: skip
deploy:
provider: pypi
on:
all_branches: true
user: jaraco
password:
secure: RAfz06AINvz7bfij/YhfkAreRqamgxS8a6jSRNxntYhtJke3ZszUbIDag8+n1I+G5XT2LnMhHqPNR7Plc+AeMz7VXTuy+b81Li5kse20NYlPhd7mBVmTUpXtqYQashV5J39F4qkATBLznCOrMEomM07VTXjO/o2hmQuXniab2Uo=
distributions: dists
skip_cleanup: true
install:
- python -m pip install tox tox-venv
env:
TOXENV: release
cache:
pip: true
......
v7.0.0
======
- :pr:`224`: Refactored "open URL" behavior in webtest to
rely on `retry_call
<https://jaracofunctools.readthedocs.io/en/latest/?badge=latest#jaraco.functools.retry_call>`_.
Callers can no longer pass ``raise_subcls`` or ``ssl_context``
positionally, but must pass them as keyword arguments.
v6.6.0
======
- Revisit :pr:`85` under :pr:`221`. Now
``backports.functools_lru_cache`` is only
required on Python 3.2 and earlier.
- :cp-issue:`1206` via :pr:`204`: Fix race condition in
threadpool shrink code.
v6.5.8
======
......
......@@ -36,6 +36,8 @@ def http_server():
"""Provision a server creator as a fixture."""
def start_srv():
bind_addr = yield
if bind_addr is None:
return
httpserver = make_http_server(bind_addr)
yield httpserver
yield httpserver
......
......@@ -476,5 +476,8 @@ def test_http_over_https_error(
underlying_error = ssl_err.value.args[0].args[-1]
err_text = str(underlying_error)
assert underlying_error.errno == expected_error_code
assert underlying_error.errno == expected_error_code, (
'The underlying error is {!r}'.
format(underlying_error)
)
assert expected_error_text in err_text
......@@ -26,11 +26,13 @@ import os
import json
import unittest
import warnings
import functools
from six.moves import range, http_client, map, urllib_parse
from six.moves import http_client, map, urllib_parse
import six
from more_itertools.more import always_iterable
import jaraco.functools
def interface(host):
......@@ -154,7 +156,7 @@ class WebCase(unittest.TestCase):
)
@property
def persistent(self): # noqa: D401; irrelevant for properties
def persistent(self):
"""Presense of the persistent HTTP connection."""
return hasattr(self.HTTP_CONN, '__class__')
......@@ -191,10 +193,7 @@ class WebCase(unittest.TestCase):
*args, **kwargs
)
`raise_subcls` must be a tuple with the exceptions classes
or a single exception class that are not going to be considered
a socket.error regardless that they were are subclass of a
socket.error and therefore not considered for a connection retry.
`raise_subcls` is passed through to openURL.
"""
ServerError.on = False
......@@ -209,7 +208,7 @@ class WebCase(unittest.TestCase):
result = openURL(
url, headers, method, body, self.HOST, self.PORT,
self.HTTP_CONN, protocol or self.PROTOCOL,
raise_subcls, ssl_context=self.ssl_context,
raise_subcls=raise_subcls, ssl_context=self.ssl_context,
)
self.time = time.time() - start
self.status, self.headers, self.body = result
......@@ -478,68 +477,69 @@ def shb(response):
return resp_status_line, h, response.read()
def openURL(
url, headers=None, method='GET', body=None,
host='127.0.0.1', port=8000, http_conn=http_client.HTTPConnection,
protocol='HTTP/1.1', raise_subcls=None, ssl_context=None,
):
# def openURL(*args, raise_subcls=(), **kwargs):
# py27 compatible signature:
def openURL(*args, **kwargs):
"""
Open the given HTTP resource and return status, headers, and body.
Open a URL, retrying when it fails.
`raise_subcls` must be a tuple with the exceptions classes
or a single exception class that are not going to be considered
a socket.error regardless that they were are subclass of a
socket.error and therefore not considered for a connection retry.
Specify `raise_subcls` (class or tuple of classes) to exclude
those socket.error subclasses from being suppressed and retried.
"""
headers = cleanHeaders(headers, method, body, host, port)
# Trying 10 times is simply in case of socket errors.
# Normal case--it should run once.
for trial in range(10):
try:
# Allow http_conn to be a class or an instance
if hasattr(http_conn, 'host'):
conn = http_conn
else:
kw = {}
if ssl_context:
kw['context'] = ssl_context
conn = http_conn(interface(host), port, **kw)
conn._http_vsn_str = protocol
conn._http_vsn = int(''.join([x for x in protocol if x.isdigit()]))
if not six.PY2 and isinstance(url, bytes):
url = url.decode()
conn.putrequest(
method.upper(), url, skip_host=True,
skip_accept_encoding=True,
)
raise_subcls = kwargs.pop('raise_subcls', ())
opener = functools.partial(_open_url_once, *args, **kwargs)
for key, value in headers:
conn.putheader(key, value.encode('Latin-1'))
conn.endheaders()
def on_exception():
type_, exc = sys.exc_info()[:2]
if isinstance(exc, raise_subcls):
raise
time.sleep(0.5)
if body is not None:
conn.send(body)
# Try up to 10 times
return jaraco.functools.retry_call(
opener,
retries=9,
cleanup=on_exception,
trap=socket.error,
)
# Handle response
response = conn.getresponse()
s, h, b = shb(response)
if not hasattr(http_conn, 'host'):
# We made our own conn instance. Close it.
conn.close()
def _open_url_once(
url, headers=None, method='GET', body=None,
host='127.0.0.1', port=8000, http_conn=http_client.HTTPConnection,
protocol='HTTP/1.1', ssl_context=None,
):
"""Open the given HTTP resource and return status, headers, and body."""
headers = cleanHeaders(headers, method, body, host, port)
return s, h, b
except socket.error as e:
if raise_subcls is not None and isinstance(e, raise_subcls):
raise
else:
time.sleep(0.5)
if trial == 9:
raise
# Allow http_conn to be a class or an instance
if hasattr(http_conn, 'host'):
conn = http_conn
else:
kw = {}
if ssl_context:
kw['context'] = ssl_context
conn = http_conn(interface(host), port, **kw)
conn._http_vsn_str = protocol
conn._http_vsn = int(''.join([x for x in protocol if x.isdigit()]))
if not six.PY2 and isinstance(url, bytes):
url = url.decode()
conn.putrequest(
method.upper(), url, skip_host=True,
skip_accept_encoding=True,
)
for key, value in headers:
conn.putheader(key, value.encode('Latin-1'))
conn.endheaders()
if body is not None:
conn.send(body)
# Handle response
response = conn.getresponse()
s, h, b = shb(response)
if not hasattr(http_conn, 'host'):
# We made our own conn instance. Close it.
conn.close()
return s, h, b
def strip_netloc(url):
......
......@@ -4,6 +4,7 @@ from __future__ import absolute_import, division, print_function
__metaclass__ = type
import collections
import threading
import time
import socket
......@@ -156,6 +157,7 @@ class ThreadPool:
self._queue = queue.Queue(maxsize=accepted_queue_size)
self._queue_put_timeout = accepted_queue_timeout
self.get = self._queue.get
self._pending_shutdowns = collections.deque()
def start(self):
"""Start the pool of threads."""
......@@ -171,7 +173,8 @@ class ThreadPool:
@property
def idle(self): # noqa: D401; irrelevant for properties
"""Number of worker threads which are idle. Read-only."""
return len([t for t in self._threads if t.conn is None])
idles = len([t for t in self._threads if t.conn is None])
return max(idles - len(self._pending_shutdowns), 0)
def put(self, obj):
"""Put request into queue.
......@@ -181,8 +184,15 @@ class ThreadPool:
waiting to be processed
"""
self._queue.put(obj, block=True, timeout=self._queue_put_timeout)
if obj is _SHUTDOWNREQUEST:
return
def _clear_dead_threads(self):
# Remove any dead threads from our list
for t in [t for t in self._threads if not t.is_alive()]:
self._threads.remove(t)
try:
self._pending_shutdowns.popleft()
except IndexError:
pass
def grow(self, amount):
"""Spawn new worker threads (not above self.max)."""
......@@ -209,10 +219,10 @@ class ThreadPool:
"""Kill off worker threads (not below self.min)."""
# Grow/shrink the pool if necessary.
# Remove any dead threads from our list
for t in self._threads:
if not t.is_alive():
self._threads.remove(t)
amount -= 1
amount -= len(self._pending_shutdowns)
self._clear_dead_threads()
if amount <= 0:
return
# calculate the number of threads above the minimum
n_extra = max(len(self._threads) - self.min, 0)
......@@ -224,6 +234,7 @@ class ThreadPool:
# to remove. As each request is processed by a worker, that worker
# will terminate and be culled from the list.
for n in range(n_to_remove):
self._pending_shutdowns.append(None)
self._queue.put(_SHUTDOWNREQUEST)
def stop(self, timeout=5):
......
[build-system]
requires = ["setuptools>=30.3", "wheel", "setuptools_scm>=1.15"]
requires = ["setuptools>=34.4", "wheel", "setuptools_scm>=1.15"]
build-backend = "setuptools.build_meta"
......@@ -62,9 +62,10 @@ setup_requires =
# These are required in actual runtime:
install_requires =
backports.functools_lru_cache
backports.functools_lru_cache; python_version < '3.3'
six>=1.11.0
more_itertools>=2.6
jaraco.functools
[options.extras_require]
docs =
......
......@@ -80,3 +80,18 @@ commands =
python -m setup checkdocs check --metadata --restructuredtext --strict --verbose
python -m twine check .tox/dist/*
python -m setuptools_scm ls
[testenv:release]
skip_install = True
deps =
pep517>=0.5
twine>=1.13
path.py
passenv =
TWINE_PASSWORD
setenv =
TWINE_USERNAME = {env:TWINE_USERNAME:__token__}
commands =
python -c "import path; path.Path('dist').rmtree_p()"
python -m pep517.build .
python -m twine upload dist/*
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment