Commit c4dfc7e6 authored by Julian Taylor's avatar Julian Taylor

Import pyzmq_15.2.0.orig.tar.gz

parent 92e15aa1
......@@ -18,8 +18,7 @@ before_install:
- 'if [[ $TRAVIS_PYTHON_VERSION != pypy* ]]; then pip install -q -f travis-wheels-master/wheelhouse cython; fi'
- 'if [[ ! -z "$ZMQ" && $ZMQ != bundled ]]; then wget$ZMQ/archive/ -O && unzip; fi'
- 'if [[ ! -z "$ZMQ" && $ZMQ != bundled ]]; then sh -c "set -x; cd $ZMQ-master; sh; ./configure; make -j; sudo make install; sudo ldconfig"; fi'
- 'if [[ $TRAVIS_PYTHON_VERSION = 2.7 ]]; then pip install -q -f travis-wheels-master/wheelhouse gevent; fi'
- pip install -q -f file://travis-wheels-master/wheelhouse nose tornado
- pip install -q -f file://travis-wheels-master/wheelhouse -r test-requirements.txt
- 'if [[ ! -z "$ZMQ" && $ZMQ != bundled ]]; then export ZMQ=/usr/local; fi'
......@@ -9,6 +9,17 @@ Changes in PyZMQ
This is a coarse summary of changes in pyzmq versions. For a real changelog, consult the
`git log <>`_
- FIX: handle multiple events in a single register call in :mod:`zmq.asyncio`
- FIX: unicode/bytes bug in password prompt in :mod:`zmq.ssh` on Python 3
- FIX: workaround gevent monkeypatches in garbage collection thread
- update bundled minitornado from tornado-4.3.
- improved inspection by setting ``binding=True`` in cython compile options
- add asyncio Authenticator implementation in :mod:`zmq.auth.asyncio`
- workaround overflow bug in libzmq preventing receiving messages larger than ``MAX_INT``
#!/usr/bin/env python
Ironhouse extends Stonehouse with client public key authentication.
This is the strongest security model we have today, protecting against every
attack we know about, except end-point attacks (where an attacker plants
spyware on a machine to capture data before it's encrypted, or after it's
Author: Steven Armstrong
Based on ./ by Chris Laws
import logging
import os
import sys
import asyncio
import zmq
import zmq.auth
from zmq.auth.asyncio import AsyncioAuthenticator
from zmq.asyncio import Context, Poller, ZMQEventLoop
def run():
''' Run Ironhouse example '''
# These directories are generated by the generate_certificates script
base_dir = os.path.dirname(__file__)
keys_dir = os.path.join(base_dir, 'certificates')
public_keys_dir = os.path.join(base_dir, 'public_keys')
secret_keys_dir = os.path.join(base_dir, 'private_keys')
if not (os.path.exists(keys_dir) and
os.path.exists(public_keys_dir) and
logging.critical("Certificates are missing - run script first")
ctx = Context.instance()
# Start an authenticator for this context.
auth = AsyncioAuthenticator(ctx)
# Tell authenticator to use the certificate in a directory
auth.configure_curve(domain='*', location=public_keys_dir)
server = ctx.socket(zmq.PUSH)
server_secret_file = os.path.join(secret_keys_dir, "server.key_secret")
server_public, server_secret = zmq.auth.load_certificate(server_secret_file)
server.curve_secretkey = server_secret
server.curve_publickey = server_public
server.curve_server = True # must come before bind
client = ctx.socket(zmq.PULL)
# We need two certificates, one for the client and one for
# the server. The client must know the server's public key
# to make a CURVE connection.
client_secret_file = os.path.join(secret_keys_dir, "client.key_secret")
client_public, client_secret = zmq.auth.load_certificate(client_secret_file)
client.curve_secretkey = client_secret
client.curve_publickey = client_public
server_public_file = os.path.join(public_keys_dir, "server.key")
server_public, _ = zmq.auth.load_certificate(server_public_file)
# The client must know the server's public key to make a CURVE connection.
client.curve_serverkey = server_public
yield from server.send(b"Hello")
if (yield from client.poll(1000)):
msg = yield from client.recv()
if msg == b"Hello":"Ironhouse test OK")
logging.error("Ironhouse test FAIL")
# close sockets
# stop auth task
if __name__ == '__main__':
if zmq.zmq_version_info() < (4,0):
raise RuntimeError("Security is not supported in libzmq version < 4.0. libzmq version {0}".format(zmq.zmq_version()))
if '-v' in sys.argv:
level = logging.DEBUG
level = logging.INFO
logging.basicConfig(level=level, format="[%(levelname)s] %(message)s")
loop = ZMQEventLoop()
......@@ -1120,6 +1120,11 @@ else:
class zbuild_ext(build_ext_c):
def finalize_options(self):
# set binding so that compiled methods can be inspected
self.cython_directives['binding'] = True
def build_extensions(self):
if self.compiler.compiler_type == 'mingw32':
gevent; python_version == '2.7' and platform_python_implementation != "PyPy"
aiohttp; python_version >= '3.4'
......@@ -5,15 +5,15 @@ Requires asyncio and Python 3.
# Copyright (c) PyZMQ Developers.
# Distributed under the terms of the Modified BSD License.
# Derived from Python 3.5.1 selectors._BaseSelectorImpl, used under PSF License
from functools import partial
from collections import Mapping
import zmq as _zmq
from zmq.eventloop import future as _future
# TODO: support trollius for Legacy Python? (probably not)
import sys
import asyncio
from asyncio import SelectorEventLoop, Future
......@@ -22,34 +22,124 @@ except ImportError:
from asyncio import selectors # py33
_aio2zmq = {
_aio2zmq_map = {
selectors.EVENT_READ: _zmq.POLLIN,
selectors.EVENT_WRITE: _zmq.POLLOUT,
_zmq2aio = { z:a for a,z in _aio2zmq.items() }
for aio_evt in _aio2zmq_map:
_AIO_EVENTS |= aio_evt
def _aio2zmq(aio_evt):
"""Turn AsyncIO event mask into ZMQ event mask"""
z_evt = 0
for aio_mask, z_mask in _aio2zmq_map.items():
if aio_mask & aio_evt:
z_evt |= z_mask
return z_evt
def _zmq2aio(z_evt):
"""Turn ZMQ event mask into AsyncIO event mask"""
aio_evt = 0
for aio_mask, z_mask in _aio2zmq_map.items():
if z_mask & z_evt:
aio_evt |= aio_mask
return aio_evt
class _AsyncIO(object):
_Future = Future
_WRITE = selectors.EVENT_WRITE
_READ = selectors.EVENT_READ
def _default_loop(self):
return asyncio.get_event_loop()
def _fileobj_to_fd(fileobj):
"""Return a file descriptor from a file object.
fileobj -- file object or file descriptor
corresponding file descriptor
ValueError if the object is invalid
if isinstance(fileobj, int):
fd = fileobj
fd = int(fileobj.fileno())
except (AttributeError, TypeError, ValueError):
raise ValueError("Invalid file object: "
"{!r}".format(fileobj)) from None
if fd < 0:
raise ValueError("Invalid file descriptor: {}".format(fd))
return fd
class _SelectorMapping(Mapping):
"""Mapping of file objects to selector keys."""
def __init__(self, selector):
self._selector = selector
def __len__(self):
return len(self._selector._fd_to_key)
def __getitem__(self, fileobj):
fd = self._selector._fileobj_lookup(fileobj)
return self._selector._fd_to_key[fd]
except KeyError:
raise KeyError("{!r} is not registered".format(fileobj)) from None
def __iter__(self):
return iter(self._selector._fd_to_key)
class ZMQSelector(selectors.BaseSelector):
"""zmq_poll-based selector for asyncio"""
def __init__(self):
self.poller = _zmq.Poller()
self._mapping = {}
# this maps file descriptors to keys
self._fd_to_key = {}
# read-only mapping returned by get_map()
self._map = _SelectorMapping(self)
self._zmq_poller = _zmq.Poller()
def _fileobj_lookup(self, fileobj):
"""Return a zmq socket or a file descriptor from a file object.
This wraps _fileobj_to_fd() to do an exhaustive search in case
the object is invalid but we still have it in our map. This
is used by unregister() so we can unregister an object that
was previously registered even if it is closed. It is also
used by _SelectorMapping.
if isinstance(fileobj, _zmq.Socket):
return fileobj
return _fileobj_to_fd(fileobj)
except ValueError:
# Do an exhaustive search.
for key in self._fd_to_key.values():
if key.fileobj is fileobj:
return key.fd
# Raise ValueError after all.
def register(self, fileobj, events, data=None):
"""Register a file object.
fileobj -- file object or file descriptor
fileobj -- zmq socket, file object or file descriptor
events -- events to monitor (bitwise mask of EVENT_READ|EVENT_WRITE)
data -- attached data
......@@ -65,21 +155,25 @@ class ZMQSelector(selectors.BaseSelector):
OSError may or may not be raised
if fileobj in self.poller:
raise KeyError(fileobj)
if not isinstance(events, int) or events not in _aio2zmq:
raise ValueError("Invalid events: %r" % events)
self.poller.register(fileobj, _aio2zmq[events])
key = selectors.SelectorKey(fileobj=fileobj, fd=fileobj if isinstance(fileobj, int) else None, events=events, data=data)
self._mapping[fileobj] = key
if (not events) or (events & ~(selectors.EVENT_READ | selectors.EVENT_WRITE)):
raise ValueError("Invalid events: {!r}".format(events))
key = selectors.SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data)
if key.fd in self._fd_to_key:
raise KeyError("{!r} (FD {}) is already registered"
.format(fileobj, key.fd))
self._fd_to_key[key.fd] = key
self._zmq_poller.register(key.fd, _aio2zmq(events))
return key
def unregister(self, fileobj):
"""Unregister a file object.
fileobj -- file object or file descriptor
fileobj -- zmq socket, file object or file descriptor
SelectorKey instance
......@@ -91,11 +185,27 @@ class ZMQSelector(selectors.BaseSelector):
If fileobj is registered but has since been closed this does
*not* raise OSError (even if the wrapped syscall does)
if fileobj not in self.poller:
raise KeyError(fileobj)
return self._mapping.pop(fileobj)
key = self._fd_to_key.pop(self._fileobj_lookup(fileobj))
except KeyError:
raise KeyError("{!r} is not registered".format(fileobj)) from None
return key
def modify(self, fileobj, events, data=None):
key = self._fd_to_key[self._fileobj_lookup(fileobj)]
except KeyError:
raise KeyError("{!r} is not registered".format(fileobj)) from None
if events !=
key = self.register(fileobj, events, data)
elif data !=
# Use a shortcut to update the data.
key = key._replace(data=data)
self._fd_to_key[key.fd] = key
return key
def select(self, timeout=None):
"""Perform the actual selection, until some monitored file objects are
......@@ -118,29 +228,51 @@ class ZMQSelector(selectors.BaseSelector):
timeout = 0
timeout = 1e3 * timeout
events = self.poller.poll(timeout)
return [ (self.get_key(fd), _zmq2aio[evt]) for fd, evt in events ]
fd_event_list = self._zmq_poller.poll(timeout)
ready = []
for fd, event in fd_event_list:
key = self._key_from_fd(fd)
if key:
events = _zmq2aio(event)
ready.append((key, events))
return ready
def close(self):
"""Close the selector.
This must be called to make sure that any underlying resource is freed.
self._mapping = None
self._poller = None
self._map = None
self._zmq_poller = None
def get_map(self):
"""Return a mapping of file objects to selector keys."""
return self._mapping
return self._map
def _key_from_fd(self, fd):
"""Return the key associated to a given file descriptor.
fd -- file descriptor
corresponding key, or None if not found
return self._fd_to_key[fd]
except KeyError:
return None
class Poller(_AsyncIO, _future._AsyncPoller):
"""Poller returning asyncio.Future for poll results."""
class Socket(_AsyncIO, _future._AsyncSocket):
"""Socket returning asyncio Futures for send/recv/poll methods."""
_poller_class = Poller
def _add_io_state(self, state):
......@@ -151,7 +283,7 @@ class Socket(_AsyncIO, _future._AsyncSocket):
self.io_loop.add_reader(self, self._handle_recv)
if state & self._WRITE:
self.io_loop.add_writer(self, self._handle_send)
def _drop_io_state(self, state):
"""Stop poller from watching an io_state."""
if self._state & state:
......@@ -160,7 +292,7 @@ class Socket(_AsyncIO, _future._AsyncSocket):
if state & self._WRITE:
def _init_io_state(self):
"""initialize the ioloop event handler"""
......@@ -177,11 +309,12 @@ class ZMQEventLoop(SelectorEventLoop):
selector = ZMQSelector()
return super(ZMQEventLoop, self).__init__(selector)
_loop = None
def install():
"""Install and return the global ZMQEventLoop
registers the loop with asyncio.set_event_loop
global _loop
......@@ -189,7 +322,7 @@ def install():
_loop = ZMQEventLoop()
return _loop
__all__ = [
......@@ -197,4 +330,4 @@ __all__ = [
\ No newline at end of file
......@@ -2,6 +2,7 @@
To run authentication in a background thread, see :mod:`zmq.auth.thread`.
For integration with the tornado eventloop, see :mod:`zmq.auth.ioloop`.
For integration with the asyncio event loop, see :mod:`zmq.auth.asyncio`.
.. versionadded:: 14.1
"""ZAP Authenticator integrated with the asyncio IO loop.
.. versionadded:: 15.2
# Copyright (C) PyZMQ Developers
# Distributed under the terms of the Modified BSD License.
import asyncio
import zmq
from zmq.asyncio import Poller
from .base import Authenticator
class AsyncioAuthenticator(Authenticator):
"""ZAP authentication for use in the asyncio IO loop"""
def __init__(self, context=None, loop=None):
self.loop = loop or asyncio.get_event_loop()
self.__poller = None
self.__task = None
def __handle_zap(self):
while True:
events = yield from self.__poller.poll()
if self.zap_socket in dict(events):
msg = yield from self.zap_socket.recv_multipart()
def start(self):
"""Start ZAP authentication"""
self.__poller = Poller()
self.__poller.register(self.zap_socket, zmq.POLLIN)
self.__task = asyncio.async(self.__handle_zap())
def stop(self):
"""Stop ZAP authentication"""
if self.__task:
if self.__poller:
self.__poller = None
__all__ = ['AsyncioAuthenticator']
......@@ -10,7 +10,7 @@ cdef inline int _check_rc(int rc) except -1:
cdef int errno = zmq_errno()
if rc < 0:
if rc == -1: # if rc < -1, it's a bug in libzmq. Should we warn?
if errno == EINTR:
from zmq.error import InterruptedSystemCall
raise InterruptedSystemCall(errno)
......@@ -120,7 +120,7 @@ def _check_rc(rc, errno=None):
and raising the appropriate Exception class
if rc < 0:
if rc == -1:
if errno is None:
from zmq.backend import zmq_errno
errno = zmq_errno()
......@@ -9,3 +9,6 @@ class NotImplementedFuture(object):
Future = TracebackFuture = NotImplementedFuture
def is_future(x):
return isinstance(x, Future)
This diff is collapsed.
......@@ -41,13 +41,13 @@ Example usage::
with StackContext(die_on_error):
# Any exception thrown here *or in callback and its desendents*
# Any exception thrown here *or in callback and its descendants*
# will cause the process to exit instead of spinning endlessly
# in the ioloop.
http_client.fetch(url, callback)
Most applications shouln't have to work with `StackContext` directly.
Most applications shouldn't have to work with `StackContext` directly.
Here are a few rules of thumb for when it's necessary:
* If you're writing an asynchronous library that doesn't rely on a
......@@ -266,6 +266,18 @@ def wrap(fn):
# TODO: Any other better way to store contexts and update them in wrapped function?
cap_contexts = [_state.contexts]
if not cap_contexts[0][0] and not cap_contexts[0][1]:
# Fast path when there are no active contexts.
def null_wrapper(*args, **kwargs):
current_state = _state.contexts
_state.contexts = cap_contexts[0]
return fn(*args, **kwargs)
_state.contexts = current_state
null_wrapper._wrapped = True
return null_wrapper
def wrapped(*args, **kwargs):
ret = None
......@@ -15,6 +15,25 @@ from __future__ import absolute_import, division, print_function, with_statement
import sys
# Fake unicode literal support: Python 3.2 doesn't have the u'' marker for
# literal strings, and alternative solutions like "from __future__ import
# unicode_literals" have other problems (see PEP 414). u() can be applied
# to ascii strings that include \u escapes (but they must not contain
# literal non-ascii characters).
if not isinstance(b'', type('')):
def u(s):
return s
unicode_type = str
basestring_type = str
def u(s):
return s.decode('unicode_escape')
# These names don't exist in py3, so use noqa comments to disable
# warnings in flake8.
unicode_type = unicode # noqa
basestring_type = basestring # noqa
def import_object(name):
"""Imports an object by name.
......@@ -33,6 +52,9 @@ def import_object(name):
ImportError: No module named missing_module
if isinstance(name, unicode_type) and str is not unicode_type:
# On python 2 a byte string is required.
name = name.encode('utf-8')
if name.count('.') == 0:
return __import__(name, None, None)
......@@ -44,24 +66,9 @@ def import_object(name):
raise ImportError("No module named %s" % parts[-1])
# Fake unicode literal support: Python 3.2 doesn't have the u'' marker for
# literal strings, and alternative solutions like "from __future__ import
# unicode_literals" have other problems (see PEP 414). u() can be applied
# to ascii strings that include \u escapes (but they must not contain
# literal non-ascii characters).
if type('') is not type(b''):
def u(s):
return s
bytes_type = bytes
unicode_type = str
basestring_type = str
def u(s):
return s.decode('unicode_escape')
bytes_type = str
unicode_type = unicode
basestring_type = basestring
# Deprecated alias that was used before we dropped py25 support.
# Left here in case anyone outside Tornado is using it.
bytes_type = bytes
if sys.version_info > (3,):
......@@ -87,6 +94,24 @@ def exec_in(code, glob, loc=None):
def errno_from_exception(e):
"""Provides the errno from an Exception object.
There are cases that the errno attribute was not set so we pull
the errno out of the args but if someone instantiates an Exception
without any args you will get a tuple error. So this function
abstracts all that behavior to give you a safe way to get the
if hasattr(e, 'errno'):
return e.errno
elif e.args:
return e.args[0]
return None
class Configurable(object):
"""Base class for configurable interfaces.
......@@ -110,21 +135,21 @@ class Configurable(object):
__impl_class = None
__impl_kwargs = None
def __new__(cls, **kwargs):
def __new__(cls, *args, **kwargs):
base = cls.configurable_base()
args = {}
init_kwargs = {}
if cls is base:
impl = cls.configured_class()
if base.__impl_kwargs:
impl = cls
instance = super(Configurable, cls).__new__(impl)
# initialize vs __init__ chosen for compatibility with AsyncHTTPClient
# singleton magic. If we get rid of that we can switch to __init__
# here too.
instance.initialize(*args, **init_kwargs)
return instance
......@@ -145,6 +170,9 @@ class Configurable(object):
"""Initialize a `Configurable` subclass instance.
Configurable classes should use `initialize` instead of ``__init__``.
.. versionchanged:: 4.2
Now accepts positional arguments in addition to keyword arguments.
......@@ -156,7 +184,7 @@ class Configurable(object):
some parameters.
base = cls.configurable_base()
if isinstance(impl, (unicode_type, bytes_type)):
if isinstance(impl, (unicode_type, bytes)):
impl = import_object(impl)
if impl is not None and not issubclass(impl, cls):
raise ValueError("Invalid subclass of %s" % cls)
......@@ -182,3 +210,8 @@ class Configurable(object):
base.__impl_class = saved[0]
base.__impl_kwargs = saved[1]
def timedelta_to_seconds(td):
"""Equivalent to td.total_seconds() (introduced in python 2.7)."""
return (td.microseconds + (td.seconds +