setup.py 45.2 KB
Newer Older
1
#!/usr/bin/env python
2
#-----------------------------------------------------------------------------
3 4
#  Copyright (C) PyZMQ Developers
#  Distributed under the terms of the Modified BSD License.
5
#
6 7
#  The `configure` subcommand is copied and adaped from h5py
#  h5py source used under the New BSD license
8
#
9
#  h5py: <http://code.google.com/p/h5py/>
10 11 12 13 14
#
#  The code to bundle libzmq as an Extension is from pyzmq-static
#  pyzmq-static source used under the New BSD license
#
#  pyzmq-static: <https://github.com/brandon-rhodes/pyzmq-static>
15
#-----------------------------------------------------------------------------
16

17
from __future__ import with_statement, print_function
18

19 20 21
import copy
import os
import shutil
22
import subprocess
23
import sys
24
import time
25
import errno
26
import platform
27
from traceback import print_exc
28

29 30 31 32
# whether any kind of bdist is happening
# do this before importing anything from distutils
doing_bdist = any(arg.startswith('bdist') for arg in sys.argv[1:])

33
if any(bdist in sys.argv for bdist in ['sdist', 'bdist_wheel', 'bdist_egg']):
34
    import setuptools
35 36

import distutils
37
from distutils.core import setup, Command
38
from distutils.ccompiler import get_default_compiler
39
from distutils.ccompiler import new_compiler
40
from distutils.extension import Extension
41
from distutils.command.build_ext import build_ext
42
from distutils.command.sdist import sdist
43
from distutils.sysconfig import customize_compiler
44
from distutils.version import LooseVersion as V
45 46

from glob import glob
47
from os.path import splitext, basename, join as pjoin
48

49
from subprocess import Popen, PIPE, check_call, CalledProcessError
50

51
# local script imports:
52
from buildutils import (
53
    discover_settings, v_str, save_config, detect_zmq, merge,
54 55
    config_from_prefix,
    info, warn, fatal, debug, line, copy_and_patch_libzmq, localpath,
56
    fetch_libzmq, stage_platform_hpp,
57
    bundled_version, customize_mingw,
58
    compile_and_forget,
59
    patch_lib_paths,
60
    )
61

62 63 64
# name of the libzmq library - can be changed by --libzmq <name>
libzmq_name = 'libzmq'

65 66 67
#-----------------------------------------------------------------------------
# Flags
#-----------------------------------------------------------------------------
68

69 70
pypy = 'PyPy' in sys.version

71
# reference points for zmq compatibility
72 73 74

min_legacy_zmq = (2,1,4)
min_good_zmq = (3,2)
75
target_zmq = bundled_version
76
dev_zmq = (target_zmq[0], target_zmq[1] + 1, 0)
77 78 79 80 81

# set dylib ext:
if sys.platform.startswith('win'):
    lib_ext = '.dll'
elif sys.platform == 'darwin':
82
    lib_ext = '.dylib'
83
else:
84
    lib_ext = '.so'
85

86 87 88 89 90 91 92 93 94 95 96
# allow `--zmq=foo` to be passed at any point,
# but always assign it to configure

configure_idx = -1
fetch_idx = -1
for idx, arg in enumerate(list(sys.argv)):
    # track index of configure and fetch_libzmq
    if arg == 'configure':
        configure_idx = idx
    elif arg == 'fetch_libzmq':
        fetch_idx = idx
97

98 99 100 101 102 103 104 105 106 107 108
    if arg.startswith('--zmq='):
        sys.argv.pop(idx)
        if configure_idx < 0:
            if fetch_idx < 0:
                configure_idx = 1
            else:
                configure_idx = fetch_idx + 1
            sys.argv.insert(configure_idx, 'configure')
        sys.argv.insert(configure_idx + 1, arg)
        break

109 110
for idx, arg in enumerate(list(sys.argv)):
    if arg.startswith('--libzmq='):
111
        sys.argv.remove(arg)
112
        libzmq_name = arg.split("=",1)[1]
113 114 115
    if arg == '--enable-drafts':
        sys.argv.remove(arg)
        os.environ['ZMQ_DRAFT_API'] = '1'
116

117
#-----------------------------------------------------------------------------
118
# Configuration (adapted from h5py: https://www.h5py.org/)
119 120
#-----------------------------------------------------------------------------

121 122
# --- compiler settings -------------------------------------------------

123
def bundled_settings(debug):
124 125 126 127 128 129
    """settings for linking extensions against bundled libzmq"""
    settings = {}
    settings['libraries'] = []
    settings['library_dirs'] = []
    settings['include_dirs'] = [pjoin("bundled", "zeromq", "include")]
    settings['runtime_library_dirs'] = []
130 131 132 133 134 135 136
    # add pthread on freebsd
    # is this necessary?
    if sys.platform.startswith('freebsd'):
        settings['libraries'].append('pthread')
    elif sys.platform.startswith('win'):
        # link against libzmq in build dir:
        plat = distutils.util.get_platform()
137 138 139 140 141 142 143 144 145 146 147 148 149
        temp = 'temp.%s-%i.%i' % (plat, sys.version_info[0], sys.version_info[1])
        suffix = ''
        if sys.version_info >= (3,5):
            # Python 3.5 adds EXT_SUFFIX to libs
            ext_suffix = distutils.sysconfig.get_config_var('EXT_SUFFIX')
            suffix = os.path.splitext(ext_suffix)[0]

        if debug:
            suffix = '_d' + suffix
            release = 'Debug'
        else:
            release = 'Release'

150
        settings['libraries'].append(libzmq_name + suffix)
151 152
        settings['library_dirs'].append(pjoin('build', temp, release, 'buildutils'))

153 154
    return settings

155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
def check_pkgconfig():
    """ pull compile / link flags from pkg-config if present. """
    pcfg = None
    zmq_config = None
    try:
        check_call(['pkg-config', '--exists', 'libzmq'])
        # this would arguably be better with --variable=libdir /
        # --variable=includedir, but would require multiple calls
        pcfg = Popen(['pkg-config', '--libs', '--cflags', 'libzmq'],
                     stdout=subprocess.PIPE)
    except OSError as osexception:
        if osexception.errno == errno.ENOENT:
            info('pkg-config not found')
        else:
            warn("Running pkg-config failed - %s." % osexception)
    except CalledProcessError:
        info("Did not find libzmq via pkg-config.")

    if pcfg is not None:
        output, _ = pcfg.communicate()
        output = output.decode('utf8', 'replace')
        bits = output.strip().split()
        zmq_config = {'library_dirs':[], 'include_dirs':[], 'libraries':[]}
        for tok in bits:
            if tok.startswith("-L"):
                zmq_config['library_dirs'].append(tok[2:])
            if tok.startswith("-I"):
                zmq_config['include_dirs'].append(tok[2:])
            if tok.startswith("-l"):
                zmq_config['libraries'].append(tok[2:])
        info("Settings obtained from pkg-config: %r" % zmq_config)

    return zmq_config
188

189 190
def _add_rpath(settings, path):
    """Add rpath to settings
191

192 193 194 195 196 197 198
    Implemented here because distutils runtime_library_dirs doesn't do anything on darwin
    """
    if sys.platform == 'darwin':
        settings['extra_link_args'].extend(['-Wl,-rpath','-Wl,%s' % path])
    else:
        settings['runtime_library_dirs'].append(path)

199
def settings_from_prefix(prefix=None, bundle_libzmq_dylib=False):
200
    """load appropriate library/include settings from ZMQ prefix"""
201 202 203 204 205
    settings = {}
    settings['libraries'] = []
    settings['include_dirs'] = []
    settings['library_dirs'] = []
    settings['runtime_library_dirs'] = []
206 207
    settings['extra_link_args'] = []

208
    if sys.platform.startswith('win'):
209
        settings['libraries'].append(libzmq_name)
210

211 212 213
        if prefix:
            settings['include_dirs'] += [pjoin(prefix, 'include')]
            settings['library_dirs'] += [pjoin(prefix, 'lib')]
214 215 216 217
    else:
        # add pthread on freebsd
        if sys.platform.startswith('freebsd'):
            settings['libraries'].append('pthread')
218 219

        if sys.platform.startswith('sunos'):
220 221 222 223
          if platform.architecture()[0] == '32bit':
            settings['extra_link_args'] += ['-m32']
          else:
            settings['extra_link_args'] += ['-m64']
224

225
        if prefix:
226 227
            settings['libraries'].append('zmq')

228 229
            settings['include_dirs'] += [pjoin(prefix, 'include')]
            if not bundle_libzmq_dylib:
230
                if sys.platform.startswith('sunos') and platform.architecture()[0] == '64bit':
231
                    settings['library_dirs'] += [pjoin(prefix, 'lib/amd64')]
232
                settings['library_dirs'] += [pjoin(prefix, 'lib')]
233
        else:
234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254
            # If prefix is not explicitly set, pull it from pkg-config by default.
            # this is probably applicable across platforms, but i don't have
            # sufficient test environments to confirm
            pkgcfginfo = check_pkgconfig()
            if pkgcfginfo is not None:
                # we can get all the zmq-specific values from pkgconfg
                for key, value in pkgcfginfo.items():
                    settings[key].extend(value)
            else:
                settings['libraries'].append('zmq')

                if sys.platform == 'darwin' and os.path.isdir('/opt/local/lib'):
                    # allow macports default
                    settings['include_dirs'] += ['/opt/local/include']
                    settings['library_dirs'] += ['/opt/local/lib']
                if os.environ.get('VIRTUAL_ENV', None):
                    # find libzmq installed in virtualenv
                    env = os.environ['VIRTUAL_ENV']
                    settings['include_dirs'] += [pjoin(env, 'include')]
                    settings['library_dirs'] += [pjoin(env, 'lib')]

255
        if bundle_libzmq_dylib:
256
            # bdist should link against bundled libzmq
257
            settings['library_dirs'].append('zmq')
258 259 260 261 262 263 264
            _add_rpath(settings, '$ORIGIN/..')
            if sys.platform == 'darwin' and pypy:
                settings['extra_link_args'].extend(['-undefined', 'dynamic_lookup'])
        else:
            for path in settings['library_dirs']:
                _add_rpath(settings, os.path.abspath(path))
    info(settings)
265

266 267
    return settings

268 269
class LibZMQVersionError(Exception):
    pass
270

271 272 273 274
#-----------------------------------------------------------------------------
# Extra commands
#-----------------------------------------------------------------------------

275
class Configure(build_ext):
276 277 278
    """Configure command adapted from h5py"""

    description = "Discover ZMQ version and features"
279

280 281 282
    user_options = build_ext.user_options + [
        ('zmq=', None, "libzmq install prefix"),
        ('build-base=', 'b', "base directory for build library"), # build_base from build
283

284 285 286 287 288
    ]
    def initialize_options(self):
        build_ext.initialize_options(self)
        self.zmq = None
        self.build_base = 'build'
289 290 291

    # DON'T REMOVE: distutils demands these be here even if they do nothing.
    def finalize_options(self):
292 293 294 295 296 297 298
        build_ext.finalize_options(self)
        self.tempdir = pjoin(self.build_temp, 'scratch')
        self.has_run = False
        self.config = discover_settings(self.build_base)
        if self.zmq is not None:
            merge(self.config, config_from_prefix(self.zmq))
        self.init_settings_from_config()
299

300 301 302
    def save_config(self, name, cfg):
        """write config to JSON"""
        save_config(name, cfg, self.build_base)
303 304 305 306 307 308 309
        # write to zmq.utils.[name].json
        save_config(name, cfg, os.path.join('zmq', 'utils'))
        # also write to build_lib, because we might be run after copying to
        # build_lib has already happened.
        build_lib_utils = os.path.join(self.build_lib, 'zmq', 'utils')
        if os.path.exists(build_lib_utils):
            save_config(name, cfg, build_lib_utils)
310

311 312 313
    def init_settings_from_config(self):
        """set up compiler settings, based on config"""
        cfg = self.config
314 315 316 317 318 319 320 321 322 323 324 325

        if sys.platform == 'win32' and cfg.get('bundle_msvcp') is None:
            # default bundle_msvcp=True on:
            # Windows Python 3.5 bdist *without* DISTUTILS_USE_SDK
            if os.environ.get("PYZMQ_BUNDLE_CRT") or (
                sys.version_info >= (3,5)
                and self.compiler_type == 'msvc'
                and not os.environ.get('DISTUTILS_USE_SDK')
                and doing_bdist
            ):
                cfg['bundle_msvcp'] = True

326
        if cfg['libzmq_extension']:
327
            settings = bundled_settings(self.debug)
328 329
        else:
            settings = settings_from_prefix(cfg['zmq_prefix'], self.bundle_libzmq_dylib)
330

331 332 333 334 335 336 337 338
        if 'have_sys_un_h' not in cfg:
            # don't link against anything when checking for sys/un.h
            minus_zmq = copy.deepcopy(settings)
            try:
                minus_zmq['libraries'] = []
            except Exception:
                pass
            try:
339
                compile_and_forget(self.tempdir,
340 341 342 343 344 345 346 347
                    pjoin('buildutils', 'check_sys_un.c'),
                    **minus_zmq
                )
            except Exception as e:
                warn("No sys/un.h, IPC_PATH_MAX_LEN will be undefined: %s" % e)
                cfg['have_sys_un_h'] = False
            else:
                cfg['have_sys_un_h'] = True
348

349
            self.save_config('config', cfg)
350

351
        settings.setdefault('define_macros', [])
352
        if cfg['have_sys_un_h']:
353
            settings['define_macros'].append(('HAVE_SYS_UN_H', 1))
354 355 356 357

        if cfg.get('zmq_draft_api'):
            settings['define_macros'].append(('ZMQ_BUILD_DRAFT_API', 1))

358 359 360 361
        use_static_zmq = cfg.get('use_static_zmq', 'False').upper()
        if use_static_zmq in ('TRUE', '1'):
            settings['define_macros'].append(('ZMQ_STATIC', '1'))

362 363
        # include internal directories
        settings.setdefault('include_dirs', [])
364 365 366 367 368
        settings['include_dirs'] += [pjoin('zmq', sub) for sub in (
            'utils',
            pjoin('backend', 'cython'),
            'devices',
        )]
369 370
        if sys.platform.startswith('win') and sys.version_info < (3, 3):
            settings['include_dirs'].insert(0, pjoin('buildutils', 'include_win32'))
371 372 373 374 375 376

        settings.setdefault('libraries', [])
        # Explicitly link dependencies, not necessary if zmq is dynamic
        if sys.platform.startswith('win'):
            settings['libraries'].extend(('ws2_32', 'iphlpapi', 'advapi32'))

377
        for ext in self.distribution.ext_modules:
378
            if ext.name.startswith('zmq.lib'):
379 380 381
                continue
            for attr, value in settings.items():
                setattr(ext, attr, value)
382

383 384
        self.compiler_settings = settings
        self.save_config('compiler', settings)
385 386 387

    def create_tempdir(self):
        self.erase_tempdir()
388
        os.makedirs(self.tempdir)
389 390
        if sys.platform.startswith('win'):
            # fetch libzmq.dll into local dir
391
            local_dll = pjoin(self.tempdir, libzmq_name + '.dll')
392 393 394
            if not self.config['zmq_prefix'] and not os.path.exists(local_dll):
                fatal("ZMQ directory must be specified on Windows via setup.cfg"
                " or 'python setup.py configure --zmq=/path/to/zeromq2'")
395

396
            try:
397
                shutil.copy(pjoin(self.config['zmq_prefix'], 'lib', libzmq_name + '.dll'), local_dll)
398 399
            except Exception:
                if not os.path.exists(local_dll):
400
                    warn("Could not copy " + libzmq_name + " into zmq/, which is usually necessary on Windows."
401
                    "Please specify zmq prefix via configure --zmq=/path/to/zmq or copy "
402
                    + libzmq_name + " into zmq/ manually.")
403 404 405 406 407 408 409

    def erase_tempdir(self):
        try:
            shutil.rmtree(self.tempdir)
        except Exception:
            pass

410 411 412 413 414 415 416
    @property
    def compiler_type(self):
        compiler = self.compiler
        if compiler is None:
            return get_default_compiler()
        elif isinstance(compiler, str):
            return compiler
417
        else:
418
            return compiler.compiler_type
419

420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437
    @property
    def cross_compiling(self):
        return self.config['bdist_egg'].get('plat-name', sys.platform) != sys.platform

    @property
    def bundle_libzmq_dylib(self):
        """
        bundle_libzmq_dylib flag for whether external libzmq library will be included in pyzmq:
        only relevant when not building libzmq extension
        """
        if 'bundle_libzmq_dylib' in self.config:
            return self.config['bundle_libzmq_dylib']
        elif (sys.platform.startswith('win') or self.cross_compiling) \
                and not self.config['libzmq_extension']:
            # always bundle libzmq on Windows and cross-compilation
            return True
        else:
            return False
438

439 440 441 442
    def check_zmq_version(self):
        """check the zmq version"""
        cfg = self.config
        # build test program
443
        zmq_prefix = cfg['zmq_prefix']
444 445
        detected = self.test_build(zmq_prefix, self.compiler_settings)
        # now check the libzmq version
446

447
        vers = tuple(detected['vers'])
448
        vs = v_str(vers)
449 450 451 452
        if cfg['allow_legacy_libzmq']:
            min_zmq = min_legacy_zmq
        else:
            min_zmq = min_good_zmq
453
        if vers < min_zmq:
454 455 456 457 458 459
            msg = [
                "Detected ZMQ version: %s, but require ZMQ >= %s" % (vs, v_str(min_zmq)),
            ]
            if zmq_prefix:
                msg.append("    ZMQ_PREFIX=%s" % zmq_prefix)
            if vers >= min_legacy_zmq:
460

461
                msg.append("    Explicitly allow legacy zmq by specifying `--zmq=/zmq/prefix`")
462

463 464 465 466 467 468 469 470 471
            raise LibZMQVersionError('\n'.join(msg))
        if vers < min_good_zmq:
            msg = [
                "Detected legacy ZMQ version: %s. It is STRONGLY recommended to use ZMQ >= %s" % (vs, v_str(min_good_zmq)),
            ]
            if zmq_prefix:
                msg.append("    ZMQ_PREFIX=%s" % zmq_prefix)
            warn('\n'.join(msg))
        elif vers < target_zmq:
472 473 474 475
            warn("Detected ZMQ version: %s, but pyzmq targets ZMQ %s." % (
                    vs, v_str(target_zmq))
            )
            warn("libzmq features and fixes introduced after %s will be unavailable." % vs)
476
            line()
477
        elif vers >= dev_zmq:
478
            warn("Detected ZMQ version: %s. Some new features in libzmq may not be exposed by pyzmq." % vs)
479
            line()
480 481 482

        if sys.platform.startswith('win'):
            # fetch libzmq.dll into local dir
483
            local_dll = localpath('zmq', libzmq_name + '.dll')
484
            if not zmq_prefix and not os.path.exists(local_dll):
485 486
                fatal("ZMQ directory must be specified on Windows via setup.cfg or 'python setup.py configure --zmq=/path/to/zeromq2'")
            try:
487
                shutil.copy(pjoin(zmq_prefix, 'lib', libzmq_name + '.dll'), local_dll)
488 489
            except Exception:
                if not os.path.exists(local_dll):
490
                    warn("Could not copy " + libzmq_name + " into zmq/, which is usually necessary on Windows."
491
                    "Please specify zmq prefix via configure --zmq=/path/to/zmq or copy "
492
                    + libzmq_name + " into zmq/ manually.")
493

494 495
    def bundle_libzmq_extension(self):
        bundledir = "bundled"
496
        ext_modules = self.distribution.ext_modules
497
        if ext_modules and any(m.name == 'zmq.libzmq' for m in ext_modules):
498 499
            # I've already been run
            return
500

501
        line()
502
        info("Using bundled libzmq")
503

504 505 506
        # fetch sources for libzmq extension:
        if not os.path.exists(bundledir):
            os.makedirs(bundledir)
507

508
        fetch_libzmq(bundledir)
509

510
        stage_platform_hpp(pjoin(bundledir, 'zeromq'))
511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531

        sources = [pjoin('buildutils', 'initlibzmq.c')]
        sources += glob(pjoin(bundledir, 'zeromq', 'src', '*.cpp'))

        includes = [
            pjoin(bundledir, 'zeromq', 'include')
        ]

        if bundled_version < (4, 2, 0):
            tweetnacl = pjoin(bundledir, 'zeromq', 'tweetnacl')
            tweetnacl_sources = glob(pjoin(tweetnacl, 'src', '*.c'))

            randombytes = pjoin(tweetnacl, 'contrib', 'randombytes')
            if sys.platform.startswith('win'):
                tweetnacl_sources.append(pjoin(randombytes, 'winrandom.c'))
            else:
                tweetnacl_sources.append(pjoin(randombytes, 'devurandom.c'))

            sources += tweetnacl_sources
            includes.append(pjoin(tweetnacl, 'src'))
            includes.append(randombytes)
532
        else:
533 534 535
            # >= 4.2
            sources += glob(pjoin(bundledir, 'zeromq', 'src', 'tweetnacl.c'))

536 537
        # construct the Extensions:
        libzmq = Extension(
538
            'zmq.libzmq',
539 540
            sources=sources,
            include_dirs=includes,
541
        )
542

543 544
        # register the extension:
        self.distribution.ext_modules.insert(0, libzmq)
545

546 547 548
        # use tweetnacl to provide CURVE support
        libzmq.define_macros.append(('ZMQ_HAVE_CURVE', 1))
        libzmq.define_macros.append(('ZMQ_USE_TWEETNACL', 1))
549

550 551 552 553 554 555 556 557 558 559
        # select polling subsystem based on platform
        if sys.platform  == 'darwin' or 'bsd' in sys.platform:
            libzmq.define_macros.append(('ZMQ_USE_KQUEUE', 1))
        elif 'linux' in sys.platform:
            libzmq.define_macros.append(('ZMQ_USE_EPOLL', 1))
        elif sys.platform.startswith('win'):
            libzmq.define_macros.append(('ZMQ_USE_SELECT', 1))
        else:
            # this may not be sufficiently precise
            libzmq.define_macros.append(('ZMQ_USE_POLL', 1))
560

561 562
        if sys.platform.startswith('win'):
            # include defines from zeromq msvc project:
563
            libzmq.define_macros.append(('FD_SETSIZE', 16384))
564
            libzmq.define_macros.append(('DLL_EXPORT', 1))
565
            libzmq.define_macros.append(('_CRT_SECURE_NO_WARNINGS', 1))
566

567 568 569
            # When compiling the C++ code inside of libzmq itself, we want to
            # avoid "warning C4530: C++ exception handler used, but unwind
            # semantics are not enabled. Specify /EHsc".
570
            if self.compiler_type == 'msvc':
571
                libzmq.extra_compile_args.append('/EHsc')
572
            elif self.compiler_type == 'mingw32':
573
                libzmq.define_macros.append(('ZMQ_HAVE_MINGW32', 1))
574 575

            # And things like sockets come from libraries that must be named.
576 577
            libzmq.libraries.extend(['rpcrt4', 'ws2_32', 'advapi32', 'iphlpapi'])

578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599
            # bundle MSCVP redist
            if self.config['bundle_msvcp']:
                cc = new_compiler(compiler=self.compiler_type)
                cc.initialize()
                # get vc_redist location via private API
                try:
                    cc._vcruntime_redist
                except AttributeError:
                    # fatal error if env set, warn otherwise
                    msg = fatal if os.environ.get("PYZMQ_BUNDLE_CRT") else warn
                    msg("Failed to get cc._vcruntime via private API, not bundling CRT")
                if cc._vcruntime_redist:
                    redist_dir, dll = os.path.split(cc._vcruntime_redist)
                    to_bundle = [
                        pjoin(redist_dir, dll.replace('vcruntime', name))
                        for name in ('msvcp', 'concrt')
                    ]
                    for src in to_bundle:
                        dest = localpath('zmq', basename(src))
                        info("Copying %s -> %s" % (src, dest))
                        # copyfile to avoid permission issues
                        shutil.copyfile(src, dest)
600

601
        else:
602
            libzmq.include_dirs.append(bundledir)
603

604 605
            # check if we need to link against Realtime Extensions library
            cc = new_compiler(compiler=self.compiler_type)
606
            customize_compiler(cc)
607
            cc.output_dir = self.build_temp
608 609 610 611 612 613 614 615
            if not sys.platform.startswith(('darwin', 'freebsd')):
                line()
                info("checking for timer_create")
                if not cc.has_function('timer_create'):
                    info("no timer_create, linking librt")
                    libzmq.libraries.append('rt')
                else:
                    info("ok")
616

617 618 619 620 621 622
        # copy the header files to the source tree.
        bundledincludedir = pjoin('zmq', 'include')
        if not os.path.exists(bundledincludedir):
            os.makedirs(bundledincludedir)
        if not os.path.exists(pjoin(self.build_lib, bundledincludedir)):
            os.makedirs(pjoin(self.build_lib, bundledincludedir))
623

624 625 626
        for header in glob(pjoin(bundledir, 'zeromq', 'include', '*.h')):
            shutil.copyfile(header, pjoin(bundledincludedir, basename(header)))
            shutil.copyfile(header, pjoin(self.build_lib, bundledincludedir, basename(header)))
627

628
        # update other extensions, with bundled settings
629 630 631
        self.config['libzmq_extension'] = True
        self.init_settings_from_config()
        self.save_config('config', self.config)
632 633


634 635
    def fallback_on_bundled(self):
        """Couldn't build, fallback after waiting a while"""
636

637
        line()
638

639
        warn('\n'.join([
640
        "Couldn't find an acceptable libzmq on the system.",
641 642 643 644 645
        "",
        "If you expected pyzmq to link against an installed libzmq, please check to make sure:",
        "",
        "    * You have a C compiler installed",
        "    * A development version of Python is installed (including headers)",
646
        "    * A development version of ZMQ >= %s is installed (including headers)" % v_str(min_good_zmq),
647 648 649 650 651 652
        "    * If ZMQ is not in a default location, supply the argument --zmq=<path>",
        "    * If you did recently install ZMQ to a default location,",
        "      try rebuilding the ld cache with `sudo ldconfig`",
        "      or specify zmq's location with `--zmq=/usr/local`",
        "",
        ]))
653

654
        info('\n'.join([
655 656 657 658 659 660 661 662 663
            "You can skip all this detection/waiting nonsense if you know",
            "you want pyzmq to bundle libzmq as an extension by passing:",
            "",
            "    `--zmq=bundled`",
            "",
            "I will now try to build libzmq as a Python extension",
            "unless you interrupt me (^C) in the next 10 seconds...",
            "",
        ]))
664

665 666 667 668
        for i in range(10,0,-1):
            sys.stdout.write('\r%2i...' % i)
            sys.stdout.flush()
            time.sleep(1)
669

670
        info("")
671

672
        return self.bundle_libzmq_extension()
673 674


675 676
    def test_build(self, prefix, settings):
        """do a test build ob libzmq"""
677
        self.create_tempdir()
678 679
        settings = settings.copy()
        if self.bundle_libzmq_dylib and not sys.platform.startswith('win'):
680 681
            # rpath slightly differently here, because libzmq not in .. but ../zmq:
            settings['library_dirs'] = ['zmq']
682
            _add_rpath(settings, os.path.abspath(pjoin('.', 'zmq')))
683
        line()
684 685
        info("Configure: Autodetecting ZMQ settings...")
        info("    Custom ZMQ dir:       %s" % prefix)
686
        try:
687
            detected = detect_zmq(self.tempdir, compiler=self.compiler_type, **settings)
688 689
        finally:
            self.erase_tempdir()
690

691
        info("    ZMQ version detected: %s" % v_str(detected['vers']))
692

693
        return detected
694

695

696 697 698
    def finish_run(self):
        self.save_config('config', self.config)
        line()
699

700
    def run(self):
701
        cfg = self.config
702

703 704 705 706
        if cfg['libzmq_extension']:
            self.bundle_libzmq_extension()
            self.finish_run()
            return
707

708 709 710 711 712 713
        # When cross-compiling and zmq is given explicitly, we can't testbuild
        # (as we can't testrun the binary), we assume things are alright.
        if cfg['skip_check_zmq'] or self.cross_compiling:
            warn("Skipping zmq version check")
            self.finish_run()
            return
714

715
        zmq_prefix = cfg['zmq_prefix']
716
        # There is no available default on Windows, so start with fallback unless
717 718 719 720 721 722 723
        # zmq was given explicitly, or libzmq extension was explicitly prohibited.
        if sys.platform.startswith("win") and \
                not cfg['no_libzmq_extension'] and \
                not zmq_prefix:
            self.fallback_on_bundled()
            self.finish_run()
            return
724

725
        if zmq_prefix and self.bundle_libzmq_dylib and not sys.platform.startswith('win'):
726
            copy_and_patch_libzmq(zmq_prefix, libzmq_name+lib_ext)
727

728 729 730
        # first try with given config or defaults
        try:
            self.check_zmq_version()
731 732
        except LibZMQVersionError as e:
            info("\nBad libzmq version: %s\n" % e)
733
        except Exception as e:
734
            # print the error as distutils would if we let it raise:
735
            info("\nerror: %s\n" % e)
736 737 738
        else:
            self.finish_run()
            return
739

740 741 742
        # try fallback on /usr/local on *ix if no prefix is given
        if not zmq_prefix and not sys.platform.startswith('win'):
            info("Failed with default libzmq, trying again with /usr/local")
743
            time.sleep(1)
744 745
            zmq_prefix = cfg['zmq_prefix'] = '/usr/local'
            self.init_settings_from_config()
746
            try:
747
                self.check_zmq_version()
748 749
            except LibZMQVersionError as e:
                info("\nBad libzmq version: %s\n" % e)
750
            except Exception as e:
751
                # print the error as distutils would if we let it raise:
752
                info("\nerror: %s\n" % e)
753
            else:
754 755
                # if we get here the second run succeeded, so we need to update compiler
                # settings for the extensions with /usr/local prefix
756 757
                self.finish_run()
                return
758

759
        # finally, fallback on bundled
760

761 762
        if cfg['no_libzmq_extension']:
            fatal("Falling back on bundled libzmq,"
763
            " but config has explicitly prohibited building the libzmq extension."
764
            )
765

766
        self.fallback_on_bundled()
767

768
        self.finish_run()
769 770 771


class FetchCommand(Command):
772
    """Fetch libzmq sources, that's it."""
773

774
    description = "Fetch libzmq sources into bundled/zeromq"
775

776
    user_options = [ ]
777

778 779
    def initialize_options(self):
        pass
780

781 782
    def finalize_options(self):
        pass
783

784 785 786
    def run(self):
        # fetch sources for libzmq extension:
        bundledir = "bundled"
787 788 789
        if os.path.exists(bundledir):
            info("Scrubbing directory: %s" % bundledir)
            shutil.rmtree(bundledir)
790 791 792 793 794
        if not os.path.exists(bundledir):
            os.makedirs(bundledir)
        fetch_libzmq(bundledir)
        for tarball in glob(pjoin(bundledir, '*.tar.gz')):
            os.remove(tarball)
795

796

797

798 799 800
class TestCommand(Command):
    """Custom distutils command to run the test suite."""

801 802
    description = "Test PyZMQ (must have been built inplace: `setup.py build_ext --inplace`)"

803 804 805 806 807 808 809
    user_options = [ ]

    def initialize_options(self):
        self._dir = os.getcwd()

    def finalize_options(self):
        pass
810

811
    def run(self):
812
        """Run the test suite with py.test"""
813 814 815 816 817
        # crude check for inplace build:
        try:
            import zmq
        except ImportError:
            print_exc()
818 819 820
            fatal('\n       '.join(["Could not import zmq!",
            "You must build pyzmq with 'python setup.py build_ext --inplace' for 'python setup.py test' to work.",
            "If you did build pyzmq in-place, then this is a real error."]))
821
            sys.exit(1)
822

823
        info("Testing pyzmq-%s with libzmq-%s" % (zmq.pyzmq_version(), zmq.zmq_version()))
824 825 826
        p = Popen([sys.executable, '-m', 'pytest', '-v', os.path.join('zmq', 'tests')])
        p.wait()
        sys.exit(p.returncode)
827

828
class GitRevisionCommand(Command):
829
    """find the current git revision and add it to zmq.sugar.version.__revision__"""
830

831
    description = "Store current git revision in version.py"
832

833
    user_options = [ ]
834

835
    def initialize_options(self):
836
        self.version_py = pjoin('zmq','sugar','version.py')
837

838 839 840 841
    def run(self):
        try:
            p = Popen('git log -1'.split(), stdin=PIPE, stdout=PIPE, stderr=PIPE)
        except IOError:
842
            warn("No git found, skipping git revision")
843
            return
844

845
        if p.wait():
846 847
            warn("checking git branch failed")
            info(p.stderr.read())
848
            return
849

850 851
        line = p.stdout.readline().decode().strip()
        if not line.startswith('commit'):
852
            warn("bad commit line: %r" % line)
853
            return
854

855
        rev = line.split()[-1]
856

857 858
        # now that we have the git revision, we can apply it to version.py
        with open(self.version_py) as f:
859
            lines = f.readlines()
860

861 862 863 864
        for i,line in enumerate(lines):
            if line.startswith('__revision__'):
                lines[i] = "__revision__ = '%s'\n"%rev
                break
865
        with open(self.version_py, 'w') as f:
866
            f.writelines(lines)
867

868 869
    def finalize_options(self):
        pass
870 871 872

class CleanCommand(Command):
    """Custom distutils command to clean the .so and .pyc files."""
873 874 875
    user_options = [('all', 'a',
         "remove all build output, not just temporary by-products")
    ]
876

877
    boolean_options = ['all']
878 879

    def initialize_options(self):
880 881 882 883 884 885
        self.all = None

    def finalize_options(self):
        pass

    def run(self):
886 887
        _clean_me = []
        _clean_trees = []
888

889 890
        for d in ('build', 'dist', 'conf'):
            if os.path.exists(d):
891
                _clean_trees.append(d)
892

893
        for root, dirs, files in os.walk('buildutils'):
894
            if any(root.startswith(pre) for pre in _clean_trees):
895
                continue
896 897
            for f in files:
                if os.path.splitext(f)[-1] == '.pyc':
898
                    _clean_me.append(pjoin(root, f))
899

900
            if '__pycache__' in dirs:
901
                _clean_trees.append(pjoin(root, '__pycache__'))
902

903
        for root, dirs, files in os.walk('zmq'):
904
            if any(root.startswith(pre) for pre in _clean_trees):
905
                continue
906

907
            for f in files:
908
                if os.path.splitext(f)[-1] in ('.pyc', '.so', '.o', '.pyd', '.json'):
909
                    _clean_me.append(pjoin(root, f))
910

911 912 913 914 915 916 917
            # remove generated cython files
            if self.all:
                for f in files:
                    f2 = os.path.splitext(f)
                    if f2[1] == '.c' and os.path.isfile(os.path.join(root, f2[0]) + '.pyx'):
                        _clean_me.append(pjoin(root, f))

918 919
            for d in dirs:
                if d == '__pycache__':
920
                    _clean_trees.append(pjoin(root, d))
921

922
        bundled = glob(pjoin('zmq', 'libzmq*'))
923
        _clean_me.extend([ b for b in bundled if b not in _clean_me ])
924

925 926
        bundled_headers = glob(pjoin('zmq', 'include', '*.h'))
        _clean_me.extend([ h for h in bundled_headers if h not in _clean_me])
927

928
        for clean_me in _clean_me:
929
            print("removing %s" % clean_me)
930 931
            try:
                os.unlink(clean_me)
932 933
            except Exception as e:
                print(e, file=sys.stderr)
934
        for clean_tree in _clean_trees:
935
            print("removing %s/" % clean_tree)
936 937
            try:
                shutil.rmtree(clean_tree)
938 939
            except Exception as e:
                print(e, file=sys.stderr)
940

941 942 943 944 945 946 947

class CheckSDist(sdist):
    """Custom sdist that ensures Cython has compiled all pyx files to c."""

    def initialize_options(self):
        sdist.initialize_options(self)
        self._pyxfiles = []
948
        for root, dirs, files in os.walk('zmq'):
949 950 951 952
            for f in files:
                if f.endswith('.pyx'):
                    self._pyxfiles.append(pjoin(root, f))
    def run(self):
953
        self.run_command('fetch_libzmq')
954 955 956 957 958 959 960 961
        if 'cython' in cmdclass:
            self.run_command('cython')
        else:
            for pyxfile in self._pyxfiles:
                cfile = pyxfile[:-3]+'c'
                msg = "C-source file '%s' not found."%(cfile)+\
                " Run 'setup.py cython' before sdist."
                assert os.path.isfile(cfile), msg
962 963 964
        sdist.run(self)

class CheckingBuildExt(build_ext):
965
    """Subclass build_ext to get clearer report if Cython is necessary."""
966

967 968 969 970
    def check_cython_extensions(self, extensions):
        for ext in extensions:
          for src in ext.sources:
            if not os.path.exists(src):
971
                fatal("""Cython-generated file '%s' not found.
972
                Cython >= %s is required to compile pyzmq from a development branch.
973
                Please install Cython or download a release package of pyzmq.
974
                """ % (src, min_cython_version))
975

976 977 978
    def build_extensions(self):
        self.check_cython_extensions(self.extensions)
        self.check_extensions_list(self.extensions)
979

980 981
        if self.compiler.compiler_type == 'mingw32':
            customize_mingw(self.compiler)
982

983 984
        for ext in self.extensions:
            self.build_extension(ext)
985

986 987 988 989
    def build_extension(self, ext):
        build_ext.build_extension(self, ext)
        ext_path = self.get_ext_fullpath(ext.name)
        patch_lib_paths(ext_path, self.compiler.library_dirs)
990

991 992
    def run(self):
        # check version, to prevent confusing undefined constant errors
993
        self.distribution.run_command('configure')
994
        build_ext.run(self)
995

996

997 998
class ConstantsCommand(Command):
    """Rebuild templated files for constants
999

1000 1001 1002 1003
    To be run after adding new constants to `utils/constant_names`.
    """
    user_options = []
    def initialize_options(self):
1004 1005
        return

1006 1007
    def finalize_options(self):
        pass
1008

1009 1010 1011 1012
    def run(self):
        from buildutils.constants import render_constants
        render_constants()

1013 1014 1015 1016
#-----------------------------------------------------------------------------
# Extensions
#-----------------------------------------------------------------------------

1017
cmdclass = {'test':TestCommand, 'clean':CleanCommand, 'revision':GitRevisionCommand,
1018
            'configure': Configure, 'fetch_libzmq': FetchCommand,
1019
            'sdist': CheckSDist, 'constants': ConstantsCommand,
1020
        }
1021

1022

1023 1024
def makename(path, ext):
    return os.path.abspath(pjoin('zmq', *path)) + ext
1025

1026 1027 1028 1029
pxd = lambda *path: makename(path, '.pxd')
pxi = lambda *path: makename(path, '.pxi')
pyx = lambda *path: makename(path, '.pyx')
dotc = lambda *path: makename(path, '.c')
1030
doth = lambda *path: makename(path, '.h')
1031

1032
libzmq = pxd('backend', 'cython', 'libzmq')
1033
buffers = pxd('utils', 'buffers')
1034 1035 1036 1037
message = pxd('backend', 'cython', 'message')
context = pxd('backend', 'cython', 'context')
socket = pxd('backend', 'cython', 'socket')
checkrc = pxd('backend', 'cython', 'checkrc')
1038
monqueue = pxd('devices', 'monitoredqueue')
1039
mutex = doth('utils', 'mutex')
1040

1041 1042
submodules = {
    'backend.cython' : {'constants': [libzmq, pxi('backend', 'cython', 'constants')],
1043 1044
            'error':[libzmq, checkrc],
            '_poll':[libzmq, socket, context, checkrc],
1045
            'utils':[libzmq, checkrc],
1046
            'context':[context, libzmq, checkrc],
1047
            'message':[libzmq, buffers, message, checkrc, mutex],
1048 1049
            'socket':[context, message, socket, libzmq, buffers, checkrc],
            '_device':[libzmq, socket, context, checkrc],
1050
            '_version':[libzmq],
1051
    },
1052
    'devices' : {
1053
            'monitoredqueue':[buffers, libzmq, monqueue, socket, context, checkrc],
1054
    },
1055
}
1056

1057
min_cython_version = '0.20'
1058
try:
1059
    import Cython
1060 1061 1062
    if V(Cython.__version__) < V(min_cython_version):
        raise ImportError("Cython >= %s required for cython build, found %s" % (
            min_cython_version, Cython.__version__))
1063
    from Cython.Distutils import build_ext as build_ext_c
1064 1065
    from Cython.Distutils import Extension
    cython = True
1066
except Exception:
1067
    cython = False
1068 1069
    suffix = '.c'
    cmdclass['build_ext'] = CheckingBuildExt
1070

1071
    class MissingCython(Command):
1072

1073
        user_options = []
1074

1075 1076
        def initialize_options(self):
            pass
1077

1078 1079
        def finalize_options(self):
            pass
1080

1081 1082 1083 1084 1085 1086 1087
        def run(self):
            try:
                import Cython
            except ImportError:
                warn("Cython is missing")
            else:
                cv = getattr(Cython, "__version__", None)
1088
                if cv is None or V(cv) < V(min_cython_version):
1089
                    warn(
1090 1091
                        "Cython >= %s is required for compiling Cython sources, "
                        "found: %s" % (min_cython_version, cv or Cython)
1092 1093
                    )
    cmdclass['cython'] = MissingCython
1094

1095
else:
1096

1097
    suffix = '.pyx'
1098

1099
    class CythonCommand(build_ext_c):
1100
        """Custom distutils command subclassed from Cython.Distutils.build_ext
1101
        to compile pyx->c, and stop there. All this does is override the
1102
        C-compile method build_extension() with a no-op."""
1103

1104
        description = "Compile Cython sources to C"
1105

1106 1107
        def build_extension(self, ext):
            pass
1108

1109
    class zbuild_ext(build_ext_c):
1110

1111 1112 1113 1114
        def build_extensions(self):
            if self.compiler.compiler_type == 'mingw32':
                customize_mingw(self.compiler)
            return build_ext_c.build_extensions(self)
1115

1116 1117 1118 1119
        def build_extension(self, ext):
            build_ext_c.build_extension(self, ext)
            ext_path = self.get_ext_fullpath(ext.name)
            patch_lib_paths(ext_path, self.compiler.library_dirs)
1120