pybuild

#! /usr/bin/python3
# vim: et ts=4 sw=4
# Copyright © 2012-2013 Piotr Ożarowski <piotr@debian.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import logging
import argparse
import re
import sys
from os import environ, getcwd, makedirs, remove
from os.path import abspath, exists, isdir, join
from shutil import rmtree

INTERP_VERSION_RE = re.compile(r'^python(?P<version>3\.\d+)(?P<dbg>-dbg)?$')
logging.basicConfig(format='%(levelname).1s: pybuild '
                           '%(module)s:%(lineno)d: %(message)s')
log = logging.getLogger('dhpython')


def main(cfg):
    log.debug('cfg: %s', cfg)
    from dhpython import build, PKG_PREFIX_MAP
    from dhpython.debhelper import DebHelper, build_options
    from dhpython.version import Version, build_sorted, get_requested_versions
    from dhpython.interpreter import Interpreter
    from dhpython.tools import execute, move_matching_files

    if cfg.list_systems:
        for name, Plugin in sorted(build.plugins.items()):
            print(name, '\t', Plugin.DESCRIPTION)
        exit(0)

    nocheck = False
    if 'DEB_BUILD_OPTIONS' in environ:
        nocheck = 'nocheck' in environ['DEB_BUILD_OPTIONS']
    if not nocheck and 'DEB_BUILD_PROFILES' in environ:
        nocheck = 'nocheck' in environ['DEB_BUILD_PROFILES']

    env = environ.copy()
    # set some defaults in environ to make the build reproducible
    env.setdefault('LC_ALL', 'C.UTF-8')
    env.setdefault('CCACHE_DIR', abspath('.pybuild/ccache'))
    env.setdefault('no_proxy', 'localhost')
    if 'http_proxy' not in env:
        env['http_proxy'] = 'http://127.0.0.1:9/'
    elif not env['http_proxy']:
        del env['http_proxy']  # some tools don't like empty var.
    if 'https_proxy' not in env:
        env['https_proxy'] = 'https://127.0.0.1:9/'
    elif not env['https_proxy']:
        del env['https_proxy']  # some tools don't like empty var.
    if 'DEB_PYTHON_INSTALL_LAYOUT' not in env:
        env['DEB_PYTHON_INSTALL_LAYOUT'] = 'deb'

    arch_data = {}
    if exists('/usr/bin/dpkg-architecture'):
        res = execute('/usr/bin/dpkg-architecture')
        for line in res['stdout'].splitlines():
            key, value = line.strip().split('=', 1)
            arch_data[key] = value

        # Set _PYTHON_HOST_PLATFORM to ensure debugging symbols on, f.e. i386
        # emded a constant name regardless of the 32/64-bit kernel.
        host_platform = '{DEB_HOST_ARCH_OS}-{DEB_HOST_ARCH}'.format(**arch_data)
        # it's not called amd64 in Python
        host_platform = host_platform.replace('amd64', 'x86_64')
        env.setdefault('_PYTHON_HOST_PLATFORM', host_platform)

        if arch_data['DEB_BUILD_ARCH'] != arch_data['DEB_HOST_ARCH']:
            # support cross compiling Python 3.X extensions, see #892931
            env.setdefault('_PYTHON_SYSCONFIGDATA_NAME',
                           '_sysconfigdata__' + arch_data["DEB_HOST_MULTIARCH"])

    # Selected on command line?
    selected_plugin = cfg.system

    # Selected by build_dep?
    if not selected_plugin:
        dh = DebHelper(build_options())
        for build_dep in dh.build_depends:
            if build_dep.startswith('pybuild-plugin-'):
                selected_plugin = build_dep.split('-', 2)[2]
                break

    if selected_plugin:
        certainty = 99
        Plugin = build.plugins.get(selected_plugin)
        if not Plugin:
            log.error('unrecognized build system: %s', selected_plugin)
            exit(10)
        plugin = Plugin(cfg)
        context = {'ENV': env, 'args': {}, 'dir': cfg.dir}
        plugin.detect(context)
    else:
        plugin, certainty, context = None, 0, None
        for Plugin in build.plugins.values():
            try:
                tmp_plugin = Plugin(cfg)
            except Exception as err:
                log.warn('cannot initialize %s plugin: %s', Plugin.NAME,
                         err, exc_info=cfg.verbose)
                continue
            tmp_context = {'ENV': env, 'args': {}, 'dir': cfg.dir}
            tmp_certainty = tmp_plugin.detect(tmp_context)
            log.debug('Plugin %s: certainty %i', Plugin.NAME, tmp_certainty)
            if tmp_certainty and tmp_certainty > certainty:
                plugin, certainty, context = tmp_plugin, tmp_certainty, tmp_context
        del Plugin
        if not plugin:
            log.error('cannot detect build system, please use --system option'
                      ' or set PYBUILD_SYSTEM env. variable')
            exit(11)

    if plugin.SUPPORTED_INTERPRETERS is not True:
        # if versioned interpreter was requested and selected plugin lists
        # versioned ones as supported: extend list of supported interpreters
        # with this interpreter
        tpls = {i for i in plugin.SUPPORTED_INTERPRETERS if '{version}' in i}
        if tpls:
            for ipreter in cfg.interpreter:
                m = INTERP_VERSION_RE.match(ipreter)
                if m:
                    ver = m.group('version')
                    updated = set(tpl.format(version=ver) for tpl in tpls)
                    updated and plugin.SUPPORTED_INTERPRETERS.update(updated)

    for interpreter in cfg.interpreter:
        if plugin.SUPPORTED_INTERPRETERS is not True and interpreter not in plugin.SUPPORTED_INTERPRETERS:
            log.error('interpreter %s not supported by %s', interpreter, plugin)
            exit(12)
    log.debug('detected build system: %s (certainty: %s%%)', plugin.NAME, certainty)

    if cfg.detect_only:
        if not cfg.really_quiet:
            print(plugin.NAME)
        exit(0)

    versions = cfg.versions
    if not versions:
        if len(cfg.interpreter) == 1:
            i = cfg.interpreter[0]
            m = INTERP_VERSION_RE.match(i)
            if m:
                log.debug('defaulting to version hardcoded in interpreter name')
                versions = [m.group('version')]
            else:
                IMAP = {v: k for k, v in PKG_PREFIX_MAP.items()}
                if i in IMAP:
                    versions = build_sorted(get_requested_versions(
                        IMAP[i], available=True), impl=IMAP[i])
                if versions and '{version}' not in i:
                    versions = versions[-1:]  # last one, the default one
        if not versions:  # still no luck
            log.debug('defaulting to all supported Python 3.X versions')
            versions = build_sorted(get_requested_versions(
                'cpython3', available=True), impl='cpython3')
    versions = [Version(v) for v in versions]

    def get_option(name, interpreter=None, version=None, default=None):
        if interpreter:
            # try PYBUILD_NAME_python3.3-dbg (or hardcoded interpreter)
            i = interpreter.format(version=version or '')
            opt = "PYBUILD_{}_{}".format(name.upper(), i)
            if opt in environ:
                return environ[opt]
            # try PYBUILD_NAME_python3-dbg (if not checked above)
            if '{version}' in interpreter and version:
                i = interpreter.format(version=version.major)
                opt = "PYBUILD_{}_{}".format(name.upper(), i)
                if opt in environ:
                    return environ[opt]
        # try PYBUILD_NAME
        opt = "PYBUILD_{}".format(name.upper())
        if opt in environ:
            return environ[opt]
        # try command line args
        return getattr(cfg, name, default) or default

    def get_args(context, step, version, interpreter):
        i = interpreter.format(version=version)
        ipreter = Interpreter(i)

        home_dir = [ipreter.impl, str(version)]
        if ipreter.debug:
            home_dir.append('dbg')
        if cfg.name:
            home_dir.append(cfg.name)
        home_dir = '.pybuild/{}'.format('_'.join(home_dir))

        build_dir = get_option('build_dir', interpreter, version,
                               default=join(home_dir, 'build'))

        destdir = context['destdir'].format(version=version, interpreter=i)
        if cfg.name:
            package = ipreter.suggest_pkg_name(cfg.name)
        else:
            package = 'PYBUILD_NAME_not_set'
        if cfg.name and destdir.rstrip('/').endswith('debian/tmp'):
            destdir = "debian/{}".format(package)
        destdir = abspath(destdir)

        args = dict(context['args'])
        args.update({
            'package': package,
            'interpreter': ipreter,
            'version': version,
            'args': get_option("%s_args" % step, interpreter, version, ''),
            'dir': abspath(context['dir'].format(version=version, interpreter=i)),
            'destdir': destdir,
            'build_dir': abspath(build_dir.format(version=version, interpreter=i)),
            # versioned dist-packages even for Python 3.X - dh_python3 will fix it later
            # (and will have a chance to compare files)
            'install_dir': get_option('install_dir', interpreter, version,
                                      '/usr/lib/python{version}/dist-packages'
                                      ).format(version=version, interpreter=i),
            'home_dir': abspath(home_dir)})
        if interpreter == 'pypy':
            args['install_dir'] = '/usr/lib/pypy/dist-packages/'
        env = dict(args.get('ENV', {}))
        pp = env.get('PYTHONPATH', context['ENV'].get('PYTHONPATH'))
        pp = pp.split(':') if pp else []
        if step in {'build', 'test'}:
            if step == 'test':
                args['test_dir'] = join(args['destdir'], args['install_dir'].lstrip('/'))
                if args['test_dir'] not in pp:
                    pp.append(args['test_dir'])
            if args['build_dir'] not in pp:
                pp.append(args['build_dir'])
        # cross compilation support for Python 2.x
        if (version.major == 2 and
            arch_data.get('DEB_BUILD_ARCH') != arch_data.get('DEB_HOST_ARCH')):
            pp.insert(0, ('/usr/lib/python{0}/plat-{1[DEB_HOST_MULTIARCH]}'
                         ).format(version, arch_data))
        env['PYTHONPATH'] = ':'.join(pp)
        # cross compilation support for Python <= 3.8 (see above)
        if version.major == 3:
            name = '_PYTHON_SYSCONFIGDATA_NAME'
            value = env.get(name, context['ENV'].get(name, ''))
            if version << '3.8' and value.startswith('_sysconfigdata_')\
               and not value.startswith('_sysconfigdata_m'):
                value = env[name] = "_sysconfigdata_m%s" % value[15:]
            # update default from main() for -dbg interpreter
            if value and ipreter.debug and not value.startswith('_sysconfigdata_d'):
                env[name] = "_sysconfigdata_d%s" % value[15:]
        args['ENV'] = env

        if not exists(args['build_dir']):
            makedirs(args['build_dir'])

        return args

    def is_disabled(step, interpreter, version):
        i = interpreter
        prefix = "{}/".format(step)
        disabled = (get_option('disable', i, version) or '').split()
        for item in disabled:
            if item in (step, '1'):
                log.debug('disabling {} step for {} {}'.format(step, i, version))
                return True
            if item.startswith(prefix):
                disabled.append(item[len(prefix):])
        if i in disabled or str(version) in disabled or \
                i.format(version=version) in disabled or \
                i.format(version=version.major) in disabled:
            log.debug('disabling {} step for {} {}'.format(step, i, version))
            return True
        return False

    def run(func, interpreter, version, context):
        step = func.__func__.__name__
        args = get_args(context, step, version, interpreter)
        env = dict(context['ENV'])
        if 'ENV' in args:
            env.update(args['ENV'])

        before_cmd = get_option('before_{}'.format(step), interpreter, version)
        if before_cmd:
            if cfg.quiet:
                log_file = join(args['home_dir'], 'before_{}_cmd.log'.format(step))
            else:
                log_file = False
            command = before_cmd.format(**args)
            log.info(command)
            output = execute(command, context['dir'], env, log_file)
            if output['returncode'] != 0:
                msg = 'exit code={}: {}'.format(output['returncode'], command)
                raise Exception(msg)

        fpath = join(args['home_dir'], 'testfiles_to_rm_before_install')
        if step == 'install' and exists(fpath):
            with open(fpath) as fp:
                for line in fp:
                    path = line.strip('\n')
                    if exists(path):
                        if isdir(path):
                            rmtree(path)
                        else:
                            remove(path)
            remove(fpath)
        result = func(context, args)

        after_cmd = get_option('after_{}'.format(step), interpreter, version)
        if after_cmd:
            if cfg.quiet:
                log_file = join(args['home_dir'], 'after_{}_cmd.log'.format(step))
            else:
                log_file = False
            command = after_cmd.format(**args)
            log.info(command)
            output = execute(command, context['dir'], env, log_file)
            if output['returncode'] != 0:
                msg = 'exit code={}: {}'.format(output['returncode'], command)
                raise Exception(msg)
        return result

    def move_to_ext_destdir(i, version, context):
        """Move built C extensions from the general destdir to ext_destdir"""
        args = get_args(context, 'install', version, interpreter)
        ext_destdir = get_option('ext_destdir', i, version)
        if ext_destdir:
            move_matching_files(args['destdir'], ext_destdir,
                                get_option('ext_pattern', i, version),
                                get_option('ext_sub_pattern', i, version),
                                get_option('ext_sub_repl', i, version))

    func = None
    if cfg.clean_only:
        func = plugin.clean
    elif cfg.configure_only:
        func = plugin.configure
    elif cfg.build_only:
        func = plugin.build
    elif cfg.install_only:
        func = plugin.install
    elif cfg.test_only:
        func = plugin.test
    elif cfg.print_args:
        func = plugin.print_args

    ### one function for each interpreter at a time mode ###
    if func:
        step = func.__func__.__name__
        if step == 'test' and nocheck:
            exit(0)
        failure = False
        for i in cfg.interpreter:
            ipreter = Interpreter(interpreter.format(version=versions[0]))
            iversions = build_sorted(versions, impl=ipreter.impl)
            if '{version}' not in i and len(versions) > 1:
                log.info('limiting Python versions to %s due to missing {version}'
                         ' in interpreter string', str(versions[-1]))
                iversions = versions[-1:]  # just the default or closest to default
            for version in iversions:
                if is_disabled(step, i, version):
                    continue
                c = dict(context)
                c['dir'] = get_option('dir', i, version, cfg.dir)
                c['destdir'] = get_option('destdir', i, version, cfg.destdir)
                try:
                    run(func, i, version, c)
                except Exception as err:
                    log.error('%s: plugin %s failed with: %s',
                              step, plugin.NAME, err, exc_info=cfg.verbose)
                    # try to build/test other interpreters/versions even if
                    # one of them fails to make build logs more verbose:
                    failure = True
                    if step not in ('build', 'test'):
                        exit(13)
                if step == 'install':
                    move_to_ext_destdir(i, version, c)
        if failure:
            # exit with a non-zero return code if at least one build/test failed
            exit(13)
        exit(0)

    ### all functions for interpreters in batches mode ###
    try:
        context_map = {}
        for i in cfg.interpreter:
            ipreter = Interpreter(interpreter.format(version=versions[0]))
            iversions = build_sorted(versions, impl=ipreter.impl)
            if '{version}' not in i and len(versions) > 1:
                log.info('limiting Python versions to %s due to missing {version}'
                         ' in interpreter string', str(versions[-1]))
                iversions = versions[-1:]  # just the default or closest to default
            for version in iversions:
                key = (i, version)
                if key in context_map:
                    c = context_map[key]
                else:
                    c = dict(context)
                    c['dir'] = get_option('dir', i, version, cfg.dir)
                    c['destdir'] = get_option('destdir', i, version, cfg.destdir)
                    context_map[key] = c

                if not is_disabled('clean', i, version):
                    run(plugin.clean, i, version, c)
                if not is_disabled('configure', i, version):
                    run(plugin.configure, i, version, c)
                if not is_disabled('build', i, version):
                    run(plugin.build, i, version, c)
                if not is_disabled('install', i, version):
                    run(plugin.install, i, version, c)
                    move_to_ext_destdir(i, version, c)
                if not nocheck and not is_disabled('test', i, version):
                    run(plugin.test, i, version, c)
    except Exception as err:
        log.error('plugin %s failed: %s', plugin.NAME, err,
                  exc_info=cfg.verbose)
        exit(14)


def parse_args(argv):
    usage = '%(prog)s [ACTION] [BUILD SYSTEM ARGS] [DIRECTORIES] [OPTIONS]'
    parser = argparse.ArgumentParser(usage=usage)
    parser.add_argument('-v', '--verbose', action='store_true',
                        default=environ.get('PYBUILD_VERBOSE') == '1',
                        help='turn verbose mode on')
    parser.add_argument('-q', '--quiet', action='store_true',
                        default=environ.get('PYBUILD_QUIET') == '1',
                        help='doesn\'t show external command\'s output')
    parser.add_argument('-qq', '--really-quiet', action='store_true',
                        default=environ.get('PYBUILD_RQUIET') == '1',
                        help='be quiet')
    parser.add_argument('--version', action='version', version='%(prog)s DEVELV')

    action = parser.add_argument_group('ACTION', '''The default is to build,
        install and test the library using detected build system version by
        version. Selecting one of following actions, will invoke given action
        for all versions - one by one - which (contrary to the default action)
        in some build systems can overwrite previous results.''')
    action.add_argument('--detect', action='store_true', dest='detect_only',
                        help='return the name of detected build system')
    action.add_argument('--clean', action='store_true', dest='clean_only',
                        help='clean files using auto-detected build system specific methods')
    action.add_argument('--configure', action='store_true', dest='configure_only',
                        help='invoke configure step for all requested Python versions')
    action.add_argument('--build', action='store_true', dest='build_only',
                        help='invoke build step for all requested Python versions')
    action.add_argument('--install', action='store_true', dest='install_only',
                        help='invoke install step for all requested Python versions')
    action.add_argument('--test', action='store_true', dest='test_only',
                        help='invoke tests for auto-detected build system')
    action.add_argument('--list-systems', action='store_true',
                        help='list available build systems and exit')
    action.add_argument('--print', action='append', dest='print_args',
                        help="print pybuild's internal parameters")

    arguments = parser.add_argument_group('BUILD SYSTEM ARGS', '''
        Additional arguments passed to the build system.
        --system=custom requires complete command.''')
    arguments.add_argument('--before-clean', metavar='CMD',
                           help='invoked before the clean command')
    arguments.add_argument('--clean-args', metavar='ARGS')
    arguments.add_argument('--after-clean', metavar='CMD',
                           help='invoked after the clean command')

    arguments.add_argument('--before-configure', metavar='CMD',
                           help='invoked before the configure command')
    arguments.add_argument('--configure-args', metavar='ARGS')
    arguments.add_argument('--after-configure', metavar='CMD',
                           help='invoked after the configure command')

    arguments.add_argument('--before-build', metavar='CMD',
                           help='invoked before the build command')
    arguments.add_argument('--build-args', metavar='ARGS')
    arguments.add_argument('--after-build', metavar='CMD',
                           help='invoked after the build command')

    arguments.add_argument('--before-install', metavar='CMD',
                           help='invoked before the install command')
    arguments.add_argument('--install-args', metavar='ARGS')
    arguments.add_argument('--after-install', metavar='CMD',
                           help='invoked after the install command')

    arguments.add_argument('--before-test', metavar='CMD',
                           help='invoked before the test command')
    arguments.add_argument('--test-args', metavar='ARGS')
    arguments.add_argument('--after-test', metavar='CMD',
                           help='invoked after the test command')

    tests = parser.add_argument_group('TESTS', '''\
        unittest\'s discover is used by default (if available)''')
    tests.add_argument('--test-nose', action='store_true',
                       default=environ.get('PYBUILD_TEST_NOSE') == '1',
                       help='use nose module in --test step')
    tests.add_argument('--test-nose2', action='store_true',
                       default=environ.get('PYBUILD_TEST_NOSE2') == '1',
                       help='use nose2 module in --test step')
    tests.add_argument('--test-pytest', action='store_true',
                       default=environ.get('PYBUILD_TEST_PYTEST') == '1',
                       help='use pytest module in --test step')
    tests.add_argument('--test-tox', action='store_true',
                       default=environ.get('PYBUILD_TEST_TOX') == '1',
                       help='use tox in --test step')
    tests.add_argument('--test-custom', action='store_true',
                       default=environ.get('PYBUILD_TEST_CUSTOM') == '1',
                       help='use custom command in --test step')

    dirs = parser.add_argument_group('DIRECTORIES')
    dirs.add_argument('-d', '--dir', action='store', metavar='DIR',
                      default=environ.get('PYBUILD_DIR', getcwd()),
                      help='source files directory - base for other relative dirs [default: CWD]')
    dirs.add_argument('--dest-dir', action='store', metavar='DIR', dest='destdir',
                      default=environ.get('DESTDIR', 'debian/tmp'),
                      help='destination directory [default: debian/tmp]')
    dirs.add_argument('--ext-dest-dir', action='store', metavar='DIR', dest='ext_destdir',
                      default=environ.get('PYBUILD_EXT_DESTDIR'),
                      help='destination directory for .so files')
    dirs.add_argument('--ext-pattern', action='store', metavar='PATTERN',
                      default=environ.get('PYBUILD_EXT_PATTERN', r'\.so(\.[^/]*)?$'),
                      help='regular expression for files that should be moved'
                      ' if --ext-dest-dir is set [default: .so files]')
    dirs.add_argument('--ext-sub-pattern', action='store', metavar='PATTERN',
                      default=environ.get('PYBUILD_EXT_SUB_PATTERN'),
                      help='pattern to change --ext-pattern\'s filename or path')
    dirs.add_argument('--ext-sub-repl', action='store', metavar='PATTERN',
                      default=environ.get('PYBUILD_EXT_SUB_REPL'),
                      help='replacement for match from --ext-sub-pattern,'
                      ' empty string by default')
    dirs.add_argument('--install-dir', action='store', metavar='DIR',
                      help='installation directory [default: .../dist-packages]')
    dirs.add_argument('--name', action='store',
                      default=environ.get('PYBUILD_NAME'),
                      help='use this name to guess destination directories')

    limit = parser.add_argument_group('LIMITATIONS')
    limit.add_argument('-s', '--system',
                       default=environ.get('PYBUILD_SYSTEM'),
                       help='select a build system [default: auto-detection]')
    limit.add_argument('-p', '--pyver', action='append', dest='versions',
                       help='''build for Python VERSION.
                               This option can be used multiple times
                               [default: all supported Python 3.X versions]''')
    limit.add_argument('-i', '--interpreter', action='append',
                       help='change interpreter [default: python{version}]')
    limit.add_argument('--disable', metavar='ITEMS',
                       help='disable action, interpreter or version')

    args = parser.parse_args()
    if not args.interpreter:
        args.interpreter = environ.get('PYBUILD_INTERPRETERS', 'python{version}').split()
    if not args.versions:
        args.versions = environ.get('PYBUILD_VERSIONS', '').split()
    else:
        # add support for -p `pyversions -rv`
        versions = []
        for version in args.versions:
            versions.extend(version.split())
        args.versions = versions

    if args.test_nose or args.test_nose2 or args.test_pytest or args.test_tox\
       or args.system == 'custom':
        args.custom_tests = True
    else:
        args.custom_tests = False

    return args


if __name__ == '__main__':
    cfg = parse_args(sys.argv)
    if cfg.really_quiet:
        cfg.quiet = True
        log.setLevel(logging.CRITICAL)
    elif cfg.verbose:
        log.setLevel(logging.DEBUG)
    else:
        log.setLevel(logging.INFO)
    log.debug('version: DEVELV')
    log.debug(sys.argv)
    main(cfg)
    # let dh/cdbs clean the .pybuild dir
    # rmtree(join(cfg.dir, '.pybuild'))