Makefile

@@ -74,7 +74,7 @@ test:
 	    --source . /usr/bin/nosetests
 	coverage2 report
 	coverage2 html
-	@echo Full coverage report in htmlcov/index.html
+	@echo Full coverage report at file://$(CURDIR)/htmlcov/index.html
 
 test3:
 	coverage3 erase
@@ -83,7 +83,7 @@ test3:
 	    /usr/bin/nosetests-3
 	coverage3 report --rcfile .coveragerc3
 	coverage3 html --rcfile .coveragerc3
-	@echo Full coverage report at file://${PWD}/htmlcov/index.html
+	@echo Full coverage report at file://$(CURDIR)/htmlcov/index.html
 
 test-tarball:
 	@rm -rf .koji-$(VERSION)

README.md

@@ -51,7 +51,7 @@ Related Software
 
 * [Mock](https://fedoraproject.org/wiki/Projects/Mock): The tool Koji uses to generate buildroots
 * [Yum](http://yum.baseurl.org/)
-* [Mash](https://pagure.io/mash)
+* [Pungi](https://pagure.io/pungi): Use Pungi to "compose" Koji builds into highly customizable Yum repositories.
 * [Koji Tools](https://pagure.io/koji-tools): Various utilities for Koji
 * [Kojiji](https://github.com/release-engineering/kojiji): Koji Java Interface
 * [txkoji](https://github.com/ktdreyer/txkoji): Async interface to Koji, using Twisted

builder/kojid

@@ -47,7 +47,7 @@ import time
 import traceback
 import xml.dom.minidom
 import zipfile
-from six.moves.configparser import ConfigParser
+from six.moves.configparser import RawConfigParser
 from fnmatch import fnmatch
 from gzip import GzipFile
 from optparse import OptionParser, SUPPRESS_HELP
@@ -272,8 +272,15 @@ class BuildRoot(object):
         opts['target_arch'] = self.target_arch
         if 'mock.package_manager' in self.config['extra']:
             opts['package_manager'] = self.config['extra']['mock.package_manager']
+        if 'mock.yum.module_hotfixes' in self.config['extra']:
+            opts['module_hotfixes'] = self.config['extra']['mock.yum.module_hotfixes']
         if self.internal_dev_setup is not None:
             opts['internal_dev_setup'] = bool(self.internal_dev_setup)
+        opts['tag_macros'] = {}
+        for key in self.config['extra']:
+            if key.startswith('rpm.macro.'):
+                macro = '%' + key[10:]
+                opts['tag_macros'][macro] = self.config['extra'][key]
         output = koji.genMockConfig(self.name, self.br_arch, managed=True, **opts)
 
         #write config
@@ -461,7 +468,7 @@ class BuildRoot(object):
                                 self.logger.info('Rereading %s, inode: %s -> %s, size: %s -> %s' %
                                                  (fpath, inode, stat_info.st_ino, size, stat_info.st_size))
                                 fd.close()
-                            fd = open(fpath, 'r')
+                            fd = open(fpath, 'rb')
                         logs[fname] = (fd, stat_info.st_ino, stat_info.st_size or size, fpath)
                     except:
                         self.logger.error("Error reading mock log: %s", fpath)
@@ -525,6 +532,7 @@ class BuildRoot(object):
         append '.' + suffix to the filenames, so that successive uploads
         of the same directory won't overwrite each other, if the files have
         the same name but different contents."""
+        koji.util.deprecated('BuildRoot.uploadDir method is deprecated and will be removed in 1.19')
         if not os.path.isdir(dirpath):
             return
         uploadpath = self.getUploadPath()
@@ -553,13 +561,41 @@ class BuildRoot(object):
         msg = '; see %s for more information' % logfile
         return parseStatus(rv, 'mock') + msg
 
+    def rebuild_srpm(self, srpm):
+        self.session.host.setBuildRootState(self.id,'BUILDING')
+
+        # unpack SRPM to tempdir
+        srpm_dir = os.path.join(self.tmpdir(), 'srpm_unpacked')
+        koji.ensuredir(srpm_dir)
+        top_dir = self.path_without_to_within(srpm_dir)
+        args = ['--no-clean', '--target', 'noarch', '--chroot', '--',
+                'rpm', '--define', '_topdir %s' % top_dir, '-iv', srpm]
+        rv = self.mock(args)
+
+        # find specfile
+        spec_files = glob.glob("%s/SPECS/*.spec" % srpm_dir)
+        if len(spec_files) == 0:
+            raise koji.BuildError("No spec file found")
+        elif len(spec_files) > 1:
+            raise koji.BuildError("Multiple spec files found: %s" % spec_files)
+        spec_file = os.path.join(top_dir, "SPECS", os.path.basename(spec_files[0]))
+
+        # rebuild SRPM from spec + sources
+        args = ['--no-clean', '--target', 'noarch', '--chroot', '--',
+                'rpmbuild', '--define', '_topdir %s' % top_dir, '-bs', '--nodeps', spec_file]
+        rv = self.mock(args)
+
+        result_dir = os.path.join(srpm_dir, 'SRPMS')
+        for fn in glob.glob('%s/*.src.rpm' % result_dir):
+            shutil.move(os.path.join(result_dir, fn), self.resultdir())
+
+        if rv:
+            self.expire()
+            raise koji.BuildError("error building srpm, %s" % self._mockResult(rv))
+
+
     def build_srpm(self, specfile, sourcedir, source_cmd):
         self.session.host.setBuildRootState(self.id,'BUILDING')
-        alt_sources_dir = "%s/SOURCES" % sourcedir
-        if self.options.support_rpm_source_layout and os.path.isdir(alt_sources_dir):
-            sources_dir = alt_sources_dir
-        else:
-            sources_dir = sourcedir
         if source_cmd:
             # call the command defined by source_cmd in the chroot so any required files not stored in
             # the SCM can be retrieved
@@ -571,6 +607,12 @@ class BuildRoot(object):
                 self.expire()
                 raise koji.BuildError("error retrieving sources, %s" % self._mockResult(rv))
 
+        alt_sources_dir = "%s/SOURCES" % sourcedir
+        if self.options.support_rpm_source_layout and os.path.isdir(alt_sources_dir):
+            sources_dir = alt_sources_dir
+        else:
+            sources_dir = sourcedir
+
         args = ['--no-clean', '--buildsrpm', '--spec', specfile, '--sources', sources_dir,
                 '--target', 'noarch']
 
@@ -739,6 +781,9 @@ class BuildRoot(object):
             h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
             h.setopt(librepo.LRO_URLS, [repo_url])
             h.setopt(librepo.LRO_DESTDIR, tmpdir)
+            # We are using this just to find out location of 'origin',
+            # we don't even need to download it since we use openRemoteFile
+            h.setopt(librepo.LRO_YUMDLIST, [])
             h.perform(r)
             pkgorigins = r.getinfo(librepo.LRR_YUM_REPOMD)['origin']['location_href']
             koji.util.rmtree(tmpdir)
@@ -761,16 +806,18 @@ class BuildRoot(object):
             #at this point we know there were external repos at the create event,
             #so there should be an origins file.
             origin_idx = {}
-            with GzipFile(fileobj=fo, mode='r') as fo2:
-                if six.PY3:
-                    fo2 = io.TextIOWrapper(fo2, encoding='utf-8')
-                for line in fo2:
-                    parts=line.split(None, 2)
-                    if len(parts) < 2:
-                        continue
-                    #first field is formated by yum as [e:]n-v-r.a
-                    nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" %  koji.parse_NVRA(parts[0])
-                    origin_idx[nvra] = parts[1]
+            # don't use 'with GzipFile' as it is not supported on py2.6
+            fo2 = GzipFile(fileobj=fo, mode='r')
+            if six.PY3:
+                fo2 = io.TextIOWrapper(fo2, encoding='utf-8')
+            for line in fo2:
+                parts=line.split(None, 2)
+                if len(parts) < 2:
+                    continue
+                #first field is formated by yum as [e:]n-v-r.a
+                nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" %  koji.parse_NVRA(parts[0])
+                origin_idx[nvra] = parts[1]
+            fo2.close()
         # mergerepo starts from a local repo in the task workdir, so internal
         # rpms have an odd-looking origin that we need to look for
         localtail = '/repo_%s_premerge/' % self.repo_info['id']
@@ -1018,12 +1065,31 @@ class BuildTask(BaseTaskHandler):
             if SCM.is_scm_url(src):
                 return self.getSRPMFromSCM(src, build_tag, repo_id)
             else:
-                #assume this is a path under uploads
-                return src
+                buildconfig = self.session.getBuildConfig(build_tag, event=self.event_id)
+                if buildconfig['extra'].get('rebuild_srpm', True):
+                    # default is to always rebuild
+                    return self.getSRPMFromSRPM(src, build_tag, repo_id)
+                else:
+                    return src
         else:
             raise koji.BuildError('Invalid source specification: %s' % src)
             #XXX - other methods?
 
+    def getSRPMFromSRPM(self, src, build_tag, repo_id):
+        # rebuild srpm in mock, so it gets correct disttag, rpm version, etc.
+        task_id = self.session.host.subtask(method='rebuildSRPM',
+                                            arglist=[src, build_tag, {'repo_id': repo_id, 'scratch': self.opts.get('scratch')}],
+                                            label='srpm',
+                                            parent=self.id)
+        # wait for subtask to finish
+        result = self.wait(task_id)[task_id]
+        if 'source' in result:
+            self.source = result['source']
+        else:
+            self.logger.warning('subtask did not provide source data')
+        srpm = result['srpm']
+        return srpm
+
     def getSRPMFromSCM(self, url, build_tag, repo_id):
         #TODO - allow different ways to get the srpm
         task_id = self.session.host.subtask(method='buildSRPMFromSCM',
@@ -2135,7 +2201,7 @@ class ChainMavenTask(MultiPlatformTask):
                 del todo[package]
             try:
                 results = self.wait(to_list(running.keys()))
-            except (six.moves.xmlrpc_client.Fault, koji.GenericError) as e:
+            except (six.moves.xmlrpc_client.Fault, koji.GenericError):
                 # One task has failed, wait for the rest to complete before the
                 # chainmaven task fails.  self.wait(all=True) should thrown an exception.
                 self.wait(all=True)
@@ -4122,6 +4188,9 @@ class BaseImageTask(OzImageTask):
         return {'image': newimg}
 
     def handler(self, name, version, release, arch, target_info, build_tag, repo_info, inst_tree, opts=None):
+        if not ozif_enabled:
+            self.logger.error("ImageFactory features require the following dependencies: pykickstart, imagefactory, oz and possibly python-hashlib")
+            raise koji.ApplianceError('ImageFactory functions not available')
 
         if opts == None:
             opts = {}
@@ -4572,6 +4641,88 @@ class BuildIndirectionImageTask(OzImageTask):
         return report
 
 
+class RebuildSRPM(BaseBuildTask):
+
+    Methods = ['rebuildSRPM']
+    _taskWeight = 1.0
+
+    def checkHost(self, hostdata):
+        tag = self.params[1]
+        return self.checkHostArch(tag, hostdata)
+
+    def handler(self, srpm, build_tag, opts=None):
+        if opts is None:
+            opts = {}
+        repo_id = opts.get('repo_id')
+        if not repo_id:
+            raise koji.BuildError("A repo id must be provided")
+
+        repo_info = self.session.repoInfo(repo_id, strict=True)
+        event_id = repo_info['create_event']
+        build_tag = self.session.getTag(build_tag, strict=True, event=event_id)
+
+        rootopts = {'install_group': 'srpm-build', 'repo_id': repo_id}
+        br_arch = self.find_arch('noarch', self.session.host.getHost(), self.session.getBuildConfig(build_tag['id'], event=event_id))
+        broot = BuildRoot(self.session, self.options, build_tag['id'], br_arch, self.id, **rootopts)
+        broot.workdir = self.workdir
+
+        self.logger.debug("Initializing buildroot")
+        broot.init()
+
+        # Setup files and directories for SRPM rebuild
+        # We can't put this under the mock homedir because that directory
+        # is completely blown away and recreated on every mock invocation
+        srpmdir = broot.tmpdir() + '/srpm'
+        koji.ensuredir(srpmdir)
+        uploadpath = self.getUploadDir()
+
+        fn = self.localPath("work/%s" % srpm)
+        if not os.path.exists(fn):
+            raise koji.BuildError("Input SRPM file missing: %s" % fn)
+        shutil.copy(fn, srpmdir)
+
+        # rebuild srpm
+        self.logger.debug("Running srpm rebuild")
+        br_srpm_path = os.path.join(broot.path_without_to_within(srpmdir), os.path.basename(srpm))
+        broot.rebuild_srpm(br_srpm_path)
+
+        srpms = glob.glob('%s/*.src.rpm' % broot.resultdir())
+        if len(srpms) == 0:
+            raise koji.BuildError("No srpms found in %s" % srpmdir)
+        elif len(srpms) > 1:
+            raise koji.BuildError("Multiple srpms found in %s: %s" % (srpmdir, ", ".join(srpms)))
+        else:
+            srpm = srpms[0]
+
+        # check srpm name
+        h = koji.get_rpm_header(srpm)
+        name = koji.get_header_field(h, 'name')
+        version = koji.get_header_field(h, 'version')
+        release = koji.get_header_field(h, 'release')
+        srpm_name = "%(name)s-%(version)s-%(release)s.src.rpm" % locals()
+        if srpm_name != os.path.basename(srpm):
+            raise koji.BuildError('srpm name mismatch: %s != %s' % (srpm_name, os.path.basename(srpm)))
+
+        # upload srpm and return
+        self.uploadFile(srpm)
+
+        brootid = broot.id
+        log_files = glob.glob('%s/*.log' % broot.resultdir())
+
+        broot.expire()
+
+        return {
+            'srpm': "%s/%s" % (uploadpath, srpm_name),
+            'logs': ["%s/%s" % (uploadpath, os.path.basename(f))
+                     for f in log_files],
+            'brootid': brootid,
+            'source': {
+                'source': os.path.basename(srpm),
+                'url': os.path.basename(srpm),
+            }
+        }
+
+
 class BuildSRPMFromSCMTask(BaseBuildTask):
 
     Methods = ['buildSRPMFromSCM']
@@ -5002,13 +5153,15 @@ class NewRepoTask(BaseTaskHandler):
     Methods = ['newRepo']
     _taskWeight = 0.1
 
-    def handler(self, tag, event=None, src=False, debuginfo=False):
+    def handler(self, tag, event=None, src=False, debuginfo=False, separate_src=False):
         tinfo = self.session.getTag(tag, strict=True, event=event)
         kwargs = {}
         if event is not None:
             kwargs['event'] = event
         if src:
             kwargs['with_src'] = True
+        if separate_src:
+            kwargs['with_separate_src'] = True
         if debuginfo:
             kwargs['with_debuginfo'] = True
         repo_id, event_id = self.session.host.repoInit(tinfo['id'], **kwargs)
@@ -5023,9 +5176,21 @@ class NewRepoTask(BaseTaskHandler):
         #only shadowbuild tags should start with SHADOWBUILD, their repos are auto
         #expired.  so lets get the most recent expired tag for newRepo shadowbuild tasks.
         if tinfo['name'].startswith('SHADOWBUILD'):
-            oldrepo = self.session.getRepo(tinfo['id'], state=koji.REPO_EXPIRED)
+            oldrepo_state = koji.REPO_EXPIRED
         else:
-            oldrepo = self.session.getRepo(tinfo['id'], state=koji.REPO_READY)
+            oldrepo_state = koji.REPO_READY
+        oldrepo = self.session.getRepo(tinfo['id'], state=oldrepo_state)
+        # If there is no old repo, try to find first usable repo in
+        # inheritance chain and use it as a source. oldrepo is not used if
+        # createrepo_update is not set, so don't waste call in such case.
+        if not oldrepo and self.options.createrepo_update:
+            tags = self.session.getFullInheritance(tinfo['id'])
+            # we care about best candidate which should be (not necessarily)
+            # something on higher levels. Sort tags according to depth.
+            for tag in sorted(tags, key=lambda x: x['currdepth']):
+                oldrepo = self.session.getRepo(tag['parent_id'], state=oldrepo_state)
+                if oldrepo:
+                    break
         subtasks = {}
         for arch in arches:
             arglist = [repo_id, arch, oldrepo]
@@ -5103,7 +5268,10 @@ class CreaterepoTask(BaseTaskHandler):
             cmd.extend(['-g', groupdata])
         #attempt to recycle repodata from last repo
         if pkglist and oldrepo and self.options.createrepo_update:
-            oldpath = self.pathinfo.repo(oldrepo['id'], rinfo['tag_name'])
+            # old repo could be from inherited tag, so path needs to be
+            # composed from that tag, not rinfo['tag_name']
+            oldrepo = self.session.repoInfo(oldrepo['id'], strict=True)
+            oldpath = self.pathinfo.repo(oldrepo['id'], oldrepo['tag_name'])
             olddatadir = '%s/%s/repodata' % (oldpath, arch)
             if not os.path.isdir(olddatadir):
                 self.logger.warn("old repodata is missing: %s" % olddatadir)
@@ -5165,13 +5333,19 @@ class CreaterepoTask(BaseTaskHandler):
             cmd = ['/usr/libexec/kojid/mergerepos',
                    '--mode', 'simple',
                    '--tempdir', self.workdir]
+        elif merge_mode == 'bare':
+            # "bare" merge mode for repos with modular metadata
+            # forces use of mergerepo_c
+            cmd = ['/usr/bin/mergerepo_c', '--pkgorigins', '--all']
         elif self.options.use_createrepo_c:
             cmd = ['/usr/bin/mergerepo_c', '--koji']
         else:
             cmd = ['/usr/libexec/kojid/mergerepos']
             cmd.extend(['--tempdir', self.workdir])
-        blocklist = self.repodir + '/blocklist'
-        cmd.extend(['-a', arch, '-b', blocklist, '-o', self.outdir])
+        if merge_mode != 'bare':
+            blocklist = self.repodir + '/blocklist'
+            cmd.extend(['-b', blocklist])
+        cmd.extend(['-a', arch, '-o', self.outdir])
         if os.path.isfile(groupdata):
             cmd.extend(['-g', groupdata])
         for repo in repos:
@@ -5932,7 +6106,7 @@ def get_options():
         assert False  # pragma: no cover
 
     # load local config
-    config = ConfigParser()
+    config = RawConfigParser()
     config.read(options.configFile)
     for x in config.sections():
         if x != 'kojid':
@@ -5970,12 +6144,11 @@ def get_options():
                 'max_retries': 120,
                 'offline_retry': True,
                 'offline_retry_interval': 120,
-                'keepalive': True,
                 'log_timestamps': False,
                 'timeout': None,
                 'no_ssl_verify': False,
                 'use_fast_upload': True,
-                'use_createrepo_c': False,
+                'use_createrepo_c': True,
                 'createrepo_skip_stat': True,
                 'createrepo_update': True,
                 'pkgurl': None,
@@ -6002,9 +6175,9 @@ def get_options():
                 except ValueError:
                     quit("value for %s option must be a valid integer" % name)
             elif name in ['offline_retry', 'use_createrepo_c',  'createrepo_skip_stat',
-                          'createrepo_update', 'keepalive', 'use_fast_upload',
-                          'support_rpm_source_layout', 'krb_rdns', 'krb_canon_host',
-                          'build_arch_can_fail', 'no_ssl_verify', 'log_timestamps']:
+                          'createrepo_update', 'use_fast_upload', 'support_rpm_source_layout',
+                          'krb_rdns', 'krb_canon_host', 'build_arch_can_fail', 'no_ssl_verify',
+                          'log_timestamps']:
                 defaults[name] = config.getboolean('kojid', name)
             elif name in ['plugin', 'plugins']:
                 defaults['plugin'] = value.split()

builder/kojid.conf

@@ -50,7 +50,7 @@ server=http://hub.example.com/kojihub
 topurl=http://hub.example.com/kojifiles
 
 ; use createrepo_c rather than createrepo
-; use_createrepo_c=False
+; use_createrepo_c=True
 
 ; A space-separated list of tuples from which kojid is allowed to checkout.
 ; The format of those tuples is:

cli/koji

@@ -27,12 +27,12 @@
 from __future__ import absolute_import
 from __future__ import division
 import logging
-import optparse
 import os
 import re
 import six
 import sys
-import types
+
+from optparse import OptionParser, SUPPRESS_HELP
 
 import six.moves.configparser
 import six.moves.xmlrpc_client
@@ -41,7 +41,7 @@ import koji
 import koji.util
 import koji.plugin
 
-from koji_cli.lib import _, OptionParser, get_epilog_str, greetings, \
+from koji_cli.lib import _, get_epilog_str, greetings, \
         warn, categories
 from koji_cli.commands import *
 
@@ -65,27 +65,47 @@ def register_plugin(plugin):
                 globals()[name] = v
 
 
-def load_plugins(options, path):
-    """Load plugins specified by our configuration plus system plugins. Order
-    is that system plugins are first, so they can be overridden by
-    user-specified ones with same name."""
+def load_plugins(plugin_paths):
+    """Load plugins specified by input paths, ~/.koji/plugins, system plugins.
+    Loading order is descending, so they can be overridden by user-specified
+    ones.
+    Notice that:
+    - plugin file should end with .py extension
+    - non-directory is not acceptable by plugin_paths
+    - all plugin files and the exported handlers inside will be loaded, and
+      handler with the same name will override the one has already been loaded
+      before"""
+
     logger = logging.getLogger('koji.plugins')
-    if os.path.exists(path):
-        tracker = koji.plugin.PluginTracker(path=path)
-        for name in sorted(os.listdir(path)):
-            if not name.endswith('.py'):
-                continue
-            name = name[:-3]
-            logger.info('Loading plugin: %s', name)
-            tracker.load(name)
-            register_plugin(tracker.get(name))
+    paths = []
+    # first, always load plugins from koji_cli_plugins module
+    paths.append(
+        '%s/lib/python%s.%s/site-packages/koji_cli_plugins' %
+        (sys.prefix, sys.version_info[0], sys.version_info[1]))
+    # second, always load plugins from ~/.koji/plugins
+    paths.append(os.path.expanduser('~/.koji/plugins'))
+    # finally, update plugin_paths to the list
+    if plugin_paths:
+        if not isinstance(plugin_paths, (list, tuple)):
+            plugin_paths = plugin_paths.split(':')
+        paths.extend([os.path.expanduser(p) for p in reversed(plugin_paths)])
+    tracker = koji.plugin.PluginTracker()
+    for path in paths:
+        if os.path.exists(path) and os.path.isdir(path):
+            for name in sorted(os.listdir(path)):
+                fullname = os.path.join(path, name)
+                if not (os.path.isfile(fullname) and name.endswith('.py')):
+                    continue
+                name = name[:-3]
+                logger.info('Loading plugin: %s', fullname)
+                register_plugin(tracker.load(name, path=path, reload=True))
 
 
 def get_options():
     """process options from command line and config file"""
 
     common_commands = ['build', 'help', 'download-build',
-                       'latest-pkg', 'search', 'list-targets']
+                       'latest-build', 'search', 'list-targets']
     usage = _("%%prog [global-options] command [command-options-and-arguments]"
                 "\n\nCommon commands: %s" % ', '.join(sorted(common_commands)))
     parser = OptionParser(usage=usage)
@@ -103,6 +123,8 @@ def get_options():
     parser.add_option("--keytab", help=_("specify a Kerberos keytab to use"), metavar="FILE")
     parser.add_option("--principal", help=_("specify a Kerberos principal to use"))
     parser.add_option("--krbservice", help=_("specify the Kerberos service name for the hub"))
+    parser.add_option("--cert", help=_("specify a SSL cert to use"), metavar="FILE")
+    parser.add_option("--ca", help=_("specify a SSL CA to use"), metavar="FILE")
     parser.add_option("--runas", help=_("run as the specified user (requires special privileges)"))
     parser.add_option("--user", help=_("specify user"))
     parser.add_option("--password", help=_("specify password"))
@@ -123,7 +145,9 @@ def get_options():
     parser.add_option("--topdir", help=_("specify topdir"))
     parser.add_option("--weburl", help=_("url of the Koji web interface"))
     parser.add_option("--topurl", help=_("url for Koji file access"))
-    parser.add_option("--pkgurl", help=optparse.SUPPRESS_HELP)
+    parser.add_option("--pkgurl", help=SUPPRESS_HELP)
+    parser.add_option("--plugin-paths", metavar='PATHS',
+            help=_("specify additional plugin paths (colon separated)"))
     parser.add_option("--help-commands", action="store_true", default=False, help=_("list commands"))
     (options, args) = parser.parse_args()
 
@@ -163,16 +187,13 @@ def get_options():
             else:
                 warn("Warning: The pkgurl option is obsolete, please use topurl instead")
 
-    plugins_path = '%s/lib/python%s.%s/site-packages/koji_cli_plugins' % \
-                   (sys.prefix, sys.version_info[0], sys.version_info[1])
-    load_plugins(options, plugins_path)
+    load_plugins(options.plugin_paths)
 
-    if options.help_commands:
-        list_commands()
-        sys.exit(0)
     if not args:
-        list_commands()
-        sys.exit(0)
+        options.help_commands = True
+    if options.help_commands:
+        # hijack args to [return_code, message]
+        return options, '_list_commands', [0, '']
 
     aliases = {
         'cancel-task' : 'cancel',
@@ -199,9 +220,8 @@ def get_options():
     elif ('handle_' + cmd) in globals():
         cmd = 'handle_' + cmd
     else:
-        list_commands()
-        parser.error('Unknown command: %s' % args[0])
-        assert False  # pragma: no cover
+        # hijack args to [return_code, message]
+        return options, '_list_commands', [1, 'Unknown command: %s' % args[0]]
 
     return options, cmd, args[1:]
 
@@ -212,7 +232,7 @@ def handle_help(options, session, args):
     usage += _("\n(Specify the --help global option for a list of other help options)")
     parser = OptionParser(usage=usage)
     # the --admin opt is for backwards compatibility. It is equivalent to: koji help admin
-    parser.add_option("--admin", action="store_true", help=optparse.SUPPRESS_HELP)
+    parser.add_option("--admin", action="store_true", help=SUPPRESS_HELP)
 
     (options, args) = parser.parse_args(args)
 
@@ -305,6 +325,12 @@ if __name__ == "__main__":
 
     session_opts = koji.grab_session_options(options)
     session = koji.ClientSession(options.server, session_opts)
+    if command == '_list_commands':
+        list_commands()
+        if args[0] != 0:
+            logger.error(args[1])
+        sys.exit(args[0])
+    # run handler
     rv = 0
     try:
         rv = locals()[command].__call__(options, session, args)

cli/koji.conf

@@ -37,6 +37,12 @@
 ;certificate of the CA that issued the HTTP server certificate
 ;serverca = ~/.koji/serverca.crt
 
+;plugin paths, separated by ':' as the same as the shell's PATH
+;koji_cli_plugins module and ~/.koji/plugins are always loaded in advance,
+;and then be overridden by this option
+;plugin_paths = ~/.koji/plugins
+
+;[not_implemented_yet]
 ;enabled plugins for CLI, runroot and save_failed_tree are available
 ;plugins =
 

cli/koji_cli/lib.py

@@ -20,18 +20,12 @@ except ImportError:  # pragma: no cover
 
 import koji
 from koji.util import to_list
+# import parse_arches to current namespace for backward compatibility
+from koji import parse_arches
 
-# fix OptionParser for python 2.3 (optparse verion 1.4.1+)
-# code taken from optparse version 1.5a2
+# for compatibility with plugins based on older version of lib
+# Use optparse imports directly in new code.
 OptionParser = optparse.OptionParser
-if optparse.__version__ == "1.4.1+":  # pragma: no cover
-    def _op_error(self, msg):
-        self.print_usage(sys.stderr)
-        msg = "%s: error: %s\n" % (self._get_prog_name(), msg)
-        if msg:
-            sys.stderr.write(msg)
-        sys.exit(2)
-    OptionParser.error = _op_error
 
 greetings = ('hello', 'hi', 'yo', "what's up", "g'day", 'back to work',
              'bonjour',
@@ -140,26 +134,6 @@ def print_task_recurse(task,depth=0):
         print_task_recurse(child,depth+1)
 
 
-def parse_arches(arches, to_list=False):
-    """Normalize user input for a list of arches.
-
-    This method parses a single comma- or space-separated string of arches and
-    returns a space-separated string.
-
-    :param str arches: comma- or space-separated string of arches, eg.
-                       "x86_64,ppc64le", or "x86_64 ppc64le"
-    :param bool to_list: return a list of each arch, instead of a single
-                         string. This is False by default.
-    :returns: a space-separated string like "x86_64 ppc64le", or a list like
-              ['x86_64', 'ppc64le'].
-    """
-    arches = arches.replace(',', ' ').split()
-    if to_list:
-        return arches
-    else:
-        return ' '.join(arches)
-
-
 class TaskWatcher(object):
 
     def __init__(self,task_id,session,level=0,quiet=False):
@@ -343,20 +317,13 @@ Running Tasks:
     return rv
 
 
-def write_to_stdout(contents):
-    """Helper function to write str/bytes to stdout
-
-    https://docs.python.org/3/library/sys.html#sys.displayhook
-    """
-    try:
+def bytes_to_stdout(contents):
+    """Helper function for writing bytes to stdout"""
+    if six.PY2:
         sys.stdout.write(contents)
-    except UnicodeEncodeError:
-        bytes = contents.encode(sys.stdout.encoding, 'backslashreplace')
-        if hasattr(sys.stdout, 'buffer'):
-            sys.stdout.buffer.write(bytes)
-        else:
-            contents = bytes.decode(sys.stdout.encoding, 'strict')
-            sys.stdout.write(contents)
+    else:
+        sys.stdout.buffer.write(contents)
+
 
 def watch_logs(session, tasklist, opts, poll_interval):
     print("Watching logs (this may be safely interrupted)...")
@@ -406,7 +373,7 @@ def watch_logs(session, tasklist, opts, poll_interval):
                                 sys.stdout.write("\n")
                             sys.stdout.write("==> %s <==\n" % currlog)
                             lastlog = currlog
-                        write_to_stdout(contents)
+                        bytes_to_stdout(contents)
 
 
             if opts.follow:

debian/changelog

+koji (1.18.0-1) UNRELEASED; urgency=medium
+
+  * New upstream release.
+  * Refresh patches, drop 0005-posix-sh-syntax.sh as this has been fixed
+    upstream.
+
+ -- Holger Levsen <holger@debian.org>  Sun, 11 Aug 2019 14:30:40 +0200
+
+koji (1.16.2-1) unstable; urgency=medium
+
+  * New upstream version, fixing CVE-2018-1002161. Closes: #922922
+
+ -- Holger Levsen <holger@debian.org>  Sun, 24 Feb 2019 19:44:30 +0100
+
+koji (1.16.1-1) unstable; urgency=medium
+
+  * New upstream version.
+
+ -- Holger Levsen <holger@debian.org>  Thu, 07 Feb 2019 14:22:11 +0100
+
+koji (1.16.0-3) unstable; urgency=medium
+
+  * koji-client: add depends on python-requests and python-dateutil, thanks
+    to Juhani Numminen for the bug report. Closes: #918780
+  * d/control:
+    - bump debhelper compat to 11 so we get nice koji* service handling in
+      postinst and postrm.
+    - use the new debhelper-compat(=11) notation and drop d/compat.
+    - bump standards version to 4.3.0, no changes needed.
+    - add "Rules-Requires-Root: no" to support building as non-root.
+    - add myself to uploaders.
+  * d/copyright:
+    - use https for Format: URL.
+    - add myself.
+
+ -- Holger Levsen <holger@debian.org>  Wed, 09 Jan 2019 12:11:40 +0100
+
+koji (1.16.0-2) unstable; urgency=medium
+
+  * Install a conf file for fedora's current main koji hub.
+
+ -- Ximin Luo <infinity0@debian.org>  Sun, 08 Jul 2018 12:37:16 -0700
+
+koji (1.16.0-1) unstable; urgency=medium
+
+  * New upstream release. (Closes: #877921, #877886, #877887)
+
+ -- Ximin Luo <infinity0@debian.org>  Sun, 08 Jul 2018 12:06:37 -0700
+
+koji (1.10.0-1) unstable; urgency=medium
+
+  [ Marek Marczykowski-Górecki ]
+  * Initial release. (Closes: #806953)
+
+ -- Ximin Luo <infinity0@debian.org>  Fri, 04 Dec 2015 11:20:57 +0100

debian/control

+Source: koji
+Maintainer: Reproducible Builds Maintainers <reproducible-builds@lists.alioth.debian.org>
+Uploaders: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>,
+ Ximin Luo <infinity0@debian.org>,
+ Holger Levsen <holger@debian.org>
+Section: devel
+Priority: optional
+Standards-Version: 4.3.0
+Rules-Requires-Root: no
+Build-Depends:
+ debhelper-compat (= 11),
+ dh-python,
+ createrepo,
+ help2man,
+ pkg-config,
+ python-all,
+ python-cheetah,
+ python-dateutil,
+ python-libvirt,
+ python-libxml2,
+ python-openssl,
+ python-qpid,
+ python-requests,
+ python-requests-kerberos,
+ python-rpm,
+ python-simplejson,
+ python-six,
+ python-nose <!nocheck>,
+ python-mock <!nocheck>,
+ python-qpid-proton <!nocheck>,
+ python-krbv <!nocheck>,
+ python-psycopg2 <!nocheck>,
+ rpm,
+ systemd,
+ yum,
+Homepage: https://pagure.io/koji
+Vcs-Git: https://salsa.debian.org/reproducible-builds/koji.git
+Vcs-Browser: https://salsa.debian.org/reproducible-builds/koji
+
+Package: koji-common
+Architecture: all
+Section: python
+Depends: ${misc:Depends}, ${python:Depends},
+ python-krbv,
+ python-openssl,
+ python-rpm,
+Description: RPM-based build system - common library
+ The Fedora Project uses Koji for their build system, as do several other
+ projects.
+ .
+ Koji's goal is to provide a flexible, secure, and reproducible way to build
+ software.
+ .
+ Key features:
+ .
+  -  New buildroot for each build
+  -  Robust XML-RPC APIs for easy integration with other tools
+  -  Web interface with SSL and Kerberos authentication
+  -  Thin, portable command line client
+  -  Users can create local buildroots
+  -  Buildroot contents are tracked in the database
+  -  Versioned data
+ .
+ This package contains the common Python library used by other components.
+
+Package: koji-client
+Architecture: all
+Depends: python, ${misc:Depends}, ${python:Depends}, koji-common (= ${binary:Version}),
+ python-requests,
+ python-dateutil,
+ yum,
+Description: RPM-based build system - client
+ The Fedora Project uses Koji for their build system, as do several other
+ projects.
+ .
+ Koji's goal is to provide a flexible, secure, and reproducible way to build
+ software.
+ .
+ Key features:
+ .
+  -  New buildroot for each build
+  -  Robust XML-RPC APIs for easy integration with other tools
+  -  Web interface with SSL and Kerberos authentication
+  -  Thin, portable command line client
+  -  Users can create local buildroots
+  -  Buildroot contents are tracked in the database
+  -  Versioned data
+ .
+ This package contains the koji client that queries remote build systems as
+ well as perform actions such as adding users and initiating build requests.
+
+Package: koji-servers
+Architecture: all
+Depends: python, ${misc:Depends}, ${python:Depends}, koji-common (= ${binary:Version}),
+ createrepo,
+ python-cheetah,
+ python-krbv,
+ python-libvirt,
+ python-libxml2,
+ python-qpid,
+ python-rpm,
+ python-simplejson,
+ yum,
+Recommends: httpd
+Suggests: systemd
+Description: RPM-based build system - server components
+ The Fedora Project uses Koji for their build system, as do several other
+ projects.
+ .
+ Koji's goal is to provide a flexible, secure, and reproducible way to build
+ software.
+ .
+ Key features:
+ .
+  -  New buildroot for each build
+  -  Robust XML-RPC APIs for easy integration with other tools
+  -  Web interface with SSL and Kerberos authentication
+  -  Thin, portable command line client
+  -  Users can create local buildroots
+  -  Buildroot contents are tracked in the database
+  -  Versioned data
+ .
+ This package contains the server components of the koji build system.
+ .
+  - koji-hub, the center of all Koji operations. It is an XML-RPC server
+    running under mod_wsgi in Apache.
+  - kojid is the build daemon that runs on each of the build machines.
+  - koji-web is a set of scripts that run in mod_wsgi and use the Cheetah
+    templating engine to provide a web interface to Koji.
+  - kojira is a daemon that keeps the build root repodata updated.

debian/copyright

+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: koji
+Source: https://pagure.io/koji
+
+Files: *
+Copyright: 2007-2014 Red Hat, Inc.
+License: LGPL-2.1
+
+Files: debian/*
+Copyright: 2015 Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
+           2019 Holger Levsen <holger@debian.org>
+License: LGPL-2.1
+
+License: LGPL-2.1
+ This program is free software; you can redistribute it
+ and/or modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License.
+ .
+ This program is distributed in the hope that it will be
+ useful, but WITHOUT ANY WARRANTY; without even the implied
+ warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+ PURPOSE.  See the GNU General Public License for more
+ details.
+ .
+ You should have received a copy of the GNU General Public
+ License along with this package; if not, write to the Free
+ Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ Boston, MA  02110-1301 USA
+ .
+ On Debian systems, the full text of the GNU Lesser General Public
+ License version 2.1 can be found in the file
+ `/usr/share/common-licenses/LGPL-2.1'.

debian/fedora.conf

+[fedora]
+
+;configuration for koji cli tool
+
+;url of XMLRPC server
+server = https://koji.fedoraproject.org/kojihub
+
+;url of web interface
+weburl = https://koji.fedoraproject.org/koji
+
+;url of package download site
+topurl = https://kojipkgs.fedoraproject.org/
+
+authtype = kerberos
+krb_rdns = false
+
+use_fast_upload = yes

debian/koji-client.install

+/etc/koji.conf
+/etc/koji.conf.d/
+/usr/bin/koji
+/usr/lib/python2.7/site-packages/koji_cli_plugins/
+/usr/lib/python2.7/site-packages/koji_cli/
+debian/fedora.conf /etc/koji.conf.d/

debian/koji-client.manpages

+debian/koji.1

debian/koji-common.install

+/usr/lib/python2.7/site-packages/koji

debian/koji-servers.install

+/lib/systemd/system/
+/usr/lib/koji-hub-plugins/
+/usr/lib/koji-builder-plugins/
+/usr/lib/koji/libexec
+/usr/sbin
+/etc/kojivmd/
+/etc/httpd/
+/etc/kojira/
+/etc/koji-hub/
+/etc/koji-shadow/
+/etc/mock/koji/
+/etc/kojid/
+/etc/kojiweb/
+/etc/koji-gc/
+/usr/share/kojivmd/
+/usr/share/koji-hub/
+/usr/share/koji-web/

debian/koji-servers.manpages

+debian/koji-gc.1
+debian/koji-shadow.1
+debian/kojid.1
+debian/kojira.1
+debian/kojivmd.1

debian/patches/0001-Fail-the-build-when-pkg-config-is-missing.patch

+Description: Fail the build when pkg-config is missing or fails
+ This prevents systemd service files being accidentally installed to /
+Author: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
+Forwarded: no
+---
+This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
+Index: koji/Makefile
+===================================================================
+--- koji.orig/Makefile
++++ koji/Makefile
+@@ -1,6 +1,7 @@
+ NAME=koji
+ SPECFILE = $(firstword $(wildcard *.spec))
+ SUBDIRS = hub builder koji cli util www plugins vm
++SYSTEMDSYSTEMUNITDIR = $(shell pkg-config systemd --variable=systemdsystemunitdir)
+ 
+ ifndef PYTHON
+ export PYTHON=python2
+@@ -152,4 +153,5 @@ install:
+ 	mkdir -p $(DESTDIR)
+ 
+ 	for d in $(SUBDIRS); do make DESTDIR=`cd $(DESTDIR); pwd` \
+-		-C $$d install TYPE=$(TYPE); [ $$? = 0 ] || exit 1; done
++		-C $$d install SYSTEMDSYSTEMUNITDIR=$(SYSTEMDSYSTEMUNITDIR) \
++		TYPE=$(TYPE); [ $$? = 0 ] || exit 1; done
+Index: koji/builder/Makefile
+===================================================================
+--- koji.orig/builder/Makefile
++++ koji/builder/Makefile
+@@ -1,6 +1,5 @@
+ BINFILES = kojid
+ LIBEXECFILES = mergerepos
+-SYSTEMDSYSTEMUNITDIR = $(shell pkg-config systemd --variable=systemdsystemunitdir)
+ TYPE = systemd
+ 
+ _default:
+@@ -29,6 +28,7 @@ _install:
+ 	install -p -m 644 kojid.conf $(DESTDIR)/etc/kojid/kojid.conf
+ 
+ install-systemd: _install
++	test -n $(SYSTEMDSYSTEMUNITDIR)
+ 	mkdir -p $(DESTDIR)$(SYSTEMDSYSTEMUNITDIR)
+ 	install -p -m 644 kojid.service $(DESTDIR)$(SYSTEMDSYSTEMUNITDIR)
+ 
+Index: koji/util/Makefile
+===================================================================
+--- koji.orig/util/Makefile
++++ koji/util/Makefile
+@@ -1,5 +1,4 @@
+ BINFILES = kojira koji-gc koji-shadow
+-SYSTEMDSYSTEMUNITDIR = $(shell pkg-config systemd --variable=systemdsystemunitdir)
+ TYPE = systemd
+ 
+ _default:
+@@ -27,6 +26,7 @@ _install:
+ 	install -p -m 644 koji-shadow.conf $(DESTDIR)/etc/koji-shadow/koji-shadow.conf
+ 
+ install-systemd: _install
++	test -n $(SYSTEMDSYSTEMUNITDIR)
+ 	mkdir -p $(DESTDIR)$(SYSTEMDSYSTEMUNITDIR)
+ 	install -p -m 644 kojira.service $(DESTDIR)$(SYSTEMDSYSTEMUNITDIR)
+ 
+Index: koji/vm/Makefile
+===================================================================
+--- koji.orig/vm/Makefile
++++ koji/vm/Makefile
+@@ -1,6 +1,5 @@
+ BINFILES = kojivmd
+ SHAREFILES = kojikamid
+-SYSTEMDSYSTEMUNITDIR = $(shell pkg-config systemd --variable=systemdsystemunitdir)
+ TYPE = systemd
+ 
+ _default:
+@@ -29,6 +28,7 @@ _install: kojikamid
+ 	install -p -m 644 kojivmd.conf $(DESTDIR)/etc/kojivmd/kojivmd.conf
+ 
+ install-systemd: _install
++	test -n $(SYSTEMDSYSTEMUNITDIR)
+ 	mkdir -p $(DESTDIR)$(SYSTEMDSYSTEMUNITDIR)
+ 	install -p -m 644 kojivmd.service $(DESTDIR)$(SYSTEMDSYSTEMUNITDIR)
+ 

debian/patches/0002-Move-files-from-usr-libexec-to-usr-lib-koji-libexec.patch

+Description: Use Debian FHS paths
+Author: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
+Forwarded: not-needed
+---
+This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
+Index: koji/builder/Makefile
+===================================================================
+--- koji.orig/builder/Makefile
++++ koji/builder/Makefile
+@@ -19,8 +19,8 @@ _install:
+ 	mkdir -p $(DESTDIR)/usr/sbin
+ 	install -p -m 755 $(BINFILES) $(DESTDIR)/usr/sbin
+ 
+-	mkdir -p $(DESTDIR)/usr/libexec/kojid
+-	install -p -m 755 $(LIBEXECFILES) $(DESTDIR)/usr/libexec/kojid
++	mkdir -p $(DESTDIR)/usr/lib/koji/libexec/kojid
++	install -p -m 755 $(LIBEXECFILES) $(DESTDIR)/usr/lib/koji/libexec/kojid
+ 
+ 	mkdir -p $(DESTDIR)/etc/mock/koji
+ 
+Index: koji/builder/kojid
+===================================================================
+--- koji.orig/builder/kojid
++++ koji/builder/kojid
+@@ -5340,7 +5340,7 @@ class CreaterepoTask(BaseTaskHandler):
+         elif self.options.use_createrepo_c:
+             cmd = ['/usr/bin/mergerepo_c', '--koji']
+         else:
+-            cmd = ['/usr/libexec/kojid/mergerepos']
++            cmd = ['/usr/lib/koji/libexec/kojid/mergerepos']
+             cmd.extend(['--tempdir', self.workdir])
+         if merge_mode != 'bare':
+             blocklist = self.repodir + '/blocklist'

debian/patches/0003-Make-kojikamid-executable.patch

+Description: Make kojikamid executable
+ Files with a shebang are meant to be executed directly.
+Author: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
+Forwarded: no
+---
+This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
+--- a/vm/Makefile
++++ b/vm/Makefile
+@@ -22,7 +22,7 @@
+ 	install -p -m 755 $(BINFILES) $(DESTDIR)/usr/sbin
+ 
+ 	mkdir -p $(DESTDIR)/usr/share/kojivmd
+-	install -p -m 644 $(SHAREFILES) $(DESTDIR)/usr/share/kojivmd
++	install -p -m 755 $(SHAREFILES) $(DESTDIR)/usr/share/kojivmd
+ 
+ 	mkdir -p $(DESTDIR)/etc/kojivmd
+ 	install -p -m 644 kojivmd.conf $(DESTDIR)/etc/kojivmd/kojivmd.conf