.coveragerc

+[run]
+
+omit =
+    /usr/*
+    tests/*
+
+[report]
+exclude_lines =
+    pragma: no cover
+    no cover: 2.x

.coveragerc3

+[run]
+
+; extra omissions for py3 for now
+
+omit =
+    /usr/*
+    tests/*
+    hub/*
+    util/*
+    koji/daemon.py
+    koji/tasks.py
+
+[report]
+exclude_lines =
+    pragma: no cover
+    no cover 3.x
+    if six.PY2

.gitignore

+#
+# python compilation artifacts
 *.pyc
 *.pyo
-tests/test.py
+cli/kojic
+builder/kojidc
+#
+# coverage artifacts
+.coverage
+htmlcov/
+*,cover
+#
+# build artifacts
+koji-[0-9]*.tar.bz2
+koji-[0-9]*.src.rpm
+koji-[0-9]*[0-9]/
+noarch/
+docs/build/
+#
+# misc
+*.swp
+.idea
+.DS_STORE
+.vagrant
+.gitreview
+devtools/*.conf

Makefile

 NAME=koji
 SPECFILE = $(firstword $(wildcard *.spec))
-SUBDIRS = hub builder koji cli docs util www plugins vm
+SUBDIRS = hub builder koji cli util www plugins vm
 
 ifdef DIST
 DIST_DEFINES := --define "dist $(DIST)"
@@ -59,12 +59,28 @@ clean:
 	rm -f *.o *.so *.pyc *~ koji*.bz2 koji*.src.rpm
 	rm -rf koji-$(VERSION)
 	for d in $(SUBDIRS); do make -s -C $$d clean; done
+	coverage erase ||:
 
 git-clean:
 	@git clean -d -q -x
 
-subdirs:
-	for d in $(SUBDIRS); do make -C $$d; [ $$? = 0 ] || exit 1; done
+test:
+	coverage erase
+	PYTHONPATH=hub/.:plugins/hub/.:plugins/builder/.:plugins/cli/.:cli/.:www/lib coverage run \
+	    --source . /usr/bin/nosetests
+	coverage report
+	coverage html
+	@echo Full coverage report in htmlcov/index.html
+
+test3:
+	coverage erase
+	PYTHONPATH=hub/.:plugins/hub/.:plugins/builder/.:plugins/cli/.:cli/. coverage3 run \
+	    --rcfile .coveragerc3 --source . \
+	    /usr/bin/nosetests-3 \
+	    tests/test_lib tests/test_cli
+	coverage report --rcfile .coveragerc3
+	coverage html --rcfile .coveragerc3
+	@echo Full coverage report at file://${PWD}/htmlcov/index.html
 
 test-tarball:
 	@rm -rf .koji-$(VERSION)
@@ -89,6 +105,23 @@ rpm: tarball
 test-rpm: tarball
 	$(RPM_WITH_DIRS) $(DIST_DEFINES) --define "testbuild 1" -bb $(SPECFILE)
 
+pypi:
+	rm -rf dist
+	python setup.py sdist
+	# py2
+	virtualenv build_py2
+	build_py2/bin/pip install --upgrade pip setuptools wheel virtualenv
+	build_py2/bin/python setup.py bdist_wheel
+	rm -rf build_py2
+	# py3
+	python3 -m venv build_py3
+	build_py3/bin/pip install --upgrade pip setuptools wheel virtualenv
+	build_py3/bin/python setup.py bdist_wheel
+	rm -rf build_py3
+
+pypi-upload:
+	twine upload dist/*
+
 tag::
 	git tag -a $(TAG)
 	@echo "Tagged with: $(TAG)"

README.md

+koji - RPM building and tracking system
+=======================================
+
+Koji is an RPM-based build system. The Fedora Project uses Koji for [their build system](https://koji.fedoraproject.org/koji/), as do [several other projects](https://fedoraproject.org/wiki/Koji/RunsHere).
+
+Koji's goal is to provide a flexible, secure, and reproducible way to build software.
+
+Key features:
+
+* New buildroot for each build
+* Robust XML-RPC APIs for easy integration with other tools
+* Web interface with SSL and Kerberos authentication
+* Thin, portable command line client
+* Users can create local buildroots
+* Buildroot contents are tracked in the database
+* Versioned data
+
+Communicate
+-----------
+
+* Comments, questions, bugs, feedback, ideas, help requests? We'd love to hear from you.
+* Mailing lists:
+  * Development: [koji-devel AT lists.fedorahosted.org](https://lists.fedorahosted.org/archives/list/koji-devel@lists.fedorahosted.org/)
+  * User discussion and Fedora-specific topics: [buildsys AT lists.fedoraproject.org](https://lists.fedoraproject.org/archives/list/buildsys@lists.fedoraproject.org/)
+* IRC chat: #koji on irc.freenode.net
+
+Bugs/RFEs
+---------
+
+If you have found a bug or would like to request a new feature, please [report an issue in Pagure](https://pagure.io/koji/issues).
+
+Download
+--------
+
+The koji source code can be downloaded with git via:
+
+    git clone https://pagure.io/koji.git
+
+You may browse code at https://pagure.io/koji
+
+Archived releases can be found at https://pagure.io/koji/releases
+
+Documentation
+-------------
+
+See: https://docs.pagure.org/koji/
+
+
+Related Software
+----------------
+
+* [Mock](https://fedoraproject.org/wiki/Projects/Mock): The tool Koji uses to generate buildroots
+* [Yum](http://yum.baseurl.org/)
+* [Mash](https://pagure.io/mash)
+* [Koji Tools](https://pagure.io/koji-tools): Various utilities for Koji
+* [Kojiji](https://github.com/release-engineering/kojiji): Koji Java Interface
+* [txkoji](https://github.com/ktdreyer/txkoji): Async interface to Koji, using Twisted

builder/kojid.conf

@@ -14,6 +14,9 @@
 ; The directory root for temporary storage
 ; workdir=/tmp/koji
 
+; The temporary directory in buildroot
+; chroot_tmpdir = /chroot_tmpdir
+
 ; The directory root for mock
 ; mockdir=/var/lib/mock
 
@@ -35,12 +38,20 @@
 ; Timeout for build duration (24 hours)
 ; rpmbuild_timeout=86400
 
+; Install timeout(seconds) for image build
+; Default value is 0, which means using the number in /etc/oz/oz.cfg,
+; supported since oz-0.16.0
+; oz_install_timeout=7200
+
 ; The URL for the xmlrpc server
 server=http://hub.example.com/kojihub
 
 ; The URL for the file access
 topurl=http://hub.example.com/kojifiles
 
+; use createrepo_c rather than createrepo
+; use_createrepo_c=False
+
 ; A space-separated list of tuples from which kojid is allowed to checkout.
 ; The format of those tuples is:
 ;
@@ -81,8 +92,13 @@ from_addr=Koji Build System <buildsys@example.com>
 ;client certificate
 ;cert = /etc/kojid/client.crt
 
-;certificate of the CA that issued the client certificate
-;ca = /etc/kojid/clientca.crt
-
 ;certificate of the CA that issued the HTTP server certificate
 ;serverca = /etc/kojid/serverca.crt
+
+;if set to True, failing subtask will not automatically cancel other siblings
+;build_arch_can_fail = False
+
+;if set to True additional logs with timestamps will get created and uploaded
+;to hub. It could be useful for debugging purposes, but creates twice as many
+;log files
+;log_timestamps = False

builder/kojid.init

@@ -11,7 +11,7 @@
 # Source function library.
 . /etc/init.d/functions
 
-# Check that we're a priviledged user
+# Check that we're a privileged user
 [ `id -u` = 0 ] || exit 0
 
 [ -f /etc/sysconfig/kojid ] && . /etc/sysconfig/kojid
@@ -37,7 +37,7 @@ start() {
     [ "$KOJID_VERBOSE" == "Y" ] && ARGS="$ARGS --verbose"
     # XXX Fix for make download-checks in kernel builds
     # Remove once we're running the buildSRPMFromSCM task
-    # as an unpriviledged user with their own environment
+    # as an unprivileged user with their own environment
     export HOME="/root"
     daemon /usr/sbin/kojid $ARGS
     RETVAL=$?
@@ -82,15 +82,15 @@ case "$1" in
   status)
         status $prog
         ;;
+  reload|graceful)
+        graceful
+        ;;
   restart|reload|force-reload)
         restart
         ;;
   condrestart|try-restart)
         [ -f /var/lock/subsys/kojid ] && restart || :
         ;;
-  graceful)
-        graceful
-        ;;
   *)
         echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload|graceful}"
         exit 1

builder/kojid.service

 [Unit]
 Description=Koji build server
-Documentation=https://fedoraproject.org/wiki/Koji/ServerHowTo
+Documentation=https://docs.pagure.org/koji/server_howto/
 
 After=network.target
 
@@ -9,6 +9,7 @@ ExecStart=/usr/sbin/kojid \
        --fg \
        --force-lock \
        --verbose
+ExecReload=/bin/kill -USR1 $MAINPID
 
 [Install]
 WantedBy=multi-user.target

builder/mergerepos

@@ -43,7 +43,7 @@ EXPAND_ARCHES = {
     'sparc': ['sparcv8', 'sparcv9', 'sparcv9v', 'sparcv9v2'],
     'alpha': ['alphaev4', 'alphaev45', 'alphaev5', 'alphaev56',
               'alphapca56', 'alphaev6', 'alphaev67', 'alphaev68', 'alphaev7'],
-    'armhfp': ['armv7hl', 'armv7hnl'],
+    'armhfp': ['armv7hl', 'armv7hnl', 'armv6hl', 'armv6hnl'],
     'arm': ['armv5tel', 'armv5tejl', 'armv6l','armv7l'],
     'sh4': ['sh4a']
     }
@@ -73,6 +73,8 @@ def parse_args(args):
                       help="A file containing a list of srpm names to exclude from the merged repo")
     parser.add_option("-o", "--outputdir", default=None,
                       help="Location to create the repository")
+    parser.add_option("--tempdir", default=None,
+                      help="Location for temporary files")
     (opts, argsleft) = parser.parse_args(args)
 
     if len(opts.repos) < 1:
@@ -81,7 +83,7 @@ def parse_args(args):
 
     # expand arches
     for arch in opts.arches[:]:
-        if EXPAND_ARCHES.has_key(arch):
+        if arch in EXPAND_ARCHES:
             opts.arches.extend(EXPAND_ARCHES[arch])
 
     # support multilib repos
@@ -109,9 +111,10 @@ def make_const_func(value):
 
 
 class RepoMerge(object):
-    def __init__(self, repolist, arches, groupfile, blocked, outputdir):
+    def __init__(self, repolist, arches, groupfile, blocked, outputdir, tempdir=None):
         self.repolist = repolist
         self.outputdir = outputdir
+        self.tempdir = tempdir
         self.mdconf = createrepo.MetaDataConfig()
         # explicitly request sha1 for backward compatibility with older yum
         self.mdconf.sumtype = 'sha1'
@@ -125,7 +128,7 @@ class RepoMerge(object):
             self.yumbase.preconf.debuglevel = 2
         else:
             self.yumbase._getConfig('/dev/null', init_plugins=False, debuglevel=2)
-        self.yumbase.conf.cachedir = tempfile.mkdtemp()
+        self.yumbase.conf.cachedir = tempfile.mkdtemp(dir=self.tempdir)
         self.yumbase.conf.cache = 0
         self.archlist = arches
         self.mdconf.groupfile = groupfile
@@ -151,7 +154,7 @@ class RepoMerge(object):
         for r in self.repolist:
             count +=1
             rid = 'repo%s' % count
-            print >> sys.stderr, 'Adding repo: ' + r
+            sys.stderr.write('Adding repo: %s\n' % r)
             n = self.yumbase.add_enable_repo(rid, baseurls=[r])
             n._merge_rank = count
 
@@ -186,6 +189,7 @@ class RepoMerge(object):
         include_srpms = {}
 
         # calculating what "builds" (srpms) we're allowing into the repo
+        seen_srpms = {}
         for reponum, repo in enumerate(repos):
             for pkg in repo.sack:
                 if reponum == 0 and not pkg.basepath:
@@ -195,12 +199,20 @@ class RepoMerge(object):
                     #force the pkg object to report a relative location
                     loc = """<location href="%s"/>\n""" % yum.misc.to_xml(pkg.remote_path, attrib=True)
                     pkg._return_remote_location = make_const_func(loc)
+                if pkg.sourcerpm in seen_srpms:
+                    # we're just looking at sourcerpms this pass and we've
+                    # already seen this one
+                    continue
+                seen_srpms[pkg.sourcerpm] = 1
                 srpm_name, ver, rel, epoch, arch = rpmUtils.miscutils.splitFilename(pkg.sourcerpm)
-                if include_srpms.has_key(srpm_name):
+                if srpm_name in include_srpms:
                     other_srpm, other_repoid = include_srpms[srpm_name]
                     if pkg.repoid != other_repoid:
                         # We found a rpm built from an srpm with the same name in a previous repo.
-                        # The previous repo takes precendence, so ignore the srpm found here.
+                        # The previous repo takes precedence, so ignore the srpm found here.
+                        sys.stderr.write('Package %s already provided by repo %s'
+                                ' (at %s in repo %s)\n'
+                                % (srpm_name, other_repoid, str(pkg), pkg.repoid))
                         continue
                     else:
                         # We're in the same repo, so compare srpm NVRs
@@ -212,7 +224,15 @@ class RepoMerge(object):
                             # The current package we're processing is from a newer srpm than the
                             # existing srpm in the dict, so update the dict
                             include_srpms[srpm_name] = (pkg.sourcerpm, pkg.repoid)
-                elif self.blocked.has_key(srpm_name):
+                            sys.stderr.write('Replacing older source nvr: '
+                                    '%s > %s\n' % (pkg.sourcerpm, other_srpm))
+                        elif cmp < 0:
+                            sys.stderr.write('Ignoring older source nvr: '
+                                    '%s < %s\n' % (pkg.sourcerpm, other_srpm))
+                        # otherwise same, so we already have it
+                elif srpm_name in self.blocked:
+                    sys.stderr.write('Ignoring blocked package: %s\n\n' % \
+                                     pkg.sourcerpm)
                     continue
                 else:
                     include_srpms[srpm_name] = (pkg.sourcerpm, pkg.repoid)
@@ -226,12 +246,18 @@ class RepoMerge(object):
                 srpm_name, ver, rel, epoch, arch = rpmUtils.miscutils.splitFilename(pkg.sourcerpm)
                 incl_srpm, incl_repoid = include_srpms.get(srpm_name, (None, None))
                 pkg_nvra = str(pkg)
-                if incl_srpm == pkg.sourcerpm and not seen_rpms.has_key(pkg_nvra):
+                if pkg_nvra in seen_rpms:
+                    sys.stderr.write('Duplicate rpm: %s\n' % pkg_nvra)
+                elif incl_srpm is None:
+                    sys.stderr.write('Excluding %s (%s is blocked)\n'
+                            % (pkg_nvra, srpm_name))
+                    repo.sack.delPackage(pkg)
+                elif incl_srpm == pkg.sourcerpm:
                     origins.write('%s\t%s\n' % (pkg_nvra, repo.urls[0]))
                     seen_rpms[pkg_nvra] = 1
                 else:
-                    # Either the srpm is in the block list, it is not built from the srpm we
-                    # identified above, or it's a duplicate, so exclude it
+                    sys.stderr.write('Excluding %s (wrong srpm version '
+                            '%s != %s)\n' % (pkg_nvra, pkg.sourcerpm, incl_srpm))
                     repo.sack.delPackage(pkg)
 
         origins.close()
@@ -264,7 +290,8 @@ def main(args):
     else:
         blocked = {}
 
-    merge = RepoMerge(opts.repos, opts.arches, opts.groupfile, blocked, opts.outputdir)
+    merge = RepoMerge(opts.repos, opts.arches, opts.groupfile, blocked,
+                      opts.outputdir, opts.tempdir)
 
     try:
         merge.merge_repos()

cli/Makefile

+SUBDIRS=koji_cli
+
+PYTHON=python
+PACKAGE = $(shell basename `pwd`)
+PYVER := $(shell $(PYTHON) -c 'import sys; print("%.3s" %(sys.version))')
+PYSYSDIR := $(shell $(PYTHON) -c 'import sys; print(sys.prefix)')
+PYLIBDIR = $(PYSYSDIR)/lib/python$(PYVER)
+PKGDIR = $(PYLIBDIR)/site-packages
+
 FILES = koji
 
 _default:
@@ -13,7 +22,11 @@ install:
 		exit 1; \
 	fi
 
+	for d in $(SUBDIRS); do make DESTDIR=$(DESTDIR) \
+		-C $$d install; [ $$? = 0 ] || exit 1; done
+
 	mkdir -p $(DESTDIR)/usr/bin
 	install -p -m 755 $(FILES) $(DESTDIR)/usr/bin
-	install -p -m 644 koji.conf $(DESTDIR)/etc/koji.conf
 	mkdir -p $(DESTDIR)/etc/koji.conf.d
+	install -p -m 644 koji.conf $(DESTDIR)/etc/koji.conf
+	install -p -m 644 koji.conf $(DESTDIR)/etc/koji.conf

cli/koji.conf

@@ -19,13 +19,23 @@
 ;the service name of the principal being used by the hub
 ;krbservice = host
 
+;the principal to auth as for automated clients
+;principal = client@EXAMPLE.COM
+
+;the keytab to auth as for automated clients
+;keytab = /etc/krb5.keytab
+
+;enable to lookup dns canonical hostname for krb auth
+;krb_canon_host = no
+
+
 ;configuration for SSL authentication
 
 ;client certificate
 ;cert = ~/.koji/client.crt
 
-;certificate of the CA that issued the client certificate
-;ca = ~/.koji/clientca.crt
-
 ;certificate of the CA that issued the HTTP server certificate
 ;serverca = ~/.koji/serverca.crt
+
+;enabled plugins for CLI, runroot and save_failed_tree are available
+;plugins =

koji/ssl/Makefile → cli/koji_cli/Makefile

-FILES = $(wildcard *.py)
+PYFILES = $(wildcard *.py)
 
 PYTHON=python
+PACKAGE = $(shell basename `pwd`)
+PYVER := $(shell $(PYTHON) -c 'import sys; print("%.3s" % (sys.version))')
+PYSYSDIR := $(shell $(PYTHON) -c 'import sys; print(sys.prefix)')
+PYLIBDIR = $(PYSYSDIR)/lib/python$(PYVER)
+PKGDIR = $(PYLIBDIR)/site-packages/$(PACKAGE)
 
 _default:
 	@echo "nothing to make.  try make install"
@@ -15,7 +20,9 @@ install:
 		exit 1; \
 	fi
 
-	mkdir -p $(DESTDIR)
-	install -p -m 644 $(FILES) $(DESTDIR)
+	mkdir -p $(DESTDIR)/$(PKGDIR)
+	for p in $(PYFILES) ; do \
+		install -p -m 644 $$p $(DESTDIR)/$(PKGDIR)/$$p; \
+	done
 
-	$(PYTHON) -c "import compileall; compileall.compile_dir('$(DESTDIR)', 1, '$(PYDIR)', 1)"
+	$(PYTHON) -c "import compileall; compileall.compile_dir('$(DESTDIR)/$(PKGDIR)', 1, '$(PYDIR)', 1)"

debian/changelog

+koji (1.16.0-1) unstable; urgency=medium
+
+  * New upstream release. (Closes: #877921, #877886, #877887)
+
+ -- Ximin Luo <infinity0@debian.org>  Sun, 08 Jul 2018 12:06:37 -0700
+
 koji (1.10.0-1) unstable; urgency=medium
 
   [ Marek Marczykowski-Górecki ]

debian/control

@@ -3,28 +3,35 @@ Maintainer: Reproducible Builds Maintainers <reproducible-builds@lists.alioth.de
 Uploaders: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>,
  Ximin Luo <infinity0@debian.org>
 Section: devel
-Priority: extra
-Standards-Version: 3.9.6
+Priority: optional
+Standards-Version: 4.1.5
 Build-Depends:
- debhelper (>= 9.0),
+ debhelper (>= 9.20160709),
  dh-python,
- dh-systemd,
  createrepo,
  help2man,
  pkg-config,
  python-all,
  python-cheetah,
+ python-dateutil,
  python-libvirt,
  python-libxml2,
  python-openssl,
  python-qpid,
+ python-requests,
+ python-requests-kerberos,
  python-rpm,
  python-simplejson,
- python-urlgrabber,
+ python-six,
+ python-nose <!nocheck>,
+ python-mock <!nocheck>,
+ python-qpid-proton <!nocheck>,
+ python-krbv <!nocheck>,
+ python-psycopg2 <!nocheck>,
  rpm,
  systemd,
  yum,
-Homepage: https://fedorahosted.org/koji/
+Homepage: https://pagure.io/koji
 Vcs-Git: https://salsa.debian.org/reproducible-builds/koji.git
 Vcs-Browser: https://salsa.debian.org/reproducible-builds/koji
 
@@ -32,7 +39,7 @@ Package: koji-common
 Architecture: all
 Section: python
 Depends: ${misc:Depends}, ${python:Depends},
-#python-krbV,
+ python-krbv,
  python-openssl,
  python-rpm,
 Description: RPM-based build system - common library
@@ -57,7 +64,6 @@ Description: RPM-based build system - common library
 Package: koji-client
 Architecture: all
 Depends: python, ${misc:Depends}, ${python:Depends}, koji-common (= ${binary:Version}),
- python-urlgrabber,
  yum,
 Description: RPM-based build system - client
  The Fedora Project uses Koji for their build system, as do several other
@@ -84,13 +90,12 @@ Architecture: all
 Depends: python, ${misc:Depends}, ${python:Depends}, koji-common (= ${binary:Version}),
  createrepo,
  python-cheetah,
-#python-krbV,
+ python-krbv,
  python-libvirt,
  python-libxml2,
  python-qpid,
  python-rpm,
  python-simplejson,
- python-urlgrabber,
  yum,
 Recommends: httpd
 Suggests: systemd

debian/copyright

 Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Name: koji
-Source: https://fedorahosted.org/koji/
+Source: https://pagure.io/koji
 
 Files: *
 Copyright: 2007-2014 Red Hat, Inc.