.gitlab-ci.yml

@@ -21,11 +21,10 @@ test buster python3:
   image: debian:buster
   script:
   - apt-get update
-  - apt-get install -y --no-install-recommends 2to3 python3-apt python3-coverage python3-debian python3-debianbts python3-ldap python3-psycopg2 python3-pytest python3-pytest-cov python3-rrdtool python3-sqlalchemy python3-tabulate python3-yaml
+  - apt-get install -y --no-install-recommends 2to3 bzip2 gnupg gzip python3-apt python3-coverage python3-debian python3-debianbts python3-ldap python3-psycopg2 python3-pytest python3-pytest-cov python3-rrdtool python3-sqlalchemy python3-tabulate python3-yaml xz-utils
   - mkdir test-gnupghome
-  - 2to3 -wn -x import dak/*.py daklib/*.py
+  - 2to3 -wn -x future -x import dak/*.py daklib/*.py
   - GNUPGHOME=$(pwd)/test-gnupghome py.test-3 -v daklib tests
-  allow_failure: true
 
 flake8:
   <<: *template

config/debian/dak.conf

@@ -22,7 +22,6 @@ Dinstall
    LintianTags "/srv/ftp-master.debian.org/dak/config/debian/lintian.tags";
    ReleaseTransitions "/srv/ftp.debian.org/web/transitions.yaml";
    AllowSourceOnlyUploads true;
-   AllowSourceOnlyNewKeys { "F275CBDDEE2EDD382EE95A7AAA06688AEF8AF8D5"; };
    // if you setup an own dak repository and want to upload Debian packages you most possibly want
    // to set the following option to a real path/filename and then enter those mail addresses that
    // you want to be able to receive mails generated by your dak installation. This avoids spamming

config/debian/external-signatures.conf

@@ -55,6 +55,6 @@ External-Signature-Requests {
   };
 };
 
-Dinstall::Dinstall::AllowSourceOnlyNewKeys {
+Dinstall::AllowSourceOnlyNewKeys {
   "F275CBDDEE2EDD382EE95A7AAA06688AEF8AF8D5";
 };
\ No newline at end of file

dak/add_user.py

@@ -127,7 +127,7 @@ def main():
 
         # Lets add user to the email-whitelist file if its configured.
         if "Dinstall::MailWhiteList" in Cnf and Cnf["Dinstall::MailWhiteList"] != "":
-            f = utils.open_file(Cnf["Dinstall::MailWhiteList"], "a")
+            f = open(Cnf["Dinstall::MailWhiteList"], "a")
             for mail in emails:
                 f.write(mail + '\n')
             f.close()

dak/check_archive.py

@@ -42,7 +42,7 @@ import apt_inst
 from daklib.dbconn import *
 from daklib import utils
 from daklib.config import Config
-from daklib.dak_exceptions import InvalidDscError, ChangesUnicodeError, CantOpenError
+from daklib.dak_exceptions import InvalidDscError
 
 ################################################################################
 
@@ -186,12 +186,15 @@ def check_dscs():
         except InvalidDscError:
             utils.warn("syntax error in .dsc file %s" % f)
             count += 1
-        except ChangesUnicodeError:
+        except UnicodeDecodeError:
             utils.warn("found invalid dsc file (%s), not properly utf-8 encoded" % f)
             count += 1
-        except CantOpenError:
+        except IOError as e:
+            if e.errno == errno.ENOENT:
                 utils.warn("missing dsc file (%s)" % f)
                 count += 1
+            else:
+                raise
         except Exception as e:
             utils.warn("miscellaneous error parsing dsc file (%s): %s" % (f, str(e)))
             count += 1
@@ -285,7 +288,7 @@ def check_checksums():
         filename = f.fullpath
 
         try:
-            fi = utils.open_file(filename)
+            fi = open(filename)
         except:
             utils.warn("can't open '%s'." % (filename))
             continue
@@ -338,7 +341,7 @@ def check_timestamps():
     for pf in q.all():
         filename = os.path.abspath(os.path.join(pf.location.path, pf.filename))
         if os.access(filename, os.R_OK):
-            f = utils.open_file(filename)
+            f = open(filename)
             current_file = filename
             print("Processing %s." % (filename), file=sys.stderr)
             apt_inst.debExtract(f, Ent, "control.tar.gz")

dak/control_overrides.py

@@ -364,7 +364,7 @@ def main():
         Logger = daklog.Logger("control-overrides", mode)
         if file_list:
             for f in file_list:
-                process_file(utils.open_file(f), suite, component, otype, mode, action, session)
+                process_file(open(f), suite, component, otype, mode, action, session)
         else:
             process_file(sys.stdin, suite, component, otype, mode, action, session)
         Logger.close()

dak/control_suite.py

@@ -142,7 +142,7 @@ def britney_changelog(packages, suite, session):
     q = session.execute(query)
 
     pu = None
-    brit = utils.open_file(brit_file, 'w')
+    brit = open(brit_file, 'w')
 
     for u in q:
         if pu and pu != u[0]:
@@ -465,7 +465,7 @@ def main():
 
             if file_list:
                 for f in file_list:
-                    process_file(utils.open_file(f), suite, action, transaction, britney, force)
+                    process_file(open(f), suite, action, transaction, britney, force)
             else:
                 process_file(sys.stdin, suite, action, transaction, britney, force)
 

dak/cruft_report.py

@@ -180,7 +180,7 @@ def parse_nfu(architecture):
     # Not all architectures may have a wanna-build dump, so we want to ignore missin
     # files
     if os.path.exists(filename):
-        f = utils.open_file(filename)
+        f = open(filename)
         for line in f:
             if line[0] == ' ':
                 continue

dak/dak.py

@@ -35,13 +35,16 @@ G{importgraph}
 
 from __future__ import print_function
 
+import importlib
 import os
 import sys
 import traceback
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
 import daklib.utils
 
 from daklib.daklog import Logger
-from daklib.dak_exceptions import CantOpenError
 
 ################################################################################
 
@@ -184,7 +187,7 @@ def main():
 
     try:
         logger = Logger('dak top-level', print_starting=False)
-    except CantOpenError:
+    except IOError:
         logger = None
 
     functionality = init()
@@ -222,7 +225,7 @@ def main():
                 usage(functionality, 1)
 
     # Invoke the module
-    module = __import__(cmdname.replace("-", "_"))
+    module = importlib.import_module("dak.{}".format(cmdname.replace("-", "_")))
 
     try:
         module.main()

dak/daklib

-../daklib
\ No newline at end of file

dak/examine_package.py

@@ -63,7 +63,6 @@ from daklib.gpg import SignedFile
 from daklib.regexes import html_escaping, re_html_escaping, re_version, re_spacestrip, \
                            re_contrib, re_nonfree, re_localhost, re_newlinespace, \
                            re_package, re_doc_directory, re_file_binary
-from daklib.dak_exceptions import ChangesUnicodeError
 import daklib.daksubprocess
 
 ################################################################################
@@ -251,7 +250,7 @@ def read_control(filename):
     maintainer = ''
     arch = ''
 
-    deb_file = utils.open_file(filename)
+    deb_file = open(filename)
     try:
         extracts = utils.deb_extract_control(deb_file)
         control = apt_pkg.TagSection(extracts)
@@ -310,7 +309,7 @@ def read_control(filename):
 def read_changes_or_dsc(suite, filename, session=None):
     dsc = {}
 
-    dsc_file = utils.open_file(filename)
+    dsc_file = open(filename)
     try:
         dsc = utils.parse_changes(filename, dsc_file=1)
     except:
@@ -643,7 +642,7 @@ def check_deb(suite, deb_filename, session=None):
 
 
 def strip_pgp_signature(filename):
-    with utils.open_file(filename) as f:
+    with open(filename) as f:
         data = f.read()
         signedfile = SignedFile(data, keyrings=(), require_signature=False)
         return signedfile.contents
@@ -659,7 +658,7 @@ def display_changes(suite, changes_filename):
 def check_changes(changes_filename):
     try:
         changes = utils.parse_changes(changes_filename)
-    except ChangesUnicodeError:
+    except UnicodeDecodeError:
         utils.warn("Encoding problem with changes file %s" % (changes_filename))
     print(display_changes(changes['distribution'], changes_filename))
 

dak/generate_releases.py

@@ -408,7 +408,7 @@ class ReleaseWriter(object):
                 else:
                     continue
 
-                contents = open(filename, 'r').read()
+                contents = open(filename, 'rb').read()
 
                 # If we find a file for which we have a compressed version and
                 # haven't yet seen the uncompressed one, store the possibility

dak/init_dirs.py

@@ -103,7 +103,7 @@ def process_keyring(fullpath, secret=False):
 
     # Touch the file
     print("Creating {} ...".format(fullpath))
-    file(fullpath, 'w')
+    open(fullpath, 'w')
     if secret:
         os.chmod(fullpath, 0o600)
     else:

dak/make_maintainers.py

@@ -159,7 +159,7 @@ SELECT
     # Process any additional Maintainer files (e.g. from pseudo
     # packages)
     for filename in extra_files:
-        extrafile = utils.open_file(filename)
+        extrafile = open(filename)
         for line in extrafile.readlines():
             line = re_comments.sub('', line).strip()
             if line == "":

dak/make_overrides.py

@@ -146,7 +146,7 @@ def main():
                 cname = cname.replace('/', '_')
                 filename = os.path.join(cnf["Dir::Override"], "override.%s.%s%s" % (override_suite, cname, suffix))
 
-                output_file = utils.open_file(filename, 'w')
+                output_file = open(filename, 'w')
                 do_list(output_file, suite, component, otype, session)
                 output_file.close()
 

dak/manage_build_queues.py

@@ -126,6 +126,9 @@ def clean(build_queue, transaction, now=None):
         Logger.log(["removed source from build queue", build_queue.queue_name, source.source, source.version])
         transaction.remove_source(source, suite)
 
+    if binaries or sources:
+        suite.update_last_changed()
+
 
 def main():
     global Options, Logger

dak/new_security_install.py

@@ -197,7 +197,7 @@ def main():
         raw_input("Press Enter to continue")
 
     for acceptfilename in acceptfiles.keys():
-        accept_file = file(acceptfilename, "w")
+        accept_file = open(acceptfilename, "w")
         accept_file.write("OK\n")
         accept_file.close()
 

dak/process_new.py

@@ -42,7 +42,7 @@
 
 ################################################################################
 
-from __future__ import print_function
+from __future__ import absolute_import, print_function
 
 import errno
 import os
@@ -52,7 +52,7 @@ import sys
 import contextlib
 import pwd
 import apt_pkg
-import examine_package
+import dak.examine_package
 import subprocess
 import daklib.daksubprocess
 from sqlalchemy import or_
@@ -62,7 +62,7 @@ from daklib.queue import *
 from daklib import daklog
 from daklib import utils
 from daklib.regexes import re_default_answer, re_isanum
-from daklib.dak_exceptions import CantOpenError, AlreadyLockedError
+from daklib.dak_exceptions import AlreadyLockedError
 from daklib.summarystats import SummaryStats
 from daklib.config import Config
 from daklib.policy import UploadCopy, PolicyQueueUploadHandler
@@ -211,7 +211,7 @@ def edit_new(overrides, upload, session):
     if result != 0:
         utils.fubar("%s invocation failed for %s." % (editor, temp_filename), result)
     # Read the edited data back in
-    temp_file = utils.open_file(temp_filename)
+    temp_file = open(temp_filename)
     lines = temp_file.readlines()
     temp_file.close()
     os.unlink(temp_filename)
@@ -369,22 +369,22 @@ def check_pkg(upload, upload_copy, session):
     less_process = daklib.daksubprocess.Popen(less_cmd, bufsize=0, stdin=subprocess.PIPE)
     try:
         sys.stdout = less_process.stdin
-        print(examine_package.display_changes(suite_name, changes))
+        print(dak.examine_package.display_changes(suite_name, changes))
 
         source = upload.source
         if source is not None:
             source_file = os.path.join(upload_copy.directory, os.path.basename(source.poolfile.filename))
-            print(examine_package.check_dsc(suite_name, source_file))
+            print(dak.examine_package.check_dsc(suite_name, source_file))
 
         for binary in upload.binaries:
             binary_file = os.path.join(upload_copy.directory, os.path.basename(binary.poolfile.filename))
-            examined = examine_package.check_deb(suite_name, binary_file)
+            examined = dak.examine_package.check_deb(suite_name, binary_file)
             # We always need to call check_deb to display package relations for every binary,
             # but we print its output only if new overrides are being added.
             if ("deb", binary.package) in missing:
                 print(examined)
 
-        print(examine_package.output_package_relations())
+        print(dak.examine_package.output_package_relations())
         less_process.stdin.close()
     except IOError as e:
         if e.errno == errno.EPIPE:
@@ -824,7 +824,7 @@ def main():
     if not Options["No-Action"]:
         try:
             Logger = daklog.Logger("process-new")
-        except CantOpenError as e:
+        except IOError:
             Options["Trainee"] = "True"
 
     Sections = Section_Completer(session)

dak/rm.py

@@ -336,7 +336,7 @@ def main():
         result = os.system("%s %s" % (editor, temp_filename))
         if result != 0:
             utils.fubar("vi invocation failed for `%s'!" % (temp_filename), result)
-        temp_file = utils.open_file(temp_filename)
+        temp_file = open(temp_filename)
         for line in temp_file.readlines():
             Options["Reason"] += line
         temp_file.close()

dak/show_deferred.py

@@ -175,7 +175,7 @@ RRA:MAX:0.5:288:795
 
 
 def get_upload_data(changesfn):
-    achanges = deb822.Changes(file(changesfn))
+    achanges = deb822.Changes(open(changesfn))
     changesname = os.path.basename(changesfn)
     delay = os.path.basename(os.path.dirname(changesfn))
     m = re.match(r'([0-9]+)-day', delay)
@@ -189,7 +189,7 @@ def get_upload_data(changesfn):
 
     uploader = achanges.get('changed-by')
     uploader = re.sub(r'^\s*(\S.*)\s+<.*>', r'\1', uploader)
-    with utils.open_file(changesfn) as f:
+    with open(changesfn) as f:
         fingerprint = SignedFile(f.read(), keyrings=get_active_keyring_paths(), require_signature=False).fingerprint
     if "Show-Deferred::LinkPath" in Cnf:
         isnew = False