Fix a nasty bug with the orig.tar.gz handling. Make return status of...

Fix a nasty bug with the orig.tar.gz handling. Make return status of find_orig_tarball() more obvious, optimize the importer not to copy the orig.tar.gz twice, fix the getorigtarball plugin handling

debexpo/lib/constants.py

@@ -36,6 +36,11 @@ __copyright__ = 'Copyright © 2008 Jonny Lamb'
 __license__ = 'MIT'
 
 
+# Importer related constants
+ORIG_TARBALL_LOCATION_NOT_FOUND = 0
+ORIG_TARBALL_LOCATION_LOCAL = 1
+ORIG_TARBALL_LOCATION_REPOSITORY = 2
+
 # User constants
 USER_TYPE_NORMAL = 1
 USER_TYPE_ADMIN = 2

debexpo/lib/filesystem.py

@@ -106,8 +106,8 @@ class CheckFiles(object):
     def find_orig_tarball(self, changes_file):
         """
         Look to see whether there is an orig tarball present, if the dsc refers to one.
-        If it is present or not necessary, this returns True. Otherwise, it returns the
-        name of the file required.
+        This method returns a triple (filename, file_found, location_hint), returning the (expected)
+        name of the original tarball, and whether it was found in the local repository
 
         ```changes_file```
             The changes file to parse for the orig.tar (note the dsc file referenced must exist)
@@ -119,12 +119,22 @@ class CheckFiles(object):
             if (file['name'].endswith('orig.tar.gz') or
                 file['name'].endswith('orig.tar.bz2') or
                 file['name'].endswith('orig.tar.xz')):
-                if os.path.isfile(file['name']):
-                    return (file['name'], True)
-                else:
-                    return (file['name'], False)
-
-        return (None, False)
+                    full_filename = os.path.join(pylons.config['debexpo.repository'], changes_file.get_pool_path(), file['name'])
+                    # tar.gz was found in the local directory
+                    if os.path.isfile(file['name']):
+                        sum = md5sum(file['name'])
+                        if sum == file['md5sum']:
+                            return (file['name'], constants.ORIG_TARBALL_LOCATION_LOCAL)
+                        # tar.gz was found, but does not seem to be the same file
+                        break
+                    # tar.gz was found in the repository
+                    elif os.path.isfile(full_filename):
+                        return (file['name'], constants.ORIG_TARBALL_LOCATION_REPOSITORY)
+                    # tar.gz was expected but not found at all
+                    else:
+                        return (file['name'], constants.ORIG_TARBALL_LOCATION_NOT_FOUND)
+
+        return (None, constants.ORIG_TARBALL_LOCATION_NOT_FOUND)
 
 
     def find_files_for_package(self, package, absolute_path=False):

debexpo/lib/plugins.py

@@ -141,8 +141,11 @@ class Plugins(object):
         for item in dsc['Files']:
             if item['name'] not in self.changes.get_files():
                 src_file = os.path.join(self.config['debexpo.upload.incoming'], item['name'])
+                repository_src_file = os.path.join(self.config['debexpo.repository'], self.changes.get_pool_path(), item['name'])
                 if os.path.exists(src_file):
                     shutil.copy(src_file, self.tempdir)
+                elif os.path.exists(repository_src_file):
+                    shutil.copy(repository_src_file, self.tempdir)
                 else:
                     log.critical("Trying to copy non-existing file %s" % (src_file))
 

debexpo/plugins/getorigtarball.py

@@ -6,6 +6,7 @@
 #
 #   Copyright © 2008 Jonny Lamb <jonny@debian.org>
 #   Copyright © 2010 Jan Dittberner <jandd@debian.org>
+#   Copyright © 2011 Arno Töll <debian@toell.net>
 #
 #   Permission is hereby granted, free of charge, to any person
 #   obtaining a copy of this software and associated documentation
@@ -33,7 +34,7 @@ Holds the getorigtarball plugin.
 """
 
 __author__ = 'Jonny Lamb'
-__copyright__ = 'Copyright © 2008 Jonny Lamb, Copyright © 2010 Jan Dittberner'
+__copyright__ = 'Copyright © 2008 Jonny Lamb, Copyright © 2010 Jan Dittberner, Copyright © 2011 Arno Töll'
 __license__ = 'MIT'
 
 from debian import deb822
@@ -42,7 +43,9 @@ import os
 import urllib
 import re
 
+from debexpo.lib import constants
 from debexpo.lib.utils import md5sum
+from debexpo.lib.filesystem import CheckFiles
 from debexpo.plugins import BasePlugin
 
 import pylons
@@ -67,27 +70,27 @@ class GetOrigTarballPlugin(BasePlugin):
         size = 15728640
         log.debug('Checking whether an orig tarball mentioned in the dsc is missing')
         dsc = deb822.Dsc(file(self.changes.get_dsc()))
+        filecheck = CheckFiles()
+
+        if filecheck.is_native_package(self.changes):
+            log.debug('No orig.tar.gz file found; native package?')
+            return
+
+        # An orig.tar.gz was found in the dsc, and also in the upload.
+        (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
+        if orig_file_found > constants.ORIG_TARBALL_LOCATION_NOT_FOUND:
+            log.debug('%s found successfully', orig)
+            return
 
-        orig = None
         for dscfile in dsc['Files']:
             dscfile['size'] = int(dscfile['size'])
-            if re.search('(orig\.tar\.(gz|bz2|xz))$', dscfile['name']):
-                orig = dscfile
+            if orig == dscfile['name']:
                 if dscfile['size'] > size:
                         log.warning("Skipping eventual download of orig.tar.gz %s: size %d > %d" % (dscfile['name'], dscfile['size'], size))
                         return
+                orig = dscfile
                 break
 
-        # There is no orig.tar.gz file in the dsc file. This is probably a native package.
-        if orig is None:
-            log.debug('No orig.tar.gz file found; native package?')
-            return
-
-        # An orig.tar.gz was found in the dsc, and also in the upload.
-        if os.path.isfile(orig['name']):
-            log.debug('%s found successfully', orig['name'])
-            return
-
         log.debug('Could not find %s; looking in Debian for it', orig['name'])
 
         url = os.path.join(pylons.config['debexpo.debian_mirror'], self.changes.get_pool_path(), orig['name'])

debexpo/scripts/debexpo_importer.py

@@ -395,13 +395,14 @@ class Importer(object):
         # Look whether the orig tarball is present, and if not, try and get it from
         # the repository.
         (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
-        if orig and not orig_file_found:
+        if orig and not orig_file_found == constants.ORIG_TARBALL_LOCATION_NOT_FOUND:
             log.debug("Upload does not contain orig.tar.gz - trying to find it elsewhere")
             filename = os.path.join(pylons.config['debexpo.repository'],
                 self.changes.get_pool_path(), orig)
             if os.path.isfile(filename):
+                log.debug("Found tar.gz in repository as %s" % (filename))
                 shutil.copy(filename, pylons.config['debexpo.upload.incoming'])
-                self.files.append(orig)
+                #self.files.append(orig)
 
         destdir = pylons.config['debexpo.repository']
 
@@ -431,11 +432,14 @@ class Importer(object):
         pool_dir = os.path.join(destdir, self.changes.get_pool_path())
         log.debug("Pool directory: %s", pool_dir)
         for file in self.files:
-            if os.path.isfile(os.path.join(pool_dir, file)):
+            if os.path.isfile(file) and os.path.isfile(os.path.join(pool_dir, file)):
                 log.warning('%s is being installed even though it already exists' % file)
-            else:
+                toinstall.append(file)
+            elif os.path.isfile(file):
                 log.debug('File %s is safe to install' % os.path.join(pool_dir, file))
-            toinstall.append(file)
+                toinstall.append(file)
+            # skip another corner case, where the dsc contains a orig.tar.gz but wasn't uploaded
+            # by doing nothing here for that case
 
         # Run post-upload plugins.
         post_upload = Plugins('post-upload', self.changes, self.changes_file,
@@ -448,7 +452,7 @@ class Importer(object):
         # Check whether a post-upload plugin has got the orig tarball from somewhere.
         if not orig_file_found and not filecheck.is_native_package(self.changes):
             (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
-            if not orig_file_found:
+            if orig_file_found == constants.ORIG_TARBALL_LOCATION_NOT_FOUND:
                 # When coming here it means:
                 # a) The uploader did not include a orig.tar.gz in his upload
                 # b) We couldn't find a orig.tar.gz in our repository
@@ -456,8 +460,10 @@ class Importer(object):
                 # ... time to give up
                 self._remove_changes()
                 self._reject("Rejecting incomplate upload. "
-                    "You did not upload %s and we didn't find it on either one of our backup resources" %
+                    "You did not upload %s and we didn't find it on either one of our alternative resources" %
                     ( orig ))
+            else:
+                toinstall.append(orig)
 
         # Check whether the debexpo.repository variable is set
         if 'debexpo.repository' not in pylons.config: