.gitlab-ci.yml

+before_script:
+  - apt-get -q update
+  - env DEBIAN_FRONTEND=noninteractive apt-get -q -y install --no-install-recommends aspcud apt-cudf
+  - env DEBIAN_FRONTEND=noninteractive apt-get -q -y --solver aspcud -o APT::Solver::Strict-Pinning=0 -o Debug::pkgProblemResolver=yes build-dep .
+
+.test_template: &test
+  script:
+    - py.test-3 -vv -l -r a
+
+test:unstable:
+  <<: *test
+  image: debian:unstable
+
+test:testing:
+  <<: *test
+  image: debian:testing
+
+test:stable-bpo:
+  <<: *test
+  image: debian:stable-backports
+
+test:ubuntu-devel:
+  <<: *test
+  image: ubuntu:devel

CONTRIBUTING.rst

@@ -17,13 +17,24 @@ as follows:
 
 
 There are `more detailed instructions available
-<https://www.debian.org/Bugs/Reporting>`__ about reporting a bug in the Debian bug tracker.
+<https://www.debian.org/Bugs/Reporting>`__ about reporting a bug in the Debian
+bug tracker.
 
 If you're on a Debian-based system, you can install and use the ``reportbug``
 package to help walk you through the process.
 
+You can also submit patches via *merge request* to Salsa, Debian's Gitlab. Start
+by forking the `diffoscope Git
+repository <https://salsa.debian.org/reproducible-builds/diffoscope>`__
+(see
+`documentation <https://salsa.debian.org/help/gitlab-basics/fork-project.md>__`),
+make your changes and commit them as you normally would. You can then push your
+changes and submit a *merge request* via Salsa.  See `Gitlab documentation
+<https://salsa.debian.org/help/gitlab-basics/add-merge-request.md>`__ about
+*merge requests*.
+
 You can also submit patches to the Debian bug tracker. Start by cloning the `Git
-repository <https://salsa.debian.org/reproducible-builds/diffoscope.git/>`__,
+repository <https://salsa.debian.org/reproducible-builds/diffoscope>`__,
 make your changes and commit them as you normally would. You can then use
 Git's ``format-patch`` command to save your changes as a series of patches that
 can be attached to the report you submit. For example:

debian/changelog

-diffoscope (96) UNRELEASED; urgency=medium
+diffoscope (99) UNRELEASED; urgency=medium
 
-  * WIP.
+  * WIP
 
- -- Mattia Rizzolo <mattia@debian.org>  Sun, 20 May 2018 17:55:35 +0200
+ -- Chris Lamb <lamby@debian.org>  Fri, 29 Jun 2018 08:20:42 +0100
+
+diffoscope (98) unstable; urgency=medium
+
+  * Fix compatibility with Python 3.7. (Closes: #902650)
+
+ -- Chris Lamb <lamby@debian.org>  Fri, 29 Jun 2018 08:19:25 +0100
+
+diffoscope (97) unstable; urgency=medium
+
+  * Create all temporary directories within a top-level dir. (Closes: #902627)
+  * tests/conftest.py: Fix compatibility with pytest 3.6.2-1, currently in
+    Debian experimental.
+
+ -- Chris Lamb <lamby@debian.org>  Thu, 28 Jun 2018 21:38:50 +0100
+
+diffoscope (96) unstable; urgency=medium
+
+  [ Chris Lamb ]
+  * Drop dependency on pdftk as it relies on GCJ, relying on the pdftotext
+    fallback. (Closes: #893702)
+  * Change the "No file format specific differences found inside, yet data
+    differs" message to be clearer that diffoscope "knows" about this file
+    format yet could not be helpful in this case.
+  * Don't append a rather useless "(data)" suffix from file(1).
+  * Comply with a number of PEP8 recommendations:
+    - E226 - Add missing whitespaces around operators.
+    - E241 - Fix extraneous whitespaces around keywords.
+    - E251 - Remove whitespace around parameter '=' signs.
+    - E302 - Add missing 2 blank lines.
+    - E501 - Try to make lines fit to length.
+    - E502 - Remove extraneous escape of newline.
+    - E731 - Don't assign lambda expressions.
+    - E121, E122, E126, E128 - Fix badly indented lines.
+
+  [ Xavier Briand ]
+  * Add merge request details to contributing documentation.
+
+ -- Chris Lamb <lamby@debian.org>  Sat, 16 Jun 2018 22:19:28 +0200
 
 diffoscope (95) unstable; urgency=medium
 

debian/control

@@ -41,11 +41,11 @@ Build-Depends:
  libjs-jquery-tablesorter <!nocheck>,
  libjs-jquery-throttle-debounce <!nocheck>,
  llvm <!nocheck>,
+ lz4 <!nocheck>,
  mono-utils <!nocheck>,
  odt2txt <!nocheck>,
  oggvideotools <!nocheck>,
  openssh-client <!nocheck>,
- pdftk <!nocheck>,
  pgpdump <!nocheck>,
  poppler-utils <!nocheck>,
  python-argcomplete,

diffoscope/__init__.py

@@ -17,4 +17,4 @@
 # You should have received a copy of the GNU General Public License
 # along with diffoscope.  If not, see <https://www.gnu.org/licenses/>.
 
-VERSION = "95"
+VERSION = "98"

diffoscope/comparators/__init__.py

@@ -64,6 +64,7 @@ class ComparatorManager(object):
         ('icc.IccFile',),
         ('iso9660.Iso9660File',),
         ('java.ClassFile',),
+        ('lz4.Lz4File',),
         ('mono.MonoExeFile',),
         ('pdf.PdfFile',),
         ('png.PngFile',),

diffoscope/comparators/json.py

@@ -39,7 +39,7 @@ class JSONFile(File):
             # Try fuzzy matching for JSON files
             is_text = any(
                 file.magic_file_type.startswith(x)
-                for x in ('ASCII text', 'UTF-8 Unicode text'),
+                for x in ('ASCII text', 'UTF-8 Unicode text')
             )
             if is_text and not file.name.endswith('.json'):
                 buf = f.read(10)

diffoscope/comparators/lz4.py

+# -*- coding: utf-8 -*-
+#
+# diffoscope: in-depth comparison of files, archives, and directories
+#
+# Copyright © 2018 Xavier Briand <xavierbriand@gmail.com>
+#
+# diffoscope is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# diffoscope is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with diffoscope.  If not, see <https://www.gnu.org/licenses/>.
+
+import re
+import os.path
+import logging
+import subprocess
+
+from diffoscope.tools import tool_required
+
+from .utils.file import File
+from .utils.archive import Archive
+
+logger = logging.getLogger(__name__)
+
+
+class Lz4Container(Archive):
+    def open_archive(self):
+        return self
+
+    def close_archive(self):
+        pass
+
+    def get_member_names(self):
+        return [self.get_compressed_content_name('.lz4')]
+
+    @tool_required('lz4')
+    def extract(self, member_name, dest_dir):
+        dest_path = os.path.join(dest_dir, member_name)
+        logger.debug('lz4 extracting to %s', dest_path)
+        with open(dest_path, 'wb') as fp:
+            subprocess.check_call(
+                ["lz4", "-d", "-c", self.source.path],
+                shell=False, stdout=fp, stderr=None)
+        return dest_path
+
+
+class Lz4File(File):
+    DESCRIPTION = "LZ4 compressed files"
+    CONTAINER_CLASS = Lz4Container
+    FILE_TYPE_RE = re.compile(r'^LZ4 compressed data \([^\)]+\)$')
+
+    # Work around file(1) Debian bug #876316
+    FALLBACK_FILE_EXTENSION_SUFFIX = ".lz4"
+    FALLBACK_FILE_TYPE_HEADER_PREFIX = b"\x04\x22M\x18"

diffoscope/comparators/pdf.py

@@ -32,19 +32,9 @@ class Pdftotext(Command):
         return ['pdftotext', self.path, '-']
 
 
-class Pdftk(Command):
-    @tool_required('pdftk')
-    def cmdline(self):
-        return ['pdftk', self.path, 'output', '-', 'uncompress']
-
-    def filter(self, line):
-        return line.decode('latin-1').encode('utf-8')
-
-
 class PdfFile(File):
     DESCRIPTION = "PDF documents"
     FILE_TYPE_RE = re.compile(r'^PDF document\b')
 
     def compare_details(self, other, source=None):
-        return [Difference.from_command(Pdftotext, self.path, other.path),
-                Difference.from_command(Pdftk, self.path, other.path)]
+        return [Difference.from_command(Pdftotext, self.path, other.path)]

diffoscope/comparators/utils/libarchive.py

@@ -80,6 +80,14 @@ if not hasattr(libarchive.ffi, 'entry_gname'):
         'entry_gname', [libarchive.ffi.c_archive_entry_p], ctypes.c_char_p)
     libarchive.ArchiveEntry.gname = property(
         lambda self: libarchive.ffi.entry_gname(self._entry_p))
+# Monkeypatch libarchive-c (>= 2.8)
+# Wire mtime_nsec attribute as some libarchive versions (>=2.8) don't expose it
+# for ArchiveEntry. Doing this allows a unified API no matter which version is
+# available.
+if not hasattr(libarchive.ArchiveEntry, 'mtime_nsec') and hasattr(libarchive.ffi, 'entry_mtime_nsec'):
+    libarchive.ArchiveEntry.mtime_nsec = property(
+        lambda self: libarchive.ffi.entry_mtime_nsec(self._entry_p))
+
 
 # Monkeypatch libarchive-c so we always get pathname as (Unicode) str
 # Otherwise, we'll get sometimes str and sometimes bytes and always pain.

diffoscope/external_tools.py

@@ -145,6 +145,10 @@ EXTERNAL_TOOLS = {
         'arch': 'e2fsprogs',
         'FreeBSD': 'e2fsprogs',
     },
+    'lz4': {
+        'debian': 'lz4',
+        'FreeBSD': 'lz4',
+    },
     'msgunfmt': {
         'debian': 'gettext',
         'arch': 'gettext',
@@ -177,10 +181,6 @@ EXTERNAL_TOOLS = {
         'debian': 'pgpdump',
         'arch': 'pgpdump',
     },
-    'pdftk': {
-        'debian': 'pdftk',
-        'FreeBSD': 'pdftk',
-    },
     'pdftotext': {
         'debian': 'poppler-utils',
         'arch': 'poppler',

diffoscope/main.py

@@ -459,6 +459,8 @@ def main(args=None):
             post_parse(parsed_args)
             sys.exit(run_diffoscope(parsed_args))
     except KeyboardInterrupt:
+        if log_handler:
+            log_handler.progressbar.bar.erase_line()
         logger.info('Keyboard Interrupt')
         sys.exit(2)
     except BrokenPipeError:

diffoscope/presenters/formats.py

@@ -109,5 +109,5 @@ class PresenterManager(object):
         """
 
         return any(
-            x['klass'].supports_visual_diffs for x in self.config.values(),
+            x['klass'].supports_visual_diffs for x in self.config.values()
         )

diffoscope/progress.py

@@ -3,6 +3,7 @@
 # diffoscope: in-depth comparison of files, archives, and directories
 #
 # Copyright © 2016 Chris Lamb <lamby@debian.org>
+# Copyright © 2018 Paul Wise <pabs@debian.org>
 #
 # diffoscope is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,6 +21,7 @@
 import os
 import sys
 import json
+import signal
 import logging
 
 logger = logging.getLogger(__name__)
@@ -32,9 +34,6 @@ class ProgressLoggingHandler(logging.StreamHandler):
 
     def emit(self, record):
         try:
-            # Delete the current line (i.e. the progress bar)
-            self.stream.write("\r\033[K")
-            self.flush()
             super().emit(record)
             if not self.progressbar.bar.finished:
                 self.progressbar.bar.update()
@@ -215,10 +214,36 @@ class ProgressBar(object):
                 # Remove after https://github.com/niltonvolpato/python-progressbar/pull/57 is fixed.
                 kwargs.setdefault('fd', sys.stderr)
                 super().__init__(*args, **kwargs)
+                # Terminal handling after parent init since that sets self.fd
+                if self.fd.isatty():
+                    from curses import tigetstr, setupterm
+                    setupterm()
+                    self.erase_to_eol = tigetstr('el')
+                else:
+                    self.erase_to_eol = None
 
             def _need_update(self):
                 return True
 
+            def erase_line(self):
+                if self.erase_to_eol:
+                    self.fd.buffer.write(self.erase_to_eol)
+                elif self.fd.isatty():
+                    print(end='\r', file=self.fd)
+                    print(' ' * self.term_width, end='', file=self.fd)
+                else:
+                    # Do not flush if nothing was written
+                    return
+                self.fd.flush()
+
+            def finish(self):
+                self.finished = True
+                self.update(self.maxval)
+                # Clear the progress bar after completion
+                self.erase_line()
+                if self.signal_set:
+                    signal.signal(signal.SIGWINCH, signal.SIG_DFL)
+
         self.bar = OurProgressBar(widgets=(
             ' ',
             progressbar.Bar(),

diffoscope/tempfiles.py

@@ -3,6 +3,7 @@
 # diffoscope: in-depth comparison of files, archives, and directories
 #
 # Copyright © 2016 Chris Lamb <lamby@debian.org>
+#           © 2018 Mattia Rizzolo <mattia@debian.org>
 #
 # diffoscope is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -27,7 +28,7 @@ logger = logging.getLogger(__name__)
 
 
 def get_named_temporary_file(*args, **kwargs):
-    kwargs['suffix'] = kwargs.pop('suffix', '_diffoscope')
+    kwargs['dir'] = kwargs.pop('dir', _get_base_temporary_directory())
 
     f = tempfile.NamedTemporaryFile(*args, **kwargs)
     _FILES.append(f.name)
@@ -36,7 +37,7 @@ def get_named_temporary_file(*args, **kwargs):
 
 
 def get_temporary_directory(*args, **kwargs):
-    kwargs['suffix'] = kwargs.pop('suffix', '_diffoscope')
+    kwargs['dir'] = kwargs.pop('dir', _get_base_temporary_directory())
 
     d = tempfile.TemporaryDirectory(*args, **kwargs)
     _DIRS.append(d)
@@ -54,10 +55,12 @@ def clean_all_temp_files():
             pass
         except:
             logger.exception("Unable to delete %s", x)
+    _FILES.clear()
 
     logger.debug("Cleaning %d temporary directories", len(_DIRS))
 
-    for x in _DIRS:
+    # Reverse so we delete the top-level directory last.
+    for x in reversed(_DIRS):
         try:
             x.cleanup()
         except PermissionError:
@@ -73,3 +76,18 @@ def clean_all_temp_files():
             pass
         except:
             logger.exception("Unable to delete %s", x)
+    _DIRS.clear()
+
+
+def _get_base_temporary_directory():
+    if not _DIRS:
+        d = tempfile.TemporaryDirectory(
+            dir=tempfile.gettempdir(),
+            prefix='diffoscope_',
+        )
+
+        logger.debug("Created top-level temporary directory: %s", d.name)
+
+        _DIRS.append(d)
+
+    return _DIRS[0].name

tests/comparators/test_lz4.py

+# -*- coding: utf-8 -*-
+#
+# diffoscope: in-depth comparison of files, archives, and directories
+#
+# Copyright © 2018 Xavier Briand <xavierbriand@gmail.com>
+#
+# diffoscope is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# diffoscope is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with diffoscope.  If not, see <https://www.gnu.org/licenses/>.
+
+import shutil
+import pytest
+
+from diffoscope.comparators.lz4 import Lz4File
+from diffoscope.comparators.binary import FilesystemFile
+from diffoscope.comparators.utils.specialize import specialize
+
+from ..utils.data import load_fixture, get_data
+from ..utils.tools import skip_unless_tools_exist
+from ..utils.nonexisting import assert_non_existing
+
+lz41 = load_fixture('test1.lz4')
+lz42 = load_fixture('test2.lz4')
+
+
+def test_identification(lz41):
+    assert isinstance(lz41, Lz4File)
+
+
+def test_no_differences(lz41):
+    difference = lz41.compare(lz41)
+    assert difference is None
+
+
+@pytest.fixture
+def differences(lz41, lz42):
+    return lz41.compare(lz42).details
+
+
+@skip_unless_tools_exist('lz4')
+def test_content_source(differences):
+    assert differences[0].source1 == 'test1'
+    assert differences[0].source2 == 'test2'
+
+
+@skip_unless_tools_exist('lz4')
+def test_content_source_without_extension(tmpdir, lz41, lz42):
+    path1 = str(tmpdir.join('test1'))
+    path2 = str(tmpdir.join('test2'))
+    shutil.copy(lz41.path, path1)
+    shutil.copy(lz42.path, path2)
+    lz41 = specialize(FilesystemFile(path1))
+    lz42 = specialize(FilesystemFile(path2))
+    difference = lz41.compare(lz42).details
+    assert difference[0].source1 == 'test1-content'
+    assert difference[0].source2 == 'test2-content'
+
+
+@skip_unless_tools_exist('lz4')
+def test_content_diff(differences):
+    expected_diff = get_data('text_ascii_expected_diff')
+    assert differences[0].unified_diff == expected_diff
+
+
+@skip_unless_tools_exist('lz4')
+def test_compare_non_existing(monkeypatch, lz41):
+    assert_non_existing(monkeypatch, lz41)

tests/comparators/test_pdf.py

@@ -52,18 +52,12 @@ def differences(pdf1, pdf2):
     return pdf1.compare(pdf2).details
 
 
-@skip_unless_tools_exist('pdftk', 'pdftotext')
+@skip_unless_tools_exist('pdftotext')
 def test_text_diff(differences):
     expected_diff = get_data('pdf_text_expected_diff')
     assert differences[0].unified_diff == expected_diff
 
 
-@skip_unless_tools_exist('pdftk', 'pdftotext')
-def test_internal_diff(differences):
-    expected_diff = get_data('pdf_internal_expected_diff')
-    assert differences[1].unified_diff == expected_diff
-
-
-@skip_unless_tools_exist('pdftk', 'pdftotext')
+@skip_unless_tools_exist('pdftotext')
 def test_compare_non_existing(monkeypatch, pdf1):
     assert_non_existing(monkeypatch, pdf1, has_null_source=False)

tests/conftest.py

@@ -59,6 +59,6 @@ def pytest_report_header(config):
             'dpkg-query',
             '-W',
             '-f', '${db:Status-Abbrev}\t${binary:Package} (${Version})\n'
-        ))]
+        )).decode('utf-8')]
     except:
         pass

tests/data/pdf_internal_expected_diff

-@@ -11,23 +11,23 @@
- /Kids [3 0 R]
- /Type /Pages
- /Count 1
- >>
- endobj 
- 4 0 obj 
- <<
--/Length 85
-+/Length 109
- >>
- stream
- q
- 
- BT
- 36.0 747.384 Td
- /F1.0 12 Tf
--[<48656c6c6f2057> 30 <6f72> -15 <6c6421>] TJ
-+[<48656c6c6f20536963> 20 <6b205361642057> 30 <6f72> -15 <6c6421>] TJ
- ET
- 
- Q
- 
- endstream 
- endobj 
- 3 0 obj 
-@@ -61,21 +61,21 @@
- /Producer <feff0050007200610077006e>
- >>
- endobj xref
- 0 7
- 0000000000 65535 f 
- 0000000015 00000 n 
- 0000000066 00000 n 
--0000000263 00000 n 
-+0000000288 00000 n 
- 0000000125 00000 n 
--0000000462 00000 n 
--0000000561 00000 n 
-+0000000487 00000 n 
-+0000000586 00000 n 
- trailer
- 
- <<
- /Info 6 0 R
- /Root 1 0 R
- /Size 7
- >>
- startxref
--656
-+681
- %%EOF