debian/changelog

-diffoscope (158) UNRELEASED; urgency=medium
-
-  * WIP (generated upon release).
-
- -- Chris Lamb <lamby@debian.org>  Fri, 21 Aug 2020 12:34:27 +0100
+diffoscope (158) unstable; urgency=medium
+
+  * Improve PGP support:
+    - Support extracting of files within PGP signed data.
+      (Closes: reproducible-builds/diffoscope#214)
+    - pgpdump(1) can successfully parse some unrelated, non-PGP binary files,
+      so check that the parsed output contains something remotely sensible
+      before identifying it as a PGP file.
+  * Don't use Python's repr(...)-style output in "Calling external command"
+    logging output.
+  * Correct a typo of "output" in an internal comment.
+
+ -- Chris Lamb <lamby@debian.org>  Fri, 28 Aug 2020 11:53:10 +0100
 
 diffoscope (157) unstable; urgency=medium
 

diffoscope/__init__.py

@@ -18,4 +18,4 @@
 # You should have received a copy of the GNU General Public License
 # along with diffoscope.  If not, see <https://www.gnu.org/licenses/>.
 
-VERSION = "157"
+VERSION = "158"

diffoscope/comparators/pgp.py

@@ -17,14 +17,17 @@
 # You should have received a copy of the GNU General Public License
 # along with diffoscope.  If not, see <https://www.gnu.org/licenses/>.
 
+import os
 import re
 import logging
 import subprocess
 
 from diffoscope.tools import tool_required
+from diffoscope.tempfiles import get_temporary_directory
 from diffoscope.difference import Difference
 
 from .text import TextFile
+from .utils.archive import Archive
 from .utils.file import File
 from .utils.command import Command, our_check_output
 
@@ -37,7 +40,6 @@ class Pgpdump(Command):
         return (
             "pgpdump",
             "-i",  # Dump integer packets
-            "-l",  # Dump literal packets
             "-m",  # Dump marker packets
             "-p",  # Dump private packets
             "-u",  # Display UTC time
@@ -45,9 +47,46 @@ class Pgpdump(Command):
         )
 
 
+class PGPContainer(Archive):
+    @tool_required("gpg")
+    def open_archive(self):
+        # Extract to a fresh temporary directory so that we can use the
+        # embedded filename.
+
+        self._temp_dir = get_temporary_directory()
+
+        try:
+            our_check_output(
+                (
+                    "gpg",
+                    "--use-embedded-filename",
+                    "--decrypt",
+                    "--no-keyring",
+                    os.path.abspath(self.source.path),
+                ),
+                cwd=self._temp_dir.name,
+                stderr=subprocess.DEVNULL,
+            )
+        except subprocess.CalledProcessError:
+            return False
+
+        return self
+
+    def close_archive(self):
+        self._temp_dir.cleanup()
+
+    def get_member_names(self):
+        # Will only return one filename, taken from the signature file itself.
+        return os.listdir(self._temp_dir.name)
+
+    def extract(self, member_name, dest_dir):
+        return os.path.join(self._temp_dir.name, member_name)
+
+
 class PgpFile(File):
     DESCRIPTION = "PGP signed/encrypted messages"
     FILE_TYPE_RE = re.compile(r"^PGP message\b")
+    CONTAINER_CLASSES = [PGPContainer]
     FALLBACK_FILE_EXTENSION_SUFFIX = {".pgp", ".asc", ".pub", ".sec", ".gpg"}
 
     @classmethod
@@ -84,7 +123,7 @@ class PgpSignature(TextFile):
         # Don't display signatures as hexdumps; use TextFile's comparisons...
         difference = super().compare(other, source)
 
-        # ... but attach pgpdump of outout
+        # ... but attach pgpdump of output
         difference.add_details(
             [
                 Difference.from_command(

diffoscope/comparators/utils/command.py

@@ -121,6 +121,6 @@ class Command(metaclass=abc.ABCMeta):
 
 
 def our_check_output(cmd, *args, **kwargs):
-    logger.debug("Calling external command %r", cmd)
+    logger.debug("Calling external command: %s", " ".join(cmd))
 
     return subprocess.check_output(cmd, *args, **kwargs)

diffoscope/comparators/utils/compare.py

@@ -141,7 +141,7 @@ def call_difftool(file1, file2):
         return
 
     cmd = " ".join((Config().difftool, shlex.quote(a), shlex.quote(b)))
-    logger.debug("Calling external command %r", cmd)
+    logger.debug("Calling external command: %s", " ".join(cmd))
     subprocess.call(cmd, shell=True)
 
 

tests/data/pgp_signed_expected_diff

@@ -4,13 +4,13 @@
  	Pub alg - RSA Encrypt or Sign(pub 1)
  	Key ID - 0x1E953E27D4311E58
  	Next packet - other than one pass signature
- New: Literal Data Packet(tag 11)(7 bytes)
+ Old: Literal Data Packet(tag 11)(10255 bytes)
  	Format - binary
--	Filename - a
--	File modified time - Wed Aug 19 10:45:54 UTC 2020
-+	Filename - b
-+	File modified time - Wed Aug 19 10:45:57 UTC 2020
- 	Literal - 
+-	Filename - test1.tar
+-	File modified time - Tue Aug 25 11:47:35 UTC 2020
++	Filename - test2.tar
++	File modified time - Tue Aug 25 11:47:38 UTC 2020
+ 	Literal - ...
  Old: Signature Packet(tag 2)(563 bytes)
  	Ver 4 - new
  	Sig type - Signature of a binary document(0x00).
@@ -19,12 +19,12 @@
  	Hashed Sub: issuer fingerprint(sub 33)(21 bytes)
  	 v4 -	Fingerprint - c2 fe 4b d2 71 c1 39 b8 6c 53 3e 46 1e 95 3e 27 d4 31 1e 58 
  	Hashed Sub: signature creation time(sub 2)(4 bytes)
--		Time - Wed Aug 19 10:45:54 UTC 2020
-+		Time - Wed Aug 19 10:45:57 UTC 2020
+-		Time - Tue Aug 25 11:47:35 UTC 2020
++		Time - Tue Aug 25 11:47:38 UTC 2020
  	Sub: issuer key ID(sub 16)(8 bytes)
  		Key ID - 0x1E953E27D4311E58
--	Hash left 2 bytes - 48 80 
--	RSA m^d mod n(4094 bits) - 23 dc ce ae ac fb 75 90 03 23 97 03 8a d1 cd a6 3f bd 81 30 d8 8a 11 03 b8 35 20 88 ec 71 f3 02 e1 1e 31 05 db 1a b3 85 60 dd 42 e3 46 4d 6f f5 5e 85 c1 2a 5f 4c a9 da fa 6a 5b f8 19 04 1a 6c ec 98 f4 97 f7 ca ac 6e 4c 88 c6 10 7b 2c 6f 4a a9 2a c1 6e c0 61 10 92 7f 03 f5 82 f3 9e ad 39 34 a3 86 f1 45 67 8c e7 92 80 69 20 c5 b3 78 dd 28 bb a0 9a 83 e3 29 80 5b 6f 55 f7 1d bc d4 a3 80 48 8a a3 86 e3 ba 1c c1 44 ec 07 7c c9 d6 d6 06 bc 21 ff a6 fd 8f 5b 37 c6 78 7f 6c 96 64 dc 7e 78 6a e2 9d 01 75 57 79 0d d8 53 54 8d 17 c0 45 3a 5a 25 c8 6a b6 39 c4 94 da d9 dc 55 85 4f 5e 99 f8 57 23 aa e3 4e 7f 65 f9 fc b2 cc b7 ff 29 46 cc 78 6b ae 6d 85 70 78 8a eb fa ba 15 7e 40 e8 e0 93 0b be e9 fe b8 3b bd 49 1f ac cd 28 ca ba 68 e6 e9 48 41 f6 7c f7 47 23 36 5e a7 03 08 9b 7e 13 a2 d1 72 36 2c ad b5 74 64 52 31 ca 32 46 fc 82 08 84 ed 32 ae d3 ec d2 c3 71 bc 27 e8 59 5e 90 86 37 65 f0 0a a6 e2 46 bf 10 f2 b5 f0 42 11 09 eb e7 b3 88 ac 36 90 1e be 22 0a fe b0 ce de 2a 16 cf ea 80 2c 97 ba 1d b0 52 47 09 d5 7d 1b 18 2f 01 d4 02 b6 b1 e9 93 64 03 30 b8 67 2c 34 5a 4b eb 26 2e a9 d9 23 cf 9b b6 88 0b e4 7b f9 6e 0f d1 fa 01 82 80 e9 4f c5 ac 51 6d 4a 19 52 b5 a1 5f f7 7c 7b 90 c7 57 bc eb b2 8b b5 18 7e 01 20 6a 81 8b c6 f5 11 aa a4 6e 93 4e a5 23 fe 5e af c1 86 26 bf 29 31 09 09 c8 16 62 73 48 b4 1b d9 72 89 f4 4a 13 37 00 0f 29 c3 56 0b 99 01 57 33 f5 97 f7 91 18 f1 86 36 11 37 e5 be d7 e4 d7 6b 1d 16 e3 04 71 68 5a 5e f4 6c 44 af 44 47 b7 ac 71 48 25 10 82 1f 74 d8 ae bf e8 c4 37 a5 47 cc 11 ca 05 fa 6f d3 4e 06 d0 45 aa 
-+	Hash left 2 bytes - c6 c2 
-+	RSA m^d mod n(4092 bits) - 0d 37 12 58 c3 ad 4a 6a e1 92 7f 3a 30 66 3a de 5a 72 09 09 40 39 7e 59 77 83 3b 1d 14 b0 2f 1f d2 50 a1 5f 7a 73 36 fe 76 96 76 62 b8 14 17 c8 db b3 78 c1 2b e0 6a 4a c1 b0 57 8a 72 12 34 45 ec cf 04 dd 4f cd 8f 80 a4 94 9a c0 cc b2 65 c4 ba 37 1b 31 c4 ec b8 34 4b f5 02 17 19 d5 93 b6 71 06 9d 20 e9 ee 39 65 1f 1f f3 40 4b 68 18 34 3b 8c 3c 1e 14 08 61 a8 65 18 5e 72 9a 7f 04 84 7f 7b b8 d3 d3 0b 75 4b 71 1f b7 c8 c8 b5 8f 36 ba 9b d6 46 ef 0d 1d 56 de 37 9d 17 d0 31 33 c7 a1 39 d0 a6 2e 12 83 1b 95 f7 70 0e ea c6 ad 0a de 7b c1 11 05 9b c5 a8 10 a2 0b 62 2b d7 72 11 ef 8a e3 e6 1d e0 13 18 69 16 0a 42 1e 9f 58 76 cd 57 f1 b5 98 fa ae 76 8e ed 12 67 5d 19 69 cc b2 c2 b9 61 a9 8f 4a 9a 40 9e 57 86 45 60 a1 e9 88 b3 b2 aa 9e 4e 02 64 e4 15 29 50 a1 be 09 e2 e0 80 cd a4 b9 39 96 4f 1f 62 2f 95 58 73 c5 f8 38 4a 0e 6e fa 09 0f 04 7c 63 a8 ae 21 57 0a a4 0a a0 f9 99 dd df 16 8f de bc 62 04 77 1e 0c 37 37 1e 1f 3d 16 43 2b a8 29 5e be 3f 87 20 41 57 ee d9 4d 37 3b 65 ff eb 13 4b 50 b2 9c a8 cf 04 84 bf d2 64 be 03 78 fb f0 85 c9 26 73 f1 c6 c9 76 20 0c 94 29 07 3a 86 72 0a 6f 0a 86 7f 10 1e 9d 5f 16 e6 2d cf f3 61 8f de 2e 12 1d 6f e1 0e 00 5a 2e ef 13 ff d4 74 ff 72 af 9e 13 46 0a 37 65 e0 b9 1b f0 34 86 ac 89 d7 09 3c f9 99 dc c0 55 be 9a bb b1 d2 90 a5 37 82 75 b0 c5 3f b8 92 24 e6 ff a8 39 3b 87 e7 9b 8a 0b bc 9e 51 a8 1f 7a 51 4e 2f 03 7d f6 df 86 f4 4f 41 9e 9f b9 e2 31 a4 f0 a7 8f 9a df 84 76 b0 0e 28 ec f9 50 02 7d 63 2f dc 7c 31 ac 37 ff 72 2b ae 50 60 1d 6b 6d 61 ac 29 0c cb 3b d4 fa ed 4d 35 d6 ed 68 3b 
+-	Hash left 2 bytes - 60 c1 
+-	RSA m^d mod n(4095 bits) - 4b 34 6a df 2e 9c 0c c7 b3 c3 36 68 16 73 8e 03 22 9d e2 c8 14 70 e8 ba 55 6c fb b8 7f 19 03 2f 46 18 05 8b 0e da 59 05 05 ff dd 7b 1d 83 d1 ee dc da 13 ea e5 7f ee 3e cd 5f 2b c5 f0 fb 7b 4f c5 cc aa 90 fe 0a 7c d6 d6 4d f7 43 fc e4 e0 b7 15 66 23 b5 2a ff a0 0c 88 f1 f0 e2 f1 74 33 dd 68 9e 3a 9f 86 46 88 d6 4f b3 e3 0a 5e d8 97 05 68 d8 67 7a b9 0b 92 df ab b7 3f df a9 ad e1 e0 1f 1b c7 98 c8 2d 72 08 c8 ef 10 64 4f 7e ae c2 c7 d9 0d 2c 08 df d0 08 ea e0 fa 2e aa e3 fc fc 86 f4 6c bc e3 83 7c 70 83 8c 02 62 62 71 eb 77 b3 23 81 26 2f bb 8b aa d8 8c 36 f8 e2 01 10 b3 c6 9a c2 83 78 fb 71 d3 58 6d 74 c6 6b 5a a8 28 9b 2c 2e ed f7 10 88 55 01 ca 63 2b b6 00 58 ca 18 64 88 2e 01 cd 77 19 ba b3 16 ec 32 b7 1b b7 41 d6 b6 ce 0e 08 f4 b1 2d e8 39 00 76 7b f0 f4 75 20 86 1a ab ba 0c 15 aa f9 c8 ce 90 84 a2 d9 4e 77 37 69 fc a1 a4 2e a3 72 2b b3 45 e9 85 8e 18 4b e9 bc 68 a4 f8 bc 12 7f 9e 70 19 d2 22 70 d6 18 5e ff e0 82 40 80 1a cc d9 2b 39 30 0a 2b f8 05 7f 50 93 31 f8 d9 b6 9b 7e b5 e1 6b db 93 cf 7e e5 a7 b4 a8 7d cf 1e ba 25 a4 c7 af 2f 88 69 16 17 50 20 43 1c b6 a1 73 7b 38 5a 7f 39 b8 4d 2a c2 fa bf 9c 62 37 b2 f3 96 74 bb a0 4b 57 26 1b 21 c6 09 a9 bc cf 27 f7 4f 62 8e 28 bb cc c6 5c 1c 9c 56 da 3d c9 d2 7b 45 d5 01 f3 fe 52 bb a7 51 d4 d2 12 c1 2c 4e 08 06 2d 35 a9 01 ff 43 ce 43 7c fc 53 fe c7 ba 16 d5 20 9d d8 5e 69 82 0b 85 47 dd 7b 75 30 db c5 dc d6 38 c5 bf 7f 6a 7c 84 09 62 3a 2b cb 4f 20 72 86 c6 ce 56 19 3c df f4 90 d6 4a a6 c1 5e 04 8c 7e 8e e4 4c 74 e5 e3 0b 58 26 ff 06 f3 46 bd 91 7b 14 63 85 65 
++	Hash left 2 bytes - 1c 0b 
++	RSA m^d mod n(4096 bits) - 97 c6 77 99 13 8d 41 39 d8 4b 37 25 47 d3 9e 94 7e 32 94 db 6f 90 8a 64 60 68 28 15 a1 84 fb 42 aa d1 9e 1f 12 19 8d 42 cd 72 0c 1b 1f e8 af 55 dd f9 0d 49 b6 9d 99 d8 a6 86 64 a1 56 68 4b 64 d5 5c 57 a1 cf a9 ed c6 5a 79 40 bc b4 2b 95 5b e6 8d 18 60 30 ac 04 9a ea bc 2f fb c5 d6 ad 5c e9 14 61 b1 ce a4 75 29 b8 1c df 26 bd 43 7c 87 ac 4d 1f 3d cb a2 c0 fb 04 a0 5f ee e7 6e fd 57 c2 38 96 60 90 ca 52 ea 52 cf 28 0d b2 78 13 e3 c4 74 f2 fb e9 a4 06 27 fd 19 14 85 cb 21 4d d5 5b 91 43 76 be ba 00 44 aa 67 a7 5c 36 d9 2c b6 3d cc 0a 2d c0 8c c6 6a 15 21 78 48 41 33 c0 97 ac 36 4e 54 8c 44 9e 4d bb 30 b0 fb 2a 02 a5 0c c9 36 b7 d5 da 8c 3d c5 aa 08 61 79 1c c5 1a da 24 14 db c9 c0 3a d1 12 6a cd e5 77 42 d9 83 f1 ad 88 8c 70 06 4b 7a 54 50 90 a7 d9 5a 0d 2f 2d fc e3 10 36 13 2a 92 ba 6e ef 15 32 d6 cf 99 79 32 fd af 7a b9 33 a3 41 5e 0d fa a6 63 11 68 a9 1d 11 e0 62 3f 54 52 a0 ba 1c 30 66 a3 bd 89 0b d1 c5 22 d8 89 f5 b3 b9 30 c4 b8 75 dd b8 67 5e cd 57 96 d8 09 f5 44 1a 10 96 f1 68 d2 d0 da ee d7 b0 fa c2 f4 e3 03 c1 c8 a5 05 6c 31 20 0f 65 c3 60 9a 4d f7 a7 76 06 82 4c 46 1e bb 73 d2 df f8 14 3c 96 e5 24 3f 3a d3 51 8d b7 0e f2 ac e2 75 bf d7 a3 f0 dc b2 ae b9 69 95 92 0e 4b 94 69 f7 14 b6 45 ef 60 35 a7 d0 58 2b 8b 08 98 fc 55 aa 8c d7 dc cb 27 49 ea cd bc 61 9b 99 8b 18 27 bc 3b 72 49 82 06 42 bc e9 26 2a 1a f9 58 9b 94 8f 4b 55 14 46 42 93 4d f2 d9 4a 7d 6c 3e 25 5e de 37 40 f2 c2 bf 62 02 14 5a 93 30 50 76 81 fc 19 92 d3 96 ba e8 78 44 fe 3a aa 85 0c ad 8d d2 03 7d 41 e9 a5 ed 68 45 64 16 8c c1 af 64 29 be 
  		-> PKCS-1