Commit d768db1e authored by Chris Lamb's avatar Chris Lamb 💬

diffoscope/comparators: Split out as many utilities from (eg.) comparators.binary.

Signed-off-by: Chris Lamb's avatarChris Lamb <lamby@debian.org>
parent 114df3d6
......@@ -18,21 +18,9 @@
# You should have received a copy of the GNU General Public License
# along with diffoscope. If not, see <https://www.gnu.org/licenses/>.
import re
import sys
import magic
import os.path
import operator
from diffoscope import logger, tool_required
from diffoscope.config import Config
from diffoscope.profiling import profile
from diffoscope.difference import Difference
COMPARATORS = (
('directory.Directory',),
('binary.MissingFile',),
('missing_file.MissingFile',),
('symlink.Symlink',),
('device.Device',),
('debian.DotChangesFile', 'debian_fallback.DotChangesFile'),
......@@ -81,20 +69,3 @@ COMPARATORS = (
('git.GitIndexFile',),
('openssh.PublicKeyFile',),
)
def specialize(file):
for cls in FILE_CLASSES:
if isinstance(file, cls):
return file
with profile('recognizes', file):
if cls.recognizes(file):
logger.debug("Using %s for %s", cls.__name__, file.name)
new_cls = type(cls.__name__, (cls, type(file)), {})
file.__class__ = new_cls
return file
logger.debug('Unidentified file. Magic says: %s', file.magic_file_type)
return file
from .utils.loading import import_comparators
FILE_CLASSES = import_comparators(COMPARATORS)
......@@ -24,7 +24,8 @@ import subprocess
from diffoscope import logger, tool_required, get_temporary_directory
from .binary import File
from .utils import Archive, get_compressed_content_name
from .utils.archive import Archive
from .utils.filenames import get_compressed_content_name
class ApkContainer(Archive):
@property
......
......@@ -23,8 +23,8 @@ import re
from diffoscope import logger, tool_required
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.command import Command
from .utils.libarchive import LibarchiveContainer, list_libarchive
......
This diff is collapsed.
......@@ -24,8 +24,9 @@ import collections
from diffoscope import logger, tool_required
from .utils import Archive, get_compressed_content_name
from .binary import File
from .utils.file import File
from .utils.archive import Archive
from .utils.filenames import get_compressed_content_name
class Bzip2Container(Archive):
......
......@@ -26,8 +26,9 @@ import subprocess
from diffoscope import logger, tool_required
from diffoscope.difference import Difference
from .utils import Archive, Command
from .binary import File
from .utils.archive import Archive
from .utils.command import Command
class CbfsListing(Command):
......
......@@ -23,8 +23,8 @@ import re
from diffoscope import tool_required
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.command import Command
from .utils.libarchive import LibarchiveContainer, list_libarchive
......
......@@ -22,11 +22,11 @@ import re
from diffoscope import logger
from diffoscope.difference import Difference
from . import specialize
from .tar import TarContainer
from .utils import ArchiveMember
from .binary import File
from .utils.file import File
from .utils.archive import ArchiveMember
from .utils.libarchive import LibarchiveContainer, list_libarchive
from .utils.specialize import specialize
try:
from debian import deb822
......
......@@ -29,8 +29,8 @@ from diffoscope import logger
from diffoscope.changes import Changes
from diffoscope.difference import Difference
from .utils import Container
from .binary import File
from .utils.container import Container
DOT_CHANGES_FIELDS = [
......
......@@ -25,7 +25,8 @@ import collections
from diffoscope import logger, tool_required
from .binary import File
from .utils import Archive, get_compressed_content_name
from .utils.archive import Archive
from .utils.filenames import get_compressed_content_name
class DexContainer(Archive):
......
......@@ -26,9 +26,9 @@ from diffoscope.exc import RequiredToolNotFound
from diffoscope.progress import Progress
from diffoscope.difference import Difference
from .utils import Container, Command
from .binary import FilesystemFile
from .utils.compare import compare_files
from .utils.command import Command
from .utils.container import Container
def list_files(path):
......@@ -147,6 +147,8 @@ class FilesystemDirectory(object):
return False
def compare(self, other, source=None):
from .utils.compare import compare_files
differences = []
try:
listing_diff = Difference.from_text('\n'.join(list_files(self.path)),
......
......@@ -27,8 +27,9 @@ from diffoscope.exc import OutputParsingError
from diffoscope.difference import Difference
from .deb import DebFile, get_build_id_map
from .utils import Command, Container
from .binary import File
from .utils.command import Command
from .utils.container import Container
from .utils.libarchive import list_libarchive
......
......@@ -22,8 +22,8 @@ import re
from diffoscope import tool_required
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.command import Command
class Showttf(Command):
......
......@@ -24,8 +24,8 @@ import collections
from diffoscope import logger
from diffoscope.difference import Difference
from .utils import Archive
from .binary import File
from .utils.archive import Archive
try:
import guestfs
......
......@@ -24,8 +24,8 @@ from diffoscope import logger
from diffoscope import tool_required
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.command import Command
class Msgunfmt(Command):
......
......@@ -25,7 +25,8 @@ import collections
from diffoscope import logger, tool_required
from diffoscope.difference import Difference
from .utils import Archive, get_compressed_content_name
from .utils.archive import Archive
from .utils.filenames import get_compressed_content_name
class GzipContainer(Archive):
......
......@@ -26,8 +26,8 @@ from diffoscope import tool_required, logger
from diffoscope.profiling import profile
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.command import Command
class ShowIface(Command):
......
......@@ -22,8 +22,8 @@ import re
from diffoscope import tool_required
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.command import Command
class Iccdump(Command):
......
......@@ -22,8 +22,8 @@ import re
from diffoscope import tool_required
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.command import Command
re_ansi_escapes = re.compile(r'\x1b[^m]*m')
......
......@@ -23,8 +23,8 @@ import subprocess
from diffoscope import tool_required
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.command import Command
from .utils.libarchive import LibarchiveContainer
......
......@@ -24,8 +24,8 @@ import os.path
from diffoscope import tool_required
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.command import Command
class Javap(Command):
......
......@@ -22,8 +22,8 @@ import re
from diffoscope import tool_required
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.file import File
from .utils.command import Command
class JavaScriptBeautify(Command):
......
......@@ -21,8 +21,8 @@
from diffoscope import tool_required
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.command import Command
class LlvmBcAnalyzer(Command):
......
......@@ -24,8 +24,8 @@ import subprocess
from diffoscope import tool_required
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.command import Command
class Otool(Command):
......
# -*- coding: utf-8 -*-
#
# diffoscope: in-depth comparison of files, archives, and directories
#
# Copyright © 2016 Chris Lamb <lamby@debian.org>
#
# diffoscope is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# diffoscope is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with diffoscope. If not, see <https://www.gnu.org/licenses/>.
import os
from diffoscope import logger
from diffoscope.config import Config
from diffoscope.difference import Difference
from .utils.file import File
from .binary import FilesystemFile
class MissingFile(File):
"""Represents a missing file when comparing containers"""
@staticmethod
def recognizes(file):
if isinstance(file, FilesystemFile) and not os.path.lexists(file.name):
assert Config().new_file, '%s does not exist' % file.name
return True
return False
def __init__(self, path, other_file=None):
self._name = path
self._other_file = other_file
@property
def path(self):
return '/dev/null'
@property
def other_file(self):
return self._other_file
@other_file.setter
def other_file(self, value):
self._other_file = value
def has_same_content_as(self, other):
return False
def is_directory(self):
return False
def is_symlink(self):
return False
def is_device(self):
return False
def compare(self, other, source=None):
# So now that comparators are all object-oriented, we don't have any clue on how to
# perform a meaningful comparison right here. So we are good do the comparison backward
# (where knowledge of the file format lies) and and then reverse it.
if isinstance(other, MissingFile):
return Difference(None, self.name, other.name, comment='Trying to compare two non-existing files.')
logger.debug('Performing backward comparison')
backward_diff = other.compare(self, source)
if not backward_diff:
return None
return backward_diff.get_reverse()
# Be nice to text comparisons
@property
def encoding(self):
return self._other_file.encoding
# Be nice to device comparisons
def get_device(self):
return ''
# Be nice to metadata comparisons
@property
def magic_file_type(self):
return self._other_file.magic_file_type
# Be nice to .changes and .dsc comparisons
@property
def deb822(self):
class DummyChanges(dict):
get_as_string = lambda self, _: ''
return DummyChanges(Files=[], Version='')
......@@ -23,8 +23,8 @@ import re
from diffoscope import tool_required
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.command import Command
class Pedump(Command):
......
......@@ -22,8 +22,8 @@ import re
from diffoscope import tool_required
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.command import Command
class SSHKeyList(Command):
......
......@@ -22,8 +22,8 @@ import re
from diffoscope import tool_required
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.command import Command
class Pdftotext(Command):
......
......@@ -23,8 +23,8 @@ import functools
from diffoscope import tool_required
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.command import Command
class Sng(Command):
......
......@@ -27,8 +27,8 @@ from diffoscope import tool_required, logger
from diffoscope.profiling import profile
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.command import Command
class Ppudump(Command):
......
......@@ -24,7 +24,7 @@ from diffoscope.exc import RequiredToolNotFound
from diffoscope.difference import Difference
from .text import TextFile
from .utils import Command
from .utils.command import Command
class Pstotext(Command):
......
......@@ -27,8 +27,8 @@ import subprocess
from diffoscope import logger, tool_required, get_temporary_directory
from diffoscope.difference import Difference
from .utils import Archive
from .rpm_fallback import AbstractRpmFile
from .utils.archive import Archive
def convert_header_field(io, header):
......
......@@ -25,7 +25,8 @@ import collections
from diffoscope import logger, tool_required
from diffoscope.difference import Difference
from .utils import Archive, get_compressed_content_name
from .utils.archive import Archive
from .utils.filenames import get_compressed_content_name
RLIB_BYTECODE_OBJECT_V1_DATASIZE_OFFSET = 15
RLIB_BYTECODE_OBJECT_V1_DATA_OFFSET = 23
......
......@@ -20,8 +20,8 @@
from diffoscope import tool_required
from diffoscope.difference import Difference
from .utils import Command
from .binary import File
from .utils.command import Command
class Sqlite3Dump(Command):
......
......@@ -26,11 +26,12 @@ import collections
from diffoscope import logger, tool_required
from diffoscope.difference import Difference
from .utils import Archive, ArchiveMember, Command
from .binary import File
from .device import Device
from .symlink import Symlink
from .directory import Directory
from .utils.archive import Archive, ArchiveMember
from .utils.command import Command
class SquashfsSuperblock(Command):
......
This diff is collapsed.
# -*- coding: utf-8 -*-
#
# diffoscope: in-depth comparison of files, archives, and directories
#
# Copyright © 2016 Chris Lamb <lamby@debian.org>
#
# diffoscope is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# diffoscope is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with diffoscope. If not, see <https://www.gnu.org/licenses/>.
import abc
from diffoscope import logger, get_temporary_directory
from diffoscope.profiling import profile
from ..missing_file import MissingFile
from .file import File
from .container import Container
class Archive(Container, metaclass=abc.ABCMeta):
def __new__(cls, source, *args, **kwargs):
if isinstance(source, MissingFile):
return super(Container, MissingArchive).__new__(MissingArchive)
else:
return super(Container, cls).__new__(cls)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
with profile('open_archive', self):
self._archive = self.open_archive()
def __del__(self):
with profile('close_archive', self):
self.close_archive()
@property
def archive(self):
return self._archive
@abc.abstractmethod
def open_archive(self):
raise NotImplementedError()
@abc.abstractmethod
def close_archive(self):
raise NotImplementedError()
@abc.abstractmethod
def get_member_names(self):
raise NotImplementedError()
@abc.abstractmethod
def extract(self, member_name, dest_dir):
raise NotImplementedError()
def get_member(self, member_name):
return ArchiveMember(self, member_name)
class ArchiveMember(File):
def __init__(self, container, member_name):
super().__init__(container=container)
self._name = member_name
self._temp_dir = None
self._path = None
@property
def path(self):
if self._path is None:
logger.debug('unpacking %s', self._name)
assert self._temp_dir is None
self._temp_dir = get_temporary_directory()
with profile('container_extract', self.container):
self._path = self.container.extract(self._name, self._temp_dir.name)
return self._path
def cleanup(self):
if self._path is not None:
self._path = None
if self._temp_dir is not None:
self._temp_dir.cleanup()
self._temp_dir = None
super().cleanup()
def is_directory(self):
return False
def is_symlink(self):
return False
def is_device(self):
return False
class MissingArchiveLikeObject(object):
def getnames(self):
return []
def list(self, *args, **kwargs):
return ''
def close(self):
pass
class MissingArchive(Archive):
@property
def source(self):
return None
def open_archive(self):
return MissingArchiveLikeObject()
def close_archive(self):
pass
def get_member_names(self):
return []
def extract(self, member_name, dest_dir):
# should never be called
raise NotImplementedError()
def get_member(self, member_name):
return MissingFile('/dev/null')
# Be nice to gzip and the likes
@property
def path(self):
return '/dev/null'
# -*- coding: utf-8 -*-
#
# diffoscope: in-depth comparison of files, archives, and directories
#
# Copyright © 2016 Chris Lamb <lamby@debian.org>
#
# diffoscope is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# diffoscope is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with diffoscope. If not, see <https://www.gnu.org/licenses/>.
import io
import abc
import subprocess
import threading
from diffoscope import logger
class Command(object, metaclass=abc.ABCMeta):
def __init__(self, path):
self._path = path
logger.debug('running %s', self.cmdline())
self._process = subprocess.Popen(self.cmdline(),
shell=False, close_fds=True,
env=self.env(),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if hasattr(self, 'feed_stdin'):
self._stdin_feeder = threading.Thread(target=self._feed_stdin, args=(self._process.stdin,))
self._stdin_feeder.daemon = True
self._stdin_feeder.start()
else:
self._stdin_feeder = None
self._process.stdin.close()
self._stderr = io.BytesIO()
self._stderr_line_count = 0
self._stderr_reader = threading.Thread(target=self._read_stderr)
self._stderr_reader.daemon = True
self._stderr_reader.start()