-
Zach Welch authoredZach Welch authored
container.py 7.83 KiB
#
# diffoscope: in-depth comparison of files, archives, and directories
#
# Copyright © 2016-2021 Chris Lamb <lamby@debian.org>
#
# diffoscope is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# diffoscope is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with diffoscope. If not, see <https://www.gnu.org/licenses/>.
import abc
import uuid
import os.path
import logging
import itertools
from collections import OrderedDict
from diffoscope.config import Config
from diffoscope.difference import Difference
from diffoscope.excludes import filter_excludes
from diffoscope.progress import Progress
from ..missing_file import MissingFile, AbstractMissingType
from .file import path_apparent_size
from .fuzzy import perform_fuzzy_matching
NO_COMMENT = None
logger = logging.getLogger(__name__)
class Container(metaclass=abc.ABCMeta):
auto_diff_metadata = True
def __new__(cls, source):
if isinstance(source, MissingFile):
new = super(Container, MissingContainer).__new__(MissingContainer)
new.__init__(source)
return new
return super(Container, cls).__new__(cls)
def __init__(self, source):
self._source = source
# Keep a count of how "nested" we are
self.depth = 0
if hasattr(source, "container") and source.container is not None:
self.depth = source.container.depth + 1
@property
def source(self):
return self._source
@abc.abstractmethod
def get_member_names(self):
raise NotImplementedError()
@abc.abstractmethod
def get_member(self, member_name):
raise NotImplementedError()
def get_path_name(self, dest_dir):
return os.path.join(dest_dir, str(uuid.uuid4()))
def get_filtered_members(self):
# If your get_member implementation is O(n) then this will be O(n^2)
# cost. In such cases it is recommended to override this as well
for name in filter_excludes(self.get_member_names()):
yield name, self.get_member(name)
def perform_fuzzy_matching(self, my_members, other_members):
return perform_fuzzy_matching(my_members, other_members)
def get_adjusted_members(self):
"""
Returns an iterable of pairs. The key is what is used to match when
comparing containers. This may be used to e.g. strip off version
numbers, hashes, etc, efficiently for known file formats, so that we
don't need to use the expensive tlsh "fuzzy-hashing" logic.
Note that containers with 1 element are already force-compared against
other containers with 1 element, so you don't need to override this
method for those cases.
"""
return self.get_filtered_members()
def lookup_file(self, *names):
"""
Try to fetch a specific file by digging in containers.
"""
from .specialize import specialize
name, remainings = names[0], names[1:]
try:
file = self.get_member(name)
except KeyError:
return None
logger.debug("lookup_file(%s) -> %s", names, file)
specialize(file)
if not remainings:
return file
container = file.as_container
if not container:
return None
return container.lookup_file(*remainings)
def get_adjusted_members_sizes(self):
for name, member in self.get_adjusted_members():
if member.is_directory():
size = 4096 # default "size" of a directory
else:
size = path_apparent_size(member.path)
yield name, (member, size)
def comparisons(self, other):
my_members = OrderedDict(self.get_adjusted_members_sizes())
other_members = OrderedDict(other.get_adjusted_members_sizes())
total_size = sum(
x[1]
for x in itertools.chain(
my_members.values(), other_members.values()
)
)
# TODO: progress could be a bit more accurate here, give more weight to fuzzy-hashed files
with Progress(total_size) as p:
def prep_yield(my_name, other_name, comment=NO_COMMENT):
my_member, my_size = my_members.pop(my_name)
other_member, other_size = other_members.pop(other_name)
p.begin_step(my_size + other_size, msg=my_member.progress_name)
return my_member, other_member, comment
# if both containers contain 1 element, compare these
if len(my_members) == 1 and len(other_members) == 1:
yield prep_yield(
next(iter(my_members.keys())),
next(iter(other_members.keys())),
)
return
other_names = set(other_members.keys())
# keep it sorted like my_members
both_names = [
name for name in my_members.keys() if name in other_names
]
for name in both_names:
yield prep_yield(name, name)
for my_name, other_name, score in self.perform_fuzzy_matching(
my_members, other_members
):
score_display = score / 400.0 * 100
comment = f"Files {score_display:.0f}% similar despite different names"
if score == 0:
comment = "Files identical despite different names"
yield prep_yield(my_name, other_name, comment)
if Config().new_file:
for my_member, my_size in my_members.values():
p.begin_step(my_size, msg=my_member.progress_name)
yield my_member, MissingFile(
"/dev/null", my_member
), NO_COMMENT
for other_member, other_size in other_members.values():
p.begin_step(other_size, msg=other_member.progress_name)
yield MissingFile(
"/dev/null", other_member
), other_member, NO_COMMENT
def compare(self, other, source=None, no_recurse=False):
from .compare import compare_files
from ..directory import compare_meta
def compare_pair(file1, file2, comment):
difference = compare_files(
file1, file2, source=None, diff_content_only=no_recurse
)
if isinstance(file1, AbstractMissingType) or isinstance(
file2, AbstractMissingType
):
# There is no need to compare metadata with a missing file,
# as it doesn't make much sense
meta_differences = []
else:
meta_differences = compare_meta(file1.name, file2.name)
if meta_differences and not difference:
difference = Difference(file1.path, file2.path)
if difference:
difference.add_details(meta_differences)
if comment:
if difference is None:
difference = Difference(file1.name, file2.name)
difference.add_comment(comment)
return difference
return filter(
None, itertools.starmap(compare_pair, self.comparisons(other))
)
class MissingContainer(Container, AbstractMissingType):
def get_member_names(self):
return self.source.other_file.as_container.get_member_names()
def get_member(self, member_name):
return MissingFile("/dev/null")