Commit 7e3fcb77 authored by Simon McVittie's avatar Simon McVittie

Move progress reporting to gdp.command_line

parent 7a116e47
......@@ -121,15 +121,6 @@ class DownloadNotAllowed(Exception):
class CDRipFailed(Exception):
pass
def _ensure_hashes(hashes, path, size):
if hashes is not None:
return hashes
if size > QUITE_LARGE:
logger.info('identifying %s', path)
return HashedFile.from_file(path, open(path, 'rb'), size=size,
progress=(size > QUITE_LARGE))
def choose_mirror(wanted):
mirrors = []
mirror = os.environ.get('GDP_MIRROR')
......@@ -252,6 +243,9 @@ class PackagingTask(object):
# list: arbitrary options (e.g. -z9 -Zgz -Sfixed)
self.compress_deb = game.compress_deb
# Factory for a progress report (or None).
self.progress_factory = lambda: None
self.game.load_file_data()
def __del__(self):
......@@ -307,8 +301,12 @@ class PackagingTask(object):
if hashes is None:
if size > QUITE_LARGE:
logger.info('checking %s', path)
progress = self.progress_factory()
else:
progress = None
hashes = HashedFile.from_file(path, open(path, 'rb'), size=size,
progress=(size > QUITE_LARGE))
progress=progress)
for wanted in remaining:
if not wanted.skip_hash_matching and not hashes.matches(wanted):
......@@ -379,7 +377,7 @@ class PackagingTask(object):
# if a file (as opposed to a directory) is specified on the
# command-line, try harder to match it to something
if really_should_match_something:
hashes = _ensure_hashes(None, path, size)
hashes = self.__ensure_hashes(None, path, size)
else:
hashes = None
......@@ -387,7 +385,7 @@ class PackagingTask(object):
if match_path.endswith('/' + look_for):
candidates = [self.game.files[c] for c in candidates]
if candidates:
hashes = _ensure_hashes(hashes, path, size)
hashes = self.__ensure_hashes(hashes, path, size)
if self.use_file('possible "%s"' % look_for, candidates,
path, hashes):
return
......@@ -395,7 +393,7 @@ class PackagingTask(object):
if size in self.game.known_sizes:
candidates = self.game.known_sizes[size]
if candidates:
hashes = _ensure_hashes(hashes, path, size)
hashes = self.__ensure_hashes(hashes, path, size)
candidates = [self.game.files[c] for c in candidates]
if self.use_file('file of size %d' % size,
candidates, path, hashes):
......@@ -622,14 +620,15 @@ class PackagingTask(object):
wf = open(tmp, 'wb')
if entry.size is not None and entry.size > QUITE_LARGE:
large = True
progress = self.progress_factory()
logger.info('extracting %s from %s', entry.name, name)
else:
large = False
progress = None
logger.debug('extracting %s from %s', entry.name, name)
hf = HashedFile.from_file(
name + '//' + entry.name, entryfile, wf,
size=entry.size, progress=large)
size=entry.size, progress=progress)
wf.close()
if entry.mtime is not None:
......@@ -668,9 +667,15 @@ class PackagingTask(object):
yield open(self.found[provider.name], 'rb')
for p in other_parts:
yield open(self.found[p], 'rb')
if wanted.size >= QUITE_LARGE:
progress = self.progress_factory()
else:
progress = None
hasher = HashedFile.from_concatenated_files(wanted.name,
open_files(), writer, size=wanted.size,
progress=(wanted.size > QUITE_LARGE))
progress=progress)
orig_time = os.stat(self.found[provider.name]).st_mtime
os.utime(path, (orig_time, orig_time))
self.use_file(wanted.name, (wanted,), path, hasher)
......@@ -772,7 +777,8 @@ class PackagingTask(object):
wf = open(tmp, 'wb')
logger.info('downloading %s', url)
hf = HashedFile.from_file(url, rf, wf,
size=wanted.size, progress=True)
size=wanted.size,
progress=self.progress_factory())
wf.close()
if self.use_file(wanted.name, (wanted,), tmp, hf):
......@@ -2659,3 +2665,16 @@ class PackagingTask(object):
logger.warning('installing these packages might help:\n' +
'%s %s', ' '.join(self.packaging.INSTALL_CMD),
' '.join(sorted(packages)))
def __ensure_hashes(self, hashes, path, size):
if hashes is not None:
return hashes
if size > QUITE_LARGE:
logger.info('identifying %s', path)
progress = self.progress_factory()
else:
progress = None
return HashedFile.from_file(path, open(path, 'rb'), size=size,
progress=progress)
......@@ -20,13 +20,16 @@ import argparse
import logging
import os
import sys
import time
import zipfile
from . import (load_games)
from .config import (read_config)
from .data import (ProgressCallback)
from .gog import (run_gog_meta_mode)
from .paths import (DATADIR)
from .steam import (run_steam_meta_mode)
from .util import (human_size)
logging.basicConfig()
logger = logging.getLogger(__name__)
......@@ -38,6 +41,45 @@ if os.environ.get('DEBUG') or os.environ.get('GDP_DEBUG'):
else:
logging.getLogger().setLevel(logging.INFO)
class TerminalProgress(ProgressCallback):
def __init__(self, interval=0.2):
"""Constructor.
Progress will update at most once per @interval seconds, unless we
are at a "checkpoint".
"""
self.pad = ' '
self.interval = interval
self.ts = time.time()
def __call__(self, done=None, total=None, checkpoint=False):
ts = time.time()
if done is None or (ts < self.ts + self.interval and not checkpoint):
return
if done is None or total == 0:
s = ''
elif total is None or total == 0:
s = human_size(done)
else:
s = '%.0f%% %s/%s' % (100 * done / total,
human_size(done),
human_size(total))
self.ts = ts
if len(self.pad) <= len(s):
self.pad = ' ' * len(s)
print(' %s \r %s\r' % (self.pad, s), end='', file=sys.stderr)
def __enter__(self):
return self
def __exit__(self, et=None, ev=None, tb=None):
self()
def run_command_line():
logger.debug('Arguments: %r', sys.argv)
......@@ -244,6 +286,9 @@ def run_command_line():
raise AssertionError('could not find %s' % parsed.shortname)
with game.construct_task() as task:
if sys.stderr.isatty():
task.progress_factory = TerminalProgress()
task.run_command_line(parsed)
if __name__ == '__main__':
......
......@@ -18,10 +18,24 @@
import hashlib
import io
import sys
import time
from .util import (human_size)
class ProgressCallback:
"""API for a progress report."""
def __call__(self, done, total=None, checkpoint=False):
"""Update progress: we have done @done bytes out of @total
(None if unknown).
If @checkpoint is True, it is a hint that this particular
update is important (for instance the end of a file).
"""
pass
def __enter__(self):
return self
def __exit__(self, et=None, ev=None, tb=None):
pass
class HashedFile:
def __init__(self, name):
......@@ -32,53 +46,35 @@ class HashedFile:
self.skip_hash_matching = False
@classmethod
def from_file(cls, name, f, write_to=None, size=None, progress=False):
def from_file(cls, name, f, write_to=None, size=None, progress=None):
return cls.from_concatenated_files(name, [f], write_to, size, progress)
@classmethod
def from_concatenated_files(cls, name, fs, write_to=None, size=None,
progress=False):
progress=None):
md5 = hashlib.new('md5')
sha1 = hashlib.new('sha1')
sha256 = hashlib.new('sha256')
done = 0
if progress and sys.stderr.isatty():
pad = [' ']
def update_progress(s):
ts = time.time()
if ts < update_progress.ts + 0.2 and not s.startswith('100%'):
return
update_progress.ts = ts
if len(pad[0]) <= len(s):
pad[0] = ' ' * len(s)
print(' %s \r %s\r' % (pad[0], s), end='', file=sys.stderr)
update_progress.ts = time.time()
else:
update_progress = lambda s: None
with (progress or ProgressCallback()) as update_progress:
for f in fs:
while True:
update_progress(done, size)
for f in fs:
while True:
if size is None:
update_progress(human_size(done))
else:
update_progress('%.0f%% %s/%s' % (
100 * done / size if size != 0 else 100,
human_size(done),
human_size(size)))
blob = f.read(io.DEFAULT_BUFFER_SIZE)
if not blob:
update_progress('')
break
done += len(blob)
md5.update(blob)
sha1.update(blob)
sha256.update(blob)
if write_to is not None:
write_to.write(blob)
blob = f.read(io.DEFAULT_BUFFER_SIZE)
if not blob:
update_progress(done, size, checkpoint=True)
break
done += len(blob)
md5.update(blob)
sha1.update(blob)
sha256.update(blob)
if write_to is not None:
write_to.write(blob)
self = cls(name)
self.md5 = md5.hexdigest()
......
......@@ -17,9 +17,23 @@
import hashlib
import io
import sys
import unittest
from game_data_packager.build import HashedFile
from game_data_packager.command_line import (TerminalProgress)
from game_data_packager.data import (HashedFile)
class ZeroReader:
def __init__(self, total):
self.done = 0
self.total = total
def read(self, max_bytes):
ret = min(max_bytes, self.total - self.done)
self.total -= ret
return b'\x00' * ret
SIZE = 30 * 1024 * 1024
class HashedFileTestCase(unittest.TestCase):
def setUp(self):
......@@ -82,6 +96,37 @@ class HashedFileTestCase(unittest.TestCase):
self.assertIs(first.matches(second), False)
self.assertIs(second.matches(first), False)
def test_progress(self):
print('', file=sys.stderr)
HashedFile.from_file('progress.bin', ZeroReader(SIZE),
size=SIZE,
progress=TerminalProgress(interval=0.1))
print('', file=sys.stderr)
HashedFile.from_file('progress.bin', ZeroReader(SIZE),
progress=TerminalProgress(interval=0.1))
print('', file=sys.stderr)
HashedFile.from_file('progress.bin', ZeroReader(SIZE),
size=SIZE)
HashedFile.from_file('progress.bin', ZeroReader(SIZE))
print('', file=sys.stderr)
HashedFile.from_concatenated_files('concatenated.bin',
[ZeroReader(SIZE), ZeroReader(SIZE)],
size=2 * SIZE,
progress=TerminalProgress(interval=0.1))
print('', file=sys.stderr)
HashedFile.from_concatenated_files('concatenated.bin',
[ZeroReader(SIZE), ZeroReader(SIZE)],
progress=TerminalProgress(interval=0.1))
print('', file=sys.stderr)
HashedFile.from_concatenated_files('concatenated.bin',
[ZeroReader(SIZE), ZeroReader(SIZE)],
size=2 * SIZE)
HashedFile.from_concatenated_files('concatenated.bin',
[ZeroReader(SIZE), ZeroReader(SIZE)])
def tearDown(self):
pass
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment