Commit fd81e454 authored by Guido Günther's avatar Guido Günther

Turn on several threads when creating xz/lzma tarballs

and not using pristine-tar.

Gives a 50% speedup on my machine:

$ time gbp export-orig --no-pristine-tar --force-create
gbp:info: Creating thunderbird_52.6.0.orig.tar.xz from 'upstream/52.6.0'
gbp:info: Creating additional tarball 'thunderbird_52.6.0.orig-thunderbird-l10n.tar.xz' from 'dea43c2b49f11d68f05edac2bd9d6c5d1e1f4bcb'
gbp:info: Creating additional tarball 'thunderbird_52.6.0.orig-lightning-l10n.tar.xz' from '9adcd261726ccb160c4fd6947f7cbfe9df2e581b'

real	8m27,153s
user	7m23,717s
sys	0m3,015s

0|agx@bogon:/var/scratch/debian/icedove/icedove (debian/wheezy)$ time gbp-from-sourcetree export-orig --no-pristine-tar --force-create
gbp:info: Creating thunderbird_52.6.0.orig.tar.xz from 'upstream/52.6.0'
gbp:info: Creating additional tarball 'thunderbird_52.6.0.orig-thunderbird-l10n.tar.xz' from 'dea43c2b49f11d68f05edac2bd9d6c5d1e1f4bcb'
gbp:info: Creating additional tarball 'thunderbird_52.6.0.orig-lightning-l10n.tar.xz' from '9adcd261726ccb160c4fd6947f7cbfe9df2e581b'

real	4m3,080s
user	11m18,733s
sys	0m3,499s

Thanks: Carsten Schönert for the suggestion
Closes: #888572
parent 5f2120b3
......@@ -26,14 +26,21 @@ class Compressor(object):
'lzma': '',
'xz': ''}
ParallelOpts = {'lzma': '-T0',
'xz': '-T0'}
Exts = {'gzip': 'gz',
'bzip2': 'bz2',
'lzma': 'lzma',
'xz': 'xz'}
def __init__(self, type_, level=None):
def __init__(self, type_, level=None, parallel=False):
self._type = type_
self._level = int(level) if level not in [None, ''] else None
self._more_opts = self.Opts.get(self._type, '')
self._parallel = parallel
if parallel:
self._more_opts += self.ParallelOpts.get(self._type, '')
def is_known(self):
return self.type in self.Opts.keys()
......@@ -47,12 +54,12 @@ class Compressor(object):
return self._level
@property
def _level_opt(self):
return '-%d' % self.level if self.level is not None else ''
def parallel(self):
return self._parallel
@property
def _more_opts(self):
return self.Opts.get(self._type, '')
def _level_opt(self):
return '-%d' % self.level if self.level is not None else ''
def cmdline(self, stdout=True):
"""
......@@ -67,9 +74,12 @@ class Compressor(object):
def __repr__(self):
"""
>>> Compressor('gzip').__repr__()
"<compressor type='gzip' >"
"<compressor type='gzip'>"
>>> Compressor('gzip', 9).__repr__()
"<compressor type='gzip' level=9>"
>>> Compressor('xz', 9, True).__repr__()
"<compressor type='xz' level=9 parallel=True>"
"""
level_str = "level=%s" % self.level if self.level is not None else ''
return "<compressor type='%s' %s>" % (self.type, level_str)
opts_str = " level=%s" % self.level if self.level is not None else ""
opts_str += " parallel=True" if self.parallel else ""
return "<compressor type='%s'%s>" % (self.type, opts_str)
......@@ -202,7 +202,8 @@ def git_archive_build_origs(repo, source, output_dir, options):
@param options: the parsed options
@type options: C{dict} of options
"""
comp = Compressor(options.comp_type, options.comp_level)
parallel = not options.pristine_tar
comp = Compressor(options.comp_type, options.comp_level, parallel)
upstream_tree = git_archive_get_upstream_tree(repo, source, options)
gbp.log.info("Creating %s from '%s'" % (source.upstream_tarball_name(comp.type),
upstream_tree))
......
......@@ -44,16 +44,19 @@ class TestExportOrig(ComponentTestBase):
tarballs = ["../%s_2.8.orig-foo.tar.gz" % pkg,
"../%s_2.8.orig.tar.gz" % pkg]
assert import_dsc(['arg0', '--no-pristine-tar', dsc]) == 0
assert import_dsc(['arg0', '--no-pristine-tar', '--verbose', dsc]) == 0
repo = ComponentTestGitRepository(pkg)
os.chdir(pkg)
assert_false(repo.has_branch('pristine-tar'), "Pristine-tar branch must not exist")
for t in tarballs:
self.assertFalse(os.path.exists(t), "Tarball %s must not exist" % t)
ret = export_orig(['arg0',
'--verbose',
'--component=foo',
'--no-pristine-tar'])
ok_(ret == 0, "Exporting tarballs failed")
self._check_in_info_log(".*Building upstream tarball with compression "
"<compressor type='gzip' parallel=True>")
for t in tarballs:
self.assertTrue(os.path.exists(t), "Tarball %s not found" % t)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment