make_changelog.py 13.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51
#!/usr/bin/env python

"""
Generate changelog entry between two suites

@contact: Debian FTP Master <ftpmaster@debian.org>
@copyright: 2010 Luca Falavigna <dktrkranz@debian.org>
@license: GNU General Public License version 2 or later
"""

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

################################################################################

# <bdefreese> !dinstall
# <dak> bdefreese: I guess the next dinstall will be in 0hr 1min 35sec
# <bdefreese> Wow I have great timing
# <DktrKranz> dating with dinstall, part II
# <bdefreese> heh
# <Ganneff> dating with that monster? do you have good combat armor?
# <bdefreese> +5 Plate :)
# <Ganneff> not a good one then
# <Ganneff> so you wont even manage to bypass the lesser monster in front, unchecked
# <DktrKranz> asbesto belt
# <Ganneff> helps only a step
# <DktrKranz> the Ultimate Weapon: cron_turned_off
# <bdefreese> heh
# <Ganneff> thats debadmin limited
# <Ganneff> no option for you
# <DktrKranz> bdefreese: it seems ftp-masters want dinstall to sexual harass us, are you good in running?
# <Ganneff> you can run but you can not hide
# <bdefreese> No, I'm old and fat :)
# <Ganneff> you can roll but you can not hide
# <Ganneff> :)
# <bdefreese> haha
# <DktrKranz> damn dinstall, you racist bastard

################################################################################

52 53
from __future__ import print_function

54
import os
55 56
import sys
import apt_pkg
57 58
from glob import glob
from shutil import rmtree
59
from yaml import safe_dump
60 61
from daklib.dbconn import *
from daklib import utils
62
from daklib.contents import UnpackedSource
63
from daklib.regexes import re_no_epoch
64 65 66

################################################################################

67 68
filelist = 'filelist.yaml'

69

70
def usage(exit_code=0):
71
    print("""Generate changelog between two suites
72 73 74

       Usage:
       make-changelog -s <suite> -b <base_suite> [OPTION]...
75
       make-changelog -e -a <archive>
76 77 78 79 80

Options:

  -h, --help                show this help and exit
  -s, --suite               suite providing packages to compare
81
  -b, --base-suite          suite to be taken as reference for comparison
82
  -n, --binnmu              display binNMUs uploads instead of source ones
83

84
  -e, --export              export interesting files from source packages
85
  -a, --archive             archive to fetch data from
86
  -p, --progress            display progress status""")
87 88 89

    sys.exit(exit_code)

90

91
def get_source_uploads(suite, base_suite, session):
92
    """
93
    Returns changelogs for source uploads where version is newer than base.
94 95
    """

96 97 98 99 100
    query = """WITH base AS (
                 SELECT source, max(version) AS version
                 FROM source_suite
                 WHERE suite_name = :base_suite
                 GROUP BY source
101
                 UNION (SELECT source, CAST(0 AS debversion) AS version
102 103 104 105 106
                 FROM source_suite
                 WHERE suite_name = :suite
                 EXCEPT SELECT source, CAST(0 AS debversion) AS version
                 FROM source_suite
                 WHERE suite_name = :base_suite
107
                 ORDER BY source)),
108 109 110 111 112 113 114
               cur_suite AS (
                 SELECT source, max(version) AS version
                 FROM source_suite
                 WHERE suite_name = :suite
                 GROUP BY source)
               SELECT DISTINCT c.source, c.version, c.changelog
               FROM changelogs c
115
               JOIN base b ON b.source = c.source
116 117 118 119 120 121 122
               JOIN cur_suite cs ON cs.source = c.source
               WHERE c.version > b.version
               AND c.version <= cs.version
               AND c.architecture LIKE '%source%'
               ORDER BY c.source, c.version DESC"""

    return session.execute(query, {'suite': suite, 'base_suite': base_suite})
123

124

125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
def get_binary_uploads(suite, base_suite, session):
    """
    Returns changelogs for binary uploads where version is newer than base.
    """

    query = """WITH base as (
                 SELECT s.source, max(b.version) AS version, a.arch_string
                 FROM source s
                 JOIN binaries b ON b.source = s.id
                 JOIN bin_associations ba ON ba.bin = b.id
                 JOIN architecture a ON a.id = b.architecture
                 WHERE ba.suite = (
                   SELECT id
                   FROM suite
                   WHERE suite_name = :base_suite)
                 GROUP BY s.source, a.arch_string),
               cur_suite as (
                 SELECT s.source, max(b.version) AS version, a.arch_string
                 FROM source s
                 JOIN binaries b ON b.source = s.id
                 JOIN bin_associations ba ON ba.bin = b.id
                 JOIN architecture a ON a.id = b.architecture
                 WHERE ba.suite = (
                   SELECT id
                   FROM suite
                   WHERE suite_name = :suite)
                 GROUP BY s.source, a.arch_string)
               SELECT DISTINCT c.source, c.version, c.architecture, c.changelog
               FROM changelogs c
               JOIN base b on b.source = c.source
               JOIN cur_suite cs ON cs.source = c.source
               WHERE c.version > b.version
157
               AND c.version <= cs.version
158 159 160 161 162 163
               AND c.architecture = b.arch_string
               AND c.architecture = cs.arch_string
               ORDER BY c.source, c.version DESC, c.architecture"""

    return session.execute(query, {'suite': suite, 'base_suite': base_suite})

164

165 166 167 168
def display_changes(uploads, index):
    prev_upload = None
    for upload in uploads:
        if prev_upload and prev_upload != upload[0]:
169 170
            print()
        print(upload[index])
171 172
        prev_upload = upload[0]

173

174
def export_files(session, archive, clpool, progress=False):
175 176 177
    """
    Export interesting files from source packages.
    """
178
    pool = os.path.join(archive.path, 'pool')
179 180

    sources = {}
181
    unpack = {}
182
    files = ('changelog', 'copyright', 'NEWS', 'NEWS.Debian', 'README.Debian')
183
    stats = {'unpack': 0, 'created': 0, 'removed': 0, 'errors': 0, 'files': 0}
184
    query = """SELECT DISTINCT s.source, su.suite_name AS suite, s.version, c.name || '/' || f.filename AS filename
185
               FROM source s
186
               JOIN newest_source n ON n.source = s.source AND n.version = s.version
187 188 189
               JOIN src_associations sa ON sa.source = s.id
               JOIN suite su ON su.id = sa.suite
               JOIN files f ON f.id = s.file
190
               JOIN files_archive_map fam ON f.id = fam.file_id AND fam.archive_id = su.archive_id
191
               JOIN component c ON fam.component_id = c.id
192
               WHERE su.archive_id = :archive_id
193 194
               ORDER BY s.source, suite"""

195
    for p in session.execute(query, {'archive_id': archive.archive_id}):
196
        if p[0] not in sources:
197
            sources[p[0]] = {}
198
        sources[p[0]][p[1]] = (re_no_epoch.sub('', p[2]), p[3])
199 200 201

    for p in sources.keys():
        for s in sources[p].keys():
202
            path = os.path.join(clpool, '/'.join(sources[p][s][1].split('/')[:-1]))
203 204
            if not os.path.exists(path):
                os.makedirs(path)
205
            if not os.path.exists(os.path.join(path,
206
                   '%s_%s_changelog' % (p, sources[p][s][0]))):
207
                if os.path.join(pool, sources[p][s][1]) not in unpack:
208 209 210
                    unpack[os.path.join(pool, sources[p][s][1])] = (path, set())
                unpack[os.path.join(pool, sources[p][s][1])][1].add(s)
            else:
211
                for file in glob('%s/%s_%s_*' % (path, p, sources[p][s][0])):
212
                    link = '%s%s' % (s, file.split('%s_%s'
213 214 215 216 217 218 219 220 221
                                      % (p, sources[p][s][0]))[1])
                    try:
                        os.unlink(os.path.join(path, link))
                    except OSError:
                        pass
                    os.link(os.path.join(path, file), os.path.join(path, link))

    for p in unpack.keys():
        package = os.path.splitext(os.path.basename(p))[0].split('_')
222
        try:
223
            unpacked = UnpackedSource(p, clpool)
224
            tempdir = unpacked.get_root_directory()
225
            stats['unpack'] += 1
226 227
            if progress:
                if stats['unpack'] % 100 == 0:
228
                    print('%d packages unpacked' % stats['unpack'], file=sys.stderr)
229
                elif stats['unpack'] % 10 == 0:
230
                    print('.', end='', file=sys.stderr)
231
            for file in files:
232
                for f in glob(os.path.join(tempdir, 'debian', '*%s' % file)):
233
                    for s in unpack[p][1]:
234
                        suite = os.path.join(unpack[p][0], '%s_%s'
235
                                % (s, os.path.basename(f)))
236
                        version = os.path.join(unpack[p][0], '%s_%s_%s' %
237 238 239 240
                                  (package[0], package[1], os.path.basename(f)))
                        if not os.path.exists(version):
                            os.link(f, version)
                            stats['created'] += 1
241
                        try:
242 243 244 245 246
                            os.unlink(suite)
                        except OSError:
                            pass
                        os.link(version, suite)
                        stats['created'] += 1
247
            unpacked.cleanup()
248
        except Exception as e:
249
            print('make-changelog: unable to unpack %s\n%s' % (p, e))
250 251
            stats['errors'] += 1

252
    for root, dirs, files in os.walk(clpool, topdown=False):
253
        files = [f for f in files if f != filelist]
254
        if len(files):
255
            if root != clpool:
256
                if root.split('/')[-1] not in sources:
257 258 259
                    if os.path.exists(root):
                        stats['removed'] += len(os.listdir(root))
                        rmtree(root)
260 261
            for file in files:
                if os.path.exists(os.path.join(root, file)):
262
                    if os.stat(os.path.join(root, file)).st_nlink == 1:
263
                        stats['removed'] += 1
264
                        os.unlink(os.path.join(root, file))
265 266 267 268 269
        for dir in dirs:
            try:
                os.rmdir(os.path.join(root, dir))
            except OSError:
                pass
270
        stats['files'] += len(files)
271 272
    stats['files'] -= stats['removed']

273 274 275 276 277 278
    print('make-changelog: file exporting finished')
    print('  * New packages unpacked: %d' % stats['unpack'])
    print('  * New files created: %d' % stats['created'])
    print('  * New files removed: %d' % stats['removed'])
    print('  * Unpack errors: %d' % stats['errors'])
    print('  * Files available into changelog pool: %d' % stats['files'])
279

280

281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
def generate_export_filelist(clpool):
    clfiles = {}
    for root, dirs, files in os.walk(clpool):
        for file in [f for f in files if f != filelist]:
            clpath = os.path.join(root, file).replace(clpool, '').strip('/')
            source = clpath.split('/')[2]
            elements = clpath.split('/')[3].split('_')
            if source not in clfiles:
                clfiles[source] = {}
            if elements[0] == source:
                if elements[1] not in clfiles[source]:
                    clfiles[source][elements[1]] = []
                clfiles[source][elements[1]].append(clpath)
            else:
                if elements[0] not in clfiles[source]:
                    clfiles[source][elements[0]] = []
                clfiles[source][elements[0]].append(clpath)
    with open(os.path.join(clpool, filelist), 'w+') as fd:
        safe_dump(clfiles, fd, default_flow_style=False)

301

302 303
def main():
    Cnf = utils.get_conf()
304 305 306 307 308 309 310
    Arguments = [('h', 'help', 'Make-Changelog::Options::Help'),
                 ('a', 'archive', 'Make-Changelog::Options::Archive', 'HasArg'),
                 ('s', 'suite', 'Make-Changelog::Options::Suite', 'HasArg'),
                 ('b', 'base-suite', 'Make-Changelog::Options::Base-Suite', 'HasArg'),
                 ('n', 'binnmu', 'Make-Changelog::Options::binNMU'),
                 ('e', 'export', 'Make-Changelog::Options::export'),
                 ('p', 'progress', 'Make-Changelog::Options::progress')]
311

312
    for i in ['help', 'suite', 'base-suite', 'binnmu', 'export', 'progress']:
313 314 315
        key = 'Make-Changelog::Options::%s' % i
        if key not in Cnf:
            Cnf[key] = ''
316

317 318
    apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
    Options = Cnf.subtree('Make-Changelog::Options')
319 320
    suite = Cnf['Make-Changelog::Options::Suite']
    base_suite = Cnf['Make-Changelog::Options::Base-Suite']
321
    binnmu = Cnf['Make-Changelog::Options::binNMU']
322
    export = Cnf['Make-Changelog::Options::export']
323
    progress = Cnf['Make-Changelog::Options::progress']
324

325
    if Options['help'] or not (suite and base_suite) and not export:
326 327 328
        usage()

    for s in suite, base_suite:
329
        if not export and not get_suite(s):
330 331
            utils.fubar('Invalid suite "%s"' % s)

332 333
    session = DBConn().session()

334
    if export:
335 336
        archive = session.query(Archive).filter_by(archive_name=Options['Archive']).one()
        exportpath = archive.changelog
337
        if exportpath:
338
            export_files(session, archive, exportpath, progress)
339
            generate_export_filelist(exportpath)
340 341
        else:
            utils.fubar('No changelog export path defined')
342
    elif binnmu:
343
        display_changes(get_binary_uploads(suite, base_suite, session), 3)
344
    else:
345
        display_changes(get_source_uploads(suite, base_suite, session), 2)
346 347

    session.commit()
348

349

350 351
if __name__ == '__main__':
    main()