utils.py 5.38 KB
Newer Older
Chris Lamb's avatar
Chris Lamb committed
1 2 3 4
import re
import hashlib

from debian import deb822
5
from dateutil.parser import parse
Chris Lamb's avatar
Chris Lamb committed
6

7
from django.db import transaction, IntegrityError
8 9
from django.core.files.base import ContentFile
from django.core.files.storage import default_storage
Chris Lamb's avatar
Chris Lamb committed
10

11
from bidb.keys.models import Key
Chris Lamb's avatar
Chris Lamb committed
12
from bidb.packages.models import Source, Architecture, Binary
13
from bidb.buildinfo.models import Buildinfo, Origin
Chris Lamb's avatar
Chris Lamb committed
14

15
SUPPORTED_FORMATS = {'0.2', '1.0'}
16

17
re_binary = re.compile(
18
    r'^(?P<name>[^_]+)_(?P<version>[^_]+)_(?P<architecture>[^\.]+)\.u?deb$'
Chris Lamb's avatar
Chris Lamb committed
19 20 21 22 23 24 25 26 27
)
re_installed_build_depends = re.compile(
    r'^(?P<package>[^ ]+) \(= (?P<version>.+)\)'
)


class InvalidSubmission(Exception):
    pass

28

29
@transaction.atomic
Chris Lamb's avatar
Chris Lamb committed
30 31 32
def parse_submission(request):
    raw_text = request.read()

33 34 35 36
    try:
        data = deb822.Deb822(raw_text)
    except TypeError:
        raise InvalidSubmission("Could not parse RFC-822 format data.")
37

Chris Lamb's avatar
Chris Lamb committed
38 39
    raw_text_gpg_stripped = data.dump()

Chris Lamb's avatar
Chris Lamb committed
40
    ## Parse GPG info #########################################################
41 42

    uid = None
Chris Lamb's avatar
Chris Lamb committed
43 44
    data.raw_text = raw_text
    gpg_info = data.get_gpg_info()
45 46 47 48 49 50 51 52 53 54

    for x in ('VALIDSIG', 'NO_PUBKEY'):
        try:
            uid = gpg_info[x][0]
            break
        except (KeyError, IndexError):
            pass

    if uid is None:
        raise InvalidSubmission("Could not determine GPG uid")
Chris Lamb's avatar
Chris Lamb committed
55

Chris Lamb's avatar
Chris Lamb committed
56 57
    ## Check whether .buildinfo already exists ################################

58
    def create_submission(buildinfo):
59
        submission = buildinfo.submissions.create(
60
            key=Key.objects.get_or_create(uid=uid)[0]
Chris Lamb's avatar
Chris Lamb committed
61 62
        )

63
        default_storage.save(
64
            submission.get_storage_name(), ContentFile(raw_text)
65 66 67 68
        )

        return submission

Chris Lamb's avatar
Chris Lamb committed
69
    ## Parse new .buildinfo ###################################################
Chris Lamb's avatar
Chris Lamb committed
70 71 72 73 74

    def get_or_create(model, field):
        try:
            return model.objects.get_or_create(name=data[field])[0]
        except KeyError:
75
            raise InvalidSubmission("Missing required field: {}".format(field))
Chris Lamb's avatar
Chris Lamb committed
76

77 78 79
    if data.get('Format') not in SUPPORTED_FORMATS:
        raise InvalidSubmission(
            "Only {} 'Format:'  versions are supported".format(
80
                ', '.join(sorted(SUPPORTED_FORMATS))
81 82
            )
        )
Chris Lamb's avatar
Chris Lamb committed
83

84 85 86 87 88 89 90 91 92 93 94 95
    sha1 = hashlib.sha1(raw_text_gpg_stripped.encode('utf-8')).hexdigest()

    try:
        with transaction.atomic():
            buildinfo = Buildinfo.objects.create(
                sha1=sha1,
                source=get_or_create(Source, 'Source'),
                architecture=get_or_create(Architecture, 'Architecture'),
                version=data['version'],
                build_path=data.get('Build-Path', ''),
                build_date=parse(data.get('Build-Date', '')),
                build_origin=get_or_create(Origin, 'Build-Origin'),
96 97 98
                build_architecture=get_or_create(
                    Architecture, 'Build-Architecture'
                ),
99 100 101 102 103
                environment=data.get('Environment', ''),
            )
    except IntegrityError:
        # Already exists; just attach a new Submission instance
        return create_submission(Buildinfo.objects.get(sha1=sha1)), False
Chris Lamb's avatar
Chris Lamb committed
104

105
    default_storage.save(
106
        buildinfo.get_storage_name(), ContentFile(raw_text_gpg_stripped)
107 108
    )

Chris Lamb's avatar
Chris Lamb committed
109 110 111 112 113
    ## Parse binaries #########################################################

    try:
        binary_names = set(data['Binary'].split(' '))
    except KeyError:
114
        raise InvalidSubmission("Missing 'Binary' field")
Chris Lamb's avatar
Chris Lamb committed
115 116

    if not binary_names:
117
        raise InvalidSubmission("Invalid 'Binary' field")
Chris Lamb's avatar
Chris Lamb committed
118 119 120 121 122

    binaries = {}
    for x in binary_names:
        # Save instances for lookup later
        binaries[x] = buildinfo.binaries.create(
123
            binary=Binary.objects.get_or_create(name=x)[0]
Chris Lamb's avatar
Chris Lamb committed
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
        )

    ## Parse checksums ########################################################

    hashes = ('Md5', 'Sha1', 'Sha256')

    checksums = {}
    for x in hashes:
        for y in data['Checksums-%s' % x].strip().splitlines():
            checksum, size, filename = y.strip().split()

            # Check size
            try:
                size = int(size)
                if size < 0:
                    raise ValueError()
            except ValueError:
141
                raise InvalidSubmission(
142
                    "Invalid size for {}: {}".format(filename, size)
Chris Lamb's avatar
Chris Lamb committed
143 144
                )

145 146 147
            checksums.setdefault(filename, {'size': size, 'binary': None})[
                'checksum_{}'.format(x.lower())
            ] = checksum
Chris Lamb's avatar
Chris Lamb committed
148 149 150

            existing = checksums[filename]['size']
            if size != existing:
151 152 153 154
                raise InvalidSubmission(
                    "Mismatched file size in "
                    "Checksums-{}: {} != {}".format(x, existing, size)
                )
Chris Lamb's avatar
Chris Lamb committed
155

Chris Lamb's avatar
Chris Lamb committed
156
    ## Create Checksum instances ##############################################
Chris Lamb's avatar
Chris Lamb committed
157 158

    for k, v in sorted(checksums.items()):
159 160 161 162
        # Match with Binary instances if possible
        m = re_binary.match(k)
        if m is not None:
            v['binary'] = binaries.get(m.group('name'))
Chris Lamb's avatar
Chris Lamb committed
163 164 165

        buildinfo.checksums.create(filename=k, **v)

166
    ## Validate Installed-Build-Depends #######################################
Chris Lamb's avatar
Chris Lamb committed
167 168 169 170

    for x in data['Installed-Build-Depends'].strip().splitlines():
        m = re_installed_build_depends.match(x.strip())

171 172
        if m is None:
            raise InvalidSubmission(
173
                "Invalid entry in Installed-Build-Depends: {}".format(x)
174 175
            )

176
    return create_submission(buildinfo), True