Commit d39e3053 authored by Chris Lamb's avatar Chris Lamb 💬

Initial commit.

Signed-off-by: 's avatarChris Lamb <chris@chris-lamb.co.uk>
parents
[run]
omit =
.venv/*
*/wsgi.py
*/tests.py
*/manage.py
*/tests/*
*/tests/*
*/settings/*
*/migrations/*
*.deb
*.pyc
.coverage
/bidb/settings/custom.py
/static/
/storage/
/coverage.xml
/pep8.txt
This diff is collapsed.
bidb / buildinfo.debian.net
===========================
Local database setup
--------------------
#. Create PostgreSQL user with id matching your UNIX username::
$ sudo -u postgres createuser $(whoami) -SDR
#. Create a database owned by this user::
$ sudo -u postgres createdb -E UTF-8 -O $(whoami) bidb
#. Run any initial migrations::
$ ./manage.py migrate
This source diff could not be displayed because it is too large. You can view the blob instead.
from django.conf.urls import url
from . import views
urlpatterns = (
url(r'^api/submit$', views.submit,
name='submit'),
)
import re
import hashlib
from debian import deb822
from django.db import IntegrityError
from bidb.packages.models import Source, Architecture, Binary
from bidb.buildinfo.models import Buildinfo
re_filename = re.compile(
r'^(?P<name>[^_]+)_(?P<version>[^_]+)_(?P<architecture>[^\.]+)\.deb$',
)
re_installed_build_depends = re.compile(
r'^(?P<package>[^ ]+) \(= (?P<version>.+)\)'
)
class InvalidSubmission(Exception):
pass
def parse_submission(request):
raw_text = request.read()
data = deb822.Deb822(raw_text)
raw_text_gpg_stripped = data.dump()
# Parse GPG info
uid = ''
data.raw_text = raw_text
gpg_info = data.get_gpg_info()
if 'NODATA' not in gpg_info:
try:
uid = gpg_info['NO_PUBKEY'][0]
except (KeyError, IndexError):
raise InvalidBuildinfo("Could not determine GPG uid")
sha1 = hashlib.sha1().hexdigest()
try:
submission = Buildinfo.objects.get(sha1=sha1).submissions.create(
uid=uid,
node=request.GET.get('node', ''),
raw_text=raw_text,
)
return submission, False
except Buildinfo.DoesNotExist:
pass
if data.get('Format') != '0.1':
raise InvalidBuildinfo("Only Format: 1.0 is supported")
def get_or_create(model, field):
try:
return model.objects.get_or_create(name=data[field])[0]
except KeyError:
raise InvalidBuildinfo("Missing required field: {}".format(field))
buildinfo = Buildinfo.objects.create(
sha1=sha1,
raw_text=raw_text_gpg_stripped,
source=get_or_create(Source, 'Source'),
architecture=get_or_create(Architecture, 'Architecture'),
version=data['Version'],
build_path=data.get('Build-Path', ''),
build_architecture=get_or_create(Architecture, 'Build-Architecture'),
)
## Parse binaries #########################################################
try:
binary_names = set(data['Binary'].split(' '))
except KeyError:
raise InvalidBuildinfo("Missing 'Binary' field")
if not binary_names:
raise InvalidBuildinfo("Invalid 'Binary' field")
binaries = {}
for x in binary_names:
# Save instances for lookup later
binaries[x] = buildinfo.binaries.create(
binary=Binary.objects.get_or_create(name=x)[0],
)
## Parse checksums ########################################################
hashes = ('Md5', 'Sha1', 'Sha256')
checksums = {}
for x in hashes:
for y in data['Checksums-%s' % x].strip().splitlines():
checksum, size, filename = y.strip().split()
# Check filename
if re_filename.match(filename) is None:
raise InvalidBuildinfo("Invalid filename: {}".format(filename))
# Check size
try:
size = int(size)
if size < 0:
raise ValueError()
except ValueError:
raise InvalidBuildinfo(
"Invalid size for {}: {}".format(filename, size),
)
checksums.setdefault(filename, {
'size': size,
})['checksum_{}'.format(x.lower())] = checksum
existing = checksums[filename]['size']
if size != existing:
raise InvalidBuildinfo("Mismatched file size in "
"Checksums-{}: {} != {}".format(x, existing, size))
## Create Binary instances ################################################
for k, v in sorted(checksums.items()):
try:
v['binary'] = binaries[re_filename.match(k).group('name')]
except KeyError:
v['binary'] = None
buildinfo.checksums.create(filename=k, **v)
## Create InstalledBuildDepends instances #################################
for x in data['Installed-Build-Depends'].strip().splitlines():
m = re_installed_build_depends.match(x.strip())
binary = Binary.objects.get_or_create(name=m.group('package'))[0]
buildinfo.installed_build_depends.get_or_create(
binary=binary,
version=m.group('version'),
)
return buildinfo.submissions.create(
uid=uid,
node=request.GET.get('node', ''),
raw_text=raw_text,
), True
from django.conf import settings
from django.http import HttpResponse, HttpResponseBadRequest
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from .utils import parse_submission, InvalidSubmission
@csrf_exempt
@require_http_methods(['PUT'])
def submit(request):
try:
submission, created = parse_submission(request)
except InvalidSubmission as exc:
return HttpResponseBadRequest("{}\n".format(exc))
return HttpResponse('{}{}\n'.format(
settings.SITE_URL,
submission.buildinfo.get_absolute_url(),
), status=201 if created else 200)
# -*- coding: utf-8 -*-
# Generated by Django 1.10.2 on 2016-10-22 13:37
from __future__ import unicode_literals
import datetime
from django.db import migrations, models
import django.db.models.deletion
import django.utils.crypto
import functools
class Migration(migrations.Migration):
initial = True
dependencies = [
('buildinfo', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='Submission',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('slug', models.CharField(default=functools.partial(django.utils.crypto.get_random_string, *(8, b'3479abcdefghijkmnopqrstuvwxyz'), **{}), max_length=8, unique=True)),
('uid', models.CharField(max_length=512)),
('node', models.CharField(max_length=512)),
('raw_text', models.TextField()),
('created', models.DateTimeField(default=datetime.datetime.utcnow)),
('buildinfo', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='submissions', to='buildinfo.Buildinfo')),
],
options={
'ordering': ('created',),
'get_latest_by': 'created',
},
),
]
import datetime
import functools
from django.db import models
from django.utils.crypto import get_random_string
class Submission(models.Model):
buildinfo = models.ForeignKey(
'buildinfo.Buildinfo',
related_name='submissions',
)
slug = models.CharField(
unique=True,
default=functools.partial(
get_random_string, 8, '3479abcdefghijkmnopqrstuvwxyz',
),
max_length=8,
)
uid = models.CharField(max_length=512)
node = models.CharField(max_length=512)
raw_text = models.TextField()
created = models.DateTimeField(default=datetime.datetime.utcnow)
class Meta:
ordering = ('created',)
get_latest_by = 'created'
def __unicode__(self):
return u"pk=%d buildinfo=%r" % (
self.pk,
self.buildinfo,
)
@models.permalink
def get_absolute_url(self):
return 'buildinfo:submissions:view', (
self.buildinfo.sha1,
self.buildinfo.get_filename(),
self.slug,
)
from django.conf.urls import url
from . import views
urlpatterns = (
url(r'^(?P<sha1>\w{40})/(?P<filename>[^/]+)/(?P<slug>\w+).buildinfo$', views.view,
name='view'),
)
from django.http import HttpResponse
from django.shortcuts import redirect, get_object_or_404
from .models import Submission
def view(request, sha1, filename, slug):
submission = get_object_or_404(
Submission,
slug=slug,
buildinfo__sha1=sha1,
)
if submission.buildinfo.get_filename() != filename:
return redirect(submission)
return HttpResponse(submission.raw_text, content_type='text/plain')
# -*- coding: utf-8 -*-
# Generated by Django 1.10.2 on 2016-10-22 13:37
from __future__ import unicode_literals
import datetime
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
initial = True
dependencies = [
('packages', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='Binary',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created', models.DateTimeField(default=datetime.datetime.utcnow)),
('binary', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='generated_binaries', to='packages.Binary')),
],
options={
'ordering': ('binary__name',),
'get_latest_by': 'created',
},
),
migrations.CreateModel(
name='Buildinfo',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('sha1', models.CharField(max_length=40, unique=True)),
('version', models.CharField(max_length=200)),
('build_path', models.CharField(max_length=512)),
('raw_text', models.TextField()),
('created', models.DateTimeField(default=datetime.datetime.utcnow)),
('architecture', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='buildinfos', to='packages.Architecture')),
('build_architecture', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='buildinfos_build', to='packages.Architecture')),
('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='buildinfos', to='packages.Source')),
],
options={
'ordering': ('-created',),
'get_latest_by': 'created',
},
),
migrations.CreateModel(
name='Checksum',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('filename', models.CharField(max_length=255)),
('size', models.IntegerField()),
('checksum_md5', models.CharField(max_length=100)),
('checksum_sha1', models.CharField(max_length=100)),
('checksum_sha256', models.CharField(max_length=100)),
('created', models.DateTimeField(default=datetime.datetime.utcnow)),
('binary', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='checksum', to='buildinfo.Binary')),
('buildinfo', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='checksums', to='buildinfo.Buildinfo')),
],
options={
'ordering': ('-created',),
'get_latest_by': 'created',
},
),
migrations.CreateModel(
name='InstalledBuildDepends',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('version', models.CharField(max_length=200)),
('binary', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='build_depends', to='packages.Binary')),
('buildinfo', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='installed_build_depends', to='buildinfo.Buildinfo')),
],
options={
'ordering': ('binary__name',),
'get_latest_by': 'created',
},
),
migrations.AddField(
model_name='binary',
name='buildinfo',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='binaries', to='buildinfo.Buildinfo'),
),
migrations.AlterUniqueTogether(
name='installedbuilddepends',
unique_together=set([('buildinfo', 'binary')]),
),
migrations.AlterUniqueTogether(
name='checksum',
unique_together=set([('buildinfo', 'filename')]),
),
migrations.AlterUniqueTogether(
name='binary',
unique_together=set([('buildinfo', 'binary')]),
),
]
import datetime
from django.db import models
class Buildinfo(models.Model):
sha1 = models.CharField(max_length=40, unique=True)
source = models.ForeignKey(
'packages.Source',
related_name='buildinfos',
)
architecture = models.ForeignKey(
'packages.Architecture',
related_name='buildinfos',
)
version = models.CharField(max_length=200)
build_path = models.CharField(max_length=512)
build_architecture = models.ForeignKey(
'packages.Architecture',
related_name='buildinfos_build',
)
raw_text = models.TextField()
created = models.DateTimeField(default=datetime.datetime.utcnow)
class Meta:
ordering = ('-created',)
get_latest_by = 'created'
def __unicode__(self):
return u"pk=%d source=%r" % (
self.pk,
self.source,
)
@models.permalink
def get_absolute_url(self):
return 'buildinfo:view', (self.sha1, self.get_filename())
def get_filename(self):
return '{}_{}_{}'.format(
self.source.name,
self.version,
self.architecture.name,
)
class Binary(models.Model):
buildinfo = models.ForeignKey(
Buildinfo,
related_name='binaries',
)
binary = models.ForeignKey(
'packages.Binary',
related_name='generated_binaries',
)
created = models.DateTimeField(default=datetime.datetime.utcnow)
class Meta:
ordering = ('binary__name',)
get_latest_by = 'created'
unique_together = (
('buildinfo', 'binary'),
)
def __unicode__(self):
return u"pk=%d binary=%r" % (
self.pk,
self.binary,
)
class Checksum(models.Model):
"""
Not the same as Binary as we could potentially have a Checksum for a
Binary, etc.
"""
buildinfo = models.ForeignKey(Buildinfo, related_name='checksums')
filename = models.CharField(max_length=255)
size = models.IntegerField()
checksum_md5 = models.CharField(max_length=100)
checksum_sha1 = models.CharField(max_length=100)
checksum_sha256 = models.CharField(max_length=100)
binary = models.OneToOneField(Binary, null=True, related_name='checksum')
created = models.DateTimeField(default=datetime.datetime.utcnow)
class Meta:
ordering = ('-created',)
get_latest_by = 'created'
unique_together = (
('buildinfo', 'filename'),
)
def __unicode__(self):
return u"pk=%d filename=%r" % (
self.pk,
self.filename,
)
class InstalledBuildDepends(models.Model):
buildinfo = models.ForeignKey(
Buildinfo,
related_name='installed_build_depends',
)
binary = models.ForeignKey(
'packages.Binary',
related_name='build_depends',
)
version = models.CharField(max_length=200)
class Meta:
ordering = ('binary__name',)
get_latest_by = 'created'
unique_together = (
('buildinfo', 'binary'),
)
def __unicode__(self):
return u"pk=%d buildinfo=%r binary=%r version=%r" % (
self.pk,
self.buildinfo,
self.binary,
self.version,
)
from django.conf.urls import url, include
from . import views
urlpatterns = (
url(r'', include('bidb.buildinfo.buildinfo_submissions.urls',
namespace='submissions')),
url(r'^(?P<sha1>\w{40})$', views.view,
name='view'),
url(r'^(?P<sha1>\w{40})/(?P<filename>.+)\.buildinfo$', views.raw_text,
name='raw-text'),
url(r'^(?P<sha1>\w{40})/(?P<filename>.+)$', views.view,
name='view'),
)
from django.http import HttpResponse
from django.shortcuts import render, get_object_or_404, redirect
from .models import Buildinfo
def view(request, sha1, filename=None):
buildinfo = get_object_or_404(Buildinfo, sha1=sha1)
if filename != buildinfo.get_filename():
return redirect(buildinfo)
return render(request, 'buildinfo/view.html', {
'buildinfo': buildinfo,
})
def raw_text(request, sha1, filename=None):
buildinfo = get_object_or_404(Buildinfo, sha1=sha1)
return HttpResponse(buildinfo.raw_text, content_type='text/plain')
#!/usr/bin/env python
import os
import sys
from os.path import join, dirname, abspath
sys.path.insert(0, dirname(dirname(abspath(__file__))))
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "bidb.settings")
from django.core.management import execute_from_command_line
execute_from_command_line(sys.argv)
# -*- coding: utf-8 -*-
# Generated by Django 1.10.2 on 2016-10-22 13:37
from __future__ import unicode_literals
import datetime
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Architecture',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=200, unique=True)),
('created', models.DateTimeField(default=datetime.datetime.utcnow)),
],
options={
'ordering': ('name',),
'get_latest_by': 'created',
},
),
migrations.CreateModel(
name='Binary',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=200, unique=True)),
],
options={
'ordering': ('name',),
'get_latest_by': 'created',
},
),
migrations.CreateModel(
name='Source',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=200, unique=True)),
('created', models.DateTimeField(default=datetime.datetime.utcnow)),
],
options={
'ordering': ('name',),
'get_latest_by': 'created',
},
),
]
import datetime
from django.db import models
class Source(models.Model):
name = models.CharField(max_length=200, unique=True)