diff --git a/debexpo/base/static/css/style.css b/debexpo/base/static/css/style.css index ed3e4fdd287b2222e1c023d003e1cce0fd40fec4..660cf4c558608108050ae0095d49b20487714632 100644 --- a/debexpo/base/static/css/style.css +++ b/debexpo/base/static/css/style.css @@ -649,6 +649,10 @@ form#pageLang { display: inline; } +.inline { + display: inline; +} + /* Extras */ input:focus { diff --git a/debexpo/base/static/img/key.png b/debexpo/base/static/img/key.png new file mode 100644 index 0000000000000000000000000000000000000000..fabe3b72e677e6ac256a517b3529b5898f901f0d Binary files /dev/null and b/debexpo/base/static/img/key.png differ diff --git a/debexpo/base/static/img/trash.png b/debexpo/base/static/img/trash.png new file mode 100644 index 0000000000000000000000000000000000000000..7f7e7f14a843839a6dc3b7b09943353bde43ab4a Binary files /dev/null and b/debexpo/base/static/img/trash.png differ diff --git a/debexpo/bugs/models.py b/debexpo/bugs/models.py index cbf25a256f7d4beb4a7c3c15047f456da3ef59d1..cdb389442389831d8b85ccb7ede6aa0ec224a3cf 100644 --- a/debexpo/bugs/models.py +++ b/debexpo/bugs/models.py @@ -130,6 +130,10 @@ class BugManager(models.Manager): return BugType.bug + def remove_bugs(self, package): + self.get_queryset().filter(sources__name=package, bugtype=BugType.bug) \ + .delete() + def _guess_packages(self, source, subject): packages = [] diff --git a/debexpo/nntp/__init__.py b/debexpo/nntp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/debexpo/nntp/apps.py b/debexpo/nntp/apps.py new file mode 100644 index 0000000000000000000000000000000000000000..9b623d4ff3ee290ae5058afca512af66ddf8f150 --- /dev/null +++ b/debexpo/nntp/apps.py @@ -0,0 +1,32 @@ +# apps.py - app definition for nntp +# +# This file is part of debexpo +# https://salsa.debian.org/mentors.debian.net-team/debexpo +# +# Copyright © 2020 Baptiste BEAUPLAT +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +from django.apps import AppConfig + + +class NntpConfig(AppConfig): + name = 'nntp' diff --git a/debexpo/nntp/locale/fr/LC_MESSAGES/django.po b/debexpo/nntp/locale/fr/LC_MESSAGES/django.po new file mode 100644 index 0000000000000000000000000000000000000000..8f841d577b51f4966f777ebd8161d07703b3621e --- /dev/null +++ b/debexpo/nntp/locale/fr/LC_MESSAGES/django.po @@ -0,0 +1,31 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2020-07-04 15:02+0000\n" +"PO-Revision-Date: 2020-07-04 17:04+0200\n" +"Language: fr\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n > 1);\n" +"Last-Translator: \n" +"Language-Team: \n" +"X-Generator: Poedit 2.3.1\n" + +#: nntp/models.py:33 +msgid "Namespace" +msgstr "Espace de nom" + +#: nntp/models.py:34 +msgid "List name" +msgstr "Nom de la liste" + +#: nntp/models.py:35 +msgid "Last message processed" +msgstr "Dernier message traité" diff --git a/debexpo/nntp/migrations/0001_initial.py b/debexpo/nntp/migrations/0001_initial.py new file mode 100644 index 0000000000000000000000000000000000000000..19a0fd1b926551b7a977f8ba0b19af04945db9c1 --- /dev/null +++ b/debexpo/nntp/migrations/0001_initial.py @@ -0,0 +1,81 @@ +# 0001_initial.py - migration script for nntp models +# +# This file is part of debexpo +# https://salsa.debian.org/mentors.debian.net-team/debexpo +# +# Copyright © 2020 Baptiste BEAUPLAT +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +from django.db import migrations, models + + +def create_gmane_lists(apps, schema_editor): + NNTPFeed = apps.get_model('nntp', 'NNTPFeed') + + # Last updated on 2020-07-01 + for namespace, name, last in ( + ( + 'remove_uploads', + 'gmane.linux.debian.backports.changes', + '40046', + ), + ( + 'remove_uploads', + 'gmane.linux.debian.devel.changes.unstable', + '583845', + ), + ( + 'remove_uploads', + 'gmane.linux.debian.devel.changes.stable', + '11992', + ), + ): + feed = NNTPFeed() + feed.namespace = namespace + feed.name = name + feed.last = last + + feed.full_clean() + feed.save() + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='NNTPFeed', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, + serialize=False, verbose_name='ID')), + ('namespace', models.TextField(verbose_name='Namespace')), + ('name', models.TextField(verbose_name='List name')), + ('last', models.TextField( + verbose_name='Last message processed')), + ], + ), + migrations.RunPython(create_gmane_lists), + ] diff --git a/debexpo/nntp/migrations/__init__.py b/debexpo/nntp/migrations/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/debexpo/nntp/models.py b/debexpo/nntp/models.py new file mode 100644 index 0000000000000000000000000000000000000000..d41cb2a66ac8c167d00b5ff2d01623ce26b222c5 --- /dev/null +++ b/debexpo/nntp/models.py @@ -0,0 +1,37 @@ +# models.py - model definition for nntp +# +# This file is part of debexpo +# https://salsa.debian.org/mentors.debian.net-team/debexpo +# +# Copyright © 2020 Baptiste BEAUPLAT +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +from django.db import models +from django.utils.translation import gettext_lazy as _ + + +class NNTPFeed(models.Model): + namespace = models.TextField(verbose_name=_('Namespace')) + name = models.TextField(verbose_name=_('List name')) + last = models.TextField(verbose_name=_('Last message processed')) + + unique_together = ('namespace', 'name',) diff --git a/debexpo/packages/locale/fr/LC_MESSAGES/django.po b/debexpo/packages/locale/fr/LC_MESSAGES/django.po index 134bec5a940394fda3f85a8e09b7151a36f79db2..b6843b15f1d8dfff9c81ba45dd4ad9bf840d9898 100644 --- a/debexpo/packages/locale/fr/LC_MESSAGES/django.po +++ b/debexpo/packages/locale/fr/LC_MESSAGES/django.po @@ -7,8 +7,8 @@ msgid "" msgstr "" "Project-Id-Version: \n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2020-06-09 20:50+0200\n" -"PO-Revision-Date: 2020-06-09 20:53+0200\n" +"POT-Creation-Date: 2020-07-05 19:51+0000\n" +"PO-Revision-Date: 2020-07-05 21:54+0200\n" "Last-Translator: \n" "Language-Team: \n" "Language: fr\n" @@ -125,84 +125,96 @@ msgstr "Supprimer ce paquet" msgid "Package uploads" msgstr "Envois du paquet" -#: packages/templates/package.html:92 +#: packages/templates/package.html:93 #, python-format -msgid "

Upload #%(index)s

" -msgstr "

Envoi #%(index)s

" +msgid "Upload #%(index)s" +msgstr "Envoi #%(index)s" -#: packages/templates/package.html:96 +#: packages/templates/package.html:103 +msgid "Admin flag" +msgstr "Drapeau admin" + +#: packages/templates/package.html:106 +msgid "Admin upload deletion" +msgstr "Suppression de l'envoi administrateur" + +#: packages/templates/package.html:110 +msgid "Delete this upload" +msgstr "Supprimer cet envoi" + +#: packages/templates/package.html:117 msgid "Information" msgstr "Informations" -#: packages/templates/package.html:100 +#: packages/templates/package.html:121 msgid "Version:" msgstr "Version :" -#: packages/templates/package.html:104 +#: packages/templates/package.html:125 msgid "View RFS template" msgstr "Voir le modèle RFS" -#: packages/templates/package.html:110 +#: packages/templates/package.html:131 msgid "Uploaded:" msgstr "Envoi :" -#: packages/templates/package.html:116 +#: packages/templates/package.html:137 msgid "Source package:" msgstr "Paquet source :" -#: packages/templates/package.html:124 +#: packages/templates/package.html:145 msgid "Distribution:" msgstr "Distribution :" -#: packages/templates/package.html:129 +#: packages/templates/package.html:150 msgid "Section:" msgstr "Section :" -#: packages/templates/package.html:134 +#: packages/templates/package.html:155 msgid "Priority:" msgstr "Priorité :" -#: packages/templates/package.html:141 +#: packages/templates/package.html:162 msgid "Homepage:" msgstr "Page d'accueil :" -#: packages/templates/package.html:168 +#: packages/templates/package.html:189 msgid "Closes bugs:" msgstr "Ferme les bugs :" -#: packages/templates/package.html:178 +#: packages/templates/package.html:199 msgid "Changelog" msgstr "Changelog" -#: packages/templates/package.html:185 +#: packages/templates/package.html:206 msgid "QA information" msgstr "Informations QA" -#: packages/templates/package.html:198 +#: packages/templates/package.html:219 msgid "Comments" msgstr "Commentaires" -#: packages/templates/package.html:207 +#: packages/templates/package.html:228 msgid "Needs work" msgstr "Nécessite des corrections" -#: packages/templates/package.html:209 +#: packages/templates/package.html:230 msgid "Ready" msgstr "Pret" -#: packages/templates/package.html:213 +#: packages/templates/package.html:234 msgid "Package has been uploaded to Debian" msgstr "Le paquet a été envoyé dans Debian" -#: packages/templates/package.html:220 +#: packages/templates/package.html:241 msgid "No comments" msgstr "Pas de commentaires" -#: packages/templates/package.html:224 +#: packages/templates/package.html:245 msgid "New comment" msgstr "Nouveau commentaire" -#: packages/templates/package.html:233 +#: packages/templates/package.html:254 msgid "Submit" msgstr "Valider" @@ -218,56 +230,59 @@ msgstr "Envoyeur" msgid "No packages" msgstr "Pas de paquets" -#: packages/views.py:121 +#: packages/views.py:124 msgid "Today" msgstr "Aujourd'hui" -#: packages/views.py:124 +#: packages/views.py:127 msgid "Yesterday" msgstr "Hier" -#: packages/views.py:128 +#: packages/views.py:131 msgid "Some days ago" msgstr "Il y a quelques jours" -#: packages/views.py:132 +#: packages/views.py:135 msgid "Older packages" msgstr "Paquets plus vieux" -#: packages/views.py:136 +#: packages/views.py:139 msgid "Uploaded long ago" msgstr "Envoyé il y a longtemps" -#: packages/views.py:197 +#: packages/views.py:229 msgid "Packages for {} {}" msgstr "Paquets pour {} {}" -#: packages/views.py:199 +#: packages/views.py:231 msgid "Package list" msgstr "Liste des paquets" -#: packages/views.py:212 +#: packages/views.py:244 #, python-format msgid "%s packages" msgstr "%s paquets" -#: packages/views.py:213 +#: packages/views.py:245 #, python-format msgid "A feed of packages on %s" msgstr "Un flux de paquets sur %s" -#: packages/views.py:235 +#: packages/views.py:267 msgid "Package {} uploaded by {}." msgstr "Paquet {} envoyé par {}." -#: packages/views.py:241 +#: packages/views.py:273 msgid "Uploader is currently looking for a sponsor." msgstr "L'envoyeur est présentement en recherche de parrain." -#: packages/views.py:243 +#: packages/views.py:275 msgid "Uploader is currently not looking for a sponsor." msgstr "L'envoyeur n'est pas présentement en recherche de parrain." +#~ msgid "

Upload #%(index)s

" +#~ msgstr "

Envoi #%(index)s

" + #~ msgid "QA plugins succeeded" #~ msgstr "Les plugins QA ont réussi" diff --git a/debexpo/packages/tasks.py b/debexpo/packages/tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..235f8c7a01790c7003eb4282d09321c5c0d7638a --- /dev/null +++ b/debexpo/packages/tasks.py @@ -0,0 +1,246 @@ +# tasks.py -- task to remove old and uploaded packages from Debexpo +# +# This file is part of debexpo - +# https://salsa.debian.org/mentors.debian.net-team/debexpo +# +# Copyright © 2011 Arno Töll +# Copyright © 2020 Baptiste BEAUPLAT +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from celery.decorators import periodic_task +from datetime import timedelta, datetime, timezone +from logging import getLogger +from debian.deb822 import Changes +from debian.debian_support import NativeVersion + +from django.conf import settings +from django.db.models import Max +from django.db import transaction + +from debexpo.packages.models import Package, PackageUpload +from debexpo.repository.models import Repository +from debexpo.tools.email import Email +from debexpo.bugs.models import Bug +from debexpo.tools.gitstorage import GitStorage +from debexpo.nntp.models import NNTPFeed +from debexpo.tools.nntp import NNTPClient +from debexpo.tools.clients import ExceptionClient +from debexpo.tools.clients.ftp_master import ClientFTPMaster + +log = getLogger(__name__) + + +@transaction.atomic +def remove_uploads(uploads): + removals = set() + repository = Repository(settings.REPOSITORY) + git_storage_path = getattr(settings, 'GIT_STORAGE', None) + + for upload in uploads: + remove_version = not PackageUpload.objects.filter( + package=upload.package, version=upload.version) \ + .exclude(id=upload.id) + remove_package = not PackageUpload.objects.filter( + package=upload.package).exclude(id=upload.id) + package = upload.package.name + distribution = upload.distribution.name + uploader = upload.uploader + + if remove_version: + repository.remove(package, upload.version) + if remove_package: + if git_storage_path: + git_storage = GitStorage(git_storage_path, package) + git_storage.remove() + + Bug.objects.remove_bugs(package) + + upload.delete() + + if remove_package: + Package.objects.get(name=package).delete() + + removals.add((package, distribution, uploader)) + + repository.update() + + for package, distribution, uploader in removals: + log.info(f'Removed package {package}, from {distribution}, ' + f'uploaded by {uploader}') + + return removals + + +@periodic_task(run_every=settings.TASK_OLD_UPLOADS_BEAT) +def remove_old_uploads(): + expiration_date = datetime.now(timezone.utc) - \ + timedelta(weeks=settings.MAX_AGE_UPLOAD_WEEKS) + packages = Package.objects \ + .values('name', 'packageupload__distribution') \ + .annotate(latest_upload=Max('packageupload__uploaded')) \ + .filter(latest_upload__lt=expiration_date) + + uploads = set() + + for package in packages: + uploads.update(PackageUpload.objects.filter( + package__name=package['name'], + distribution=package['packageupload__distribution'])) + + removals = remove_uploads(uploads) + notify_uploaders(removals, reason='Your package found no sponsor for ' + '20 weeks') + + +def notify_uploaders(removals, reason): + for package, distribution, uploader in removals: + email = Email('email-upload-removed.html') + email.send(f'{package} has been removed from ' + f'{distribution} on {settings.SITE_NAME}', + recipients=[uploader.email], package=package, + distribution=distribution, + reason=reason) + + +@periodic_task(run_every=settings.TASK_ACCEPTED_UPLOADS_BEAT) +def remove_uploaded_packages(client=None): + uploads_to_archive = set() + uploads_to_new = set() + + uploads_to_archive.update(get_packages_uploaded_to_archive(client)) + uploads_to_new.update(get_packages_uploaded_to_new()) + + removals_from_archive = remove_uploads(uploads_to_archive) + removals_from_new = remove_uploads(uploads_to_new) + + mark_packages_as_uploaded(set([removal[0] for removal in + removals_from_archive]), debian=True) + mark_packages_as_uploaded(set([removal[0] for removal in + removals_from_new])) + + notify_uploaders(removals_from_archive, + reason='Your package was uploaded to the ' + 'official Debian archive') + notify_uploaders(removals_from_new, + reason='Your package was uploaded to the ' + 'NEW queue') + + +def get_packages_uploaded_to_new(): + uploads = set() + ftp_master = ClientFTPMaster() + + try: + packages = ftp_master.get_packages_uploaded_to_new() + except ExceptionClient as e: + log.warning(f'Could not retrive package uploaded to new: {e}') + return [] + + for changes in packages: + try: + uploads.update(process_accepted_changes(changes)) + except Exception as e: + log.warning(f'Failed to process package in NEW {changes} ' + f'{e}') + + return uploads + + +def get_packages_uploaded_to_archive(client): + uploads = set() + feeds = NNTPFeed.objects.filter(namespace='remove_uploads') + + # We allow the caller to define another NNTPClient, if not fallback to the + # default one. This is for testing purposes only: easier than setting up a + # testing NNTP server. + if not client: # pragma: no cover + client = NNTPClient() + + if not client.connect_to_server(): + return [] + + for feed in feeds: + last = feed.last + + for msg in client.unread_messages(feed.name, feed.last): + try: + changes = convert_mail_to_changes(msg) + uploads.update(process_accepted_changes(changes)) + except Exception as e: + log.warning('Failed to process message after ' + f'#{last} on ' + f'{feed.name}: {e}') + else: + last = msg['X-Debexpo-Message-Number'] + + feed.last = last + + client.disconnect_from_server() + + for feed in feeds: + feed.full_clean() + feed.save() + + return uploads + + +def mark_packages_as_uploaded(packages, debian=False): + for name in packages: + try: + package = Package.objects.get(name=name) + except Package.DoesNotExist: + continue + + if debian: + package.in_debian = True + package.needs_sponsor = False + package.full_clean() + package.save() + + +def convert_mail_to_changes(mail): + if not mail or mail.is_multipart(): + return + + changes = mail.get_payload(decode=True) + changes = Changes(changes) + + return changes + + +def process_accepted_changes(changes): + if not changes or \ + 'Source' not in changes \ + or 'Distribution' not in changes \ + or 'Version' not in changes: + raise Exception(f'Cannot process accepted upload: {changes}') + + uploads = PackageUpload.objects.filter( + package__name=changes['Source'], + distribution__name__in=(changes['Distribution'], 'UNRELEASED',)) + + return [ + upload for upload in uploads + if NativeVersion(upload.version) <= NativeVersion(changes['Version']) or + upload.distribution.name == 'UNRELEASED' + ] diff --git a/debexpo/packages/templates/email-upload-removed.html b/debexpo/packages/templates/email-upload-removed.html new file mode 100644 index 0000000000000000000000000000000000000000..9bab3b6126cfdc323dcdd91f409ae34ab36e9b47 --- /dev/null +++ b/debexpo/packages/templates/email-upload-removed.html @@ -0,0 +1,8 @@ +{% extends "email-base.html" %}{% block content %}Hi, + +Your package {{ args.package }} has been removed from {{ args.distribution }} +on {{ args.settings.SITE_NAME }} for the following reason: + +{{ args.reason }} + +Thanks,{% endblock %} diff --git a/debexpo/packages/templates/package.html b/debexpo/packages/templates/package.html index 46fe07d9357383731652f91ca7e2e3bf18dd3013..8809354132fb7977b25f53eaea24db1a01fef40f 100644 --- a/debexpo/packages/templates/package.html +++ b/debexpo/packages/templates/package.html @@ -1,4 +1,4 @@ -{% extends "base.html" %}{% load i18n %} +{% extends "base.html" %}{% load i18n %}{% load static %} {% block content %} {% blocktrans with name=package.name trimmed %} @@ -71,11 +71,11 @@ {% csrf_token %} {% if user.is_superuser and user not in package.get_uploaders %} - {% else %} - {% endif %} @@ -89,9 +89,30 @@ {% for upload in package.packageupload_set.all %} +

{% blocktrans with index=forloop.revcounter trimmed %} -

Upload #{{ index }}

+Upload #{{ index }} {% endblocktrans %} + {% if user in package.get_uploaders or user.is_superuser %} +
+ {% csrf_token %} + {% if user.is_superuser and user not in package.get_uploaders %} + {% trans 'Admin flag' %} + + {% else %} + + {% endif %} +
+ +{% endif %} +

{% trans 'Information' %}

diff --git a/debexpo/packages/views.py b/debexpo/packages/views.py index f956b0a0e9adad9aa4255c20729fd3845bef05e1..23b420a81e24b6889cbdfea4afbbd90ee31d7214 100644 --- a/debexpo/packages/views.py +++ b/debexpo/packages/views.py @@ -44,6 +44,9 @@ from django.contrib.auth.decorators import login_required from debexpo.packages.models import PackageUpload, Package, SourcePackage from debexpo.comments.forms import CommentForm from debexpo.repository.tasks import remove_from_repository +from debexpo.tools.gitstorage import GitStorage +from debexpo.bugs.models import Bug +from debexpo.packages.tasks import remove_uploads log = logging.getLogger(__name__) @@ -150,6 +153,28 @@ def package(request, name): }) +@login_required +def delete_upload(request, name, upload): + if request.method != 'POST': + return HttpResponseNotAllowed(['POST']) + + upload = get_object_or_404(PackageUpload, id=upload) + package = upload.package.name + + if (request.user != upload.uploader and not + request.user.is_superuser): + return HttpResponseForbidden() + + remove_uploads([upload]) + + try: + Package.objects.get(name=package) + except Package.DoesNotExist: + return HttpResponseRedirect(reverse('packages_my')) + else: + return HttpResponseRedirect(reverse('package', args=[package])) + + @login_required def delete_package(request, name): if request.method != 'POST': @@ -164,7 +189,14 @@ def delete_package(request, name): package.delete() log.info('Package deleted: {}'.format(name)) + git_storage_path = getattr(settings, 'GIT_STORAGE', None) + + if git_storage_path: + git_storage = GitStorage(git_storage_path, name) + git_storage.remove() + remove_from_repository.delay(name) + Bug.objects.remove_bugs(package) return HttpResponseRedirect(reverse('packages_my')) diff --git a/debexpo/settings/common.py b/debexpo/settings/common.py index 119cc238e9a86c953c335a6a67be603fb710afac..54bcfad512f822faae02c261321b3576bde97356 100644 --- a/debexpo/settings/common.py +++ b/debexpo/settings/common.py @@ -53,6 +53,7 @@ INSTALLED_APPS = [ 'debexpo.repository', 'debexpo.plugins', 'debexpo.bugs', + 'debexpo.nntp', ] MIDDLEWARE = [ @@ -72,9 +73,6 @@ ROOT_URLCONF = 'debexpo.urls' TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [ - 'tests/unit/tools/templates' - ], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ @@ -168,7 +166,9 @@ CELERY_BEAT_SCHEDULER = 'django' # Tasks beats TASK_IMPORTER_BEAT = 60 * 15 # Every 15 minutes -TASK_CLEANUPACCOUNTS_BEAT = 60 * 60 +TASK_CLEANUPACCOUNTS_BEAT = 60 * 60 # Every hours +TASK_OLD_UPLOADS_BEAT = 60 * 10 # Every 10 minutes +TASK_ACCEPTED_UPLOADS_BEAT = 60 * 10 # Every 10 minutes # Account registration expiration REGISTRATION_EXPIRATION_DAYS = 7 @@ -197,3 +197,7 @@ BUGS_REPORT_NOT_OPEN = True # Debian tracker access TRACKER_URL = 'https://tracker.debian.org' +FTP_MASTER_NEW_PACKAGES_URL = 'https://ftp-master.debian.org/new.822' + +# Cleanup package older than NN weeks +MAX_AGE_UPLOAD_WEEKS = 20 diff --git a/debexpo/settings/test.py b/debexpo/settings/test.py index 080084fa2d6d1112b04e15b9ce6d3d78d6457e42..376e2da24492ea8d481ec8497a10d1c71904aab9 100644 --- a/debexpo/settings/test.py +++ b/debexpo/settings/test.py @@ -42,6 +42,12 @@ DEBUG = False ALLOWED_HOSTS = [] +# Add testing directory templates +TEMPLATES[0]['DIRS'] = [ # noqa: F405 + 'tests/unit/tools/templates', + 'tests/functional/packages/templates', +] + # Database # https://docs.djangoproject.com/en/2.2/ref/settings/#databases diff --git a/debexpo/tools/clients/ftp_master.py b/debexpo/tools/clients/ftp_master.py index c0b8270473d6aebdc67f717c3ea3ece9e6c9926f..4454261c1b53e53dbd2fce2c6d5b35984a8bbf69 100644 --- a/debexpo/tools/clients/ftp_master.py +++ b/debexpo/tools/clients/ftp_master.py @@ -26,8 +26,12 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. +from debian.deb822 import Deb822, Changes + +from django.conf import settings + from debexpo.tools.files import CheckSumedFile -from debexpo.tools.clients import ClientJsonAPI +from debexpo.tools.clients import ClientJsonAPI, ClientHTTP import debexpo.repository.models as repository @@ -50,3 +54,14 @@ class ClientFTPMasterAPI(ClientJsonAPI): origin_files.append(origin) return origin_files + + +class ClientFTPMaster(ClientHTTP): + def get_packages_uploaded_to_new(self): + packages = [] + content = self.fetch_resource(settings.FTP_MASTER_NEW_PACKAGES_URL) + + for package in Deb822.iter_paragraphs(content): + packages.append(Changes(package)) + + return packages diff --git a/debexpo/tools/gitstorage.py b/debexpo/tools/gitstorage.py index 564747f755fb97e1c7399c38a9b4ea60feab18e1..9a777bf32f9a16f07d5e82cf983698a9082d044e 100644 --- a/debexpo/tools/gitstorage.py +++ b/debexpo/tools/gitstorage.py @@ -55,8 +55,9 @@ class GitStorage(): return ref - # def remove(self, source): - # pass + def remove(self): + if isdir(self.repository): + rmtree(self.repository) # def diff(self, upload_from, upload_to): # pass diff --git a/debexpo/urls.py b/debexpo/urls.py index c08d6ddf400d589a6ceeb699ec6004706181fda5..59d8224aded2242a464f372f3f159a71ed4761b8 100644 --- a/debexpo/urls.py +++ b/debexpo/urls.py @@ -40,7 +40,7 @@ from debexpo.base.views import index, contact, intro_reviewers, \ from debexpo.accounts.views import register, profile from debexpo.accounts.forms import PasswordResetForm from debexpo.packages.views import package, packages, packages_my, \ - PackagesFeed, sponsor_package, delete_package + PackagesFeed, sponsor_package, delete_package, delete_upload from debexpo.comments.views import subscribe, unsubscribe, subscriptions, \ comment from debexpo.importer.views import upload @@ -113,6 +113,8 @@ urlpatterns = [ url(r'^packages/$', packages, name='packages'), + url(r'^package/(?P.+)/delete/(?P[0-9]+)/$', delete_upload, + name='delete_upload'), url(r'^package/(?P.+)/delete/$', delete_package, name='delete_package'), url(r'^package/(?P.+)/sponsor/$', sponsor_package, diff --git a/old/debexpo/cronjobs/removeolduploads.py b/old/debexpo/cronjobs/removeolduploads.py deleted file mode 100644 index 98aafac78b1b93eba971a6a37bac837b5eb7e19c..0000000000000000000000000000000000000000 --- a/old/debexpo/cronjobs/removeolduploads.py +++ /dev/null @@ -1,180 +0,0 @@ -# -*- coding: utf-8 -*- -# -# removeolduploads.py -- remove old and uploaded packages from Debexpo -# -# This file is part of debexpo - -# https://salsa.debian.org/mentors.debian.net-team/debexpo -# -# Copyright © 2011 Arno Töll -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -""" -Import RFS comments from debian-mentors -""" -__author__ = 'Arno Töll' -__copyright__ = 'Copyright © 2011 Arno Töll' -__license__ = 'MIT' - -from debexpo.cronjobs import BaseCronjob - -from debexpo.lib.email import Email -from debexpo.lib.filesystem import CheckFiles -from debexpo.controllers.package import PackageController -from debexpo.controllers.packages import PackagesController -from debexpo.model.users import User -from debexpo.model.data_store import DataStore -from debexpo.model.package_versions import PackageVersion -from debexpo.model import meta -from debian import deb822 - -import socket -import apt_pkg -import datetime - -__namespace__ = '_remove_uploads_' - - -class RemoveOldUploads(BaseCronjob): - - def _remove_package(self, package, version, reason): - user = meta.session.query(User).filter_by(id=package.user_id).one() - if user: - self.mailer.send([user.email, ], - package=package.name, - version=version.version, - reason=reason) - - CheckFiles().delete_files_for_packageversion(version) - meta.session.delete(version) - - if (meta.session.query(PackageVersion).filter_by( - package_id=package.id).count() == 0): - CheckFiles().delete_files_for_package(package) - meta.session.delete(package) - - meta.session.commit() - - def _process_changes(self, mail): - if mail.is_multipart(): - self.log.debug("Changes message is multipart?!") - return - changes = mail.get_payload(decode=True) - try: - changes = deb822.Changes(changes) - except Exception: - self.log.error('Could not open changes file; skipping mail "%s"' % - (mail['subject'])) - return - - if 'Source' not in changes: - # self.log.debug('Changes file "%s" seems incomplete' % - # (mail['subject'])) - return - - package = self.pkg_controller._get_package(changes['Source'], - from_controller=False) - if package is not None: - for pv in package.package_versions: - if pv.distribution == changes['Distribution']: - if (apt_pkg.version_compare(changes['Version'], pv.version) - == 0): - self.log.debug("Package %s was uploaded to Debian - " - "removing it from Expo" % - (changes['Source'])) - self._remove_package(package, pv, - "Package was uploaded to " - "official Debian repositories") - if (apt_pkg.version_compare(changes['Version'], pv.version) - > 0): - self.log.debug("More recent package %s was uploaded to " - "Debian - removing it from Expo" % - (changes['Source'])) - self._remove_package(package, pv, - "A more recent package was " - "uploaded to official Debian " - "repositories") - else: - # self.log.debug("Package %s was not uploaded to Expo before - " - # "ignoring it" % (changes['Source'])) - pass - - def _remove_uploaded_packages(self): - - if self.mailer.connection_established(): - lists = meta.session.query(DataStore) \ - .filter(DataStore.namespace == __namespace__) \ - .all() - for list_name in lists: - for message in self.mailer.unread_messages(list_name.code, - list_name.value): - self._process_changes(message) - list_name.value = message['X-Debexpo-Message-Number'] - self.log.debug("Processed all messages up to #%s on %s" % - (list_name.value, list_name.code)) - meta.session.merge(list_name) - meta.session.commit() - self.mailer.disconnect_from_server() - - def _remove_old_packages(self): - now = datetime.datetime.now() - for package in self.pkgs_controller._get_packages(): - if (now - package.package_versions[-1].uploaded) > \ - datetime.timedelta(weeks=20): - self.log.debug("Removing package %s - uploaded on %s" % - (package.name, - package.package_versions[-1].uploaded)) - for pv in package.package_versions: - self._remove_package(package, pv, - "Your package found no sponsor for " - "20 weeks") - - def setup(self): - self.mailer = Email('upload_removed_from_expo') - self.mailer.connect_to_server() - self.pkg_controller = PackageController() - self.pkgs_controller = PackagesController() - apt_pkg.init_system() - self.last_cruft_run = datetime.datetime(year=1970, month=1, day=1) - self.log.debug("%s loaded successfully" % (__name__)) - - def teardown(self): - self.mailer.disconnect_from_server() - - def invoke(self): - try: - self._remove_uploaded_packages() - except socket.error as e: - # better luck next time - self.log.debug("Socket error %s: skipping removals his time" % (e)) - pass - - # We don't need to run our garbage collection of old cruft that often - # It's ok if we purge old packages once a day. - if (datetime.datetime.now() - self.last_cruft_run) >= \ - datetime.timedelta(hours=24): - self.last_cruft_run = datetime.datetime.now() - self._remove_old_packages() - - -cronjob = RemoveOldUploads -schedule = datetime.timedelta(minutes=10) diff --git a/old/debexpo/model/data_store.py b/old/debexpo/model/data_store.py deleted file mode 100644 index b786076e9a9e404efa71d9b9b15549709b60bc9a..0000000000000000000000000000000000000000 --- a/old/debexpo/model/data_store.py +++ /dev/null @@ -1,71 +0,0 @@ -# -*- coding: utf-8 -*- -# -# data_store.py - Generic, general purpose data store -# -# This file is part of debexpo - -# https://salsa.debian.org/mentors.debian.net-team/debexpo -# -# Copyright © 2011 Arno Töll -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -""" -Holds data_store model -""" - -__author__ = 'Arno Töll' -__copyright__ = 'Copyright © 2011 Arno Töll' -__license__ = 'MIT' - -import sqlalchemy as sa -from sqlalchemy import orm - -from debexpo.model import meta, OrmObject - -t_user_countries = sa.Table( - 'data_store', meta.metadata, - sa.Column('namespace', sa.types.String(100), primary_key=True), - sa.Column('code', sa.types.String(100), primary_key=True, nullable=False), - sa.Column('value', sa.types.String(100), nullable=True) - ) - - -class DataStore(OrmObject): - pass - - -orm.mapper(DataStore, t_user_countries) - - -def fill_data_store(): - import debexpo.model.data.data_store_init - import logging - for data in debexpo.model.data.data_store_init.DATA_STORE_INIT_OBJECTS: - query = meta.session.query(DataStore) \ - .filter(DataStore.code == data.code) \ - .filter(DataStore.namespace == data.namespace) \ - .count() - if (query): - continue - logging.info("Pre-configure value %s.%s" % (data.namespace, data.code)) - meta.session.add(data) - meta.session.commit() diff --git a/old/debexpo/templates/email/upload_removed_from_expo.mako b/old/debexpo/templates/email/upload_removed_from_expo.mako deleted file mode 100644 index 71120140384cbb20ebef317c2fe6f23d10125405..0000000000000000000000000000000000000000 --- a/old/debexpo/templates/email/upload_removed_from_expo.mako +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -<%inherit file="/base.mako"/>To: ${ c.to } -Subject: ${ _('%s package has been removed from %s' % (c.package, c.config['debexpo.sitetitle'])) } - -${ _('Your package %s %s has been removed from %s for the following reason:' % (c.package, c.version, c.config['debexpo.sitename'])) } - -${ c.reason } - -${ _('Thanks,') } diff --git a/old/debexpo/tests/cronjobs/test_removeolduploads.py b/old/debexpo/tests/cronjobs/test_removeolduploads.py deleted file mode 100644 index d1a6b8e812d33c59ffdb31fdc5eaf19a97add0d2..0000000000000000000000000000000000000000 --- a/old/debexpo/tests/cronjobs/test_removeolduploads.py +++ /dev/null @@ -1,243 +0,0 @@ -# -*- coding: utf-8 -*- -# -# test_removeolduploads.py - Test the removeolduploads cronjob -# -# This file is part of debexpo -# https://salsa.debian.org/mentors.debian.net-team/debexpo -# -# Copyright © 2019 Baptiste BEAUPLAT -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. - -__author__ = 'Baptiste BEAUPLAT' -__copyright__ = 'Copyright © 2019 Baptiste BEAUPLAT' -__license__ = 'MIT' - -from mako.lookup import TemplateLookup -from mako.template import Template -from os.path import join, dirname -from pylons.test import pylonsapp - -import apt_pkg -import datetime -import email.parser -import logging -import pylons -import sqlalchemy.orm.exc as orm_exc - -from debexpo.controllers.package import PackageController -from debexpo.controllers.packages import PackagesController -from debexpo.lib.email import Email -from debexpo.model.users import User -from debexpo.model.packages import Package -from debexpo.model.package_versions import PackageVersion -from debexpo.tests.cronjobs import TestCronjob - -log = logging.getLogger(__name__) - - -class TestCronjobRemoveOldUploads(TestCronjob): - def setUp(self): - self.nntp_server = pylonsapp.config['debexpo.nntp_server'] - pylonsapp.config['debexpo.nntp_server'] = None - - self._setup_plugin('removeolduploads', run_setup=False) - self._finish_setup_plugin() - self._create_user() - self.state = [] - - def tearDown(self): - pylonsapp.config['debexpo.nntp_server'] = self.nntp_server - self._remove_packages() - self._remove_user() - - # This is needed since we do not call the plugin setup (to avoid the NNTP - # connection) - def _finish_setup_plugin(self): - self.plugin.mailer = Email('upload_removed_from_expo') - self.plugin.pkg_controller = PackageController() - self.plugin.pkgs_controller = PackagesController() - apt_pkg.init_system() - self.plugin.last_cruft_run = datetime.datetime(year=1970, - month=1, - day=1) - - def _create_user(self): - user = User(name='Test user', - email='test.user@example.org', - password='password', - lastlogin=datetime.datetime.now()) - self.db.add(user) - self.db.commit() - - def _remove_user(self): - user = self.db.query(User).one() - self.db.delete(user) - - def _setup_packages(self, include_expired=False): - packages = [ - # (package, version, distrib, expired), - ('htop', '1.0.0', 'unstable', False), - ('htop', '0.9.0', 'unstable', False), - ('htop', '0.8.0', 'buster-backports', False), - ('tmux', '1.0.0', 'unstable', False), - ('tmux', '1.0.0', 'UNRELEASED', False), - ('zsh', '1.0.0', 'unstable', False), - ('zsh', '1.0.0', 'unstable', False), - ('hello', '1.0.0', 'unstable', True), - ('hello', '0.9.0', 'unstable', True), - ] - - for (package, version, distrib, expired) in packages: - if not expired or (expired and include_expired): - self._create_package(package, version, distrib, expired) - - if not expired: - self.state.append((package, version, distrib)) - - self.db.commit() - - def _create_package(self, name, version_number, distrib, is_expired): - date = datetime.datetime.now() - user = self.db.query(User).one() - - try: - package = self.db.query(Package).filter_by(name=name).one() - except orm_exc.NoResultFound: - package = Package(name=name, - user=user) - - if is_expired: - date -= datetime.timedelta(days=365) - - version = PackageVersion(package=package, - version=version_number, - maintainer='vtime@example.org', - section='main', - distribution=distrib, - qa_status=0, - component='tools', - uploaded=date) - - self.db.add(version) - - def _remove_packages(self): - packages = self.db.query(Package) - - for package in packages: - self.db.delete(package) - - self.db.commit() - - def _assert_cronjob_success(self): - packages = self.db.query(Package) - count = 0 - - # Each package exists in state - for package in packages: - for version in package.package_versions: - log.debug('validating {}-{}/{}'.format(package.name, - version.version, - version.distribution)) - self.assertTrue((package.name, - version.version, - version.distribution) in self.state) - count += 1 - - # Each state exists in package - self.assertEquals(len(self.state), count) - - def _expect_package_removal(self, package_to_remove): - new_state = [] - - for package in self.state: - if package not in package_to_remove: - new_state.append(package) - - self.state = new_state - - def _build_email(self, package): - (name, version, distrib) = package - c = {'name': name, 'version': version, 'distrib': distrib} - template_file = join(dirname(__file__), 'email/accept.mako') - lookup = TemplateLookup(directories=[dirname(template_file)]) - template = Template( - filename=template_file, lookup=lookup, - module_directory=pylons.config['app_conf']['cache_dir']) - - return email.message_from_string(template.render_unicode(c=c)) - - def _invoke_plugin(self, uploaded): - for package in uploaded: - message = self._build_email(package) - self.plugin._process_changes(message) - - self.plugin._remove_old_packages() - - def test_remove_uploads_noop_no_packages(self): - self._invoke_plugin([]) - self._assert_cronjob_success() - - def test_remove_uploads_inexistant_package(self): - self._invoke_plugin([('inexistant_package', '1.0.0', 'unstable')]) - self._assert_cronjob_success() - - def test_remove_uploads_noop_with_packages(self): - self._setup_packages() - self._invoke_plugin([]) - self._assert_cronjob_success() - - def test_remove_uploads_expired(self): - self._setup_packages(include_expired=True) - self._invoke_plugin([]) - self._assert_cronjob_success() - - def test_remove_uploads_keep_other_dists(self): - removed_packages = [ - ('htop', '1.0.0', 'unstable'), - ('htop', '0.9.0', 'unstable'), - ] - - self._setup_packages() - self._invoke_plugin([('htop', '1.0.0', 'unstable')]) - self._expect_package_removal(removed_packages) - self._assert_cronjob_success() - - def test_remove_uploads_same_version(self): - removed_packages = [ - ('zsh', '1.0.0', 'unstable'), - ] - - self._setup_packages() - self._invoke_plugin([('zsh', '1.0.0', 'unstable')]) - self._expect_package_removal(removed_packages) - self._assert_cronjob_success() - - def test_remove_uploads_keep_newer(self): - removed_packages = [ - ('htop', '0.9.0', 'unstable'), - ] - - self._setup_packages() - self._invoke_plugin([('htop', '0.9.0', 'unstable')]) - self._expect_package_removal(removed_packages) - self._assert_cronjob_success() diff --git a/tests/__init__.py b/tests/__init__.py index c10c2a3d75af0fd83a8169828fa5dd3ae69c0985..099ae5e3eb52eb845cfc95cf209111eda1fe4014 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -153,8 +153,7 @@ Xcgnuh6Rlywt6uiaFIGYnGefYPGXRAA= """ user = User.objects.get(email='email@example.com') - package = Package(name='testpackage') - package.save() + package = Package.objects.get_or_create(name='testpackage')[0] package_upload = PackageUpload( uploader=user, @@ -182,8 +181,8 @@ Xcgnuh6Rlywt6uiaFIGYnGefYPGXRAA= binary.save() - package = Package(name='anotherpackage', in_debian=True) - package.save() + package = Package.objects.get_or_create(name='anotherpackage', + in_debian=True)[0] package_upload = PackageUpload( uploader=user, diff --git a/tests/functional/importer/__init__.py b/tests/functional/importer/__init__.py index ab312bf9958e645c607bf40d4e44c375bec6be24..ced9190cf0f2504ac3c35e10fa60007255be75dd 100644 --- a/tests/functional/importer/__init__.py +++ b/tests/functional/importer/__init__.py @@ -40,6 +40,7 @@ from django_redis.pool import ConnectionFactory from debexpo.importer.models import Importer, Spool from debexpo.packages.models import Package, PackageUpload +from debexpo.packages.tasks import remove_uploads from debexpo.accounts.models import User from debexpo.comments.models import PackageSubscription from debexpo.plugins.models import PluginResults @@ -140,6 +141,13 @@ class TestImporterController(TestController): result.append(join(root, name)) return result + def remove_package(self, package, version): + uploads = PackageUpload.objects.filter(package__name=package, + version=version) + + with self.settings(REPOSITORY=self.repository): + remove_uploads(uploads) + def import_source_package(self, package_dir, skip_gpg=False, skip_email=False, base_dir=None): source_package = TestSourcePackage(package_dir, base_dir) diff --git a/tests/functional/importer/test_importer.py b/tests/functional/importer/test_importer.py index e52b75691f42b64379311a99c5e924fbbeb87fbe..4cb43e068c8a95a2ee9df3e6b4946dce9604db69 100644 --- a/tests/functional/importer/test_importer.py +++ b/tests/functional/importer/test_importer.py @@ -360,6 +360,9 @@ r1JREXlgQRuRdd5ZWSvIxKaKGVbYCw== self.assert_package_count('hello', '1.0-1', 2) self.assert_package_in_repo('hello', '1.0-1') + self.remove_package('hello', '1.0-1') + self._assert_no_leftover(str(join(self.repository, 'pool'))) + def test_import_package_htop_download_orig(self): self.import_package('orig-from-official') self.assert_importer_succeeded() @@ -388,6 +391,9 @@ r1JREXlgQRuRdd5ZWSvIxKaKGVbYCw== 'htop_2.2.0.orig.tar.gz.asc'): self.assert_file_in_repo(filename) + self.remove_package('htop', '2.2.0-1') + self._assert_no_leftover(str(join(self.repository, 'pool'))) + # Since we cannot really make the importer fail (it is not supposed to # happend), we test that the field method can report an error to admins and # optionnaly to uploader if available. diff --git a/old/debexpo/tests/cronjobs/email/accept.mako b/tests/functional/packages/templates/test-upload-accepted.html similarity index 64% rename from old/debexpo/tests/cronjobs/email/accept.mako rename to tests/functional/packages/templates/test-upload-accepted.html index a32951e3c29e686a2cf50f3870aa2205a6111ca6..c719bf79b82e407b46a99848a1e7bd040f264afd 100644 --- a/old/debexpo/tests/cronjobs/email/accept.mako +++ b/tests/functional/packages/templates/test-upload-accepted.html @@ -19,39 +19,39 @@ From: Debian FTP Masters To: Vincent TIME X-DAK: dak process-upload X-Debian: DAK -X-Debian-Package: ${ c['name'] } +X-Debian-Package: {{ args.name }} Precedence: bulk Auto-Submitted: auto-generated MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 8bit -Subject: ${ c['name'] }_${ c['version'] }_source.changes ACCEPTED into ${ c['distrib'] } +Subject: {{ args.name }}_{{ args.version }}_source.changes ACCEPTED into {{ args.distrib }} Message-Id: Date: Fri, 19 Jul 2019 08:49:25 +0000 Received-SPF: none client-ip=2001:41b8:202:deb:6564:a62:52c3:4b72; envelope-from=envelope@ftp-master.debian.org; helo=mailly.debian.org Format: 1.8 Date: Tue, 16 Jul 2019 23:37:22 +0200 -Source: ${ c['name'] } +Source: {{ args.name }} Architecture: source -Version: ${ c['version'] } -Distribution: ${ c['distrib'] } +Version: {{ args.version }} +Distribution: {{ args.distrib }} Urgency: medium Maintainer: Vincent TIME Changed-By: Vincent TIME Changes: - ${ c['name'] } (${ c['version'] }) ${ c['distrib'] }; urgency=medium + {{ args.name }} ({{ args.version }}) {{ args.distrib }}; urgency=medium . * Rebuild, no changes Checksums-Sha1: - 36a03fa187774adfbe732dff7a3b23784647a783 1818 ${ c['name'] }_${ c['version'] }.dsc - c2a9b41fa5052c3248ad338ed39d23f7166213df 5548 ${ c['name'] }_${ c['version'] }.debian.tar.xz - 6e0a0beb343beab28d2e1215fa4c18be2b508ef7 6031 ${ c['name'] }_${ c['version'] }_amd64.buildinfo + 36a03fa187774adfbe732dff7a3b23784647a783 1818 {{ args.name }}_{{ args.version }}.dsc + c2a9b41fa5052c3248ad338ed39d23f7166213df 5548 {{ args.name }}_{{ args.version }}.debian.tar.xz + 6e0a0beb343beab28d2e1215fa4c18be2b508ef7 6031 {{ args.name }}_{{ args.version }}_amd64.buildinfo Checksums-Sha256: - 63e84b7dc83a8a263b1d0093b7d0b69d15282f4c128cf5f90cef666582ec333e 1818 ${ c['name'] }_${ c['version'] }.dsc - 12ab5db9022e5f470de823656d9706248dcb5793d330e19e3b5178d22d247c91 5548 ${ c['name'] }_${ c['version'] }.debian.tar.xz - 1173147d14de7d56bc1ecf98d2a3616f3348e3a4c055f684306f37d5b98b3bc0 6031 ${ c['name'] }_${ c['version'] }_amd64.buildinfo + 63e84b7dc83a8a263b1d0093b7d0b69d15282f4c128cf5f90cef666582ec333e 1818 {{ args.name }}_{{ args.version }}.dsc + 12ab5db9022e5f470de823656d9706248dcb5793d330e19e3b5178d22d247c91 5548 {{ args.name }}_{{ args.version }}.debian.tar.xz + 1173147d14de7d56bc1ecf98d2a3616f3348e3a4c055f684306f37d5b98b3bc0 6031 {{ args.name }}_{{ args.version }}_amd64.buildinfo Files: - 88619dc01f636b0f180cfc67653b932f 1818 utils optional ${ c['name'] }_${ c['version'] }.dsc - 4e978439dac618e92c982b75f701ddee 5548 utils optional ${ c['name'] }_${ c['version'] }.debian.tar.xz - 461ea7fb0137d8b726787561ea1a8966 6031 utils optional ${ c['name'] }_${ c['version'] }_amd64.buildinfo + 88619dc01f636b0f180cfc67653b932f 1818 utils optional {{ args.name }}_{{ args.version }}.dsc + 4e978439dac618e92c982b75f701ddee 5548 utils optional {{ args.name }}_{{ args.version }}.debian.tar.xz + 461ea7fb0137d8b726787561ea1a8966 6031 utils optional {{ args.name }}_{{ args.version }}_amd64.buildinfo diff --git a/tests/functional/packages/test_package.py b/tests/functional/packages/test_package.py index 55be8ad0a9aaae31d6c00a670c1bc9d5cb5fcd89..a2d0999f592ab12a8bebf0d4627c0ac576a274bd 100644 --- a/tests/functional/packages/test_package.py +++ b/tests/functional/packages/test_package.py @@ -44,21 +44,31 @@ class TestPackageController(TestController): self._remove_example_package() self._remove_example_user() - def _test_bad_method(self, action): + def _test_bad_method(self, action, args=None): + if not args: + args = [] + self.client.post(reverse('login'), self._AUTHDATA) response = self.client.get(reverse( - action, args=['testpackage'])) + action, args=['testpackage'] + args)) self.assertEquals(response.status_code, 405) - def _test_no_auth(self, action, redirect_login=True): + def _test_no_auth(self, action, redirect_login=True, args=None): + if not args: + args = [] + response = self.client.post(reverse( - action, args=['testpackage'])) + action, args=['testpackage'] + args)) self.assertEquals(response.status_code, 302) + if (redirect_login): self.assertIn(reverse('login'), response.url) - def _test_not_owned_package(self, action, method='get'): + def _test_not_owned_package(self, action, method='get', args=None): + if not args: + args = [] + user = User.objects.create_user(name='Another user', email='another@example.com', password='password') @@ -68,7 +78,8 @@ class TestPackageController(TestController): 'password': 'password', 'commit': 'submit'}) - response = self.client.post(reverse(action, args=['testpackage'])) + response = self.client.post(reverse(action, + args=['testpackage'] + args)) self.assertEquals(response.status_code, 403) user.delete() @@ -157,12 +168,15 @@ class TestPackageController(TestController): def test_delete_no_auth(self): self._test_no_auth('delete_package') + self._test_no_auth('delete_upload', args=['1']) def test_delete_not_owned_package(self): self._test_not_owned_package('delete_package') + self._test_not_owned_package('delete_upload', args=['1']) def test_delete_bad_method(self): self._test_bad_method('delete_package') + self._test_bad_method('delete_upload', args=['1']) # def test_delete_gitstorage_utf8(self): # gitdir = join(pylons.test.pylonsapp.config['debexpo.repository'], @@ -185,6 +199,28 @@ class TestPackageController(TestController): self.assertRaises(Package.DoesNotExist, Package.objects.get, name='testpackage') + def test_delete_upload_successful(self): + self.client.post(reverse('login'), self._AUTHDATA) + + response = self.client.post(reverse( + 'delete_upload', args=['testpackage', '1'])) + + self.assertEquals(response.status_code, 302) + self.assertTrue(reverse('packages_my'), response.url) + self.assertRaises(Package.DoesNotExist, Package.objects.get, + name='testpackage') + + def test_delete_single_upload_successful(self): + self._setup_example_package() + self.client.post(reverse('login'), self._AUTHDATA) + + response = self.client.post(reverse( + 'delete_upload', args=['testpackage', '1'])) + + self.assertEquals(response.status_code, 302) + self.assertTrue(reverse('package', args=['testpackage']), response.url) + Package.objects.get(name='testpackage') + def test_comment_no_auth(self): self._test_no_auth('comment_package') diff --git a/tests/functional/packages/test_removeolduploads.py b/tests/functional/packages/test_removeolduploads.py new file mode 100644 index 0000000000000000000000000000000000000000..d3871fd21f694984e31697b3c79a235bbe45480d --- /dev/null +++ b/tests/functional/packages/test_removeolduploads.py @@ -0,0 +1,263 @@ +# test_removeolduploads.py - Test the removeolduploads cronjob +# +# This file is part of debexpo +# https://salsa.debian.org/mentors.debian.net-team/debexpo +# +# Copyright © 2019-2020 Baptiste BEAUPLAT +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +import datetime +import logging +from email import message_from_string +from http.server import BaseHTTPRequestHandler + +# from debexpo.lib.email import Email +from debexpo.accounts.models import User +from debexpo.packages.models import Package, PackageUpload, Distribution, \ + Component +from tests import TestController, TestingHTTPServer +from debexpo.packages.tasks import remove_old_uploads, remove_uploaded_packages +from debexpo.tools.email import Email + +log = logging.getLogger(__name__) +PACKAGE_IN_NEW = '''Source: tmux +Version: 1.0.0 +Distribution: unstable + +Source: Missing-other-fields''' + + +class TestCronjobRemoveOldUploads(TestController): + def setUp(self): + self._setup_example_user() + self.state = [] + + def tearDown(self): + Package.objects.all().delete() + self._remove_example_user() + + def _setup_packages(self, include_expired=False): + packages = [ + # (package, version, distrib, expired), + ('htop', '1.0.0', 'unstable', False), + ('htop', '0.9.0', 'unstable', False), + ('htop', '0.8.0', 'buster-backports', False), + ('tmux', '1.0.0', 'unstable', False), + ('tmux', '1.0.0', 'UNRELEASED', False), + ('zsh', '1.0.0', 'unstable', False), + ('zsh', '1.0.0', 'unstable', False), + ('hello', '1.0.0', 'unstable', True), + ('hello', '0.9.0', 'unstable', True), + ] + + for (package, version, distrib, expired) in packages: + if not expired or (expired and include_expired): + self._create_package(package, version, distrib, expired) + + if not expired: + self.state.append((package, version, distrib)) + + def _create_package(self, name, version_number, distrib, is_expired): + date = datetime.datetime.now() + user = User.objects.first() + + try: + package = Package.objects.get(name=name) + except Package.DoesNotExist: + package = Package(name=name) + + if is_expired: + date -= datetime.timedelta(days=365) + + package.full_clean() + package.save() + + upload = PackageUpload( + package=package, + version=version_number, + uploader=user, + changes='changes', + distribution=Distribution.objects.get(name=distrib), + component=Component.objects.get_or_create(name='tools')[0]) + + upload.full_clean() + upload.save() + + PackageUpload.objects.filter(id=upload.id).update( + uploaded=date.replace(tzinfo=datetime.timezone.utc)) + + def _assert_cronjob_success(self): + packages = Package.objects.all() + count = 0 + + # Each package exists in state + for package in packages: + for upload in package.packageupload_set.all(): + log.info('validating {}-{}/{}'.format(package.name, + upload.version, + upload.distribution.name)) + self.assertIn((package.name, + upload.version, + upload.distribution.name), self.state) + count += 1 + + # Each state exists in package + self.assertEquals(len(self.state), count) + + def _expect_package_removal(self, package_to_remove): + new_state = [] + + for package in self.state: + if package not in package_to_remove: + new_state.append(package) + + self.state = new_state + + def remove_upload_accepted(self, uploaded, down=False, garbage=False): + removed_packages = ( + ('tmux', '1.0.0', 'unstable'), + ('tmux', '1.0.0', 'UNRELEASED'), + ) + + self._expect_package_removal(removed_packages) + with TestingHTTPServer(FTPMasterPackageInNewHTTPHandler) as httpd: + with self.settings( + FTP_MASTER_NEW_PACKAGES_URL='http://localhost:' + f'{httpd.port}'): + remove_uploaded_packages(FakeNNTPClient(uploaded, down, + garbage)) + + def test_package_in_new_server_error(self): + with TestingHTTPServer(FTPMasterPackageInNewErrorHTTPHandler) as httpd: + with self.settings( + FTP_MASTER_NEW_PACKAGES_URL='http://localhost:' + f'{httpd.port}'): + remove_uploaded_packages(FakeNNTPClient([])) + self._assert_cronjob_success() + + def test_remove_uploads_noop_no_packages(self): + remove_old_uploads() + self._assert_cronjob_success() + + def test_remove_uploads_server_down(self): + self.remove_upload_accepted([], True) + self._assert_cronjob_success() + + def test_remove_uploads_server_garbage(self): + self.remove_upload_accepted([], False, True) + self._assert_cronjob_success() + + def test_remove_uploads_inexistant_package(self): + self.remove_upload_accepted([('inexistant_package', '1.0.0', + 'unstable')]) + self._assert_cronjob_success() + + def test_remove_uploads_noop_with_packages(self): + self._setup_packages() + remove_old_uploads() + self._assert_cronjob_success() + + def test_remove_uploads_expired(self): + self._setup_packages(include_expired=True) + remove_old_uploads() + self._assert_cronjob_success() + + def test_remove_uploads_keep_other_dists(self): + removed_packages = [ + ('htop', '1.0.0', 'unstable'), + ('htop', '0.9.0', 'unstable'), + ] + + self._setup_packages() + self.remove_upload_accepted([('htop', '1.0.0', 'unstable')]) + self._expect_package_removal(removed_packages) + self._assert_cronjob_success() + + def test_remove_uploads_same_version(self): + removed_packages = [ + ('zsh', '1.0.0', 'unstable'), + ] + + self._setup_packages() + self.remove_upload_accepted([('zsh', '1.0.0', 'unstable')]) + self._expect_package_removal(removed_packages) + self._assert_cronjob_success() + + def test_remove_uploads_keep_newer(self): + removed_packages = [ + ('htop', '0.9.0', 'unstable'), + ] + + self._setup_packages() + self.remove_upload_accepted([('htop', '0.9.0', 'unstable')]) + self._expect_package_removal(removed_packages) + self._assert_cronjob_success() + + +class FTPMasterPackageInNewHTTPHandler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200, 'OK') + self.end_headers() + self.wfile.write(bytes(PACKAGE_IN_NEW, 'UTF-8')) + + +class FTPMasterPackageInNewErrorHTTPHandler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(500, 'Internal Server Error') + self.end_headers() + + +class FakeNNTPClient(): + def __init__(self, uploads, down=False, garbage=False): + self.uploads = uploads + self.iter = 0 + self.down = down + self.garbage = garbage + + def connect_to_server(self): + return not self.down + + def disconnect_from_server(self): + return True + + def unread_messages(self, name, last): + self.iter += 1 + + if self.iter == 1: + if self.garbage: + for item in [None, + message_from_string('Subject: test\n\ntest')]: + yield item + else: + for upload in self.uploads: + yield self._build_nntp_response(upload) + + def _build_nntp_response(self, upload): + email = Email('test-upload-accepted.html') + body = message_from_string(email._render_content([], **{ + 'name': upload[0], + 'version': upload[1], + 'distrib': upload[2], + })) + body['X-Debexpo-Message-Number'] = 42 + + return body