From dfba8755d7929abf7c77c969400557045672a6e9 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Mon, 10 Feb 2020 09:06:48 +0000 Subject: [PATCH 01/30] Re-adds the RemoveOldUploads task (cronjob and tests) --- .../cronjobs/removeolduploads.py => debexpo/packages/tasks.py | 0 .../packages/templates/email-upload-removed.html | 0 .../functional/packages}/test_removeolduploads.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename old/debexpo/cronjobs/removeolduploads.py => debexpo/packages/tasks.py (100%) rename old/debexpo/templates/email/upload_removed_from_expo.mako => debexpo/packages/templates/email-upload-removed.html (100%) rename {old/debexpo/tests/cronjobs => tests/functional/packages}/test_removeolduploads.py (100%) diff --git a/old/debexpo/cronjobs/removeolduploads.py b/debexpo/packages/tasks.py similarity index 100% rename from old/debexpo/cronjobs/removeolduploads.py rename to debexpo/packages/tasks.py diff --git a/old/debexpo/templates/email/upload_removed_from_expo.mako b/debexpo/packages/templates/email-upload-removed.html similarity index 100% rename from old/debexpo/templates/email/upload_removed_from_expo.mako rename to debexpo/packages/templates/email-upload-removed.html diff --git a/old/debexpo/tests/cronjobs/test_removeolduploads.py b/tests/functional/packages/test_removeolduploads.py similarity index 100% rename from old/debexpo/tests/cronjobs/test_removeolduploads.py rename to tests/functional/packages/test_removeolduploads.py -- GitLab From f27b8c6772ee652e934977d4934ce7d88d28114d Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Tue, 30 Jun 2020 22:29:57 +0200 Subject: [PATCH 02/30] Add remove_bugs method to BugManager --- debexpo/bugs/models.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/debexpo/bugs/models.py b/debexpo/bugs/models.py index cbf25a25..cdb38944 100644 --- a/debexpo/bugs/models.py +++ b/debexpo/bugs/models.py @@ -130,6 +130,10 @@ class BugManager(models.Manager): return BugType.bug + def remove_bugs(self, package): + self.get_queryset().filter(sources__name=package, bugtype=BugType.bug) \ + .delete() + def _guess_packages(self, source, subject): packages = [] -- GitLab From 327e71ca7da728fb4dc673abf69575004fc6b3a1 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Tue, 30 Jun 2020 22:30:19 +0200 Subject: [PATCH 03/30] Add remove method to GitStorage --- debexpo/tools/gitstorage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/debexpo/tools/gitstorage.py b/debexpo/tools/gitstorage.py index 564747f7..9a777bf3 100644 --- a/debexpo/tools/gitstorage.py +++ b/debexpo/tools/gitstorage.py @@ -55,8 +55,9 @@ class GitStorage(): return ref - # def remove(self, source): - # pass + def remove(self): + if isdir(self.repository): + rmtree(self.repository) # def diff(self, upload_from, upload_to): # pass -- GitLab From bacfe6da15e0c2ae9250c836c64bd777ecf1a4db Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Tue, 30 Jun 2020 22:30:48 +0200 Subject: [PATCH 04/30] Remove bugs and gitstorage on user package deletion --- debexpo/packages/views.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/debexpo/packages/views.py b/debexpo/packages/views.py index f956b0a0..123ec164 100644 --- a/debexpo/packages/views.py +++ b/debexpo/packages/views.py @@ -44,6 +44,8 @@ from django.contrib.auth.decorators import login_required from debexpo.packages.models import PackageUpload, Package, SourcePackage from debexpo.comments.forms import CommentForm from debexpo.repository.tasks import remove_from_repository +from debexpo.tools.gitstorage import GitStorage +from debexpo.bugs.models import Bug log = logging.getLogger(__name__) @@ -164,7 +166,14 @@ def delete_package(request, name): package.delete() log.info('Package deleted: {}'.format(name)) + git_storage_path = getattr(settings, 'GIT_STORAGE', None) + + if git_storage_path: + git_storage = GitStorage(git_storage_path, name) + git_storage.remove() + remove_from_repository.delay(name) + Bug.objects.remove_bugs(package) return HttpResponseRedirect(reverse('packages_my')) -- GitLab From be13269989ae705057299662d006fc2f516b833a Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Tue, 30 Jun 2020 22:32:04 +0200 Subject: [PATCH 05/30] Add settings for expired package cleanup --- debexpo/settings/common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/debexpo/settings/common.py b/debexpo/settings/common.py index 119cc238..c849b304 100644 --- a/debexpo/settings/common.py +++ b/debexpo/settings/common.py @@ -169,6 +169,7 @@ CELERY_BEAT_SCHEDULER = 'django' # Tasks beats TASK_IMPORTER_BEAT = 60 * 15 # Every 15 minutes TASK_CLEANUPACCOUNTS_BEAT = 60 * 60 +TASK_OLD_UPLOADS_BEAT = 10 * 60 # Account registration expiration REGISTRATION_EXPIRATION_DAYS = 7 @@ -197,3 +198,6 @@ BUGS_REPORT_NOT_OPEN = True # Debian tracker access TRACKER_URL = 'https://tracker.debian.org' + +# Cleanup package older than NN weeks +MAX_AGE_UPLOAD_WEEKS = 20 -- GitLab From b71b1fd7cf749be3b9a19601427f1d1c7e34264a Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Tue, 30 Jun 2020 22:32:34 +0200 Subject: [PATCH 06/30] Convert expired package cleanup code and template to django --- debexpo/packages/tasks.py | 348 ++++++++++-------- .../templates/email-upload-removed.html | 10 +- 2 files changed, 199 insertions(+), 159 deletions(-) diff --git a/debexpo/packages/tasks.py b/debexpo/packages/tasks.py index 98aafac7..9c3e8598 100644 --- a/debexpo/packages/tasks.py +++ b/debexpo/packages/tasks.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- -# -# removeolduploads.py -- remove old and uploaded packages from Debexpo +# tasks.py -- task to remove old and uploaded packages from Debexpo # # This file is part of debexpo - # https://salsa.debian.org/mentors.debian.net-team/debexpo # # Copyright © 2011 Arno Töll +# Copyright © 2020 Baptiste BEAUPLAT # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -28,153 +27,196 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -""" -Import RFS comments from debian-mentors -""" -__author__ = 'Arno Töll' -__copyright__ = 'Copyright © 2011 Arno Töll' -__license__ = 'MIT' - -from debexpo.cronjobs import BaseCronjob - -from debexpo.lib.email import Email -from debexpo.lib.filesystem import CheckFiles -from debexpo.controllers.package import PackageController -from debexpo.controllers.packages import PackagesController -from debexpo.model.users import User -from debexpo.model.data_store import DataStore -from debexpo.model.package_versions import PackageVersion -from debexpo.model import meta -from debian import deb822 - -import socket -import apt_pkg -import datetime - -__namespace__ = '_remove_uploads_' - - -class RemoveOldUploads(BaseCronjob): - - def _remove_package(self, package, version, reason): - user = meta.session.query(User).filter_by(id=package.user_id).one() - if user: - self.mailer.send([user.email, ], - package=package.name, - version=version.version, - reason=reason) - - CheckFiles().delete_files_for_packageversion(version) - meta.session.delete(version) - - if (meta.session.query(PackageVersion).filter_by( - package_id=package.id).count() == 0): - CheckFiles().delete_files_for_package(package) - meta.session.delete(package) - - meta.session.commit() - - def _process_changes(self, mail): - if mail.is_multipart(): - self.log.debug("Changes message is multipart?!") - return - changes = mail.get_payload(decode=True) - try: - changes = deb822.Changes(changes) - except Exception: - self.log.error('Could not open changes file; skipping mail "%s"' % - (mail['subject'])) - return - - if 'Source' not in changes: - # self.log.debug('Changes file "%s" seems incomplete' % - # (mail['subject'])) - return - - package = self.pkg_controller._get_package(changes['Source'], - from_controller=False) - if package is not None: - for pv in package.package_versions: - if pv.distribution == changes['Distribution']: - if (apt_pkg.version_compare(changes['Version'], pv.version) - == 0): - self.log.debug("Package %s was uploaded to Debian - " - "removing it from Expo" % - (changes['Source'])) - self._remove_package(package, pv, - "Package was uploaded to " - "official Debian repositories") - if (apt_pkg.version_compare(changes['Version'], pv.version) - > 0): - self.log.debug("More recent package %s was uploaded to " - "Debian - removing it from Expo" % - (changes['Source'])) - self._remove_package(package, pv, - "A more recent package was " - "uploaded to official Debian " - "repositories") - else: - # self.log.debug("Package %s was not uploaded to Expo before - " - # "ignoring it" % (changes['Source'])) - pass - - def _remove_uploaded_packages(self): - - if self.mailer.connection_established(): - lists = meta.session.query(DataStore) \ - .filter(DataStore.namespace == __namespace__) \ - .all() - for list_name in lists: - for message in self.mailer.unread_messages(list_name.code, - list_name.value): - self._process_changes(message) - list_name.value = message['X-Debexpo-Message-Number'] - self.log.debug("Processed all messages up to #%s on %s" % - (list_name.value, list_name.code)) - meta.session.merge(list_name) - meta.session.commit() - self.mailer.disconnect_from_server() - - def _remove_old_packages(self): - now = datetime.datetime.now() - for package in self.pkgs_controller._get_packages(): - if (now - package.package_versions[-1].uploaded) > \ - datetime.timedelta(weeks=20): - self.log.debug("Removing package %s - uploaded on %s" % - (package.name, - package.package_versions[-1].uploaded)) - for pv in package.package_versions: - self._remove_package(package, pv, - "Your package found no sponsor for " - "20 weeks") - - def setup(self): - self.mailer = Email('upload_removed_from_expo') - self.mailer.connect_to_server() - self.pkg_controller = PackageController() - self.pkgs_controller = PackagesController() - apt_pkg.init_system() - self.last_cruft_run = datetime.datetime(year=1970, month=1, day=1) - self.log.debug("%s loaded successfully" % (__name__)) - - def teardown(self): - self.mailer.disconnect_from_server() - - def invoke(self): - try: - self._remove_uploaded_packages() - except socket.error as e: - # better luck next time - self.log.debug("Socket error %s: skipping removals his time" % (e)) - pass - - # We don't need to run our garbage collection of old cruft that often - # It's ok if we purge old packages once a day. - if (datetime.datetime.now() - self.last_cruft_run) >= \ - datetime.timedelta(hours=24): - self.last_cruft_run = datetime.datetime.now() - self._remove_old_packages() - - -cronjob = RemoveOldUploads -schedule = datetime.timedelta(minutes=10) +# from celery.task import PeriodicTask +from celery.decorators import periodic_task +from datetime import timedelta, datetime, timezone + +from django.conf import settings +from django.db.models import Max + +from debexpo.packages.models import Package, PackageUpload +from debexpo.repository.models import Repository +from debexpo.tools.email import Email +from debexpo.bugs.models import Bug +from debexpo.tools.gitstorage import GitStorage +# from debexpo.comments.models import PackageSubscription +# from debexpo.accounts.models import User + + +def remove_uploads(uploads): + removals = set() + repository = Repository(settings.REPOSITORY) + git_storage_path = getattr(settings, 'GIT_STORAGE', None) + + for upload in uploads: + remove_version = not PackageUpload.objects.filter( + package=upload.package, version=upload.version) \ + .exclude(id=upload.id) + remove_package = not PackageUpload.objects.filter( + package=upload.package).exclude(id=upload.id) + package = upload.package.name + distribution = upload.distribution.name + uploader = upload.uploader + + if remove_version: + repository.remove(package, upload.version) + if remove_package: + if git_storage_path: + git_storage = GitStorage(git_storage_path, package) + git_storage.remove() + + Bug.objects.remove_bugs(package) + + upload.delete() + + if remove_package: + Package.objects.get(name=package).delete() + + removals.add((package, distribution, uploader)) + + repository.update() + + return removals + + +@periodic_task(run_every=settings.TASK_OLD_UPLOADS_BEAT) +def remove_old_uploads(): + expiration_date = datetime.now(timezone.utc) - \ + timedelta(weeks=settings.MAX_AGE_UPLOAD_WEEKS) + packages = Package.objects \ + .values('name', 'packageupload__distribution') \ + .annotate(latest_upload=Max('packageupload__uploaded')) \ + .filter(latest_upload__lt=expiration_date) + + uploads = set() + from logging import getLogger + log = getLogger(__name__) + + for package in packages: + uploads.update(PackageUpload.objects.filter( + package__name=package['name'], + distribution=package['packageupload__distribution'])) + + log.debug(f'uploads to remove: {uploads}') + + removals = remove_uploads(uploads) + + for package, distribution, uploader in removals: + email = Email('email-upload-removed.html') + email.send(f'{package} has been removed from ' + f'{distribution} on {settings.SITE_NAME}', + recipients=[uploader.email], package=package, + distribution=distribution, + reason="Your package found no sponsor for " + "20 weeks") + + +# class RemoveAcceptedUploads(PeriodicTask): +# run_every = settings.TASK_ACCEPT_UPLOADS_BEAT +# +# def _process_changes(self, mail): +# if mail.is_multipart(): +# self.log.debug("Changes message is multipart?!") +# return +# changes = mail.get_payload(decode=True) +# try: +# changes = deb822.Changes(changes) +# except Exception: +# self.log.error('Could not open changes file; skipping mail "%s"' % +# (mail['subject'])) +# return +# +# if 'Source' not in changes: +# # self.log.debug('Changes file "%s" seems incomplete' % +# # (mail['subject'])) +# return +# +# package = self.pkg_controller._get_package(changes['Source'], +# from_controller=False) +# if package is not None: +# for pv in package.package_versions: +# if pv.distribution == changes['Distribution']: +# if (apt_pkg.version_compare(changes['Version'], +# pv.version) +# == 0): +# self.log.debug("Package %s was uploaded to Debian - " +# "removing it from Expo" % +# (changes['Source'])) +# self._remove_package(package, pv, +# "Package was uploaded to " +# "official Debian repositories") +# if (apt_pkg.version_compare(changes['Version'], +# pv.version) +# > 0): +# self.log.debug("More recent package %s was uploaded " +# "to Debian - removing it from Expo" % +# (changes['Source'])) +# self._remove_package(package, pv, +# "A more recent package was " +# "uploaded to official Debian " +# "repositories") +# else: +# # self.log.debug("Package %s was not uploaded to Expo before - " +# # "ignoring it" % (changes['Source'])) +# pass +# +# def _remove_uploaded_packages(self): +# +# if self.mailer.connection_established(): +# lists = meta.session.query(DataStore) \ +# .filter(DataStore.namespace == __namespace__) \ +# .all() +# for list_name in lists: +# for message in self.mailer.unread_messages(list_name.code, +# list_name.value): +# self._process_changes(message) +# list_name.value = message['X-Debexpo-Message-Number'] +# self.log.debug("Processed all messages up to #%s on %s" % +# (list_name.value, list_name.code)) +# meta.session.merge(list_name) +# meta.session.commit() +# self.mailer.disconnect_from_server() +# +# def _remove_old_packages(self): +# now = datetime.datetime.now() +# for package in self.pkgs_controller._get_packages(): +# if (now - package.package_versions[-1].uploaded) > \ +# datetime.timedelta(weeks=20): +# self.log.debug("Removing package %s - uploaded on %s" % +# (package.name, +# package.package_versions[-1].uploaded)) +# for pv in package.package_versions: +# self._remove_package(package, pv, +# "Your package found no sponsor for " +# "20 weeks") +# +# def setup(self): +# self.mailer = Email('upload_removed_from_expo') +# self.mailer.connect_to_server() +# self.pkg_controller = PackageController() +# self.pkgs_controller = PackagesController() +# apt_pkg.init_system() +# self.log.debug("%s loaded successfully" % (__name__)) +# +# def teardown(self): +# self.mailer.disconnect_from_server() +# +# def invoke(self): +# try: +# self._remove_uploaded_packages() +# except socket.error as e: +# # better luck next time +# self.log.debug("Socket error %s: skipping removals his time" +# % (e)) +# pass +# +# # We don't need to run our garbage collection of old cruft that often +# # It's ok if we purge old packages once a day. +# if (datetime.datetime.now() - self.last_cruft_run) >= \ +# datetime.timedelta(hours=24): +# self.last_cruft_run = datetime.datetime.now() +# self._remove_old_packages() +# +# +# cronjob = RemoveOldUploads +# schedule = datetime.timedelta(minutes=10) diff --git a/debexpo/packages/templates/email-upload-removed.html b/debexpo/packages/templates/email-upload-removed.html index 71120140..20eea15a 100644 --- a/debexpo/packages/templates/email-upload-removed.html +++ b/debexpo/packages/templates/email-upload-removed.html @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- -<%inherit file="/base.mako"/>To: ${ c.to } -Subject: ${ _('%s package has been removed from %s' % (c.package, c.config['debexpo.sitetitle'])) } +{% extends "email-base.html" %}{% block content %}Hi, -${ _('Your package %s %s has been removed from %s for the following reason:' % (c.package, c.version, c.config['debexpo.sitename'])) } +Your package {{ args.package }} has been removed from {{ args.distribution }} on {{ args.settings.SITE_NAME }} for the following reason: -${ c.reason } +{{ args.reason }} -${ _('Thanks,') } +Thanks,{% endblock %} -- GitLab From 8c0cd64ad327400489abc1dbe3c6d8ad46aceccf Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Tue, 30 Jun 2020 22:32:45 +0200 Subject: [PATCH 07/30] Convert expired package cleanup tests to django --- .../packages/test_removeolduploads.py | 206 ++++++------------ 1 file changed, 70 insertions(+), 136 deletions(-) diff --git a/tests/functional/packages/test_removeolduploads.py b/tests/functional/packages/test_removeolduploads.py index d1a6b8e8..67a881dc 100644 --- a/tests/functional/packages/test_removeolduploads.py +++ b/tests/functional/packages/test_removeolduploads.py @@ -1,11 +1,9 @@ -# -*- coding: utf-8 -*- -# # test_removeolduploads.py - Test the removeolduploads cronjob # # This file is part of debexpo # https://salsa.debian.org/mentors.debian.net-team/debexpo # -# Copyright © 2019 Baptiste BEAUPLAT +# Copyright © 2019-2020 Baptiste BEAUPLAT # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -27,71 +25,27 @@ # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. - -__author__ = 'Baptiste BEAUPLAT' -__copyright__ = 'Copyright © 2019 Baptiste BEAUPLAT' -__license__ = 'MIT' - -from mako.lookup import TemplateLookup -from mako.template import Template -from os.path import join, dirname -from pylons.test import pylonsapp - -import apt_pkg import datetime -import email.parser import logging -import pylons -import sqlalchemy.orm.exc as orm_exc -from debexpo.controllers.package import PackageController -from debexpo.controllers.packages import PackagesController -from debexpo.lib.email import Email -from debexpo.model.users import User -from debexpo.model.packages import Package -from debexpo.model.package_versions import PackageVersion -from debexpo.tests.cronjobs import TestCronjob +# from debexpo.lib.email import Email +from debexpo.accounts.models import User +from debexpo.packages.models import Package, PackageUpload, Distribution, \ + Component +from tests import TestController +from debexpo.packages.tasks import remove_old_uploads log = logging.getLogger(__name__) -class TestCronjobRemoveOldUploads(TestCronjob): +class TestCronjobRemoveOldUploads(TestController): def setUp(self): - self.nntp_server = pylonsapp.config['debexpo.nntp_server'] - pylonsapp.config['debexpo.nntp_server'] = None - - self._setup_plugin('removeolduploads', run_setup=False) - self._finish_setup_plugin() - self._create_user() + self._setup_example_user() self.state = [] def tearDown(self): - pylonsapp.config['debexpo.nntp_server'] = self.nntp_server - self._remove_packages() - self._remove_user() - - # This is needed since we do not call the plugin setup (to avoid the NNTP - # connection) - def _finish_setup_plugin(self): - self.plugin.mailer = Email('upload_removed_from_expo') - self.plugin.pkg_controller = PackageController() - self.plugin.pkgs_controller = PackagesController() - apt_pkg.init_system() - self.plugin.last_cruft_run = datetime.datetime(year=1970, - month=1, - day=1) - - def _create_user(self): - user = User(name='Test user', - email='test.user@example.org', - password='password', - lastlogin=datetime.datetime.now()) - self.db.add(user) - self.db.commit() - - def _remove_user(self): - user = self.db.query(User).one() - self.db.delete(user) + Package.objects.all().delete() + self._remove_example_user() def _setup_packages(self, include_expired=False): packages = [ @@ -114,53 +68,48 @@ class TestCronjobRemoveOldUploads(TestCronjob): if not expired: self.state.append((package, version, distrib)) - self.db.commit() - def _create_package(self, name, version_number, distrib, is_expired): date = datetime.datetime.now() - user = self.db.query(User).one() + user = User.objects.first() try: - package = self.db.query(Package).filter_by(name=name).one() - except orm_exc.NoResultFound: - package = Package(name=name, - user=user) + package = Package.objects.get(name=name) + except Package.DoesNotExist: + package = Package(name=name) if is_expired: date -= datetime.timedelta(days=365) - version = PackageVersion(package=package, - version=version_number, - maintainer='vtime@example.org', - section='main', - distribution=distrib, - qa_status=0, - component='tools', - uploaded=date) + package.full_clean() + package.save() - self.db.add(version) + upload = PackageUpload( + package=package, + version=version_number, + uploader=user, + changes='changes', + distribution=Distribution.objects.get(name=distrib), + component=Component.objects.get_or_create(name='tools')[0]) - def _remove_packages(self): - packages = self.db.query(Package) + upload.full_clean() + upload.save() - for package in packages: - self.db.delete(package) - - self.db.commit() + PackageUpload.objects.filter(id=upload.id).update( + uploaded=date.replace(tzinfo=datetime.timezone.utc)) def _assert_cronjob_success(self): - packages = self.db.query(Package) + packages = Package.objects.all() count = 0 # Each package exists in state for package in packages: - for version in package.package_versions: - log.debug('validating {}-{}/{}'.format(package.name, - version.version, - version.distribution)) - self.assertTrue((package.name, - version.version, - version.distribution) in self.state) + for upload in package.packageupload_set.all(): + log.info('validating {}-{}/{}'.format(package.name, + upload.version, + upload.distribution.name)) + self.assertIn((package.name, + upload.version, + upload.distribution.name), self.state) count += 1 # Each state exists in package @@ -175,31 +124,16 @@ class TestCronjobRemoveOldUploads(TestCronjob): self.state = new_state - def _build_email(self, package): - (name, version, distrib) = package - c = {'name': name, 'version': version, 'distrib': distrib} - template_file = join(dirname(__file__), 'email/accept.mako') - lookup = TemplateLookup(directories=[dirname(template_file)]) - template = Template( - filename=template_file, lookup=lookup, - module_directory=pylons.config['app_conf']['cache_dir']) - - return email.message_from_string(template.render_unicode(c=c)) - def _invoke_plugin(self, uploaded): - for package in uploaded: - message = self._build_email(package) - self.plugin._process_changes(message) - - self.plugin._remove_old_packages() + remove_old_uploads() def test_remove_uploads_noop_no_packages(self): self._invoke_plugin([]) self._assert_cronjob_success() - def test_remove_uploads_inexistant_package(self): - self._invoke_plugin([('inexistant_package', '1.0.0', 'unstable')]) - self._assert_cronjob_success() + # def test_remove_uploads_inexistant_package(self): + # self._invoke_plugin([('inexistant_package', '1.0.0', 'unstable')]) + # self._assert_cronjob_success() def test_remove_uploads_noop_with_packages(self): self._setup_packages() @@ -211,33 +145,33 @@ class TestCronjobRemoveOldUploads(TestCronjob): self._invoke_plugin([]) self._assert_cronjob_success() - def test_remove_uploads_keep_other_dists(self): - removed_packages = [ - ('htop', '1.0.0', 'unstable'), - ('htop', '0.9.0', 'unstable'), - ] - - self._setup_packages() - self._invoke_plugin([('htop', '1.0.0', 'unstable')]) - self._expect_package_removal(removed_packages) - self._assert_cronjob_success() - - def test_remove_uploads_same_version(self): - removed_packages = [ - ('zsh', '1.0.0', 'unstable'), - ] - - self._setup_packages() - self._invoke_plugin([('zsh', '1.0.0', 'unstable')]) - self._expect_package_removal(removed_packages) - self._assert_cronjob_success() - - def test_remove_uploads_keep_newer(self): - removed_packages = [ - ('htop', '0.9.0', 'unstable'), - ] - - self._setup_packages() - self._invoke_plugin([('htop', '0.9.0', 'unstable')]) - self._expect_package_removal(removed_packages) - self._assert_cronjob_success() + # def test_remove_uploads_keep_other_dists(self): + # removed_packages = [ + # ('htop', '1.0.0', 'unstable'), + # ('htop', '0.9.0', 'unstable'), + # ] + + # self._setup_packages() + # self._invoke_plugin([('htop', '1.0.0', 'unstable')]) + # self._expect_package_removal(removed_packages) + # self._assert_cronjob_success() + + # def test_remove_uploads_same_version(self): + # removed_packages = [ + # ('zsh', '1.0.0', 'unstable'), + # ] + + # self._setup_packages() + # self._invoke_plugin([('zsh', '1.0.0', 'unstable')]) + # self._expect_package_removal(removed_packages) + # self._assert_cronjob_success() + + # def test_remove_uploads_keep_newer(self): + # removed_packages = [ + # ('htop', '0.9.0', 'unstable'), + # ] + + # self._setup_packages() + # self._invoke_plugin([('htop', '0.9.0', 'unstable')]) + # self._expect_package_removal(removed_packages) + # self._assert_cronjob_success() -- GitLab From fe35674a9aa5fe2efbaaaca804cdd3d0207a28c3 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Tue, 30 Jun 2020 23:27:04 +0200 Subject: [PATCH 08/30] Add tests to assert repository cleanup after upload removal --- tests/functional/importer/__init__.py | 8 ++++++++ tests/functional/importer/test_importer.py | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/tests/functional/importer/__init__.py b/tests/functional/importer/__init__.py index ab312bf9..ced9190c 100644 --- a/tests/functional/importer/__init__.py +++ b/tests/functional/importer/__init__.py @@ -40,6 +40,7 @@ from django_redis.pool import ConnectionFactory from debexpo.importer.models import Importer, Spool from debexpo.packages.models import Package, PackageUpload +from debexpo.packages.tasks import remove_uploads from debexpo.accounts.models import User from debexpo.comments.models import PackageSubscription from debexpo.plugins.models import PluginResults @@ -140,6 +141,13 @@ class TestImporterController(TestController): result.append(join(root, name)) return result + def remove_package(self, package, version): + uploads = PackageUpload.objects.filter(package__name=package, + version=version) + + with self.settings(REPOSITORY=self.repository): + remove_uploads(uploads) + def import_source_package(self, package_dir, skip_gpg=False, skip_email=False, base_dir=None): source_package = TestSourcePackage(package_dir, base_dir) diff --git a/tests/functional/importer/test_importer.py b/tests/functional/importer/test_importer.py index e52b7569..4cb43e06 100644 --- a/tests/functional/importer/test_importer.py +++ b/tests/functional/importer/test_importer.py @@ -360,6 +360,9 @@ r1JREXlgQRuRdd5ZWSvIxKaKGVbYCw== self.assert_package_count('hello', '1.0-1', 2) self.assert_package_in_repo('hello', '1.0-1') + self.remove_package('hello', '1.0-1') + self._assert_no_leftover(str(join(self.repository, 'pool'))) + def test_import_package_htop_download_orig(self): self.import_package('orig-from-official') self.assert_importer_succeeded() @@ -388,6 +391,9 @@ r1JREXlgQRuRdd5ZWSvIxKaKGVbYCw== 'htop_2.2.0.orig.tar.gz.asc'): self.assert_file_in_repo(filename) + self.remove_package('htop', '2.2.0-1') + self._assert_no_leftover(str(join(self.repository, 'pool'))) + # Since we cannot really make the importer fail (it is not supposed to # happend), we test that the field method can report an error to admins and # optionnaly to uploader if available. -- GitLab From fa21e4c4f80609a1fdfb4acf6a91d52856ca9e6a Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Wed, 1 Jul 2020 20:44:34 +0200 Subject: [PATCH 09/30] Add and register upload_delete view method --- debexpo/packages/views.py | 23 +++++++++++++++++++++++ debexpo/urls.py | 4 +++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/debexpo/packages/views.py b/debexpo/packages/views.py index 123ec164..23b420a8 100644 --- a/debexpo/packages/views.py +++ b/debexpo/packages/views.py @@ -46,6 +46,7 @@ from debexpo.comments.forms import CommentForm from debexpo.repository.tasks import remove_from_repository from debexpo.tools.gitstorage import GitStorage from debexpo.bugs.models import Bug +from debexpo.packages.tasks import remove_uploads log = logging.getLogger(__name__) @@ -152,6 +153,28 @@ def package(request, name): }) +@login_required +def delete_upload(request, name, upload): + if request.method != 'POST': + return HttpResponseNotAllowed(['POST']) + + upload = get_object_or_404(PackageUpload, id=upload) + package = upload.package.name + + if (request.user != upload.uploader and not + request.user.is_superuser): + return HttpResponseForbidden() + + remove_uploads([upload]) + + try: + Package.objects.get(name=package) + except Package.DoesNotExist: + return HttpResponseRedirect(reverse('packages_my')) + else: + return HttpResponseRedirect(reverse('package', args=[package])) + + @login_required def delete_package(request, name): if request.method != 'POST': diff --git a/debexpo/urls.py b/debexpo/urls.py index c08d6ddf..59d8224a 100644 --- a/debexpo/urls.py +++ b/debexpo/urls.py @@ -40,7 +40,7 @@ from debexpo.base.views import index, contact, intro_reviewers, \ from debexpo.accounts.views import register, profile from debexpo.accounts.forms import PasswordResetForm from debexpo.packages.views import package, packages, packages_my, \ - PackagesFeed, sponsor_package, delete_package + PackagesFeed, sponsor_package, delete_package, delete_upload from debexpo.comments.views import subscribe, unsubscribe, subscriptions, \ comment from debexpo.importer.views import upload @@ -113,6 +113,8 @@ urlpatterns = [ url(r'^packages/$', packages, name='packages'), + url(r'^package/(?P.+)/delete/(?P[0-9]+)/$', delete_upload, + name='delete_upload'), url(r'^package/(?P.+)/delete/$', delete_package, name='delete_package'), url(r'^package/(?P.+)/sponsor/$', sponsor_package, -- GitLab From 8ee39406922f311054cfc3c4618b0034fd4c4535 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Wed, 1 Jul 2020 20:44:53 +0200 Subject: [PATCH 10/30] Add trash and key PNG icons (16x16), colored with the debian red. Source: https://github.com/feathericons/feather (trash-2.svg and key.svg) License: MIT Copyright: (c) 2013-2017 Cole Bemi --- debexpo/base/static/img/key.png | Bin 0 -> 539 bytes debexpo/base/static/img/trash.png | Bin 0 -> 564 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 debexpo/base/static/img/key.png create mode 100644 debexpo/base/static/img/trash.png diff --git a/debexpo/base/static/img/key.png b/debexpo/base/static/img/key.png new file mode 100644 index 0000000000000000000000000000000000000000..fabe3b72e677e6ac256a517b3529b5898f901f0d GIT binary patch literal 539 zcmV+$0_6RPP)pF8FWQhbW?9;ba!ELWdL_~cP?peYja~^aAhuUa%Y?FJQ@H10jNnt zK~y-6wU9AuQ(+i|pYz=$-1Lis9SfCOs9GzugHWU{7IY8-m2gM9>L$915JYg*-=G~t zyXc?=Yn=oY#9Cvn2#S+BWRc!pgX#A=NP-lDM(|w^=Q-y+@DG>9$=+tXFb3cTH8*@R zV=g>bR5&*<;H^J3?JPV*v^KN$#eo0~pUebvtB4lHgfExQ0_bm(gSk~q>sj8edSAMN zaURp$9~<z(4)&CW-FV9{t_K!_{_l*Sz0DOM`QSpW-RqaQ#3&7s-~Fs|TE9Qbzo13;sk?2*C) z8J{Qf<@tVq0zgOK7YkNgF&Ex6eeD^6fWj>B-r-L7Gxr(*G)PL3x5Uhj2vd-ITFkOr dzYhCf?g!mUs-OCF?Ogx>002ovPDHLkV1gp#?2rHe literal 0 HcmV?d00001 diff --git a/debexpo/base/static/img/trash.png b/debexpo/base/static/img/trash.png new file mode 100644 index 0000000000000000000000000000000000000000..7f7e7f14a843839a6dc3b7b09943353bde43ab4a GIT binary patch literal 564 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`oCO|{#X#yh2s3WX6WN&PEETC=;Z}IX^cyHLrxhCo?%UuQ=68 z!PhfHN5MJ2pt2}4J)=ZHBUw|y$iUE87f6@`#dA^>oKkZ$i**!&^Gl18ff_PON(zdt z^!3v-OEOB6^pf*)_0v)lOEOZ6GL!T3ieVhjo=#Q<21XxG7sn8b-ldZb{h1sETKDg4 zC={2@T-m`Yz@415u=h{L=M6?5m(PrtIa!{uS4{2Lp#l*#5w)@#2b@&iL`*O~AggjD z{Oy_Q-=EUVm7hGA`@&((ex497hPaY7dN1UJ`>$=CRBmbTk9FZTCykTmkIb7}U>}#q zF6nNauXLp)a_PfcmkM$P(?naUF78UpYdR9+8uw@I3K_*;yhVKi>s31(<_B7aT%|A>Xt)r#MVo6}{{~8+gSh?(lH8Jglyk^EGFMWD>8fbXL-#uqbgeF5i-k_L_T{ ze9r#ZV%RzVQ;9UM!2P+iE`EKqhToykoLPe-is6Fpt~2KEy6UBKq!SIoe=>2nuv<%; z#O&S`5@!DLpWBB+91HRnol+}2we;3h7UOj5>SL;Lf6~vl?PQbdcgla3qyBbn&SPow qrHe|>mh@g%Je2Xur@T_Rp7DP`Y3q|WE=<74WbkzLb6Mw<&;$VZx7yYK literal 0 HcmV?d00001 -- GitLab From e94fb94d610dd2f081eb157a83dfb76acca939c0 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Wed, 1 Jul 2020 20:47:31 +0200 Subject: [PATCH 11/30] Add trash button next to the upload numbers on package page --- debexpo/packages/templates/package.html | 27 +++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/debexpo/packages/templates/package.html b/debexpo/packages/templates/package.html index 46fe07d9..97f6d838 100644 --- a/debexpo/packages/templates/package.html +++ b/debexpo/packages/templates/package.html @@ -1,4 +1,4 @@ -{% extends "base.html" %}{% load i18n %} +{% extends "base.html" %}{% load i18n %}{% load static %} {% block content %} {% blocktrans with name=package.name trimmed %} @@ -71,11 +71,11 @@ {% csrf_token %} {% if user.is_superuser and user not in package.get_uploaders %} - {% else %} - {% endif %} @@ -89,9 +89,28 @@ {% for upload in package.packageupload_set.all %} +

{% blocktrans with index=forloop.revcounter trimmed %} -

Upload #{{ index }}

+Upload #{{ index }} {% endblocktrans %} + {% if user in package.get_uploaders or user.is_superuser %} +
+ {% csrf_token %} + {% if user.is_superuser and user not in package.get_uploaders %} + + {% else %} + + {% endif %} +
+ +{% endif %} +

{% trans 'Information' %}

-- GitLab From ded7c5c58c314050de7a0a4911f32cfec321119d Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Wed, 1 Jul 2020 22:09:32 +0200 Subject: [PATCH 12/30] Add new nntp app --- debexpo/nntp/__init__.py | 0 debexpo/nntp/apps.py | 32 +++++++++++++++++++++++++++++ debexpo/nntp/migrations/__init__.py | 0 debexpo/settings/common.py | 1 + 4 files changed, 33 insertions(+) create mode 100644 debexpo/nntp/__init__.py create mode 100644 debexpo/nntp/apps.py create mode 100644 debexpo/nntp/migrations/__init__.py diff --git a/debexpo/nntp/__init__.py b/debexpo/nntp/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/debexpo/nntp/apps.py b/debexpo/nntp/apps.py new file mode 100644 index 00000000..9b623d4f --- /dev/null +++ b/debexpo/nntp/apps.py @@ -0,0 +1,32 @@ +# apps.py - app definition for nntp +# +# This file is part of debexpo +# https://salsa.debian.org/mentors.debian.net-team/debexpo +# +# Copyright © 2020 Baptiste BEAUPLAT +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +from django.apps import AppConfig + + +class NntpConfig(AppConfig): + name = 'nntp' diff --git a/debexpo/nntp/migrations/__init__.py b/debexpo/nntp/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/debexpo/settings/common.py b/debexpo/settings/common.py index c849b304..9b7145bf 100644 --- a/debexpo/settings/common.py +++ b/debexpo/settings/common.py @@ -53,6 +53,7 @@ INSTALLED_APPS = [ 'debexpo.repository', 'debexpo.plugins', 'debexpo.bugs', + 'debexpo.nntp', ] MIDDLEWARE = [ -- GitLab From ae1e6f9487c1136477dbdf53e50701ac08765281 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Wed, 1 Jul 2020 22:10:27 +0200 Subject: [PATCH 13/30] Re-add data_store model as nntp models --- old/debexpo/model/data_store.py => debexpo/nntp/models.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename old/debexpo/model/data_store.py => debexpo/nntp/models.py (100%) diff --git a/old/debexpo/model/data_store.py b/debexpo/nntp/models.py similarity index 100% rename from old/debexpo/model/data_store.py rename to debexpo/nntp/models.py -- GitLab From 4e0483ef2bfb45c72a3cdcae3aaa93fbd19e2f07 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Wed, 1 Jul 2020 22:27:36 +0200 Subject: [PATCH 14/30] Add NNTPFeed model --- debexpo/nntp/migrations/0001_initial.py | 81 +++++++++++++++++++++++++ debexpo/nntp/models.py | 54 +++-------------- 2 files changed, 91 insertions(+), 44 deletions(-) create mode 100644 debexpo/nntp/migrations/0001_initial.py diff --git a/debexpo/nntp/migrations/0001_initial.py b/debexpo/nntp/migrations/0001_initial.py new file mode 100644 index 00000000..19a0fd1b --- /dev/null +++ b/debexpo/nntp/migrations/0001_initial.py @@ -0,0 +1,81 @@ +# 0001_initial.py - migration script for nntp models +# +# This file is part of debexpo +# https://salsa.debian.org/mentors.debian.net-team/debexpo +# +# Copyright © 2020 Baptiste BEAUPLAT +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +from django.db import migrations, models + + +def create_gmane_lists(apps, schema_editor): + NNTPFeed = apps.get_model('nntp', 'NNTPFeed') + + # Last updated on 2020-07-01 + for namespace, name, last in ( + ( + 'remove_uploads', + 'gmane.linux.debian.backports.changes', + '40046', + ), + ( + 'remove_uploads', + 'gmane.linux.debian.devel.changes.unstable', + '583845', + ), + ( + 'remove_uploads', + 'gmane.linux.debian.devel.changes.stable', + '11992', + ), + ): + feed = NNTPFeed() + feed.namespace = namespace + feed.name = name + feed.last = last + + feed.full_clean() + feed.save() + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='NNTPFeed', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, + serialize=False, verbose_name='ID')), + ('namespace', models.TextField(verbose_name='Namespace')), + ('name', models.TextField(verbose_name='List name')), + ('last', models.TextField( + verbose_name='Last message processed')), + ], + ), + migrations.RunPython(create_gmane_lists), + ] diff --git a/debexpo/nntp/models.py b/debexpo/nntp/models.py index b786076e..d41cb2a6 100644 --- a/debexpo/nntp/models.py +++ b/debexpo/nntp/models.py @@ -1,11 +1,9 @@ -# -*- coding: utf-8 -*- +# models.py - model definition for nntp # -# data_store.py - Generic, general purpose data store -# -# This file is part of debexpo - +# This file is part of debexpo # https://salsa.debian.org/mentors.debian.net-team/debexpo # -# Copyright © 2011 Arno Töll +# Copyright © 2020 Baptiste BEAUPLAT # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -27,45 +25,13 @@ # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. +from django.db import models +from django.utils.translation import gettext_lazy as _ -""" -Holds data_store model -""" - -__author__ = 'Arno Töll' -__copyright__ = 'Copyright © 2011 Arno Töll' -__license__ = 'MIT' - -import sqlalchemy as sa -from sqlalchemy import orm - -from debexpo.model import meta, OrmObject - -t_user_countries = sa.Table( - 'data_store', meta.metadata, - sa.Column('namespace', sa.types.String(100), primary_key=True), - sa.Column('code', sa.types.String(100), primary_key=True, nullable=False), - sa.Column('value', sa.types.String(100), nullable=True) - ) - - -class DataStore(OrmObject): - pass - - -orm.mapper(DataStore, t_user_countries) +class NNTPFeed(models.Model): + namespace = models.TextField(verbose_name=_('Namespace')) + name = models.TextField(verbose_name=_('List name')) + last = models.TextField(verbose_name=_('Last message processed')) -def fill_data_store(): - import debexpo.model.data.data_store_init - import logging - for data in debexpo.model.data.data_store_init.DATA_STORE_INIT_OBJECTS: - query = meta.session.query(DataStore) \ - .filter(DataStore.code == data.code) \ - .filter(DataStore.namespace == data.namespace) \ - .count() - if (query): - continue - logging.info("Pre-configure value %s.%s" % (data.namespace, data.code)) - meta.session.add(data) - meta.session.commit() + unique_together = ('namespace', 'name',) -- GitLab From d536ef05cf604dbabdac86c22f867c39eeeff4dc Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Wed, 1 Jul 2020 22:28:20 +0200 Subject: [PATCH 15/30] Re-add cronjob to remove accepted uploads --- debexpo/packages/tasks.py | 185 ++++++++++++++----------------------- debexpo/settings/common.py | 5 +- 2 files changed, 74 insertions(+), 116 deletions(-) diff --git a/debexpo/packages/tasks.py b/debexpo/packages/tasks.py index 9c3e8598..d5b6a2f7 100644 --- a/debexpo/packages/tasks.py +++ b/debexpo/packages/tasks.py @@ -27,9 +27,11 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -# from celery.task import PeriodicTask from celery.decorators import periodic_task from datetime import timedelta, datetime, timezone +from logging import getLogger +from debian.deb822 import Changes +from debian.debian_support import NativeVersion from django.conf import settings from django.db.models import Max @@ -39,8 +41,10 @@ from debexpo.repository.models import Repository from debexpo.tools.email import Email from debexpo.bugs.models import Bug from debexpo.tools.gitstorage import GitStorage -# from debexpo.comments.models import PackageSubscription -# from debexpo.accounts.models import User +from debexpo.nntp.models import NNTPFeed +from debexpo.tools.nntp import NNTPClient + +log = getLogger(__name__) def remove_uploads(uploads): @@ -100,123 +104,76 @@ def remove_old_uploads(): log.debug(f'uploads to remove: {uploads}') removals = remove_uploads(uploads) + notify_uploaders(removals, reason='Your package found no sponsor for ' + '20 weeks') + +def notify_uploaders(removals, reason): for package, distribution, uploader in removals: email = Email('email-upload-removed.html') email.send(f'{package} has been removed from ' f'{distribution} on {settings.SITE_NAME}', recipients=[uploader.email], package=package, distribution=distribution, - reason="Your package found no sponsor for " - "20 weeks") + reason=reason) -# class RemoveAcceptedUploads(PeriodicTask): -# run_every = settings.TASK_ACCEPT_UPLOADS_BEAT -# -# def _process_changes(self, mail): -# if mail.is_multipart(): -# self.log.debug("Changes message is multipart?!") -# return -# changes = mail.get_payload(decode=True) -# try: -# changes = deb822.Changes(changes) -# except Exception: -# self.log.error('Could not open changes file; skipping mail "%s"' % -# (mail['subject'])) -# return -# -# if 'Source' not in changes: -# # self.log.debug('Changes file "%s" seems incomplete' % -# # (mail['subject'])) -# return -# -# package = self.pkg_controller._get_package(changes['Source'], -# from_controller=False) -# if package is not None: -# for pv in package.package_versions: -# if pv.distribution == changes['Distribution']: -# if (apt_pkg.version_compare(changes['Version'], -# pv.version) -# == 0): -# self.log.debug("Package %s was uploaded to Debian - " -# "removing it from Expo" % -# (changes['Source'])) -# self._remove_package(package, pv, -# "Package was uploaded to " -# "official Debian repositories") -# if (apt_pkg.version_compare(changes['Version'], -# pv.version) -# > 0): -# self.log.debug("More recent package %s was uploaded " -# "to Debian - removing it from Expo" % -# (changes['Source'])) -# self._remove_package(package, pv, -# "A more recent package was " -# "uploaded to official Debian " -# "repositories") -# else: -# # self.log.debug("Package %s was not uploaded to Expo before - " -# # "ignoring it" % (changes['Source'])) -# pass -# -# def _remove_uploaded_packages(self): -# -# if self.mailer.connection_established(): -# lists = meta.session.query(DataStore) \ -# .filter(DataStore.namespace == __namespace__) \ -# .all() -# for list_name in lists: -# for message in self.mailer.unread_messages(list_name.code, -# list_name.value): -# self._process_changes(message) -# list_name.value = message['X-Debexpo-Message-Number'] -# self.log.debug("Processed all messages up to #%s on %s" % -# (list_name.value, list_name.code)) -# meta.session.merge(list_name) -# meta.session.commit() -# self.mailer.disconnect_from_server() -# -# def _remove_old_packages(self): -# now = datetime.datetime.now() -# for package in self.pkgs_controller._get_packages(): -# if (now - package.package_versions[-1].uploaded) > \ -# datetime.timedelta(weeks=20): -# self.log.debug("Removing package %s - uploaded on %s" % -# (package.name, -# package.package_versions[-1].uploaded)) -# for pv in package.package_versions: -# self._remove_package(package, pv, -# "Your package found no sponsor for " -# "20 weeks") -# -# def setup(self): -# self.mailer = Email('upload_removed_from_expo') -# self.mailer.connect_to_server() -# self.pkg_controller = PackageController() -# self.pkgs_controller = PackagesController() -# apt_pkg.init_system() -# self.log.debug("%s loaded successfully" % (__name__)) -# -# def teardown(self): -# self.mailer.disconnect_from_server() -# -# def invoke(self): -# try: -# self._remove_uploaded_packages() -# except socket.error as e: -# # better luck next time -# self.log.debug("Socket error %s: skipping removals his time" -# % (e)) -# pass -# -# # We don't need to run our garbage collection of old cruft that often -# # It's ok if we purge old packages once a day. -# if (datetime.datetime.now() - self.last_cruft_run) >= \ -# datetime.timedelta(hours=24): -# self.last_cruft_run = datetime.datetime.now() -# self._remove_old_packages() -# -# -# cronjob = RemoveOldUploads -# schedule = datetime.timedelta(minutes=10) +@periodic_task(run_every=settings.TASK_ACCEPTED_UPLOADS_BEAT) +def remove_uploaded_packages(client=None): + feeds = NNTPFeed.objects.filter(namespace='remove_uploads') + uploads = set() + + # We allow the caller to define another NNTPClient, if not fallback to the + # default one. This is for testing purposes only: easier than setting up a + # testing NNTP server. + if not client: # pragma: no cover + client = NNTPClient() + + if not client.connect_to_server(): + return + + for feed in feeds: + last = feed.last + + for msg in client.unread_messages(feed.name, feed.last): + try: + uploads.update(process_accepted_changes(msg)) + except Exception as e: + log.warning('Failed to process message after ' + f'#{last} on ' + f'{feed.name}: {str(e)}') + else: + last = msg['X-Debexpo-Message-Number'] + + feed.last = last + + client.disconnect_from_server() + removals = remove_uploads(uploads) + notify_uploaders(removals, reason='Your package was uploaded to the ' + 'official Debian archive') + + for feed in feeds: + feed.full_clean() + feed.save() + + +def process_accepted_changes(mail): + if not mail or mail.is_multipart(): + return + + changes = mail.get_payload(decode=True) + changes = Changes(changes) + + if 'Source' not in changes \ + or 'Distribution' not in changes \ + or 'Version' not in changes: + raise Exception(f'Missing required keys in changes: {changes}') + + uploads = PackageUpload.objects.filter( + package__name=changes['Source'], + distribution__name=changes['Distribution']) + + return [ + upload for upload in uploads + if NativeVersion(upload.version) <= NativeVersion(changes['Version']) + ] diff --git a/debexpo/settings/common.py b/debexpo/settings/common.py index 9b7145bf..f3c55283 100644 --- a/debexpo/settings/common.py +++ b/debexpo/settings/common.py @@ -169,8 +169,9 @@ CELERY_BEAT_SCHEDULER = 'django' # Tasks beats TASK_IMPORTER_BEAT = 60 * 15 # Every 15 minutes -TASK_CLEANUPACCOUNTS_BEAT = 60 * 60 -TASK_OLD_UPLOADS_BEAT = 10 * 60 +TASK_CLEANUPACCOUNTS_BEAT = 60 * 60 # Every hours +TASK_OLD_UPLOADS_BEAT = 60 * 10 # Every 10 minutes +TASK_ACCEPTED_UPLOADS_BEAT = 60 * 10 # Every 10 minutes # Account registration expiration REGISTRATION_EXPIRATION_DAYS = 7 -- GitLab From b88b92e2de16272b6ec1574730eb593a4f867935 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Thu, 2 Jul 2020 20:05:08 +0200 Subject: [PATCH 16/30] Re-add template for accepted upload for testing --- .../functional/packages/templates/test-upload-accepted.html | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename old/debexpo/tests/cronjobs/email/accept.mako => tests/functional/packages/templates/test-upload-accepted.html (100%) diff --git a/old/debexpo/tests/cronjobs/email/accept.mako b/tests/functional/packages/templates/test-upload-accepted.html similarity index 100% rename from old/debexpo/tests/cronjobs/email/accept.mako rename to tests/functional/packages/templates/test-upload-accepted.html -- GitLab From e90e45f21716af971fded08ed6018125a940250a Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Thu, 2 Jul 2020 23:03:21 +0200 Subject: [PATCH 17/30] Allow multiple call for testing method _setup_example_package --- tests/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index c10c2a3d..099ae5e3 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -153,8 +153,7 @@ Xcgnuh6Rlywt6uiaFIGYnGefYPGXRAA= """ user = User.objects.get(email='email@example.com') - package = Package(name='testpackage') - package.save() + package = Package.objects.get_or_create(name='testpackage')[0] package_upload = PackageUpload( uploader=user, @@ -182,8 +181,8 @@ Xcgnuh6Rlywt6uiaFIGYnGefYPGXRAA= binary.save() - package = Package(name='anotherpackage', in_debian=True) - package.save() + package = Package.objects.get_or_create(name='anotherpackage', + in_debian=True)[0] package_upload = PackageUpload( uploader=user, -- GitLab From 8f173416a67e7f89bdf9d016caaf1e015edcdb41 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Thu, 2 Jul 2020 23:04:02 +0200 Subject: [PATCH 18/30] Convert testing template for accepted uploads to django --- .../templates/test-upload-accepted.html | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/functional/packages/templates/test-upload-accepted.html b/tests/functional/packages/templates/test-upload-accepted.html index a32951e3..c719bf79 100644 --- a/tests/functional/packages/templates/test-upload-accepted.html +++ b/tests/functional/packages/templates/test-upload-accepted.html @@ -19,39 +19,39 @@ From: Debian FTP Masters To: Vincent TIME X-DAK: dak process-upload X-Debian: DAK -X-Debian-Package: ${ c['name'] } +X-Debian-Package: {{ args.name }} Precedence: bulk Auto-Submitted: auto-generated MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 8bit -Subject: ${ c['name'] }_${ c['version'] }_source.changes ACCEPTED into ${ c['distrib'] } +Subject: {{ args.name }}_{{ args.version }}_source.changes ACCEPTED into {{ args.distrib }} Message-Id: Date: Fri, 19 Jul 2019 08:49:25 +0000 Received-SPF: none client-ip=2001:41b8:202:deb:6564:a62:52c3:4b72; envelope-from=envelope@ftp-master.debian.org; helo=mailly.debian.org Format: 1.8 Date: Tue, 16 Jul 2019 23:37:22 +0200 -Source: ${ c['name'] } +Source: {{ args.name }} Architecture: source -Version: ${ c['version'] } -Distribution: ${ c['distrib'] } +Version: {{ args.version }} +Distribution: {{ args.distrib }} Urgency: medium Maintainer: Vincent TIME Changed-By: Vincent TIME Changes: - ${ c['name'] } (${ c['version'] }) ${ c['distrib'] }; urgency=medium + {{ args.name }} ({{ args.version }}) {{ args.distrib }}; urgency=medium . * Rebuild, no changes Checksums-Sha1: - 36a03fa187774adfbe732dff7a3b23784647a783 1818 ${ c['name'] }_${ c['version'] }.dsc - c2a9b41fa5052c3248ad338ed39d23f7166213df 5548 ${ c['name'] }_${ c['version'] }.debian.tar.xz - 6e0a0beb343beab28d2e1215fa4c18be2b508ef7 6031 ${ c['name'] }_${ c['version'] }_amd64.buildinfo + 36a03fa187774adfbe732dff7a3b23784647a783 1818 {{ args.name }}_{{ args.version }}.dsc + c2a9b41fa5052c3248ad338ed39d23f7166213df 5548 {{ args.name }}_{{ args.version }}.debian.tar.xz + 6e0a0beb343beab28d2e1215fa4c18be2b508ef7 6031 {{ args.name }}_{{ args.version }}_amd64.buildinfo Checksums-Sha256: - 63e84b7dc83a8a263b1d0093b7d0b69d15282f4c128cf5f90cef666582ec333e 1818 ${ c['name'] }_${ c['version'] }.dsc - 12ab5db9022e5f470de823656d9706248dcb5793d330e19e3b5178d22d247c91 5548 ${ c['name'] }_${ c['version'] }.debian.tar.xz - 1173147d14de7d56bc1ecf98d2a3616f3348e3a4c055f684306f37d5b98b3bc0 6031 ${ c['name'] }_${ c['version'] }_amd64.buildinfo + 63e84b7dc83a8a263b1d0093b7d0b69d15282f4c128cf5f90cef666582ec333e 1818 {{ args.name }}_{{ args.version }}.dsc + 12ab5db9022e5f470de823656d9706248dcb5793d330e19e3b5178d22d247c91 5548 {{ args.name }}_{{ args.version }}.debian.tar.xz + 1173147d14de7d56bc1ecf98d2a3616f3348e3a4c055f684306f37d5b98b3bc0 6031 {{ args.name }}_{{ args.version }}_amd64.buildinfo Files: - 88619dc01f636b0f180cfc67653b932f 1818 utils optional ${ c['name'] }_${ c['version'] }.dsc - 4e978439dac618e92c982b75f701ddee 5548 utils optional ${ c['name'] }_${ c['version'] }.debian.tar.xz - 461ea7fb0137d8b726787561ea1a8966 6031 utils optional ${ c['name'] }_${ c['version'] }_amd64.buildinfo + 88619dc01f636b0f180cfc67653b932f 1818 utils optional {{ args.name }}_{{ args.version }}.dsc + 4e978439dac618e92c982b75f701ddee 5548 utils optional {{ args.name }}_{{ args.version }}.debian.tar.xz + 461ea7fb0137d8b726787561ea1a8966 6031 utils optional {{ args.name }}_{{ args.version }}_amd64.buildinfo -- GitLab From f5f2f50e5dee17f1bae3519e03681b8806115db3 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Thu, 2 Jul 2020 23:04:20 +0200 Subject: [PATCH 19/30] Register new testing template directory --- debexpo/settings/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debexpo/settings/common.py b/debexpo/settings/common.py index f3c55283..80555f2d 100644 --- a/debexpo/settings/common.py +++ b/debexpo/settings/common.py @@ -74,7 +74,8 @@ TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', 'DIRS': [ - 'tests/unit/tools/templates' + 'tests/unit/tools/templates', + 'tests/functional/packages/templates', ], 'APP_DIRS': True, 'OPTIONS': { -- GitLab From 5b30b624f376e1726a96582946f9559ede67ee93 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Thu, 2 Jul 2020 23:04:52 +0200 Subject: [PATCH 20/30] Add tests for removal of accepted uploads --- .../packages/test_removeolduploads.py | 126 ++++++++++++------ 1 file changed, 87 insertions(+), 39 deletions(-) diff --git a/tests/functional/packages/test_removeolduploads.py b/tests/functional/packages/test_removeolduploads.py index 67a881dc..a59a74b0 100644 --- a/tests/functional/packages/test_removeolduploads.py +++ b/tests/functional/packages/test_removeolduploads.py @@ -27,13 +27,15 @@ # OTHER DEALINGS IN THE SOFTWARE. import datetime import logging +from email import message_from_string # from debexpo.lib.email import Email from debexpo.accounts.models import User from debexpo.packages.models import Package, PackageUpload, Distribution, \ Component from tests import TestController -from debexpo.packages.tasks import remove_old_uploads +from debexpo.packages.tasks import remove_old_uploads, remove_uploaded_packages +from debexpo.tools.email import Email log = logging.getLogger(__name__) @@ -124,54 +126,100 @@ class TestCronjobRemoveOldUploads(TestController): self.state = new_state - def _invoke_plugin(self, uploaded): - remove_old_uploads() + def remove_upload_accepted(self, uploaded, down=False, garbage=False): + remove_uploaded_packages(FakeNNTPClient(uploaded, down, garbage)) def test_remove_uploads_noop_no_packages(self): - self._invoke_plugin([]) + remove_old_uploads() + self._assert_cronjob_success() + + def test_remove_uploads_server_down(self): + self.remove_upload_accepted([], True) self._assert_cronjob_success() - # def test_remove_uploads_inexistant_package(self): - # self._invoke_plugin([('inexistant_package', '1.0.0', 'unstable')]) - # self._assert_cronjob_success() + def test_remove_uploads_server_garbage(self): + self.remove_upload_accepted([], False, True) + self._assert_cronjob_success() + + def test_remove_uploads_inexistant_package(self): + self.remove_upload_accepted([('inexistant_package', '1.0.0', + 'unstable')]) + self._assert_cronjob_success() def test_remove_uploads_noop_with_packages(self): self._setup_packages() - self._invoke_plugin([]) + remove_old_uploads() self._assert_cronjob_success() def test_remove_uploads_expired(self): self._setup_packages(include_expired=True) - self._invoke_plugin([]) + remove_old_uploads() self._assert_cronjob_success() - # def test_remove_uploads_keep_other_dists(self): - # removed_packages = [ - # ('htop', '1.0.0', 'unstable'), - # ('htop', '0.9.0', 'unstable'), - # ] - - # self._setup_packages() - # self._invoke_plugin([('htop', '1.0.0', 'unstable')]) - # self._expect_package_removal(removed_packages) - # self._assert_cronjob_success() - - # def test_remove_uploads_same_version(self): - # removed_packages = [ - # ('zsh', '1.0.0', 'unstable'), - # ] - - # self._setup_packages() - # self._invoke_plugin([('zsh', '1.0.0', 'unstable')]) - # self._expect_package_removal(removed_packages) - # self._assert_cronjob_success() - - # def test_remove_uploads_keep_newer(self): - # removed_packages = [ - # ('htop', '0.9.0', 'unstable'), - # ] - - # self._setup_packages() - # self._invoke_plugin([('htop', '0.9.0', 'unstable')]) - # self._expect_package_removal(removed_packages) - # self._assert_cronjob_success() + def test_remove_uploads_keep_other_dists(self): + removed_packages = [ + ('htop', '1.0.0', 'unstable'), + ('htop', '0.9.0', 'unstable'), + ] + + self._setup_packages() + self.remove_upload_accepted([('htop', '1.0.0', 'unstable')]) + self._expect_package_removal(removed_packages) + self._assert_cronjob_success() + + def test_remove_uploads_same_version(self): + removed_packages = [ + ('zsh', '1.0.0', 'unstable'), + ] + + self._setup_packages() + self.remove_upload_accepted([('zsh', '1.0.0', 'unstable')]) + self._expect_package_removal(removed_packages) + self._assert_cronjob_success() + + def test_remove_uploads_keep_newer(self): + removed_packages = [ + ('htop', '0.9.0', 'unstable'), + ] + + self._setup_packages() + self.remove_upload_accepted([('htop', '0.9.0', 'unstable')]) + self._expect_package_removal(removed_packages) + self._assert_cronjob_success() + + +class FakeNNTPClient(): + def __init__(self, uploads, down=False, garbage=False): + self.uploads = uploads + self.iter = 0 + self.down = down + self.garbage = garbage + + def connect_to_server(self): + return not self.down + + def disconnect_from_server(self): + return True + + def unread_messages(self, name, last): + self.iter += 1 + + if self.iter == 1: + if self.garbage: + for item in [None, + message_from_string('Subject: test\n\ntest')]: + yield item + else: + for upload in self.uploads: + yield self._build_nntp_response(upload) + + def _build_nntp_response(self, upload): + email = Email('test-upload-accepted.html') + body = message_from_string(email._render_content([], **{ + 'name': upload[0], + 'version': upload[1], + 'distrib': upload[2], + })) + body['X-Debexpo-Message-Number'] = 42 + + return body -- GitLab From 6d33fc1bef5316bae2e754c07a902dd1b4ef0e4e Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Thu, 2 Jul 2020 23:05:16 +0200 Subject: [PATCH 21/30] Add test for remove_upload package view method --- tests/functional/packages/test_package.py | 48 ++++++++++++++++++++--- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/tests/functional/packages/test_package.py b/tests/functional/packages/test_package.py index 55be8ad0..a2d0999f 100644 --- a/tests/functional/packages/test_package.py +++ b/tests/functional/packages/test_package.py @@ -44,21 +44,31 @@ class TestPackageController(TestController): self._remove_example_package() self._remove_example_user() - def _test_bad_method(self, action): + def _test_bad_method(self, action, args=None): + if not args: + args = [] + self.client.post(reverse('login'), self._AUTHDATA) response = self.client.get(reverse( - action, args=['testpackage'])) + action, args=['testpackage'] + args)) self.assertEquals(response.status_code, 405) - def _test_no_auth(self, action, redirect_login=True): + def _test_no_auth(self, action, redirect_login=True, args=None): + if not args: + args = [] + response = self.client.post(reverse( - action, args=['testpackage'])) + action, args=['testpackage'] + args)) self.assertEquals(response.status_code, 302) + if (redirect_login): self.assertIn(reverse('login'), response.url) - def _test_not_owned_package(self, action, method='get'): + def _test_not_owned_package(self, action, method='get', args=None): + if not args: + args = [] + user = User.objects.create_user(name='Another user', email='another@example.com', password='password') @@ -68,7 +78,8 @@ class TestPackageController(TestController): 'password': 'password', 'commit': 'submit'}) - response = self.client.post(reverse(action, args=['testpackage'])) + response = self.client.post(reverse(action, + args=['testpackage'] + args)) self.assertEquals(response.status_code, 403) user.delete() @@ -157,12 +168,15 @@ class TestPackageController(TestController): def test_delete_no_auth(self): self._test_no_auth('delete_package') + self._test_no_auth('delete_upload', args=['1']) def test_delete_not_owned_package(self): self._test_not_owned_package('delete_package') + self._test_not_owned_package('delete_upload', args=['1']) def test_delete_bad_method(self): self._test_bad_method('delete_package') + self._test_bad_method('delete_upload', args=['1']) # def test_delete_gitstorage_utf8(self): # gitdir = join(pylons.test.pylonsapp.config['debexpo.repository'], @@ -185,6 +199,28 @@ class TestPackageController(TestController): self.assertRaises(Package.DoesNotExist, Package.objects.get, name='testpackage') + def test_delete_upload_successful(self): + self.client.post(reverse('login'), self._AUTHDATA) + + response = self.client.post(reverse( + 'delete_upload', args=['testpackage', '1'])) + + self.assertEquals(response.status_code, 302) + self.assertTrue(reverse('packages_my'), response.url) + self.assertRaises(Package.DoesNotExist, Package.objects.get, + name='testpackage') + + def test_delete_single_upload_successful(self): + self._setup_example_package() + self.client.post(reverse('login'), self._AUTHDATA) + + response = self.client.post(reverse( + 'delete_upload', args=['testpackage', '1'])) + + self.assertEquals(response.status_code, 302) + self.assertTrue(reverse('package', args=['testpackage']), response.url) + Package.objects.get(name='testpackage') + def test_comment_no_auth(self): self._test_no_auth('comment_package') -- GitLab From 550988a020eb8904bdf487404de3d92097608d19 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Sat, 4 Jul 2020 16:52:03 +0200 Subject: [PATCH 22/30] Add ClientFTPMaster to retrive package list from NEW --- debexpo/settings/common.py | 1 + debexpo/tools/clients/ftp_master.py | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/debexpo/settings/common.py b/debexpo/settings/common.py index 80555f2d..710a8509 100644 --- a/debexpo/settings/common.py +++ b/debexpo/settings/common.py @@ -201,6 +201,7 @@ BUGS_REPORT_NOT_OPEN = True # Debian tracker access TRACKER_URL = 'https://tracker.debian.org' +FTP_MASTER_NEW_PACKAGES_URL = 'https://ftp-master.debian.org/new.822' # Cleanup package older than NN weeks MAX_AGE_UPLOAD_WEEKS = 20 diff --git a/debexpo/tools/clients/ftp_master.py b/debexpo/tools/clients/ftp_master.py index c0b82704..4454261c 100644 --- a/debexpo/tools/clients/ftp_master.py +++ b/debexpo/tools/clients/ftp_master.py @@ -26,8 +26,12 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. +from debian.deb822 import Deb822, Changes + +from django.conf import settings + from debexpo.tools.files import CheckSumedFile -from debexpo.tools.clients import ClientJsonAPI +from debexpo.tools.clients import ClientJsonAPI, ClientHTTP import debexpo.repository.models as repository @@ -50,3 +54,14 @@ class ClientFTPMasterAPI(ClientJsonAPI): origin_files.append(origin) return origin_files + + +class ClientFTPMaster(ClientHTTP): + def get_packages_uploaded_to_new(self): + packages = [] + content = self.fetch_resource(settings.FTP_MASTER_NEW_PACKAGES_URL) + + for package in Deb822.iter_paragraphs(content): + packages.append(Changes(package)) + + return packages -- GitLab From 79582b1251dc3d5cac8a2d441f303d22a5ec790d Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Sat, 4 Jul 2020 16:52:46 +0200 Subject: [PATCH 23/30] Add support for removing package uploaded to NEW --- debexpo/packages/tasks.py | 95 ++++++++++++++++++++++++++++++++------- 1 file changed, 79 insertions(+), 16 deletions(-) diff --git a/debexpo/packages/tasks.py b/debexpo/packages/tasks.py index d5b6a2f7..260d1d12 100644 --- a/debexpo/packages/tasks.py +++ b/debexpo/packages/tasks.py @@ -35,6 +35,7 @@ from debian.debian_support import NativeVersion from django.conf import settings from django.db.models import Max +from django.db import transaction from debexpo.packages.models import Package, PackageUpload from debexpo.repository.models import Repository @@ -43,10 +44,13 @@ from debexpo.bugs.models import Bug from debexpo.tools.gitstorage import GitStorage from debexpo.nntp.models import NNTPFeed from debexpo.tools.nntp import NNTPClient +from debexpo.tools.clients import ExceptionClient +from debexpo.tools.clients.ftp_master import ClientFTPMaster log = getLogger(__name__) +@transaction.atomic def remove_uploads(uploads): removals = set() repository = Repository(settings.REPOSITORY) @@ -93,16 +97,12 @@ def remove_old_uploads(): .filter(latest_upload__lt=expiration_date) uploads = set() - from logging import getLogger - log = getLogger(__name__) for package in packages: uploads.update(PackageUpload.objects.filter( package__name=package['name'], distribution=package['packageupload__distribution'])) - log.debug(f'uploads to remove: {uploads}') - removals = remove_uploads(uploads) notify_uploaders(removals, reason='Your package found no sponsor for ' '20 weeks') @@ -120,8 +120,51 @@ def notify_uploaders(removals, reason): @periodic_task(run_every=settings.TASK_ACCEPTED_UPLOADS_BEAT) def remove_uploaded_packages(client=None): - feeds = NNTPFeed.objects.filter(namespace='remove_uploads') + uploads_to_archive = set() + uploads_to_new = set() + + uploads_to_archive.update(get_packages_uploaded_to_archive(client)) + uploads_to_new.update(get_packages_uploaded_to_new()) + + removals_from_archive = remove_uploads(uploads_to_archive) + removals_from_new = remove_uploads(uploads_to_new) + + mark_packages_as_uploaded(set([removal[0] for removal in + removals_from_archive]), debian=True) + mark_packages_as_uploaded(set([removal[0] for removal in + removals_from_new])) + + notify_uploaders(removals_from_archive, + reason='Your package was uploaded to the ' + 'official Debian archive') + notify_uploaders(removals_from_new, + reason='Your package was uploaded to the ' + 'NEW queue') + + +def get_packages_uploaded_to_new(): uploads = set() + ftp_master = ClientFTPMaster() + + try: + packages = ftp_master.get_packages_uploaded_to_new() + except ExceptionClient as e: + log.warning(f'Could not retrive package uploaded to new: {e}') + return [] + + for changes in packages: + try: + uploads.update(process_accepted_changes(changes)) + except Exception as e: + log.warning(f'Failed to process package in NEW {changes} ' + f'{e}') + + return uploads + + +def get_packages_uploaded_to_archive(client): + uploads = set() + feeds = NNTPFeed.objects.filter(namespace='remove_uploads') # We allow the caller to define another NNTPClient, if not fallback to the # default one. This is for testing purposes only: easier than setting up a @@ -130,50 +173,70 @@ def remove_uploaded_packages(client=None): client = NNTPClient() if not client.connect_to_server(): - return + return [] for feed in feeds: last = feed.last for msg in client.unread_messages(feed.name, feed.last): try: - uploads.update(process_accepted_changes(msg)) + changes = convert_mail_to_changes(msg) + uploads.update(process_accepted_changes(changes)) except Exception as e: log.warning('Failed to process message after ' f'#{last} on ' - f'{feed.name}: {str(e)}') + f'{feed.name}: {e}') else: last = msg['X-Debexpo-Message-Number'] feed.last = last client.disconnect_from_server() - removals = remove_uploads(uploads) - notify_uploaders(removals, reason='Your package was uploaded to the ' - 'official Debian archive') for feed in feeds: feed.full_clean() feed.save() + return uploads + -def process_accepted_changes(mail): +def mark_packages_as_uploaded(packages, debian=False): + for name in packages: + try: + package = Package.objects.get(name=name) + except Package.DoesNotExist: + continue + + if debian: + package.in_debian = True + package.needs_sponsor = False + package.full_clean() + package.save() + + +def convert_mail_to_changes(mail): if not mail or mail.is_multipart(): return changes = mail.get_payload(decode=True) changes = Changes(changes) - if 'Source' not in changes \ + return changes + + +def process_accepted_changes(changes): + if not changes or \ + 'Source' not in changes \ or 'Distribution' not in changes \ or 'Version' not in changes: - raise Exception(f'Missing required keys in changes: {changes}') + raise Exception(f'Cannot process accepted upload: {changes}') uploads = PackageUpload.objects.filter( package__name=changes['Source'], - distribution__name=changes['Distribution']) + distribution__name__in=(changes['Distribution'], 'UNRELEASED',)) return [ upload for upload in uploads - if NativeVersion(upload.version) <= NativeVersion(changes['Version']) + if NativeVersion(upload.version) <= NativeVersion(changes['Version']) or + upload.distribution.name == 'UNRELEASED' ] -- GitLab From 9fb7cb00206a3acf86142fc38bd969af308afa75 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Sat, 4 Jul 2020 16:53:03 +0200 Subject: [PATCH 24/30] Add tests for removing package uploaded to NEW --- .../packages/test_removeolduploads.py | 42 ++++++++++++++++++- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/tests/functional/packages/test_removeolduploads.py b/tests/functional/packages/test_removeolduploads.py index a59a74b0..d3871fd2 100644 --- a/tests/functional/packages/test_removeolduploads.py +++ b/tests/functional/packages/test_removeolduploads.py @@ -28,16 +28,22 @@ import datetime import logging from email import message_from_string +from http.server import BaseHTTPRequestHandler # from debexpo.lib.email import Email from debexpo.accounts.models import User from debexpo.packages.models import Package, PackageUpload, Distribution, \ Component -from tests import TestController +from tests import TestController, TestingHTTPServer from debexpo.packages.tasks import remove_old_uploads, remove_uploaded_packages from debexpo.tools.email import Email log = logging.getLogger(__name__) +PACKAGE_IN_NEW = '''Source: tmux +Version: 1.0.0 +Distribution: unstable + +Source: Missing-other-fields''' class TestCronjobRemoveOldUploads(TestController): @@ -127,7 +133,26 @@ class TestCronjobRemoveOldUploads(TestController): self.state = new_state def remove_upload_accepted(self, uploaded, down=False, garbage=False): - remove_uploaded_packages(FakeNNTPClient(uploaded, down, garbage)) + removed_packages = ( + ('tmux', '1.0.0', 'unstable'), + ('tmux', '1.0.0', 'UNRELEASED'), + ) + + self._expect_package_removal(removed_packages) + with TestingHTTPServer(FTPMasterPackageInNewHTTPHandler) as httpd: + with self.settings( + FTP_MASTER_NEW_PACKAGES_URL='http://localhost:' + f'{httpd.port}'): + remove_uploaded_packages(FakeNNTPClient(uploaded, down, + garbage)) + + def test_package_in_new_server_error(self): + with TestingHTTPServer(FTPMasterPackageInNewErrorHTTPHandler) as httpd: + with self.settings( + FTP_MASTER_NEW_PACKAGES_URL='http://localhost:' + f'{httpd.port}'): + remove_uploaded_packages(FakeNNTPClient([])) + self._assert_cronjob_success() def test_remove_uploads_noop_no_packages(self): remove_old_uploads() @@ -188,6 +213,19 @@ class TestCronjobRemoveOldUploads(TestController): self._assert_cronjob_success() +class FTPMasterPackageInNewHTTPHandler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200, 'OK') + self.end_headers() + self.wfile.write(bytes(PACKAGE_IN_NEW, 'UTF-8')) + + +class FTPMasterPackageInNewErrorHTTPHandler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(500, 'Internal Server Error') + self.end_headers() + + class FakeNNTPClient(): def __init__(self, uploads, down=False, garbage=False): self.uploads = uploads -- GitLab From b0497fcfae5f4a68639a2e06a2ebb37f85c08ce8 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Sun, 5 Jul 2020 21:25:18 +0200 Subject: [PATCH 25/30] Add logging on package removal --- debexpo/packages/tasks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/debexpo/packages/tasks.py b/debexpo/packages/tasks.py index 260d1d12..235f8c7a 100644 --- a/debexpo/packages/tasks.py +++ b/debexpo/packages/tasks.py @@ -84,6 +84,10 @@ def remove_uploads(uploads): repository.update() + for package, distribution, uploader in removals: + log.info(f'Removed package {package}, from {distribution}, ' + f'uploaded by {uploader}') + return removals -- GitLab From f96e8bc85cafc8364a9274ac51f0006768679383 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Sun, 5 Jul 2020 21:27:22 +0200 Subject: [PATCH 26/30] Wrap email template on upload removal --- debexpo/packages/templates/email-upload-removed.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debexpo/packages/templates/email-upload-removed.html b/debexpo/packages/templates/email-upload-removed.html index 20eea15a..9bab3b61 100644 --- a/debexpo/packages/templates/email-upload-removed.html +++ b/debexpo/packages/templates/email-upload-removed.html @@ -1,6 +1,7 @@ {% extends "email-base.html" %}{% block content %}Hi, -Your package {{ args.package }} has been removed from {{ args.distribution }} on {{ args.settings.SITE_NAME }} for the following reason: +Your package {{ args.package }} has been removed from {{ args.distribution }} +on {{ args.settings.SITE_NAME }} for the following reason: {{ args.reason }} -- GitLab From b14cae13497b7ca7a59c41dd89f4417732d12daa Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Sun, 5 Jul 2020 21:37:52 +0200 Subject: [PATCH 27/30] Move templates testing directory to test settings --- debexpo/settings/common.py | 4 ---- debexpo/settings/test.py | 6 ++++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/debexpo/settings/common.py b/debexpo/settings/common.py index 710a8509..54bcfad5 100644 --- a/debexpo/settings/common.py +++ b/debexpo/settings/common.py @@ -73,10 +73,6 @@ ROOT_URLCONF = 'debexpo.urls' TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [ - 'tests/unit/tools/templates', - 'tests/functional/packages/templates', - ], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ diff --git a/debexpo/settings/test.py b/debexpo/settings/test.py index 080084fa..376e2da2 100644 --- a/debexpo/settings/test.py +++ b/debexpo/settings/test.py @@ -42,6 +42,12 @@ DEBUG = False ALLOWED_HOSTS = [] +# Add testing directory templates +TEMPLATES[0]['DIRS'] = [ # noqa: F405 + 'tests/unit/tools/templates', + 'tests/functional/packages/templates', +] + # Database # https://docs.djangoproject.com/en/2.2/ref/settings/#databases -- GitLab From 28f24e401a46188cdf715d87d97f9ff6031cdc4a Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Sun, 5 Jul 2020 21:50:48 +0200 Subject: [PATCH 28/30] Add visual indicator that the upload removal will be an admin one. --- debexpo/packages/templates/package.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debexpo/packages/templates/package.html b/debexpo/packages/templates/package.html index 97f6d838..28117934 100644 --- a/debexpo/packages/templates/package.html +++ b/debexpo/packages/templates/package.html @@ -99,6 +99,8 @@ Upload #{{ index }} method="post"> {% csrf_token %} {% if user.is_superuser and user not in package.get_uploaders %} + {% trans 'Admin flag' %} -- GitLab From 7fcf3fbfb96b064e8a00d0891b6d3c5284264f81 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Sat, 4 Jul 2020 10:07:57 +0200 Subject: [PATCH 29/30] Update french translation --- debexpo/nntp/locale/fr/LC_MESSAGES/django.po | 31 +++++++ .../packages/locale/fr/LC_MESSAGES/django.po | 87 +++++++++++-------- 2 files changed, 82 insertions(+), 36 deletions(-) create mode 100644 debexpo/nntp/locale/fr/LC_MESSAGES/django.po diff --git a/debexpo/nntp/locale/fr/LC_MESSAGES/django.po b/debexpo/nntp/locale/fr/LC_MESSAGES/django.po new file mode 100644 index 00000000..8f841d57 --- /dev/null +++ b/debexpo/nntp/locale/fr/LC_MESSAGES/django.po @@ -0,0 +1,31 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2020-07-04 15:02+0000\n" +"PO-Revision-Date: 2020-07-04 17:04+0200\n" +"Language: fr\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n > 1);\n" +"Last-Translator: \n" +"Language-Team: \n" +"X-Generator: Poedit 2.3.1\n" + +#: nntp/models.py:33 +msgid "Namespace" +msgstr "Espace de nom" + +#: nntp/models.py:34 +msgid "List name" +msgstr "Nom de la liste" + +#: nntp/models.py:35 +msgid "Last message processed" +msgstr "Dernier message traité" diff --git a/debexpo/packages/locale/fr/LC_MESSAGES/django.po b/debexpo/packages/locale/fr/LC_MESSAGES/django.po index 134bec5a..b6843b15 100644 --- a/debexpo/packages/locale/fr/LC_MESSAGES/django.po +++ b/debexpo/packages/locale/fr/LC_MESSAGES/django.po @@ -7,8 +7,8 @@ msgid "" msgstr "" "Project-Id-Version: \n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2020-06-09 20:50+0200\n" -"PO-Revision-Date: 2020-06-09 20:53+0200\n" +"POT-Creation-Date: 2020-07-05 19:51+0000\n" +"PO-Revision-Date: 2020-07-05 21:54+0200\n" "Last-Translator: \n" "Language-Team: \n" "Language: fr\n" @@ -125,84 +125,96 @@ msgstr "Supprimer ce paquet" msgid "Package uploads" msgstr "Envois du paquet" -#: packages/templates/package.html:92 +#: packages/templates/package.html:93 #, python-format -msgid "

Upload #%(index)s

" -msgstr "

Envoi #%(index)s

" +msgid "Upload #%(index)s" +msgstr "Envoi #%(index)s" -#: packages/templates/package.html:96 +#: packages/templates/package.html:103 +msgid "Admin flag" +msgstr "Drapeau admin" + +#: packages/templates/package.html:106 +msgid "Admin upload deletion" +msgstr "Suppression de l'envoi administrateur" + +#: packages/templates/package.html:110 +msgid "Delete this upload" +msgstr "Supprimer cet envoi" + +#: packages/templates/package.html:117 msgid "Information" msgstr "Informations" -#: packages/templates/package.html:100 +#: packages/templates/package.html:121 msgid "Version:" msgstr "Version :" -#: packages/templates/package.html:104 +#: packages/templates/package.html:125 msgid "View RFS template" msgstr "Voir le modèle RFS" -#: packages/templates/package.html:110 +#: packages/templates/package.html:131 msgid "Uploaded:" msgstr "Envoi :" -#: packages/templates/package.html:116 +#: packages/templates/package.html:137 msgid "Source package:" msgstr "Paquet source :" -#: packages/templates/package.html:124 +#: packages/templates/package.html:145 msgid "Distribution:" msgstr "Distribution :" -#: packages/templates/package.html:129 +#: packages/templates/package.html:150 msgid "Section:" msgstr "Section :" -#: packages/templates/package.html:134 +#: packages/templates/package.html:155 msgid "Priority:" msgstr "Priorité :" -#: packages/templates/package.html:141 +#: packages/templates/package.html:162 msgid "Homepage:" msgstr "Page d'accueil :" -#: packages/templates/package.html:168 +#: packages/templates/package.html:189 msgid "Closes bugs:" msgstr "Ferme les bugs :" -#: packages/templates/package.html:178 +#: packages/templates/package.html:199 msgid "Changelog" msgstr "Changelog" -#: packages/templates/package.html:185 +#: packages/templates/package.html:206 msgid "QA information" msgstr "Informations QA" -#: packages/templates/package.html:198 +#: packages/templates/package.html:219 msgid "Comments" msgstr "Commentaires" -#: packages/templates/package.html:207 +#: packages/templates/package.html:228 msgid "Needs work" msgstr "Nécessite des corrections" -#: packages/templates/package.html:209 +#: packages/templates/package.html:230 msgid "Ready" msgstr "Pret" -#: packages/templates/package.html:213 +#: packages/templates/package.html:234 msgid "Package has been uploaded to Debian" msgstr "Le paquet a été envoyé dans Debian" -#: packages/templates/package.html:220 +#: packages/templates/package.html:241 msgid "No comments" msgstr "Pas de commentaires" -#: packages/templates/package.html:224 +#: packages/templates/package.html:245 msgid "New comment" msgstr "Nouveau commentaire" -#: packages/templates/package.html:233 +#: packages/templates/package.html:254 msgid "Submit" msgstr "Valider" @@ -218,56 +230,59 @@ msgstr "Envoyeur" msgid "No packages" msgstr "Pas de paquets" -#: packages/views.py:121 +#: packages/views.py:124 msgid "Today" msgstr "Aujourd'hui" -#: packages/views.py:124 +#: packages/views.py:127 msgid "Yesterday" msgstr "Hier" -#: packages/views.py:128 +#: packages/views.py:131 msgid "Some days ago" msgstr "Il y a quelques jours" -#: packages/views.py:132 +#: packages/views.py:135 msgid "Older packages" msgstr "Paquets plus vieux" -#: packages/views.py:136 +#: packages/views.py:139 msgid "Uploaded long ago" msgstr "Envoyé il y a longtemps" -#: packages/views.py:197 +#: packages/views.py:229 msgid "Packages for {} {}" msgstr "Paquets pour {} {}" -#: packages/views.py:199 +#: packages/views.py:231 msgid "Package list" msgstr "Liste des paquets" -#: packages/views.py:212 +#: packages/views.py:244 #, python-format msgid "%s packages" msgstr "%s paquets" -#: packages/views.py:213 +#: packages/views.py:245 #, python-format msgid "A feed of packages on %s" msgstr "Un flux de paquets sur %s" -#: packages/views.py:235 +#: packages/views.py:267 msgid "Package {} uploaded by {}." msgstr "Paquet {} envoyé par {}." -#: packages/views.py:241 +#: packages/views.py:273 msgid "Uploader is currently looking for a sponsor." msgstr "L'envoyeur est présentement en recherche de parrain." -#: packages/views.py:243 +#: packages/views.py:275 msgid "Uploader is currently not looking for a sponsor." msgstr "L'envoyeur n'est pas présentement en recherche de parrain." +#~ msgid "

Upload #%(index)s

" +#~ msgstr "

Envoi #%(index)s

" + #~ msgid "QA plugins succeeded" #~ msgstr "Les plugins QA ont réussi" -- GitLab From bb400920ee57322d261f7706bbc18fe2fda22ceb Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Mon, 6 Jul 2020 22:13:17 +0200 Subject: [PATCH 30/30] Use class instead of id for upload deletion form --- debexpo/base/static/css/style.css | 4 ++++ debexpo/packages/templates/package.html | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/debexpo/base/static/css/style.css b/debexpo/base/static/css/style.css index ed3e4fdd..660cf4c5 100644 --- a/debexpo/base/static/css/style.css +++ b/debexpo/base/static/css/style.css @@ -649,6 +649,10 @@ form#pageLang { display: inline; } +.inline { + display: inline; +} + /* Extras */ input:focus { diff --git a/debexpo/packages/templates/package.html b/debexpo/packages/templates/package.html index 28117934..88093541 100644 --- a/debexpo/packages/templates/package.html +++ b/debexpo/packages/templates/package.html @@ -94,7 +94,7 @@ Upload #{{ index }} {% endblocktrans %} {% if user in package.get_uploaders or user.is_superuser %} -
{% csrf_token %} -- GitLab