Rename deploy() method to invoke(), extract changes/mentoring handling...

Rename deploy() method to invoke(), extract changes/mentoring handling interfaces from actual email fetching, move email handling to lib/email. Moreover implement stub code in ImportComments._process_changes

debexpo/controllers/package.py

@@ -63,20 +63,23 @@ log = logging.getLogger(__name__)
 
 class PackageController(BaseController):
 
-    def _get_package(self, packagename):
+    def _get_package(self, packagename, from_controller=True):
         """
         """
         log.debug('Details of package "%s" requested' % packagename)
 
         package = meta.session.query(Package).filter_by(name=packagename).first()
 
-        if package is None:
+        if package is None and from_controller:
             log.error('Could not get package information')
             redirect(url(controller='packages', action='index', packagename=None))
+        if package is None and not from_controller:
+            return None
 
-        c.package = package
-        c.config = config
-        c.package_dir = get_package_dir(package.name)
+	if from_controller:
+            c.package = package
+            c.config = config
+            c.package_dir = get_package_dir(package.name)
         return package
 
     def index(self, packagename = None):

debexpo/cronjobs/importcomments.py

@@ -36,13 +36,15 @@ __license__ = 'MIT'
 
 from debexpo.cronjobs import BaseCronjob
 
-from debexpo.model.packages import Package
+from debexpo.lib.email import Email
+from debexpo.lib.filesystem import CheckFiles
+from debexpo.controllers.package import PackageController
+from debexpo.model.users import User
 from debexpo.model import meta
 from debian import deb822
 
-import imaplib
-import email.parser
 import re
+import apt_pkg
 
 class ImportComments(BaseCronjob):
     def _belongs_to_package(self, mail):
@@ -60,15 +62,7 @@ class ImportComments(BaseCronjob):
 
         return None
 
-    def _check(self, msg, err, data = None):
-        if err != 'OK':
-            if (data):
-                self.log.error("%s failed: %s" % (msg, data))
-            else:
-                self.log.error("%s failed: %s" % (msg))
-
     def _process_changes(self, mail):
-        return
         if mail.is_multipart():
             self.log.debug("Changes message is multipart?!")
             return
@@ -80,18 +74,31 @@ class ImportComments(BaseCronjob):
             return
 
         if not 'Source' in changes:
-            self.log.debug('Changes file "%s" seems incomplete' % (mail['subject']))
+            #self.log.debug('Changes file "%s" seems incomplete' % (mail['subject']))
             return
-        package = meta.session.query(Package).filter_by(name=changes['Source']).first()
+        package = self.pkg_controller._get_package(changes['Source'], from_controller=False)
         if package != None:
-            #XXX: Finish me
-            pass
+            for pv in package.package_versions:
+                if pv.distribution == changes['Distribution'] and apt_pkg.VersionCompare(changes['Version'], pv.version) >= 0:
+                    self.log.debug("Package %s was was uploaded to Debian - removing it from Expo" % (changes['Source']))
+                    user = meta.session.query(User).filter_by(id=package.user_id).one()
+                    if user:
+                        self.mailer.send([user.email, ],
+                            package=package.name,
+                            version=pv.version,
+                            reason='Package was uploaded to official Debian repositories')
+                    CheckFiles().delete_files_for_package(package)
+                    meta.session.delete(package)
+                    meta.session.commit()
+                    break
         else:
-            self.log.debug("Package %s was not uploaded to Expo before - ignoring it" % (changes['Source']))
+            #self.log.debug("Package %s was not uploaded to Expo before - ignoring it" % (changes['Source']))
+            pass
 
     def _process_mentors(self, mail):
+        return
         if mail.is_multipart():
-            self.log.debug("Changes message is multipart?!")
+            self.log.debug("Mentors message is multipart?!")
             return
         package = self._belongs_to_package(mail)
         if not package:
@@ -101,45 +108,31 @@ class ImportComments(BaseCronjob):
 
 
     def setup(self):
-        self.established = False
-        self.imap = imaplib.IMAP4(self.config['debexpo.imap_server'])
-        (err, data) = self.imap.login(self.config['debexpo.imap_user'], self.config['debexpo.imap_password'])
-        self._check("IMAP login", err, data)
-        if err == 'OK':
-            self.established = True
-
-        (err, data) = self.imap.select("INBOX", readonly=True)
-        self._check("IMAP select", err, data)
+        self.mailer = Email('upload_removed_from_expo')
+        self.mailer.connect_to_server()
+        self.pkg_controller = PackageController()
+        apt_pkg.InitSystem()
 
     def teardown(self):
-        self.imap.close()
-        self.imap.logout()
-
+        self.mailer.disconnect_from_server()
 
     def invoke(self):
-        if not self.established:
-            self.log.warning("Unable to establish IMAP connection")
-            return
-
-        print("Running ImportUpload")
-        (err, messages) = self.imap.search(None, '(UNSEEN)')
-        self._check("IMAP search messages", err)
-
-        for msg_id in messages[0].split(" "):
-            #(err, msginfo) = self.imap.fetch(msg_id, '(BODY[HEADER.FIELDS (SUBJECT FROM LIST-ID)])')
-            (err, msginfo) = self.imap.fetch(msg_id, 'RFC822')
-            self._check("IMAP fetch message", err)
-            if (err != 'OK'):
-                continue
-            ep = email.parser.Parser().parsestr(msginfo[0][1])
-            if ep["list-id"] in ('<debian-devel-changes.lists.debian.org>',
-                    '<debian-changes.lists.debian.org>',
-                    '<debian-backports-changes.lists.debian.org>'):
-                self._process_changes(ep)
-            elif ep["list-id"] == '<debian-mentors.lists.debian.org>':
-                self._process_mentors(ep)
-            else:
-                self.log.debug("Unrecognized message in mailbox: '%s'" % ep["subject"])
+        filter_pattern = ('list-id',
+            ['<debian-devel-changes.lists.debian.org>',
+            '<debian-mentors.lists.debian.org>',
+            '<debian-changes.lists.debian.org>',
+            '<debian-backports-changes.lists.debian.org>']
+        )
+
+        if self.mailer.connection_established():
+            for message in self.mailer.unread_messages(filter_pattern):
+                if 'list-id' in message:
+                    if message['list-id'] in ('<debian-devel-changes.lists.debian.org>',
+                            '<debian-changes.lists.debian.org>',
+                            '<debian-backports-changes.lists.debian.org>'):
+                        self._process_changes(message)
+                    elif message['list-id'] == '<debian-mentors.lists.debian.org>':
+                        self._process_mentors(message)
 
 cronjob = ImportComments
 schedule = 1

debexpo/lib/email.py

 # -*- coding: utf-8 -*-
 #
-#   email.py — Helper class for sending email
+#   email.py — Helper class for sending and receiving email
 #
 #   This file is part of debexpo - http://debexpo.workaround.org
 #
 #   Copyright © 2008 Jonny Lamb <jonny@debian.org>
 #   Copyright © 2010 Jan Dittberner <jandd@debian.org>
+#   Copyright © 2011 Arno Töll <debian@toell.net>
 #
 #   Permission is hereby granted, free of charge, to any person
 #   obtaining a copy of this software and associated documentation
@@ -29,11 +30,15 @@
 #   OTHER DEALINGS IN THE SOFTWARE.
 
 """
-Holds helper class for sending email.
+Holds helper class for sending and receiving email. The latter is achieved to fetch mails from an IMAP mailbox
 """
 
+# You don't like that line?
+# Come over it. Or, alternatively don't call your local modules like Python standard libraries
+from __future__ import absolute_import
+
 __author__ = 'Jonny Lamb'
-__copyright__ = 'Copyright © 2008 Jonny Lamb, Copyright © 2010 Jan Dittberner'
+__copyright__ = 'Copyright © 2008 Jonny Lamb, Copyright © 2010 Jan Dittberner, Copyright © 2011 Arno Töll'
 __license__ = 'MIT'
 
 import logging
@@ -46,6 +51,8 @@ import pylons
 import debexpo.lib.helpers as h
 from gettext import gettext
 import routes.util
+import imaplib
+import email.parser
 
 log = logging.getLogger(__name__)
 
@@ -116,3 +123,56 @@ class Email(object):
                     result[recipient][1]))
         else:
             log.debug('Successfully sent')
+
+
+    def _check_error(self, msg, err, data = None):
+        if err != 'OK':
+            if (data):
+                self.log.error("%s failed: %s" % (msg, data))
+            else:
+                self.log.error("%s failed: %s" % (msg))
+
+    def unread_messages(self, filter_pattern):
+        if not self.connection_established():
+            return
+
+        (err, messages) = self.imap.search(None, '(UNSEEN)')
+        self._check_error("IMAP search messages", err)
+
+        for msg_id in messages[0].split(" "):
+            #(err, msginfo) = self.imap.fetch(msg_id, '(BODY[HEADER.FIELDS (SUBJECT FROM LIST-ID)])')
+            (err, msginfo) = self.imap.fetch(msg_id, 'RFC822')
+            self._check_error("IMAP fetch message", err)
+            if (err != 'OK'):
+                continue
+            ep = email.parser.Parser().parsestr(msginfo[0][1])
+            if not filter_pattern[0] in ep:
+                log.debug("No such header in message: %s" % (filter_pattern[0]))
+                continue
+            if ep[filter_pattern[0]] in filter_pattern[1]:
+                yield ep
+            else:
+                self.log.debug("Unrecognized message in mailbox: '%s'" % ep["subject"])
+
+
+    def connect_to_server(self):
+        self.established = False
+        self.imap = imaplib.IMAP4(pylons.config['debexpo.imap_server'])
+        (err, data) = self.imap.login(pylons.config['debexpo.imap_user'], pylons.config['debexpo.imap_password'])
+        self._check_error("IMAP login", err, data)
+        if err == 'OK':
+            self.established = True
+
+        (err, data) = self.imap.select("INBOX", readonly=True)
+        self._check_error("IMAP select", err, data)
+
+    def disconnect_from_server(self):
+        self.imap.close()
+        self.imap.logout()
+
+
+    def connection_established(self):
+        if not self.established:
+            log.debug("Connection to IMAP server not established");
+            return False
+        return True

debexpo/lib/filesystem.py

@@ -161,7 +161,7 @@ class CheckFiles(object):
             if os.path.exists(file):
                     log.debug("Removing file '%s'" % (file))
                     os.unlink(file)
-        if os.path.isdir(path):
+        if os.path.isdir(path) and os.listdir(path) == []:
             log.debug("Remove empty package repository '%s'" % (path))
             os.rmdir(path)
 

debexpo/scripts/debexpo_worker.py

@@ -213,7 +213,7 @@ class Worker(object):
                 self.jobs[job]['last_run'] += 1
                 if self.jobs[job]['last_run'] % self.jobs[job]['schedule'] == 0:
                     log.debug("Run job %s" % (job))
-                    self.jobs[job]['module'].deploy()
+                    self.jobs[job]['module'].invoke()
                     self.jobs[job]['last_run'] = 0
             time.sleep(delay)
 

debexpo/templates/email/upload_removed_from_expo.mako

+# -*- coding: utf-8 -*-
+<%inherit file="/base.mako"/>To: ${ c.to }
+Subject: ${ _('%s package has been removed from %s' % (c.package, c.config['debexpo.sitetitle'])) }
+
+${ _('Your package %s %s has been removed from %s for the following reason:' % (c.package, c.version, c.config['debexpo.sitename'])) }
+
+${ c.reason }
+
+${ _('Thanks,') }