Merge branch 'devel' into semantic-review

Conflicts:
	bin/debexpo_importer.py

README

-running debexpo
-===============
+installing and running debexpo
+==============================
 
-First, create a virtualenv.
+For installation instructions, see docs/installing.rst
 
-  % sudo apt-get install virtualenvwrapper
-  % mkdir ~/.virtualenvs
-  % source /etc/bash_completion
-  % mkvirtualenv expo
-  % workon expo
 
-This creates a virtual Python environment in which you can install the
-dependencies for debexpo without altering your system (i.e., without
-requiring root). Once you have the virtualenv set up, run this command to
-download the dependencies and set up the debexpo code so it can be run:
-
-  % python setup.py develop
-
-You might think you are done, but wait! You have to compile the
-translations from "PO" format into "MO" format. To do that:
-
-  % python setup.py compile_catalog
+running tests
+=============
 
 Hooray! Now you're ready to run the automated tests. (The code for the
 tests lives in debexpo/tests/*. This is a test suite; you can read more at

debexpo/cronjobs/importuploads.py

@@ -39,13 +39,13 @@ from debexpo.cronjobs import BaseCronjob
 import glob
 import os
 import os.path
-import subprocess
 import time
 import datetime
 import shutil
 
 import debexpo.lib.filesystem
 from debexpo.lib.changes import Changes
+from debexpo.importer.importer import Importer
 
 class NotCompleteUpload(Exception): pass
 
@@ -96,13 +96,14 @@ class ImportUpload(BaseCronjob):
                 continue
 
             self.log.info("Import upload: %s" % (file))
-            command = [ self.config['debexpo.importer'], '-i', self.config['global_conf']['__file__'], '-c', changes.get_filename() ]
-            self.log.debug("Executing: %s" % (" ".join(command)))
-            proc = subprocess.Popen(command, close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            (istdout, istderr) = proc.communicate()
-            if proc.returncode != 0:
-                self.log.critical("Importer failed to import package %s [err=%d]." % (file, proc.returncode))
-                self.log.debug("Output was\n%s\n%s" % (istdout,istderr))
+	    importer = Importer(changes.get_filename(),
+				self.config['global_conf']['__file__'],
+				False,
+				False)
+
+	    returncode = importer.main()
+            if returncode != 0:
+                self.log.critical("Importer failed to import package %s [err=%d]." % (file, returncode))
             for filename in changes.get_files() + [ changes.get_filename(), ]:
                 destination_file = os.path.join(self.config['debexpo.upload.incoming'], filename)
                 if os.path.exists(destination_file):

debexpo/lib/filesystem.py

@@ -145,6 +145,26 @@ class CheckFiles(object):
         return (orig_name, constants.ORIG_TARBALL_LOCATION_NOT_FOUND)
 
 
+    def find_files_for_packageversion(self, packageversion, absolute_path=False):
+        """
+        Returns all unique paths for files associated with the supplied
+        package version
+
+        ```packageversion``` The package version to be scanned
+
+        ```absolute_path``` if set to True, returns the absolute path
+            instead of a path relative to the repository root
+        """
+        package_files = []
+        for attr in ('binary_packages', 'source_packages'):
+            if hasattr(packageversion, attr):
+                for bp in getattr(packageversion, attr):
+                    for files in bp.package_files:
+                        if not files.filename in package_files:
+                            package_files.append(files.filename if not absolute_path
+                                                 else pylons.config['debexpo.repository'] + files.filename)
+        return package_files
+
     def find_files_for_package(self, package, absolute_path=False):
         """
         Returns all unique paths for files associated with the supplied
@@ -157,16 +177,30 @@ class CheckFiles(object):
         """
         package_files = []
         for p in package.package_versions:
-                for attr in ('binary_packages', 'source_packages'):
-                    if hasattr(p, attr):
-                        for bp in getattr(p, attr):
-                            for files in bp.package_files:
-                                if not files.filename in package_files:
-                                    package_files.append(files.filename if not absolute_path
-                                        else pylons.config['debexpo.repository'] + files.filename)
+            package_files.extend(self.find_files_for_packageversion(p, absolute_path))
         return package_files
 
 
+
+    def delete_files_for_packageversion(self, packageversion):
+        """
+        Removes all files associated with the package version supplied
+
+        ```packageversion``` PackageVersion object whose files are supposed to be removed
+        """
+        files = self.find_files_for_packageversion(packageversion, absolute_path=True)
+        if not files:
+            return
+        path = os.path.dirname(files[0])
+        for file in files:
+            if os.path.exists(file):
+                    log.debug("Removing file '%s'" % (file))
+                    os.unlink(file)
+        if os.path.isdir(path) and os.listdir(path) == []:
+            log.debug("Remove empty package repository '%s'" % (path))
+            os.rmdir(path)
+
+
     def delete_files_for_package(self, package):
         """
         Removes all files associated with the package supplied

debexpo/lib/validators.py

@@ -222,7 +222,7 @@ class GpgSignature(formencode.validators.FieldStorageUploadConverter):
     """
 
     not_empty = True
-    
+
     def __init__(self):
         self.gnupg = GnuPG()
 
@@ -248,12 +248,16 @@ class GpgSignature(formencode.validators.FieldStorageUploadConverter):
         if not 'key_id' in gpg_out:
             log.debug("Cannot read GPG key information")
             raise formencode.Invalid_("Not signed with the user's key", value, c)
-        
+
+        # checking that the user has uploaded a gpg key
+        if user.gpg_id is None:
+            raise formencode.Invalid(_('You have not uploaded any GPG key'), value, c)
+
         # checking that the file has been signed with the right key
         if gpg_out['key_id'] != user.gpg_id.split('/', 1)[1]:
             log.debug("File has not been signed by the user's key")
             raise formencode.Invalid_("Not signed with the user's key", value, c)
 
         log.debug("The file has been signed with the user's key")
-        
+
         return formencode.validators.FieldStorageUploadConverter._to_python(self, value, c)

debexpo/templates/base.mako

@@ -93,13 +93,13 @@
 
 
 	    </span>
-        % if c.submenu.has_menu():
+        % if c.submenu and c.submenu.has_menu():
             <%include file="submenu.mako" />
         % endif
 		${next.body()}
 	</div><!-- end content -->
         <div id="footer">
-	<p>© 2008-2011 ${c.config['debexpo.sitename']}
+	<p>© 2008-2012 ${c.config['debexpo.sitename']}
 	-
     Hosting and hardware provided by <a href="http://www.wavecon.de">Wavecon</a>
     -

debexpo/templates/index/index.mako

@@ -3,13 +3,11 @@
 
 <h1>Welcome to ${ c.config['debexpo.sitename'] }</h1>
 
-<!--
 <div class="warning">
     <p>
-    This page is currently on a public beta test! All uploads and account data will be wiped upon migration to this new service. Use the old <a href="http://mentors.debian.net">mentor service</a> for your data.
+    In preparation of the upcoming Debian Wheezy release <a href="https://lists.debian.org/debian-devel-announce/2012/06/msg00009.html">Testing migration has been frozen</a>. Do not expect much sponsoring activity regarding NEW packages or packages not fixing important bugs.
     </p>
 </div>
--->
 
 <p>Only approved members of the Debian project (Debian Developers) are granted the permission to upload software packages into the Debian distribution. Still a large number of packages is maintained by non-official developers. How do they get their work into Debian when they are not allowed to upload their own packages directly? By means of a process called <em>sponsorship</em>. Sponsorship means that a Debian Developer uploads the package on behalf of the actual maintainer. The Debian Developer will also check the package for technical correctness and help the maintainer to improve the package if necessary. Therefore the sponsor is sometimes also called a mentor.</p>
 

docs/installing.rst

@@ -6,49 +6,97 @@ Installing and setting up debexpo
 
 debexpo is easy to set up on your own. Simply follow the instructions below.
 
-Installing on Debian Lenny:
----------------------------
+There are two solutions:
+ 1. install all dependencies on your system as root
+ 2. install dependencies and debexpo in an isolated environment using
+    virtualenv
 
-You need to install the required packages. Using apt, you should execute::
+Getting debexpo
+---------------
 
-    sudo aptitude install python-setuptools python-apt python-sphinx python-pylons python-debian python-sqlalchemy python-soappy lintian dpkg-dev python-nose python-pybabel
+You can clone the git repository::
 
-`lintian` and `dpkg-dev` are optional if you do not want to run any plugins,
-and `python-nose` is optional if you don't want to run the test suite.
+  git clone git://git.debian.org/debexpo/debexpo.git
 
-Getting debexpo
----------------
+You should use the 'devel' branch::
+  git checkout devel
 
-You can either download a release tarball, or clone from the Git repository.
+Dependencies needed for both methods
+------------------------------------
 
-`The debexpo website <http://debexpo.workaround.org/>`_ will contain details
-about releases.
+Whatever method you choose, these packages are required::
+  sudo apt-get install python-apt python-debian iso-codes
 
-Or you can clone the repository contents by executing::
+If you want to run qa plugins, you will need `lintian` and
+`dpkg-dev`::
+  sudo apt-get install lintian dpkg-dev
 
-    git clone git://debexpo.workaround.org/debexpo.git
+Installing on Debian Squeeze or Wheezy as root
+---------------------------------------------------
 
-Editing your configuration
+You need to install the required packages. Using apt, you should execute::
+
+    sudo apt-get install python-setuptools python-sphinx python-pylons python-sqlalchemy python-soappy python-nose python-pybabel
+
+`python-nose` is optional if you don't want to run the test suite.
+
+
+You also need python-soaplib version >= 0.8.2 :
+http://pypi.python.org/pypi/soaplib
+
+Using pip::
+  sudo pip install soaplib
+
+
+Installing in an virtualenv
 ---------------------------
 
-Create a configuration file::
+Using this method, you will create a virtual Python environment in
+which you can install dependencies for debexpo without altering your
+system (i.e., without requiring root). In addition, this will also let
+you isolate debexpo's requirements, in the event an application installed
+globally might require a conflicting version of a library, or vise versa.
+
+Virtualenv setup
+~~~~~~~~~~~~~~~~
+
+Skip this section if you already have a working virtualenv setup.
+
+Install virtualenvwrapper::
+  sudo apt-get install virtualenvwrapper
+
+Dependencies
+~~~~~~~~~~~~
+
+To install lxml from sources, you will need `gcc`, `libxml2`,
+`libxslt1.1` and `python-dev` ::
+  sudo apt-get install gcc libxml2 libxml2-dev libxslt1.1 libxslt1-dev python-dev
+
+Debexpo installation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    paster make-config debexpo debexpo.ini
+First, create a new virtualenv for debexpo, and enter it::
+  mkvirtualenv expo
+  workon expo
 
-Next you should edit the default configuration file ``debexpo.ini``.
-You should only have to look at options with the ``DEBEXPO``
-comment preceeding them. You can find explanations of the `debexpo.*` options
-on the :ref:`config-file` page.
+You can now install debexpo. This will download and install all
+required libraries::
+  python setup.py develop
 
-If you just want to get it running somehow, edit at least the path
-``debexpo.repository = /tmp/debexpo_cache/``.
+If for any reason you need to exit the virtualenv, you may enter
+`deactivate` to exit the virtualenv.
+
+Editing your configuration
+---------------------------
+
+Now edit `development.ini` to match your configuration.
 
 Setting up the application
 --------------------------
 
 Execute the following commands to setup the application::
 
-    paster setup-app debexpo.ini
+    paster setup-app development.ini
     python setup.py compile_catalog
 
 Running debexpo
@@ -80,7 +128,7 @@ Using Apache
     use = egg:PasteScript#flup_fcgi_thread
     host = 0.0.0.0
     port = 6500
- 
+
 #. Add the following to your config::
 
     <IfModule mod_fastcgi.c>
@@ -91,4 +139,3 @@ Using Apache
   Note: Parts of this may conflict with your `/etc/apache2/conf-available/fastcgi.conf`.
 
   `/some/path/to/debexpo/fcgi` need not physically exist on the webserver.
-