Merge branch 'fix/importer-hangs' of salsa.debian.org:lyknode-guest/debexpo into live

MR: !113


Signed-off-by: Mattia Rizzolo <mattia@debian.org>

bin/debexpo_worker.py

@@ -48,6 +48,7 @@ import pylons
 import optparse
 import signal
 import datetime
+from socket import setdefaulttimeout
 
 from paste.deploy import appconfig
 from debexpo.config.environment import load_environment
@@ -75,6 +76,9 @@ class Worker(object):
         self.can_continue = True
         signal.signal(signal.SIGTERM, self._on_sig_term)
 
+        # Set default socket timeout to 30s to avoid hanging indefinitely
+        setdefaulttimeout(30)
+
     def _on_sig_term(self, sig, frame):
         log.info("Received SIGTERM, shutting down worker after current run.")
         self._remove_pid()

debexpo/plugins/debianqa.py

@@ -43,6 +43,7 @@ __license__ = 'MIT'
 import logging
 import lxml.html
 import urllib2
+from socket import timeout
 
 from debexpo.model import meta
 from debexpo.model.users import User
@@ -60,7 +61,8 @@ class DebianPlugin(BasePlugin):
         try:
             self.qa_page = urllib2.urlopen('https://tracker.debian.org/%s' %
                                            self.changes['Source'])
-        except urllib2.HTTPError:
+        except (urllib2.URLError, timeout) as e:
+            log.error('Failed to contact tracker: {}'.format(e))
             return
 
         self.parsed_qa = lxml.html.fromstring(self.qa_page.read())

debexpo/plugins/getorigtarball.py

@@ -121,7 +121,7 @@ class GetOrigTarballPlugin(BasePlugin):
 
                 if not downloaded:
                     log.debug('Failed to download from debian archive')
-                    return self.failed('failed-to-download',
+                    return self.failed(outcomes['failed-to-download'],
                                        (None,
                                         list(set(self.additional_files))),
                                        constants.PLUGIN_SEVERITY_ERROR)
@@ -144,4 +144,5 @@ outcomes = {
                                      'file present in the official archives'},
     'invalid-orig': {'name': 'Could not find a valid orig tarball'},
     'found-orig': {'name': 'Found origin tarball'},
+    'failed-to-download': {'name': 'Failed to download from debian archive'},
 }

development.ini

@@ -116,7 +116,7 @@ debexpo.cronjobdir = /tmp
 debexpo.cronjob_delay = 60
 
 # NNTP server to connect to fetch mailing list comments/changes
-debexpo.nntp_server = news.gmane.org
+debexpo.nntp_server = news.gmane.io
 
 # Enable experimental and/or broken code
 debexpo.enable_experimental_code = true
@@ -146,6 +146,7 @@ handlers = console
 level = DEBUG
 handlers = console
 qualname = debexpo
+propagate = 0
 
 [handler_console]
 class = StreamHandler

live.ini

@@ -146,6 +146,7 @@ handlers = console, accesslog
 level = DEBUG
 handlers = console, accesslog
 qualname = debexpo
+propagate = 0
 
 [handler_console]
 class = StreamHandler

test.ini

@@ -116,7 +116,7 @@ debexpo.cronjobdir = /tmp
 debexpo.cronjob_delay = 60
 
 # NNTP server to connect to fetch mailing list comments/changes
-debexpo.nntp_server = news.gmane.org
+debexpo.nntp_server = news.gmane.io
 
 # Enable experimental and/or broken code
 debexpo.enable_experimental_code = true
@@ -149,6 +149,7 @@ handlers = console
 level = DEBUG
 handlers = console
 qualname = debexpo
+propagate = 0
 
 [handler_console]
 class = StreamHandler