From acc7da35a9d984c83186815b13f00eb1c7b9ccf1 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Sat, 7 Mar 2020 17:04:22 +0100 Subject: [PATCH 1/6] Add a default timeout for socket operation to the importer of 30s (Closes #94) --- bin/debexpo_worker.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/debexpo_worker.py b/bin/debexpo_worker.py index d6b738b0..551b7d34 100755 --- a/bin/debexpo_worker.py +++ b/bin/debexpo_worker.py @@ -48,6 +48,7 @@ import pylons import optparse import signal import datetime +from socket import setdefaulttimeout from paste.deploy import appconfig from debexpo.config.environment import load_environment @@ -75,6 +76,9 @@ class Worker(object): self.can_continue = True signal.signal(signal.SIGTERM, self._on_sig_term) + # Set default socket timeout to 30s to avoid hanging indefinitely + setdefaulttimeout(30) + def _on_sig_term(self, sig, frame): log.info("Received SIGTERM, shutting down worker after current run.") self._remove_pid() -- GitLab From 6326e2d70ba24b8fc8c450416f44d55f0806ff80 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Sat, 7 Mar 2020 17:55:16 +0100 Subject: [PATCH 2/6] Deduplicate log lines --- development.ini | 1 + live.ini | 1 + test.ini | 1 + 3 files changed, 3 insertions(+) diff --git a/development.ini b/development.ini index 25fe2dfe..b21f39eb 100644 --- a/development.ini +++ b/development.ini @@ -146,6 +146,7 @@ handlers = console level = DEBUG handlers = console qualname = debexpo +propagate = 0 [handler_console] class = StreamHandler diff --git a/live.ini b/live.ini index c1120e8d..e45923bf 100644 --- a/live.ini +++ b/live.ini @@ -146,6 +146,7 @@ handlers = console, accesslog level = DEBUG handlers = console, accesslog qualname = debexpo +propagate = 0 [handler_console] class = StreamHandler diff --git a/test.ini b/test.ini index 6781405d..50ab0d29 100644 --- a/test.ini +++ b/test.ini @@ -149,6 +149,7 @@ handlers = console level = DEBUG handlers = console qualname = debexpo +propagate = 0 [handler_console] class = StreamHandler -- GitLab From 1e3b29859857b34dabd9343be28639f576ed559d Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Sat, 7 Mar 2020 18:13:47 +0100 Subject: [PATCH 3/6] Catch timeout when tracker.d.o is not responding --- debexpo/plugins/debianqa.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/debexpo/plugins/debianqa.py b/debexpo/plugins/debianqa.py index 3ce3fe2a..36c84e1c 100644 --- a/debexpo/plugins/debianqa.py +++ b/debexpo/plugins/debianqa.py @@ -43,6 +43,7 @@ __license__ = 'MIT' import logging import lxml.html import urllib2 +from socket import timeout from debexpo.model import meta from debexpo.model.users import User @@ -60,7 +61,8 @@ class DebianPlugin(BasePlugin): try: self.qa_page = urllib2.urlopen('https://tracker.debian.org/%s' % self.changes['Source']) - except urllib2.HTTPError: + except (urllib2.HTTPError, timeout) as e: + log.error('Failed to contact tracker: {}'.format(e)) return self.parsed_qa = lxml.html.fromstring(self.qa_page.read()) -- GitLab From 520bce88ab71bb0d23919022993efdeb2b0633a5 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Sat, 7 Mar 2020 18:14:37 +0100 Subject: [PATCH 4/6] Fix failed to download condition in getorigtarball --- debexpo/plugins/getorigtarball.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debexpo/plugins/getorigtarball.py b/debexpo/plugins/getorigtarball.py index 25f55031..889d104a 100644 --- a/debexpo/plugins/getorigtarball.py +++ b/debexpo/plugins/getorigtarball.py @@ -121,7 +121,7 @@ class GetOrigTarballPlugin(BasePlugin): if not downloaded: log.debug('Failed to download from debian archive') - return self.failed('failed-to-download', + return self.failed(outcomes['failed-to-download'], (None, list(set(self.additional_files))), constants.PLUGIN_SEVERITY_ERROR) @@ -144,4 +144,5 @@ outcomes = { 'file present in the official archives'}, 'invalid-orig': {'name': 'Could not find a valid orig tarball'}, 'found-orig': {'name': 'Found origin tarball'}, + 'failed-to-download': {'name': 'Failed to download from debian archive'}, } -- GitLab From f77f7d42998933943d51326c570eb9881d6f9b87 Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Sat, 7 Mar 2020 18:18:54 +0100 Subject: [PATCH 5/6] In debianqa plugin, use URLError instead of HTTPError to catch more errors --- debexpo/plugins/debianqa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debexpo/plugins/debianqa.py b/debexpo/plugins/debianqa.py index 36c84e1c..2f9149ab 100644 --- a/debexpo/plugins/debianqa.py +++ b/debexpo/plugins/debianqa.py @@ -61,7 +61,7 @@ class DebianPlugin(BasePlugin): try: self.qa_page = urllib2.urlopen('https://tracker.debian.org/%s' % self.changes['Source']) - except (urllib2.HTTPError, timeout) as e: + except (urllib2.URLError, timeout) as e: log.error('Failed to contact tracker: {}'.format(e)) return -- GitLab From d261035f89f54c084369ded59b90b2905c9a54ab Mon Sep 17 00:00:00 2001 From: Baptiste BEAUPLAT Date: Sat, 7 Mar 2020 18:31:28 +0100 Subject: [PATCH 6/6] Use news.gmane.io for development and test configs --- development.ini | 2 +- test.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/development.ini b/development.ini index b21f39eb..29db0ebb 100644 --- a/development.ini +++ b/development.ini @@ -116,7 +116,7 @@ debexpo.cronjobdir = /tmp debexpo.cronjob_delay = 60 # NNTP server to connect to fetch mailing list comments/changes -debexpo.nntp_server = news.gmane.org +debexpo.nntp_server = news.gmane.io # Enable experimental and/or broken code debexpo.enable_experimental_code = true diff --git a/test.ini b/test.ini index 50ab0d29..d7fe306f 100644 --- a/test.ini +++ b/test.ini @@ -116,7 +116,7 @@ debexpo.cronjobdir = /tmp debexpo.cronjob_delay = 60 # NNTP server to connect to fetch mailing list comments/changes -debexpo.nntp_server = news.gmane.org +debexpo.nntp_server = news.gmane.io # Enable experimental and/or broken code debexpo.enable_experimental_code = true -- GitLab