Added a view to show the mailbox stats

backend/management/commands/process_mbox_stats.py

@@ -16,11 +16,13 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from django.core.management.base import BaseCommand, CommandError
+import django.db
 from django.conf import settings
 import optparse
 import sys
 import logging
 from backend import models as bmodels
+from backend import const
 import email.utils
 import mailbox
 import datetime
@@ -89,14 +91,16 @@ class Interaction(object):
                 d = cls.data['process'][process_key]
                 log.debug("response_time: %s" % d['response_time'])
                 d['mean'] = numpy.mean(d['response_time'])
-                d['mean_fancy'] = "%s" % datetime.timedelta(seconds=numpy.int_(d['mean']))
-                log.info( "%s -> mean: %s[%s]" % \
-                    (process_key, d['mean'], d['mean_fancy']))
+                d['mean_fancy'] = "%s" % datetime.timedelta(
+                    seconds=numpy.int_(d['mean']))
+                log.debug("%s -> mean: %s[%s]" %
+                         (process_key, d['mean'], d['mean_fancy']))
         else:
             for k, d in cls.data['emails'].iteritems():
                 if d['num_mails'] > 1:
                     d['mean'] = numpy.mean(d['response_time'])
-                    d['mean_fancy'] = "%s" % datetime.timedelta(seconds=numpy.int_(d['mean']))
+                    d['mean_fancy'] = "%s" % datetime.timedelta(
+                        seconds=numpy.int_(d['mean']))
 
     @classmethod
     def _median(cls, process_key):
@@ -105,14 +109,16 @@ class Interaction(object):
                 d = cls.data['process'][process_key]
                 log.debug("response_time: %s" % d['response_time'])
                 d['median'] = numpy.median(d['response_time'])
-                d['median_fancy'] = "%s" % datetime.timedelta(seconds=numpy.int_(d['median']))
-                log.info( "%s -> median: %s[%s]" % \
-                    (process_key, d['median'], d['median_fancy']))
+                d['median_fancy'] = "%s" % datetime.timedelta(
+                    seconds=numpy.int_(d['median']))
+                log.debug("%s -> median: %s[%s]" %
+                         (process_key, d['median'], d['median_fancy']))
         else:
             for k, d in cls.data['emails'].iteritems():
                 if d['num_mails'] > 1:
                     d['median'] = numpy.median(d['response_time'])
-                    d['median_fancy'] = "%s" % datetime.timedelta(seconds=numpy.int_(d['median']))
+                    d['median_fancy'] = "%s" % datetime.timedelta(
+                        seconds=numpy.int_(d['median']))
 
     def generate_stats(self, process_key):
         self.sort_data = sorted(self.unsort_data,
@@ -168,7 +174,8 @@ class Command(BaseCommand):
                 interactions.generate_stats(key)
                 log.info("%s processed", mailbox_file)
             else:
-                log.warn("skipped, no mailbox file defined")
+                log.warn("%s[%s] skiped, no mailbox file defined" %
+                         (unicode(progress), key))
 
         interactions.generate_email_stats()
         interactions.export(os.path.join(settings.DATA_DIR, 'mbox_stats.json'))

backend/models.py

@@ -764,6 +764,23 @@ class Person(PermissionsMixin, models.Model):
         except cls.DoesNotExist:
             return None
 
+    @classmethod
+    def lookup_by_email(cls, addr):
+        """
+        Return the person corresponding to an email address, or None if no such
+        person has been found.
+        """
+        try:
+            return cls.objects.get(email=addr)
+        except cls.DoesNotExist:
+            pass
+        if not addr.endswith("@debian.org"):
+            return None
+        try:
+            return cls.objects.get(uid=addr[:-11])
+        except cls.DoesNotExist:
+            return None
+
     @classmethod
     def lookup_or_404(cls, key):
         from django.http import Http404

restricted/templates/restricted/mailbox-stats.html

+{% extends "restricted/base.html" %}
+{% load nm %}
+{% load js %}
+
+{% block head_resources %}
+  {{block.super}}
+  {% jsinclude "tables" %}
+  {% jsinclude "tables,sparkline" %}
+{% endblock %}
+
+{% block head %}
+{{block.super}}
+<script type="text/javascript">
+$(function() {
+    $("#emails").tablesorter({
+        textExtraction: function(node) {
+            val = node.getAttribute("val");
+            if (val == null)
+                val = node.textContent || node.innerText;
+            return val;
+        },
+    })
+
+    $(".mbox_sparkline").sparkline("html", {
+      type: "bar",
+      barColor: "#005382",
+      negBarColor: "#823000",
+      chartRangeMin: -30,
+      chartRangeMax: 30
+    });
+});
+</script>
+{% endblock %}
+
+{% block content %}
+
+<h1>Mailbox statistics</h1>
+
+<table id="emails" class="tablesorter">
+    <thead>
+        <tr>
+            <th>E-mail</th>
+            <th>Person</th>
+            <th>First mail</th>
+            <th>Last mail</th>
+            <th>Mail count</th>
+            <th>Median</th>
+            <th>Mean</th>
+            <th>History</th>
+        </tr>
+    </thead>
+    <tbody>
+      {% for email, stats in emails %}
+      <tr>
+        <td>{{email}}</td>
+        <td>{{stats.person}}</td>
+        <td val="{{stats.date_first}}">{{stats.date_first_py|date:"Y-m-d"}}</td>
+        <td val="{{stats.date_last}}">{{stats.date_last_py|date:"Y-m-d"}}</td>
+        <td>{{stats.num_mails}}</td>
+        <td val="{{stats.median}}">
+          {% if stats.median_py %}
+            {% if stats.median_py.days %}{{stats.median_py.days}}d {% endif %}{{stats.median_hours}}h
+          {% else %}
+            -
+          {% endif %}
+        </td>
+        <td><span class="mbox_sparkline" values="{{stats.response_time|join:","}}"></span></td>
+        {#"median": 585625.0, "mean_fancy": "35 days, 0:30:08", "median_fancy": "6 days, 18:40:25"#}
+      </tr>
+      {% endfor %}
+    </tbody>
+</table>
+
+{% endblock %}

restricted/urls.py

@@ -45,4 +45,6 @@ urlpatterns = patterns('restricted.views',
     url(r'^display-mail-archive/(?P<key>[^/]+)$', views.DisplayMailArchive.as_view(), name="display_mail_archive"),
     # Assign AMs to NMs
     url(r'^assign-am/(?P<key>[^/]+)$', views.AssignAM.as_view(), name="assign_am"),
+    # Mailbox stats
+    url(r'^mailbox-stats$', views.MailboxStats.as_view(), name="mailbox_stats"),
 )

restricted/views.py

 # coding: utf8
 # nm.debian.org AM interaction
 #
-# Copyright (C) 2013--2014  Enrico Zini <enrico@debian.org>
+# Copyright (C) 2013--2015  Enrico Zini <enrico@debian.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -20,6 +20,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import unicode_literals
 from django import http, template, forms
+from django.conf import settings
 from django.shortcuts import render, render_to_response, redirect
 from django.contrib.auth.decorators import login_required
 from django.utils.translation import ugettext as _
@@ -34,6 +35,7 @@ from backend.mixins import VisitorMixin, VisitorTemplateView, VisitPersonTemplat
 import backend.email
 import json
 import datetime
+import os
 
 class AMMain(VisitorTemplateView):
     require_visitor = "am"
@@ -534,3 +536,31 @@ class AssignAM(VisitorTemplateView):
         am = bmodels.AM.lookup_or_404(am_key)
         _assign_am(request, self.visitor, process, am)
         return redirect(process.get_absolute_url())
+
+class MailboxStats(VisitorTemplateView):
+    template_name = "restricted/mailbox-stats.html"
+    require_visitor = "admin"
+
+    def get_context_data(self, **kw):
+        ctx = super(MailboxStats, self).get_context_data(**kw)
+
+        try:
+            with open(os.path.join(settings.DATA_DIR, 'mbox_stats.json'), "rt") as infd:
+                stats = json.load(infd)
+        except OSError:
+            stats = {}
+
+        for email, st in stats["emails"].items():
+            st["person"] = bmodels.Person.lookup_by_email(email)
+            st["date_first_py"] = datetime.datetime.fromtimestamp(st["date_first"])
+            st["date_last_py"] = datetime.datetime.fromtimestamp(st["date_last"])
+            if "median" not in st:
+                st["median_py"] = None
+            else:
+                st["median_py"] = datetime.timedelta(seconds=st["median"])
+                st["median_hours"] = st["median_py"].seconds // 3600
+
+        ctx.update(
+            emails=sorted(stats["emails"].items()),
+        )
+        return ctx