Skip to content
GitLab
Commit 72cfeeac authored by Enrico Zini's avatar Enrico Zini
Browse files

Consolidated import and management; import advocates; show advocates in nmstatus

parent 4bb5a355
Hide whitespace changes
Inline Side-by-side
GETTING-STARTED
View file @ 72cfeeac
......@@ -2,7 +2,7 @@
* Running this code on your own machine
# Dependencies
apt-get install python-django
apt-get install python-django python-ldap
# Configuration
ln -s settings.py.devel settings.py
......@@ -18,13 +18,9 @@ On nono.debian.org:
Fetch the resulting dump.json
./manage.py importjson dump.json
./manage.py importldap
./manage.py importkeyrings
./manage.py import dump.json
# Run database maintenance
./manage.py updateldap
./manage.py updatekeyrings
./manage.py maintenance
# Run the web server
......
dsa/management/commands/importldap.py deleted 100644 → 0
View file @ 4bb5a355
# nm.debian.org website backend
#
# Copyright (C) 2012 Enrico Zini <enrico@debian.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.core.management.base import BaseCommand, CommandError
import django.db
from django.conf import settings
import optparse
import sys
import ldap
import logging
import json
from backend import models as bmodels
from backend import const
log = logging.getLogger(__name__)
class Importer(object):
"""
Perform initial data import from LDAP
Imports cn, sn, nm, fpr, email for DDs and guest accounts.
Does not set status, that will be taken from keyrings
"""
def __init__(self):
pass
def do_import(self):
# enrico> Hi. Can you give me an official procedure to check if one is a DD from LDAP info?
# @weasel> enrico: not really, that's your decision.
# @weasel> enrico: for one, you can filter on gid 800. and then filter for having a
# fingerprint. that's usually right
# enrico> weasel: what are person accounts without fingerprints for?
# @weasel> people who screwed up their keys
# @weasel> we've had that on occasion
# enrico> weasel: ack
# @weasel> enrico: and of course retired people
# @weasel> we try to set ldap's account status nowadays, but no idea if
# that applies to all that ever retired
search_base = "dc=debian,dc=org"
l = ldap.initialize("ldap://db.debian.org")
#l = ldap.initialize("ldap://localhost:3389")
l.simple_bind_s("","")
for dn, attrs in l.search_s(search_base, ldap.SCOPE_SUBTREE, "objectclass=inetOrgPerson"):
# Try to match the person using uid
uid = attrs["uid"][0]
try:
person = bmodels.Person.objects.get(uid=uid)
if person.status == const.STATUS_MM:
person.status = const.STATUS_MM_GA
person.save()
continue
except bmodels.Person.DoesNotExist:
pass
# Try to match the person using emails
try:
person = bmodels.Person.objects.get(email=uid + "@debian.org")
person.uid = uid
if person.status == const.STATUS_MM:
person.status = const.STATUS_MM_GA,
person.save()
continue
except bmodels.Person.DoesNotExist:
pass
def get_field(f):
if f not in attrs:
return None
f = attrs[f]
if not f:
return None
return f[0]
email = get_field("emailForward")
try:
person = bmodels.Person.objects.get(email=email)
person.uid = uid
if person.status == const.STATUS_MM:
person.status = const.STATUS_MM_GA,
person.save()
continue
except bmodels.Person.DoesNotExist:
pass
person = bmodels.Person(
cn=get_field("cn"),
mn=get_field("mn"),
sn=get_field("sn"),
fpr=get_field("keyFingerPrint"),
uid=uid,
# Default to MM_GA: if they are in LDAP, they have at least a
# guest account
status=const.STATUS_MM_GA,
)
if get_field("gidNumber") == '800':
person.email = uid + "@debian.org"
else:
person.email = email
if person.email is None:
log.warning("Skipping %s because we have no email address", uid)
continue
person.save()
class Command(BaseCommand):
help = 'Import people and changes from LDAP'
option_list = BaseCommand.option_list + (
optparse.make_option("--quiet", action="store_true", dest="quiet", default=None, help="Disable progress reporting"),
)
def handle(self, *fnames, **opts):
FORMAT = "%(asctime)-15s %(levelname)s %(message)s"
if opts["quiet"]:
logging.basicConfig(level=logging.WARNING, stream=sys.stderr, format=FORMAT)
else:
logging.basicConfig(level=logging.INFO, stream=sys.stderr, format=FORMAT)
importer = Importer()
importer.do_import()
#log.info("%d patch(es) applied", len(fnames))
dsa/management/commands/updateldap.py deleted 100644 → 0
View file @ 4bb5a355
# nm.debian.org website backend
#
# Copyright (C) 2012 Enrico Zini <enrico@debian.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.core.management.base import BaseCommand, CommandError
import django.db
from django.conf import settings
import optparse
import sys
import ldap
import logging
import json
from backend import models as bmodels
from backend import const
log = logging.getLogger(__name__)
class Updater(object):
"""
Perform data updates from LDAP
Updates cn, sn, nm, email for DDs and guest accounts.
"""
def __init__(self):
pass
def do_update(self):
search_base = "dc=debian,dc=org"
l = ldap.initialize("ldap://db.debian.org")
l.simple_bind_s("","")
for dn, attrs in l.search_s(search_base, ldap.SCOPE_SUBTREE, "objectclass=inetOrgPerson"):
uid = attrs["uid"][0]
try:
person = bmodels.Person.objects.get(uid=uid)
except bmodels.Person.DoesNotExist:
log.warning("Person %s exists in LDAP but not in NM database", uid)
continue
def get_field(f):
if f not in attrs:
return None
f = attrs[f]
if not f:
return None
return f[0]
# TODO: if cn is '-', then set cn=sn and sn=None
changed = False
for field in ("cn", "mn", "sn"):
val = get_field(field)
if val is not None:
for encoding in ("utf8", "latin1"):
try:
val = val.decode(encoding)
good = True
break
except (UnicodeDecodeError, UnicodeEncodeError):
good = False
if not good:
log.warning("Field %s=%s for %s has invalid unicode information: skipping", field, repr(val), uid)
continue
old = getattr(person, field)
if old is not None:
for encoding in ("utf8", "latin1"):
try:
old = old.decode(encoding)
good = True
except (UnicodeDecodeError, UnicodeEncodeError):
good = False
if not good:
old = "<invalid encoding>"
if val != old:
try:
log.info("Person %s changed %s from %s to %s", uid, field, old, val)
except UnicodeDecodeError:
print "Problems with", uid
continue
setattr(person, field, val)
changed = True
if changed:
person.save()
class Command(BaseCommand):
help = 'Import people and changes from LDAP'
option_list = BaseCommand.option_list + (
optparse.make_option("--quiet", action="store_true", dest="quiet", default=None, help="Disable progress reporting"),
)
def handle(self, *fnames, **opts):
FORMAT = "%(asctime)-15s %(levelname)s %(message)s"
if opts["quiet"]:
logging.basicConfig(level=logging.WARNING, stream=sys.stderr, format=FORMAT)
else:
logging.basicConfig(level=logging.INFO, stream=sys.stderr, format=FORMAT)
updater = Updater()
updater.do_update()
#log.info("%d patch(es) applied", len(fnames))
keyring/management/commands/importkeyrings.py deleted 100644 → 0
View file @ 4bb5a355
# nm.debian.org website backend
#
# Copyright (C) 2012 Enrico Zini <enrico@debian.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.core.management.base import BaseCommand, CommandError
import django.db
from django.conf import settings
import optparse
import sys
import ldap
import logging
import json
from backend import models as bmodels
from backend import const
import keyring.models as kmodels
log = logging.getLogger(__name__)
class Importer(object):
"""
Perform initial import from keyring.d.o
Detects status by checking what keyring contains the fingerprint
"""
def __init__(self):
log.info("Importing dm keyring...")
self.dm = frozenset(kmodels.list_dm())
log.info("Importing dd_u keyring...")
self.dd_u = frozenset(kmodels.list_dd_u())
log.info("Importing dd_nu keyring...")
self.dd_nu = frozenset(kmodels.list_dd_nu())
log.info("Importing emeritus_dd keyring...")
self.emeritus_dd = frozenset(kmodels.list_emeritus_dd())
log.info("Importing removed_dd keyring...")
self.removed_dd = frozenset(kmodels.list_removed_dd())
def do_import(self):
for person in bmodels.Person.objects.all():
if not person.fpr:
log.info("%s/%s has no fingerprint: skipped", person.uid, person.email)
continue
old_status = person.status
if person.fpr in self.dm:
# If we have a fingerprint in the Person during the initial import,
# it means they come from LDAP, so they have a guest account
person.status = const.STATUS_DM_GA
if person.fpr in self.dd_u:
person.status = const.STATUS_DD_U
if person.fpr in self.dd_nu:
person.status = const.STATUS_DD_NU
if person.fpr in self.emeritus_dd:
person.status = const.STATUS_EMERITUS_DD
if person.fpr in self.removed_dd:
person.status = const.STATUS_REMOVED_DD
if old_status != person.status:
log.info("%s: status changed from %s to %s", person.uid, old_status, person.status)
person.save()
class Command(BaseCommand):
help = 'Import people and changes from LDAP'
option_list = BaseCommand.option_list + (
optparse.make_option("--quiet", action="store_true", dest="quiet", default=None, help="Disable progress reporting"),
)
def handle(self, *fnames, **opts):
FORMAT = "%(asctime)-15s %(levelname)s %(message)s"
if opts["quiet"]:
logging.basicConfig(level=logging.WARNING, stream=sys.stderr, format=FORMAT)
else:
logging.basicConfig(level=logging.INFO, stream=sys.stderr, format=FORMAT)
importer = Importer()
importer.do_import()
#log.info("%d patch(es) applied", len(fnames))
backend/management/commands/importjson.py → maintenance/management/commands/import.py
View file @ 72cfeeac
# nm.debian.org website backend
# nm.debian.org website maintenance
#
# Copyright (C) 2012 Enrico Zini <enrico@debian.org>
#
......@@ -22,7 +22,10 @@ import optparse
import sys
import logging
import json
import ldap
from backend import models as bmodels
from backend import const
import keyring.models as kmodels
log = logging.getLogger(__name__)
......@@ -30,6 +33,18 @@ class Importer(object):
def __init__(self):
self.people_cache_by_email = dict()
self.people_cache_by_uid = dict()
self.todo_advocates = dict()
log.info("Importing dm keyring...")
self.dm = frozenset(kmodels.list_dm())
log.info("Importing dd_u keyring...")
self.dd_u = frozenset(kmodels.list_dd_u())
log.info("Importing dd_nu keyring...")
self.dd_nu = frozenset(kmodels.list_dd_nu())
log.info("Importing emeritus_dd keyring...")
self.emeritus_dd = frozenset(kmodels.list_emeritus_dd())
log.info("Importing removed_dd keyring...")
self.removed_dd = frozenset(kmodels.list_removed_dd())
def import_person(self, person):
p = bmodels.Person(
......@@ -75,12 +90,7 @@ class Importer(object):
manager=am,
)
pr.save()
for adv in proc["advocates"]:
a = self.people_cache_by_uid.get(adv, None)
if a is None:
log.warning("advocate %s not found: skipping the DB association and leaving it just in the logs", adv)
continue
pr.advocates.add(a)
self.todo_advocates[pr.id] = proc["advocates"]
by_target[pr.applying_for] = pr
def get_person(uid):
......@@ -131,11 +141,136 @@ class Importer(object):
)
l.save()
def import_ldap(self, server):
"""
Perform initial data import from LDAP
Imports cn, sn, nm, fpr, email for DDs and guest accounts.
Does not set status, that will be taken from keyrings
"""
# enrico> Hi. Can you give me an official procedure to check if one is a DD from LDAP info?
# @weasel> enrico: not really, that's your decision.
# @weasel> enrico: for one, you can filter on gid 800. and then filter for having a
# fingerprint. that's usually right
# enrico> weasel: what are person accounts without fingerprints for?
# @weasel> people who screwed up their keys
# @weasel> we've had that on occasion
# enrico> weasel: ack
# @weasel> enrico: and of course retired people
# @weasel> we try to set ldap's account status nowadays, but no idea if
# that applies to all that ever retired
search_base = "dc=debian,dc=org"
l = ldap.initialize(server)
l.simple_bind_s("","")
for dn, attrs in l.search_s(search_base, ldap.SCOPE_SUBTREE, "objectclass=inetOrgPerson"):
# Try to match the person using uid
uid = attrs["uid"][0]
try:
person = bmodels.Person.objects.get(uid=uid)
if person.status == const.STATUS_MM:
person.status = const.STATUS_MM_GA
person.save()
continue
except bmodels.Person.DoesNotExist:
pass
# Try to match the person using emails
try:
person = bmodels.Person.objects.get(email=uid + "@debian.org")
person.uid = uid
if person.status == const.STATUS_MM:
person.status = const.STATUS_MM_GA,
person.save()
continue
except bmodels.Person.DoesNotExist:
pass
def get_field(f):
if f not in attrs:
return None
f = attrs[f]
if not f:
return None
return f[0]
email = get_field("emailForward")
try:
person = bmodels.Person.objects.get(email=email)
person.uid = uid
if person.status == const.STATUS_MM:
person.status = const.STATUS_MM_GA,
person.save()
continue
except bmodels.Person.DoesNotExist:
pass
person = bmodels.Person(
cn=get_field("cn"),
mn=get_field("mn"),
sn=get_field("sn"),
fpr=get_field("keyFingerPrint"),
uid=uid,
# Default to MM_GA: if they are in LDAP, they have at least a
# guest account
status=const.STATUS_MM_GA,
)
if get_field("gidNumber") == '800':
person.email = uid + "@debian.org"
else:
person.email = email
if person.email is None:
log.warning("Skipping %s because we have no email address", uid)
continue
person.save()
def import_advocates(self):
for id, advocates in self.todo_advocates.iteritems():
proc = bmodels.Process.objects.get(id=id)
for adv in advocates:
a = self.people_cache_by_uid.get(adv, None)
if a is None:
log.warning("advocate %s not found: skipping the DB association and leaving it just in the logs", adv)
continue
proc.advocates.add(a)
def import_keyrings(self):
"""
Perform initial import from keyring.d.o
Detects status by checking what keyring contains the fingerprint
"""
for person in bmodels.Person.objects.all():
if not person.fpr:
log.info("%s/%s has no fingerprint: skipped", person.uid, person.email)
continue
old_status = person.status
if person.fpr in self.dm:
# If we have a fingerprint in the Person during the initial import,
# it means they come from LDAP, so they have a guest account
person.status = const.STATUS_DM_GA
if person.fpr in self.dd_u:
person.status = const.STATUS_DD_U
if person.fpr in self.dd_nu:
person.status = const.STATUS_DD_NU
if person.fpr in self.emeritus_dd:
person.status = const.STATUS_EMERITUS_DD
if person.fpr in self.removed_dd:
person.status = const.STATUS_REMOVED_DD
if old_status != person.status:
log.info("%s: status changed from %s to %s", person.uid, old_status, person.status)
person.save()
class Command(BaseCommand):
help = 'Import a JSON database dump'
option_list = BaseCommand.option_list + (
optparse.make_option("--quiet", action="store_true", dest="quiet", default=None, help="Disable progress reporting"),
optparse.make_option("--ldap", action="store", default="ldap://db.debian.org", help="LDAP server to use. Default: %default"),
#l = ldap.initialize("ldap://localhost:3389")
)
def handle(self, *fnames, **opts):
......@@ -157,5 +292,8 @@ class Command(BaseCommand):
importer.import_person(v)
for k, v in people.iteritems():
importer.import_processes(v)
importer.import_ldap(opts["ldap"])
importer.import_advocates()
importer.import_keyrings()
#log.info("%d patch(es) applied", len(fnames))
backend/management/commands/maintenance.py → maintenance/management/commands/maintenance.py
View file @ 72cfeeac
# nm.debian.org website backend
# nm.debian.org website maintenance
#
# Copyright (C) 2012 Enrico Zini <enrico@debian.org>
#
......@@ -23,14 +23,75 @@ import optparse
import sys
import datetime
import logging
import json
import ldap
from backend import models as bmodels
from backend import const
log = logging.getLogger(__name__)
@transaction.commit_on_success
def compute_update_from_ldap(**kw):
search_base = "dc=debian,dc=org"
l = ldap.initialize(kw["ldap"])
l.simple_bind_s("","")
for dn, attrs in l.search_s(search_base, ldap.SCOPE_SUBTREE, "objectclass=inetOrgPerson"):
uid = attrs["uid"][0]
try:
person = bmodels.Person.objects.get(uid=uid)
except bmodels.Person.DoesNotExist:
log.warning("Person %s exists in LDAP but not in NM database", uid)
continue
def get_field(f):
if f not in attrs:
return None
f = attrs[f]
if not f:
return None
return f[0]
# TODO: if cn is '-', then set cn=sn and sn=None
changed = False
for field in ("cn", "mn", "sn"):
val = get_field(field)
if val is not None:
for encoding in ("utf8", "latin1"):
try:
val = val.decode(encoding)
good = True
break
except (UnicodeDecodeError, UnicodeEncodeError):
good = False
if not good:
log.warning("Field %s=%s for %s has invalid unicode information: skipping", field, repr(val), uid)
continue
old = getattr(person, field)
if old is not None:
for encoding in ("utf8", "latin1"):
try:
old = old.decode(encoding)
good = True
except (UnicodeDecodeError, UnicodeEncodeError):
good = False
if not good:
old = "<invalid encoding>"
if val != old:
try:
log.info("Person %s changed %s from %s to %s", uid, field, old, val)
except UnicodeDecodeError:
print "Problems with", uid
continue
setattr(person, field, val)
changed = True
if changed:
person.save()
@transaction.commit_on_success
def compute_am_ctte():
def compute_am_ctte(**kw):
from django.db.models import Max
# Set all to False
bmodels.AM.objects.update(is_am_ctte=False)
......@@ -55,7 +116,7 @@ def compute_am_ctte():
@transaction.commit_on_success
def compute_process_is_active():
def compute_process_is_active(**kw):
"""
Compute Process.is_active from Process.progress
"""
......@@ -68,7 +129,7 @@ def compute_process_is_active():
bmodels.Process.objects.filter(is_active=True).count(),
cursor.rowcount)
def check_one_process_per_person():
def check_one_process_per_person(**kw):
"""
Check that one does not have more than one open process at the current time
"""
......@@ -80,7 +141,7 @@ def check_one_process_per_person():
for idx, proc in enumerate(p.processes.filter(is_active=True)):
log.warning(" %d: %s (%s)", idx+1, proc.applying_for, proc.progress)
def check_am_must_have_uid():
def check_am_must_have_uid(**kw):
"""
Check that one does not have more than one open process at the current time
"""
......@@ -93,6 +154,7 @@ class Command(BaseCommand):
help = 'Daily maintenance of the nm.debian.org database'
option_list = BaseCommand.option_list + (
optparse.make_option("--quiet", action="store_true", dest="quiet", default=None, help="Disable progress reporting"),
optparse.make_option("--ldap", action="store", default="ldap://db.debian.org", help="LDAP server to use. Default: %default"),
)
def handle(self, *fnames, **opts):
......@@ -104,7 +166,7 @@ class Command(BaseCommand):
# Run procedures
for prefix in ("backup", "compute", "check"):
for k, v in globals().iteritems():
for k, v in sorted(globals().iteritems()):
if not k.startswith(prefix + "_"): continue
log.info("running %s", k)
v()
v(**opts)
maintenance/models.py 0 → 100644
View file @ 72cfeeac
from django.db import models
# Create your models here.
maintenance/tests.py 0 → 100644
View file @ 72cfeeac
"""
This file demonstrates writing tests using the unittest module. These will pass
when you run "manage.py test".
Replace this with more appropriate tests for your application.
"""
from django.test import TestCase
class SimpleTest(TestCase):
def test_basic_addition(self):
"""
Tests that 1 + 1 always equals 2.
"""
self.assertEqual(1 + 1, 2)
maintenance/views.py 0 → 100644
View file @ 72cfeeac
# Create your views here.
public/templates/public/nmstatus.html
View file @ 72cfeeac
......@@ -18,6 +18,14 @@
<table class="personinfo">
<tr><th>Received application</th><td>{{started.date}}</td></tr>
<tr>
<th>Applicant{{process.advocates.count|pluralize}}</th>
<td>
{% for a in process.advocates.all %}
{{a.uid}}{% if not forloop.last %},{% endif %}
{% endfor %}
</td>
</tr>
<tr><th>Account name</th><td>{{process.person.uid|default:"None yet"}}</td></tr>
<tr><th>Time of Last Action</th><td>{{last_change}}</th></tr>
<tr><th>Account created</td><td>{% if process.progress == PROGRESS_DONE %}yes{% else %}no{% endif %}</td></tr>
......
settings.py.devel
View file @ 72cfeeac
......@@ -145,6 +145,7 @@ INSTALLED_APPS = (
'backend',
'public',
'restricted',
'maintenance',
)
TEMPLATE_CONTEXT_PROCESSORS = global_settings.TEMPLATE_CONTEXT_PROCESSORS + (
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment