Serialize/deserialize/test also gpg keys and inconsistencies

backend/export.py

@@ -2,6 +2,10 @@ from . import models as bmodels
 from . import serializers as bserializers
 import process.models as pmodels
 import process.serializers as pserializers
+import keyring.models as kmodels
+import keyring.serializers as kserializers
+import sitechecks.models as smodels
+import sitechecks.serializers as sserializers
 
 
 def export_db(full=False):
@@ -44,20 +48,20 @@ def export_db(full=False):
         for key in kmodels.Key.objects.all().order_by("fpr"):
             gpgkeys.append(GpgKeySerializer(key).data)
 
-    sitechecks = []
+    inconsistencies = []
     if full:
         from sitechecks import models as smodels
         from sitechecks import serializers as sserializers
         InconsistencySerializer = sserializers.InconsistencyExportSerializer
         for key in smodels.Inconsistency.objects.all().order_by("person", "process"):
-            sitechecks.append(InconsistencySerializer(key).data)
+            inconsistencies.append(InconsistencySerializer(key).data)
 
     res = {
         "version": 1,
         "people": people,
         "apikeys": apikeys,
         "gpgkeys": gpgkeys,
-        "sitechecks": sitechecks,
+        "inconsistencies": inconsistencies,
     }
 
     return res
@@ -92,7 +96,7 @@ class ImporterV1:
         self.gpgkeys = []
 
         # Site checks
-        self.sitechecks = []
+        self.inconsistencies = []
 
         self.context = {"importer": self}
 
@@ -314,6 +318,46 @@ class ImporterV1:
 
         return ama
 
+    def import_gpgkey(self, data):
+        s = kserializers.KeyExportSerializer(data=data, context=self.context)
+        if not s.is_valid():
+            raise RuntimeError("Invalid GPG Key record {}: {}".format(data, s.errors))
+
+        fpr = s.validated_data.pop("fpr")
+
+        key = kmodels.Key(**s.validated_data)
+        key._rels = {
+            "fpr": self.fprs[fpr],
+        }
+
+        return key
+
+    def import_inconsistency(self, data):
+        s = sserializers.InconsistencyExportSerializer(data=data, context=self.context)
+        if not s.is_valid():
+            raise RuntimeError("Invalid Inconsistency record {}: {}".format(data, s.errors))
+
+        person = self.people[s.validated_data.pop("person")]
+        process = s.validated_data.pop("process")
+
+        inc = smodels.Inconsistency(**s.validated_data)
+        inc._rels = {
+            "person": person,
+            "process": None
+        }
+
+        if process is not None:
+            appfor, started = process
+            found = None
+            for p in person._rels["processes"]:
+                if p.applying_for == appfor and p.started.replace(microsecond=0) == started:
+                    found = p
+            if not found:
+                raise RuntimeError("Process for {} started {} not found for {}".format(appfor, started, person))
+            inc._rels["process"] = found
+
+        return inc
+
     def import_all(self, data):
         # We do a double pass to create all people by lookup key, so that they
         # can be referenced in dependent objects
@@ -326,10 +370,10 @@ class ImporterV1:
             pass  # TODO
 
         for gpgkey in data["gpgkeys"]:
-            pass  # TODO
+            self.gpgkeys.append(self.import_gpgkey(gpgkey))
 
-        for sitecheck in data["sitechecks"]:
-            pass  # TODO
+        for inconsistency in data["inconsistencies"]:
+            self.inconsistencies.append(self.import_inconsistency(inconsistency))
 
     def save(self):
         for person in self.people.values():
@@ -386,6 +430,15 @@ class ImporterV1:
                     le.requirement = le._rels["requirement"]
                     le.save()
 
+        for gpgkey in self.gpgkeys:
+            gpgkey.fpr = gpgkey._rels["fpr"]
+            gpgkey.save()
+
+        for inconsistency in self.inconsistencies:
+            inconsistency.person = inconsistency._rels["person"]
+            inconsistency.process = inconsistency._rels["process"]
+            inconsistency.save()
+
 
 #def parse_datetime(s):
 #    if s is None:

backend/tests/test_import_export.py

@@ -4,6 +4,9 @@ from django.forms.models import model_to_dict
 from backend import const
 import backend.models as bmodels
 import process.models as pmodels
+import keyring.models as kmodels
+import sitechecks.models as smodels
+
 import datetime
 from backend.export import export_db, load_db
 
@@ -197,6 +200,31 @@ class TestImportExport(TestCase):
             "requirement": self.requirement,
         }
 
+        self.key = kmodels.Key.objects.create(
+            fpr=self.fingerprint,
+            key="GPG KEY",
+            key_updated=datetime.datetime(2019, 8, 7, 6, 5, tzinfo=utc),
+            check_sigs="test check sigs",
+            check_sigs_updated=datetime.datetime(2019, 8, 7, 6, 6, tzinfo=utc),
+        )
+        self.key._rels = {
+            "fpr": self.fingerprint,
+        }
+
+        self.inconsistency = smodels.Inconsistency.objects.create(
+            person=self.person,
+            process=self.process,
+            first_seen=datetime.date(2019, 1, 1),
+            last_seen=datetime.date(2019, 8, 7),
+            ignore_until=datetime.date(2019, 9, 8),
+            tag="test tag",
+            text="test text",
+        )
+        self.inconsistency._rels = {
+            "person": self.inconsistency.person,
+            "process": self.inconsistency.process,
+        }
+
     def assertPersonEqual(self, expected, actual):
         if expected is None and actual is None:
             return
@@ -413,8 +441,27 @@ class TestImportExport(TestCase):
         self.assertPersonEqual(expected._rels["changed_by"], actual._rels["changed_by"])
         self.assertRequirementEqual(expected._rels["requirement"], actual._rels["requirement"])
 
+    def assertGpgKeyEqual(self, expected, actual):
+        self.maxDiff = None
+        exclude = ["id", "fpr"]
+        self.assertEqual(
+                model_to_dict(expected, exclude=exclude),
+                model_to_dict(actual, exclude=exclude))
+        self.assertFingerprintEqual(expected._rels["fpr"], actual._rels["fpr"])
+
+    def assertInconsistencyEqual(self, expected, actual):
+        self.maxDiff = None
+        exclude = ["id", "person", "process"]
+        self.assertEqual(
+                model_to_dict(expected, exclude=exclude),
+                model_to_dict(actual, exclude=exclude))
+        self.assertPersonEqual(expected._rels["person"], actual._rels["person"])
+        self.assertProcessEqual(expected._rels["process"], actual._rels["process"])
+
     def clear_database(self):
         bmodels.Person.objects.all().delete()
+        kmodels.Key.objects.all().delete()
+        smodels.Inconsistency.objects.all().delete()
 
     def assertSaves(self, loaded):
         """
@@ -469,6 +516,17 @@ class TestImportExport(TestCase):
             self.assertEqual(log[1].changed_by, fd)
             self.assertIsNone(log[1].requirement)
 
+        if loaded.gpgkeys:
+            key = loaded.gpgkeys[0]
+            self.assertIsInstance(key.pk, int)
+            self.assertEqual(key.fpr, person.fingerprint)
+
+        if loaded.inconsistencies:
+            inconsistency = loaded.inconsistencies[0]
+            self.assertIsInstance(inconsistency.pk, int)
+            self.assertEqual(inconsistency.person, person)
+            self.assertEqual(inconsistency.process, process)
+
     def test_redacted(self):
         serialized = export_db(full=False)
         self.clear_database()
@@ -484,6 +542,9 @@ class TestImportExport(TestCase):
 
         self.assertRedactedProcessEqual(self.process, person._rels["processes"][0])
 
+        self.assertEqual(len(loaded.gpgkeys), 0)
+        self.assertEqual(len(loaded.inconsistencies), 0)
+
         self.assertSaves(loaded)
 
     def test_full(self):
@@ -505,4 +566,10 @@ class TestImportExport(TestCase):
 
         self.assertProcessEqual(self.process, person._rels["processes"][0])
 
+        self.assertEqual(len(loaded.gpgkeys), 1)
+        self.assertGpgKeyEqual(loaded.gpgkeys[0], self.key)
+
+        self.assertEqual(len(loaded.inconsistencies), 1)
+        self.assertInconsistencyEqual(loaded.inconsistencies[0], self.inconsistency)
+
         self.assertSaves(loaded)

process/models.py

@@ -297,7 +297,7 @@ class Process(models.Model):
     applying_for = models.CharField("target status", max_length=20, null=False, choices=[
                                     x[1:3] for x in const.ALL_STATUS])
     started = models.DateTimeField(
-        auto_now_add=True, verbose_name='process started')
+        default=now, verbose_name='process started')
     frozen_by = models.ForeignKey(
             bmodels.Person, related_name="+", blank=True, null=True,
             help_text=_("Person that froze this process for review, or NULL if it is still being worked on"))

sitechecks/serializers.py

@@ -5,10 +5,10 @@ from . import models
 
 class ProcessField(serializers.Field):
     def to_representation(self, value):
-        return [value.person.lookup_key, value.applying_for, value.started]
+        return [value.applying_for, value.started]
 
-#    def to_internal_value(self, data):
-#        return Person.lookup(data).am
+    def to_internal_value(self, data):
+        return data
 
 
 class InconsistencyExportSerializer(serializers.ModelSerializer):