update rpki.gui.cacheview app to support new rcynic xml format. closes #106

svn path=/trunk/; revision=4054

6 files changed, 263 insertions, 241 deletions

diff --git a/rpkid/portal-gui/scripts/rpkigui-rcynic.py b/rpkid/portal-gui/scripts/rpkigui-rcynic.py
index a4d55f5e..4ec21795 100644
--- a/rpkid/portal-gui/scripts/rpkigui-rcynic.py
+++ b/rpkid/portal-gui/scripts/rpkigui-rcynic.py
@@ -24,129 +24,142 @@ from rpki.gui.cacheview import models
 from rpki.rcynic import rcynic_xml_iterator, label_iterator
 from rpki.sundial import datetime
 from django.db import transaction
+import django.db.models
 
 debug = False
+fam_map = { 'roa_prefix_set_ipv6': 6, 'roa_prefix_set_ipv4': 4 }
 
-class TransactionManager(object):
-    """
-    Context manager wrapper around the Django transaction API.
-    """
-    def __enter__(self):
-        transaction.enter_transaction_management()
-        transaction.managed()
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if exc_type is None:
-            transaction.commit()
-        else:
-            transaction.set_clean()
-        transaction.leave_transaction_management()
-        return False
-
-def process_object(obj, model_class):
-    """
-    do initial processing on a rcynic_object instance.
-
-    return value is a tuple: first element is a boolean value indicating whether
-    the object is changed/new since the last time we processed it.  second
-    element is the db instance.
-    """
-    if debug:
-        print 'processing %s at %s' % (obj.__class__.__name__, obj.uri)
+class rcynic_object(object):
 
-    q = model_class.objects.filter(uri=obj.uri)
-    if not q:
-        if debug:
-            print 'creating new db instance'
-        inst = model_class(uri=obj.uri)
-    else:
-        inst = q[0]
-
-    # metadata that is updated on every run, regardless of whether the object
-    # has changed
-    inst.ok = obj.ok
-    inst.status = models.ValidationStatus.objects.get(label=obj.status)
-    inst.timestamp = datetime.fromXMLtime(obj.timestamp).to_sql()
-
-    # determine if the object is changed/new
-    mtime = os.stat(obj.filename)[8]
-    if mtime != inst.mtime:
-        inst.mtime = mtime
-        inst.not_before = obj.notBefore.to_sql()
-        inst.not_after = obj.notAfter.to_sql()
+    def __call__(self, vs):
+        """
+        do initial processing on a rcynic_object instance.
+
+        return value is a tuple: first element is a boolean value indicating whether
+        the object is changed/new since the last time we processed it.  second
+        element is the db instance.
+        """
         if debug:
-            sys.stderr.write('name=%s ski=%s\n' % (obj.subject, obj.ski))
-        inst.name = obj.subject
-        inst.keyid = obj.ski
-
-        # look up signing cert
-        if obj.issuer == obj.subject:
-            # self-signed cert (TA)
-            inst.cert = inst
-        else:
-            q = models.Cert.objects.filter(keyid=obj.aki)
-            if q:
-                inst.issuer = q[0]
+            print 'processing %s at %s' % (vs.file_class.__name__, vs.uri)
+
+        # rcynic will generation <validation_status/> elements for objects
+        # listed in the manifest but not found on disk
+        if os.path.exists(vs.filename):
+            q = self.model_class.objects.filter(uri=vs.uri)
+            if not q:
+                if debug:
+                    print 'creating new db instance'
+                inst = self.model_class(uri=vs.uri)
             else:
-                sys.stderr.write('warning: unable to find signing cert with ski=%s (%s)\n' % (obj.aki, obj.issuer))
+                inst = q[0]
+
+            # determine if the object is changed/new
+            mtime = os.stat(vs.filename)[8]
+            if mtime != inst.mtime:
+                inst.mtime = mtime
+                obj = vs.obj # causes object to be lazily loaded
+                inst.not_before = obj.notBefore.to_sql()
+                inst.not_after = obj.notAfter.to_sql()
+                if debug:
+                    sys.stderr.write('name=%s ski=%s\n' % (obj.subject, obj.ski))
+                inst.name = obj.subject
+                inst.keyid = obj.ski
+
+                # look up signing cert
+                if obj.issuer == obj.subject:
+                    # self-signed cert (TA)
+                    inst.cert = inst
+                else:
+                    q = models.Cert.objects.filter(keyid=obj.aki)
+                    if q:
+                        inst.issuer = q[0]
+                    else:
+                        sys.stderr.write('warning: unable to find signing cert with ski=%s (%s)\n' % (obj.aki, obj.issuer))
+                        return None
 
-        return True, inst
-    elif debug:
-        print 'object is unchanged'
+                self.callback(obj, inst)
+            else:
+                if debug:
+                    print 'object is unchanged'
 
-    # metadata has changed, so a save is required
-    inst.save()
+            # save required to create new ValidationStatus object refering to it
+            inst.save()
+            inst.statuses.create(generation=models.generations_dict[vs.generation] if vs.generation else None,
+                    timestamp=datetime.fromXMLtime(vs.timestamp).to_sql(),
+                    status=models.ValidationLabel.objects.get(label=vs.status))
 
-    return False, inst
+            return inst
+        else:
+            if debug:
+                print 'ERROR - file is missing: %s' % vs.filename
+
+        return True
 
-def process_rescert(cert):
-    """
-    Process a RPKI resource certificate.
-    """
+class rcynic_cert(rcynic_object):
+    model_class = models.Cert
 
-    refresh, obj = process_object(cert, models.Cert)
+    def callback(self, cert, obj):
+        """
+        Process a RPKI resource certificate.
+        """
 
-    if refresh:
+        obj.sia = cert.sia_directory_uri
         obj.save()
 
         # resources can change when a cert is updated
         obj.asns.clear()
         obj.addresses.clear()
 
-        with TransactionManager():
-            for asr in cert.resources.asn:
+        for asr in cert.resources.asn:
+            if debug:
+                sys.stderr.write('processing %s\n' % asr)
+
+            attrs = { 'min': asr.min, 'max': asr.max }
+            q = models.ASRange.objects.filter(**attrs)
+            if not q:
+                obj.asns.create(**attrs)
+            else:
+                obj.asns.add(q[0])
+
+        for family, addrset in (4, cert.resources.v4), (6, cert.resources.v6):
+            for rng in addrset:
                 if debug:
-                    sys.stderr.write('processing %s\n' % asr)
+                    sys.stderr.write('processing %s\n' % rng)
 
-                attrs = { 'min': asr.min, 'max': asr.max }
-                q = models.ASRange.objects.filter(**attrs)
+                attrs = { 'family': family, 'min': str(rng.min), 'max': str(rng.max) }
+                q = models.AddressRange.objects.filter(**attrs)
                 if not q:
-                    obj.asns.create(**attrs)
+                    obj.addresses.create(**attrs)
                 else:
-                    obj.asns.add(q[0])
-
-            for family, addrset in (4, cert.resources.v4), (6, cert.resources.v6):
-                for rng in addrset:
-                    if debug:
-                        sys.stderr.write('processing %s\n' % rng)
+                    obj.addresses.add(q[0])
 
-                    attrs = { 'family': family, 'min': str(rng.min), 'max': str(rng.max) }
-                    q = models.AddressRange.objects.filter(**attrs)
-                    if not q:
-                        obj.addresses.create(**attrs)
-                    else:
-                        obj.addresses.add(q[0])
+        if debug:
+            print 'finished processing rescert at %s' % cert.uri
 
-    if debug:
-        print 'finished processing rescert at %s' % cert.uri
+class rcynic_roa(rcynic_object):
+    model_class = models.ROA
 
-    return obj
+    def callback(self, roa, obj):
+        obj.asid = roa.asID
+        obj.save()
+        obj.prefixes.clear()
+        for pfxset in roa.prefix_sets:
+            family = fam_map[pfxset.__class__.__name__]
+            for pfx in pfxset:
+                attrs = { 'family' : family,
+                          'prefix': str(pfx.prefix),
+                          'bits' : pfx.prefixlen,
+                          'max_length': pfx.max_prefixlen }
+                q = models.ROAPrefix.objects.filter(**attrs)
+                if not q:
+                    obj.prefixes.create(**attrs)
+                else:
+                    obj.prefixes.add(q[0])
 
-def process_ghostbuster(gbr):
-    refresh, obj = process_object(gbr, models.Ghostbuster)
+class rcynic_gbr(rcynic_object):
+    model_class = models.Ghostbuster
 
-    if refresh:
+    def callback(self, gbr, obj):
         vcard = vobject.readOne(gbr.vcard)
         if debug:
             vcard.prettyPrint()
@@ -154,91 +167,51 @@ def process_ghostbuster(gbr):
         obj.email_address = vcard.email.value if hasattr(vcard, 'email') else None
         obj.telephone = vcard.tel.value if hasattr(vcard, 'tel') else None
         obj.organization = vcard.org.value[0] if hasattr(vcard, 'org') else None
-        obj.save()
-
-fam_map = { 'roa_prefix_set_ipv6': 6, 'roa_prefix_set_ipv4': 4 }
-
-def process_roa(roa):
-    refresh, obj = process_object(roa, models.ROA)
-
-    if refresh:
-        obj.asid = roa.asID
-        obj.save()
-        with TransactionManager():
-            obj.prefixes.clear()
-            for pfxset in roa.prefix_sets:
-                family = fam_map[pfxset.__class__.__name__]
-                for pfx in pfxset:
-                    attrs = { 'family' : family,
-                              'prefix': str(pfx.prefix),
-                              'bits' : pfx.prefixlen,
-                              'max_length': pfx.max_prefixlen }
-                    q = models.ROAPrefix.objects.filter(**attrs)
-                    if not q:
-                        obj.prefixes.create(**attrs)
-                    else:
-                        obj.prefixes.add(q[0])
-
-    return obj
-
-def trydelete(seq):
-    """
-    iterate over a sequence and attempt to delete each item.  safely
-    ignore IntegrityError since the object may be referenced elsewhere.
-    """
-    for o in seq:
-        try:
-            o.delete()
-        except IntegrityError:
-            pass
-
-def garbage_collect(ts):
-    """
-    rcynic's XML output file tells us what is currently in the cache,
-    but not what has been removed.  we save the timestamp from the first
-    entry in the XML file, and remove all objects which are older.
-    """
-    if debug:
-        print 'doing garbage collection'
-
-    for roa in models.ROA.objects.filter(timestamp__lt=ts):
-        if debug:
-            sys.stderr.write('removing %s\n' % roa.uri)
-        trydelete(roa.prefixes.all())
-        roa.delete()
-
-    for cert in models.Cert.objects.filter(timestamp__lt=ts):
-        if debug:
-            sys.stderr.write('removing %s\n' % cert.uri)
-        trydelete(cert.asns.all())
-        trydelete(cert.addresses.all())
-        cert.delete()
-
-    for gbr in models.Ghostbuster.objects.filter(timestamp__lt=ts):
-        if debug:
-            sys.stderr.write('removing %s\n' % gbr.uri)
-        gbr.delete()
 
 def process_cache(root, xml_file):
     start = time.time()
 
-    # the timestamp from the first element in the rcynic xml file is saved
-    # to perform garbage collection of stale db entries
-    ts = 0
-
     dispatch = {
-      'rcynic_certificate': process_rescert,
-      'rcynic_roa'        : process_roa,
-      'rcynic_ghostbuster': process_ghostbuster
+      'rcynic_certificate': rcynic_cert(),
+      'rcynic_roa'        : rcynic_roa(),
+      'rcynic_ghostbuster': rcynic_gbr()
     }
 
+    # remove all existing ValidationStatus_* entries
+    models.ValidationStatus_Cert.objects.all().delete()
+    models.ValidationStatus_ROA.objects.all().delete()
+    models.ValidationStatus_Ghostbuster.objects.all().delete()
+
     # loop over all rcynic objects and dispatch based on the returned
     # rcynic_object subclass
-    for obj in rcynic_xml_iterator(root, xml_file):
-        r = dispatch[obj.__class__.__name__](obj)
-        if not ts:
-            ts = r.timestamp
-    garbage_collect(ts)
+    n = 1
+    defer = rcynic_xml_iterator(root, xml_file)
+    while defer:
+        if debug:
+            print 'starting iteration %d for deferred objects' % n
+            n = n + 1
+
+        elts = defer
+        defer = []
+        for vs in elts:
+            # need to defer processing this object, most likely because
+            # the <validation_status/> element for the signing cert hasn't
+            # been seen yet
+            if not dispatch[vs.file_class.__name__](vs):
+                defer.append(vs)
+
+    # garbage collection
+    # remove all objects which have no ValidationStatus references, which
+    # means they did not appear in the last XML output
+    if debug:
+        print 'performing garbage collection'
+
+    # trying to .delete() the querysets directly results in a "too many sql variables" exception
+    for qs in (models.Cert.objects.annotate(num_statuses=django.db.models.Count('statuses')).filter(num_statuses=0),
+            models.Ghostbuster.objects.annotate(num_statuses=django.db.models.Count('statuses')).filter(num_statuses=0),
+            models.ROA.objects.annotate(num_statuses=django.db.models.Count('statuses')).filter(num_statuses=0)):
+        for e in qs:
+            e.delete()
 
     if debug:
         stop = time.time()
@@ -248,21 +221,19 @@ def process_labels(xml_file):
     if debug:
         sys.stderr.write('updating labels...\n')
 
-    with TransactionManager():
-        kinds = { 'good': 0, 'warn': 1, 'bad': 2 }
-        for label, kind, desc in label_iterator(xml_file):
-            if debug:
-                sys.stderr.write('label=%s kind=%s desc=%s\n' % (label, kind, desc))
-            if kind:
-                q = models.ValidationStatus.objects.filter(label=label)
-                if not q:
-                    obj = models.ValidationStatus(label=label)
-                else:
-                    obj = q[0]
+    for label, kind, desc in label_iterator(xml_file):
+        if debug:
+            sys.stderr.write('label=%s kind=%s desc=%s\n' % (label, kind, desc))
+        if kind:
+            q = models.ValidationLabel.objects.filter(label=label)
+            if not q:
+                obj = models.ValidationLabel(label=label)
+            else:
+                obj = q[0]
 
-                obj.kind = kinds[kind]
-                obj.status = desc
-                obj.save()
+            obj.kind = models.kinds_dict[kind]
+            obj.status = desc
+            obj.save()
 
 if __name__ == '__main__':
     import optparse
@@ -280,7 +251,8 @@ if __name__ == '__main__':
     if options.debug:
         debug = True
 
-    process_labels(options.logfile)
-    process_cache(options.root, options.logfile)
+    with transaction.commit_on_success():
+        process_labels(options.logfile)
+        process_cache(options.root, options.logfile)
 
 # vim:sw=4 ts=8
diff --git a/rpkid/rpki/gui/cacheview/admin.py b/rpkid/rpki/gui/cacheview/admin.py
index 2b88c1f3..05bab881 100644
--- a/rpkid/rpki/gui/cacheview/admin.py
+++ b/rpkid/rpki/gui/cacheview/admin.py
@@ -37,8 +37,13 @@ class ROAAdmin(admin.ModelAdmin):
 class GhostbusterAdmin(admin.ModelAdmin):
     pass
 
-class ValidationStatusAdmin(admin.ModelAdmin):
-    pass
+class ValidationLabelAdmin(admin.ModelAdmin): pass
+
+class ValidationStatus_CertAdmin(admin.ModelAdmin): pass
+
+class ValidationStatus_ROAAdmin(admin.ModelAdmin): pass
+
+class ValidationStatus_GhostbusterAdmin(admin.ModelAdmin): pass
 
 admin.site.register(models.AddressRange, AddressRangeAdmin)
 admin.site.register(models.ASRange, AddressRangeAdmin)
@@ -46,6 +51,9 @@ admin.site.register(models.Cert, CertAdmin)
 admin.site.register(models.Ghostbuster, GhostbusterAdmin)
 admin.site.register(models.ROA, ROAAdmin)
 admin.site.register(models.ROAPrefix, ROAPrefixAdmin)
-admin.site.register(models.ValidationStatus, ValidationStatusAdmin)
+admin.site.register(models.ValidationLabel, ValidationLabelAdmin)
+admin.site.register(models.ValidationStatus_Cert, ValidationStatus_CertAdmin)
+admin.site.register(models.ValidationStatus_ROA, ValidationStatus_ROAAdmin)
+admin.site.register(models.ValidationStatus_Ghostbuster, ValidationStatus_GhostbusterAdmin)
 
 # vim:sw=4 ts=8
diff --git a/rpkid/rpki/gui/cacheview/models.py b/rpkid/rpki/gui/cacheview/models.py
index d68601fc..077a28ff 100644
--- a/rpkid/rpki/gui/cacheview/models.py
+++ b/rpkid/rpki/gui/cacheview/models.py
@@ -73,9 +73,10 @@ class ASRange(models.Model):
     def get_absolute_url(self):
         return ('rpki.gui.cacheview.views.asrange_detail', [str(self.pk)])
 
-kinds = ( (0, 'good'), (1, 'warn'), (2, 'bad') )
+kinds = list(enumerate(('good', 'warn', 'bad')))
+kinds_dict = dict((v,k) for k,v in kinds)
 
-class ValidationStatus(models.Model):
+class ValidationLabel(models.Model):
     """
     Represents a specific error condition defined in the rcynic XML
     output file.
@@ -87,11 +88,19 @@ class ValidationStatus(models.Model):
     def __unicode__(self):
         return self.label
 
-    def kind_as_str(self):
-        return kinds[self.kind][1]
+    class Meta:
+        verbose_name_plural = 'ValidationLabels'
+
+generations = list(enumerate(('current', 'backup')))
+generations_dict = dict((val, key) for (key, val) in generations)
+
+class ValidationStatus(models.Model):
+    timestamp  = models.DateTimeField()
+    generation = models.PositiveSmallIntegerField(choices=generations, null=True)
+    status     = models.ForeignKey('ValidationLabel')
 
     class Meta:
-        verbose_name_plural = 'ValidationStatuses'
+        abstract = True
 
 class SignedObject(models.Model):
     """
@@ -101,10 +110,8 @@ class SignedObject(models.Model):
     """
     # attributes from rcynic's output XML file
     uri        = models.URLField(unique=True, db_index=True)
-    timestamp  = models.DateTimeField()
-    ok         = models.BooleanField()
-    status     = models.ForeignKey('ValidationStatus')
 
+    # on-disk file modification time
     mtime      = models.PositiveIntegerField(default=0)
 
     # SubjectName
@@ -126,6 +133,24 @@ class SignedObject(models.Model):
         """
         return datetime.utcfromtimestamp(self.mtime + time.timezone)
 
+    def is_valid(self):
+        """
+        Returns a boolean value indicating whether this object has passed
+        validation checks.
+        """
+        return bool(self.statuses.filter(status=ValidationLabel.objects.get(label="object_accepted")))
+
+    def status_id(self):
+        """
+        Returns a HTML class selector for the current object based on its validation status.
+        The selector is chosen based on the current generation only.  If there is any bad status,
+        return bad, else if there are any warn status, return warn, else return good.
+        """
+        for x in reversed(kinds):
+            if self.statuses.filter(generation=generations_dict['current'], status__kind=x[0]):
+                return x[1]
+        return None # should not happen
+
     def __unicode__(self):
         return u'%s' % self.name
 
@@ -136,11 +161,15 @@ class Cert(SignedObject):
     addresses = models.ManyToManyField(AddressRange, related_name='certs')
     asns      = models.ManyToManyField(ASRange, related_name='certs')
     issuer    = models.ForeignKey('Cert', related_name='children', null=True, blank=True)
+    sia       = models.CharField(max_length=255)
 
     @models.permalink
     def get_absolute_url(self):
         return ('rpki.gui.cacheview.views.cert_detail', [str(self.pk)])
 
+class ValidationStatus_Cert(ValidationStatus):
+    cert = models.ForeignKey('Cert', related_name='statuses')
+
 class ROAPrefix(models.Model):
     family     = models.PositiveIntegerField()
     prefix     = models.IPAddressField()
@@ -175,6 +204,9 @@ class ROA(SignedObject):
     def get_absolute_url(self):
         return ('rpki.gui.cacheview.views.roa_detail', [str(self.pk)])
 
+class ValidationStatus_ROA(ValidationStatus):
+    roa = models.ForeignKey('ROA', related_name='statuses')
+
 class Ghostbuster(SignedObject):
     full_name     = models.CharField(max_length=40)
     email_address = models.EmailField(blank=True, null=True)
@@ -195,4 +227,7 @@ class Ghostbuster(SignedObject):
             return self.email_address
         return self.telephone
 
+class ValidationStatus_Ghostbuster(ValidationStatus):
+    gbr = models.ForeignKey('Ghostbuster', related_name='statuses')
+
 # vim:sw=4 ts=8 expandtab
diff --git a/rpkid/rpki/gui/cacheview/templates/cacheview/cert_detail.html b/rpkid/rpki/gui/cacheview/templates/cacheview/cert_detail.html
index a0291d7b..9ff304a2 100644
--- a/rpkid/rpki/gui/cacheview/templates/cacheview/cert_detail.html
+++ b/rpkid/rpki/gui/cacheview/templates/cacheview/cert_detail.html
@@ -32,6 +32,7 @@ Resource Certificate Detail
 <h2>Issued Objects</h2>
 <ul>
 
+{% if object.ghostbusters.all %}
     <li>
 <h3>Ghostbusters</h3>
 
@@ -39,26 +40,27 @@ Resource Certificate Detail
     <tr><th>Name</th><th>Valid</th><th>Until</th></tr>
 
 {% for g in object.ghostbusters.all %}
-    <tr class='{{ g.status.kind_as_str }}'>
+    <tr class='{{ g.status_id }}'>
         <td><a href="{{ g.get_absolute_url }}">{{ g }}</a></td>
-        <td>{{ g.ok }}</td>
+        <td>{{ g.is_valid }}</td>
         <td>{{ g.not_after }}</td>
     </tr>
 {% endfor %}
 
 </table>
+{% endif %}
 
+{% if object.roas.all %}
         <li>
 <h3>ROAs</h3>
-{% if object.roas.all %}
 <table>
     <tr><th>Prefix</th><th>AS</th><th>Valid</th><th>Until</th></tr>
     {% for roa in object.roas.all %}
     {% for pfx in roa.prefixes.all %}
-    <tr class='{{ roa.status.kind_as_str }}'>
+    <tr class='{{ roa.status_id }}'>
         <td>{{ pfx }}</td>
         <td>{{ roa.asid }}</td>
-        <td><a href="{{ roa.get_absolute_url }}">{{ roa.ok }}</a></td>
+        <td><a href="{{ roa.get_absolute_url }}">{{ roa.is_valid }}</a></td>
         <td>{{ roa.not_after }}</td>
     </tr>
     {% endfor %}
@@ -66,21 +68,24 @@ Resource Certificate Detail
 </table>
 {% endif %}
 
+{% if object.children.all %}
 <li>
-
 <h3>Children</h3>
 <table>
-    <tr><th>Name</th><th>Valid</th><th>Until</th></tr>
+    <tr><th>Name</th><th>SIA</th><th>Valid</th><th>Until</th><th>Ghostbuster</th></tr>
 
     {% for child in object.children.all %}
-    <tr class='{{ child.status.kind_as_str }}'>
+    <tr class='{{ child.status_id }}'>
         <td><a href="{{ child.get_absolute_url }}">{{ child.name }}</a></td>
-        <td>{{ child.ok }}</td>
+        <td>{{ child.sia }}</td>
+        <td>{{ child.is_valid }}</td>
         <td>{{ child.not_after }}</td>
+        <td><a href="{{ child.ghostbusters.all.0.get_absolute_url }}">{{ child.ghostbusters.all.0 }}</a></td>
     </tr>
     {% endfor %}
 
 </table>
+{% endif %}
 
 </ul>
 
diff --git a/rpkid/rpki/gui/cacheview/templates/cacheview/signedobject_detail.html b/rpkid/rpki/gui/cacheview/templates/cacheview/signedobject_detail.html
index 45dff81b..b5f629d8 100644
--- a/rpkid/rpki/gui/cacheview/templates/cacheview/signedobject_detail.html
+++ b/rpkid/rpki/gui/cacheview/templates/cacheview/signedobject_detail.html
@@ -8,22 +8,29 @@
 <table>
     <tr><td>URI</td><td>{{ object.uri }}</td></tr>
     <tr><td>Last Modified</td><td>{{ object.mtime_as_datetime|date:"DATETIME_FORMAT" }}</td></tr>
-    <tr><td>Timestamp</td><td>{{ object.timestamp }}</td></tr>
-    <tr><td>Valid</td><td>{{ object.ok }}</td></tr>
-    <tr><td>Status</td><td class='{{ object.status.kind_as_str }}'>{{ object.status.status }}</td></tr>
+</table>
+
+<h3>Validation Status</h3>
+<table>
+    <tr><th>Timestamp</th><th>Generation</th><th>Status</th></tr>
+    {% for status in object.statuses.all %}
+    <tr class="{{ status.status.get_kind_display }}"><td>{{ status.timestamp }}</td><td>{{ status.get_generation_display }}</td><td>{{ status.status.status }}</td></tr>
+    {% endfor %}
 </table>
 
 <h2>X.509 Certificate Chain</h2>
 
 <table>
-    <tr><th>Depth</th><th>Name</th><th>Valid</th><th>Until</th></tr>
+    <tr><th>Depth</th><th>Name</th><th>SIA</th><th>Valid</th><th>Until</th><th>Ghostbuster</th></tr>
 
 {% for cert in chain %}
-<tr class='{{ cert.1.status.kind_as_str }}'>
+<tr class='{{ cert.1.status_id }}'>
     <td>{{ cert.0 }}</td>
     <td><a href="{{ cert.1.get_absolute_url }}">{{ cert.1.name }}</a></td>
-    <td>{{ cert.1.ok }}</td>
+    <td>{{ cert.1.sia }}</td>
+    <td>{{ cert.1.is_valid }}</td>
     <td>{{ cert.1.not_after }}</td>
+    <td><a href="{{ cert.1.ghostbusters.all.0.get_absolute_url }}">{{ cert.1.ghostbusters.all.0 }}</a></td>
 </tr>
 {% endfor %}
 
diff --git a/rpkid/rpki/rcynic.py b/rpkid/rpki/rcynic.py
index c29f5605..c2562cbd 100644
--- a/rpkid/rpki/rcynic.py
+++ b/rpkid/rpki/rcynic.py
@@ -187,12 +187,25 @@ class rcynic_file_iterator(object):
         if ext in file_name_classes:
           yield file_name_classes[ext](filename)
 
+class validation_status_element(object):
+    def __init__(self, *args, **kwargs):
+        for k,v in kwargs.iteritems():
+            setattr(self, k, v)
+        self._obj = None
+
+    def get_obj(self):
+        if not self._obj:
+            self._obj = self.file_class(filename=self.filename, uri=self.uri)
+        return self._obj
+
+    obj = property(get_obj)
+
 class rcynic_xml_iterator(object):
   """
   Iterate over validation_status entries in the XML output from an
   rcynic run.  Yields a tuple for each entry:
 
-    URI, OK, status, timestamp, object
+    timestamp, generation, status, object
 
   where URI, status, and timestamp are the corresponding values from
   the XML element, OK is a boolean indicating whether validation was
@@ -209,12 +222,10 @@ class rcynic_xml_iterator(object):
   """
 
   def __init__(self, rcynic_root, xml_file,
-               authenticated_subdir = "authenticated",
                authenticated_old_subdir = "authenticated.old",
                unauthenticated_subdir = "unauthenticated"):
     self.rcynic_root = rcynic_root
     self.xml_file = xml_file
-    self.authenticated_subdir = os.path.join(rcynic_root, authenticated_subdir)
     self.authenticated_old_subdir = os.path.join(rcynic_root, authenticated_old_subdir)
     self.unauthenticated_subdir = os.path.join(rcynic_root, unauthenticated_subdir)
 
@@ -227,17 +238,20 @@ class rcynic_xml_iterator(object):
       raise NotRsyncURI, "Not an rsync URI %r" % uri
 
   def __iter__(self):
-
-    for validation_status in ElementTree(file = self.xml_file).getroot().getiterator("validation_status"):
+    for validation_status in ElementTree(file=self.xml_file).getroot().getiterator("validation_status"):
       timestamp = validation_status.get("timestamp")
       status = validation_status.get("status")
       uri = validation_status.text.strip()
       generation = validation_status.get("generation")
-      ok = status == "object_accepted"
-      filename = os.path.join(self.authenticated_subdir if ok else self.unauthenticated_subdir, self.uri_to_filename(uri))
+
+      # determine the path to this object
+      filename = os.path.join(self.authenticated_old_subdir if generation == 'backup' else self.unauthenticated_subdir,
+              self.uri_to_filename(uri))
+
       ext = os.path.splitext(filename)[1]
       if ext in file_name_classes:
-        yield file_name_classes[ext](filename = filename, uri = uri, ok = ok, status = status, timestamp = timestamp, generation = generation)
+          yield validation_status_element(timestamp=timestamp, generation=generation, uri=uri,
+                  status=status, filename=filename, file_class=file_name_classes[ext])
 
 def label_iterator(xml_file):
     """
@@ -248,22 +262,3 @@ def label_iterator(xml_file):
 
     for label in ElementTree(file=xml_file).find("labels"):
         yield label.tag, label.get("kind"), label.text.strip()
-
-
-if __name__ == "__main__":
-  rcynic_dir = os.path.normpath(os.path.join(sys.path[0], "..", "rcynic"))
-  if False:
-    try:
-      for i in rcynic_file_iterator(os.path.join(rcynic_dir, "rcynic-data")):
-        print i
-    except IOError:
-      pass
-  if True:
-    try:
-      for i in rcynic_xml_iterator(os.path.join(rcynic_dir, "rcynic-data"),
-                                   os.path.join(rcynic_dir, "rcynic.xml")):
-        #print i
-        i.show()
-        print
-    except IOError:
-      pass