Update the GUI to work with the new rcynicdb.

svn path=/branches/tk705/; revision=6365

1 files changed, 0 insertions, 435 deletions

diff --git a/rpki/gui/cacheview/util.py b/rpki/gui/cacheview/util.py
deleted file mode 100644
index 00298b2c..00000000
--- a/rpki/gui/cacheview/util.py
+++ /dev/null
@@ -1,435 +0,0 @@
-# Copyright (C) 2011  SPARTA, Inc. dba Cobham
-# Copyright (C) 2012, 2013  SPARTA, Inc. a Parsons Company
-#
-# Permission to use, copy, modify, and distribute this software for any
-# purpose with or without fee is hereby granted, provided that the above
-# copyright notice and this permission notice appear in all copies.
-#
-# THE SOFTWARE IS PROVIDED "AS IS" AND SPARTA DISCLAIMS ALL WARRANTIES WITH
-# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-# AND FITNESS.  IN NO EVENT SHALL SPARTA BE LIABLE FOR ANY SPECIAL, DIRECT,
-# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
-# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-# PERFORMANCE OF THIS SOFTWARE.
-
-__version__ = '$Id$'
-__all__ = ('import_rcynic_xml')
-
-default_logfile = '/var/rcynic/data/rcynic.xml'
-default_root = '/var/rcynic/data'
-object_accepted = None  # set by import_rcynic_xml()
-
-import time
-import vobject
-import logging
-import os
-import stat
-from socket import getfqdn
-from cStringIO import StringIO
-
-from django.db import transaction
-import django.db.models
-
-import rpki
-import rpki.left_right
-import rpki.gui.app.timestamp
-from rpki.gui.app.models import Conf, Alert
-from rpki.gui.cacheview import models
-from rpki.rcynic import rcynic_xml_iterator, label_iterator
-from rpki.sundial import datetime
-from rpki.irdb.zookeeper import Zookeeper
-
-from lxml.etree import Element, SubElement
-
-logger = logging.getLogger(__name__)
-
-
-def rcynic_cert(cert, obj):
-    obj.sia = cert.sia_directory_uri
-
-    # object must be saved for the related manager methods below to work
-    obj.save()
-
-    # for the root cert, we can't set inst.issuer = inst until
-    # after inst.save() has been called.
-    if obj.issuer is None:
-        obj.issuer = obj
-        obj.save()
-
-    # resources can change when a cert is updated
-    obj.asns.clear()
-    obj.addresses.clear()
-
-    if cert.resources.asn.inherit:
-        # FIXME: what happens when the parent's resources change and the child
-        # cert is not reissued?
-        obj.asns.add(*obj.issuer.asns.all())
-    else:
-        for asr in cert.resources.asn:
-            logger.debug('processing %s', asr)
-
-            attrs = {'min': asr.min, 'max': asr.max}
-            q = models.ASRange.objects.filter(**attrs)
-            if not q:
-                obj.asns.create(**attrs)
-            else:
-                obj.asns.add(q[0])
-
-    # obj.issuer is None the first time we process the root cert in the
-    # hierarchy, so we need to guard against dereference
-    for cls, addr_obj, addrset, parentset in (
-        models.AddressRange, obj.addresses, cert.resources.v4,
-        obj.issuer.addresses.all() if obj.issuer else []
-    ), (
-        models.AddressRangeV6, obj.addresses_v6, cert.resources.v6,
-        obj.issuer.addresses_v6.all() if obj.issuer else []
-    ):
-        if addrset.inherit:
-            addr_obj.add(*parentset)
-        else:
-            for rng in addrset:
-                logger.debug('processing %s', rng)
-
-                attrs = {'prefix_min': rng.min, 'prefix_max': rng.max}
-                q = cls.objects.filter(**attrs)
-                if not q:
-                    addr_obj.create(**attrs)
-                else:
-                    addr_obj.add(q[0])
-
-
-def rcynic_roa(roa, obj):
-    obj.asid = roa.asID
-    # object must be saved for the related manager methods below to work
-    obj.save()
-    obj.prefixes.clear()
-    obj.prefixes_v6.clear()
-    for pfxset in roa.prefix_sets:
-        if pfxset.__class__.__name__ == 'roa_prefix_set_ipv6':
-            roa_cls = models.ROAPrefixV6
-            prefix_obj = obj.prefixes_v6
-        else:
-            roa_cls = models.ROAPrefixV4
-            prefix_obj = obj.prefixes
-
-        for pfx in pfxset:
-            attrs = {'prefix_min': pfx.min(),
-                     'prefix_max': pfx.max(),
-                     'max_length': pfx.max_prefixlen}
-            q = roa_cls.objects.filter(**attrs)
-            if not q:
-                prefix_obj.create(**attrs)
-            else:
-                prefix_obj.add(q[0])
-
-
-def rcynic_gbr(gbr, obj):
-    vcard = vobject.readOne(gbr.vcard)
-    obj.full_name = vcard.fn.value if hasattr(vcard, 'fn') else None
-    obj.email_address = vcard.email.value if hasattr(vcard, 'email') else None
-    obj.telephone = vcard.tel.value if hasattr(vcard, 'tel') else None
-    obj.organization = vcard.org.value[0] if hasattr(vcard, 'org') else None
-    obj.save()
-
-LABEL_CACHE = {}
-
-# dict keeping mapping of uri to (handle, old status, new status) for objects
-# published by the local rpkid
-uris = {}
-
-dispatch = {
-    'rcynic_certificate': rcynic_cert,
-    'rcynic_roa': rcynic_roa,
-    'rcynic_ghostbuster': rcynic_gbr
-}
-
-model_class = {
-    'rcynic_certificate': models.Cert,
-    'rcynic_roa': models.ROA,
-    'rcynic_ghostbuster': models.Ghostbuster
-}
-
-
-def save_status(repo, vs):
-    timestamp = datetime.fromXMLtime(vs.timestamp).to_sql()
-    status = LABEL_CACHE[vs.status]
-    g = models.generations_dict.get(vs.generation)
-    repo.statuses.create(generation=g, timestamp=timestamp, status=status)
-
-    # if this object is in our interest set, update with the current validation
-    # status
-    if repo.uri in uris:
-        x, y, z, q = uris[repo.uri]
-        valid = z or (status is object_accepted)  # don't clobber previous True value
-        uris[repo.uri] = x, y, valid, repo
-
-    if status is not object_accepted:
-        return
-
-    cls = model_class[vs.file_class.__name__]
-    # find the instance of the signedobject subclass that is associated with
-    # this repo instance (may be empty when not accepted)
-    inst_qs = cls.objects.filter(repo=repo)
-
-    logger.debug('processing %s', vs.filename)
-
-    if not inst_qs:
-        inst = cls(repo=repo)
-        logger.debug('object not found in db, creating new object cls=%s id=%s',
-                     cls, id(inst))
-    else:
-        inst = inst_qs[0]
-
-    try:
-        # determine if the object is changed/new
-        mtime = os.stat(vs.filename)[stat.ST_MTIME]
-    except OSError as e:
-        logger.error('unable to stat %s: %s %s',
-                     vs.filename, type(e), e)
-        # treat as if missing from rcynic.xml
-        # use inst_qs rather than deleting inst so that we don't raise an
-        # exception for newly created objects (inst_qs will be empty)
-        inst_qs.delete()
-        return
-
-    if mtime != inst.mtime:
-        inst.mtime = mtime
-        try:
-            obj = vs.obj  # causes object to be lazily loaded
-        except Exception, e:
-            logger.warning('Caught %s while processing %s: %s',
-                           type(e), vs.filename, e)
-            return
-
-        inst.not_before = obj.notBefore.to_sql()
-        inst.not_after = obj.notAfter.to_sql()
-        inst.name = obj.subject
-        inst.keyid = obj.ski
-
-        # look up signing cert
-        if obj.issuer == obj.subject:
-            # self-signed cert (TA)
-            assert isinstance(inst, models.Cert)
-            inst.issuer = None
-        else:
-            # if an object has moved in the repository, the entry for
-            # the old location will still be in the database, but
-            # without any object_accepted in its validtion status
-            qs = models.Cert.objects.filter(
-                keyid=obj.aki,
-                name=obj.issuer,
-                repo__statuses__status=object_accepted
-            )
-            ncerts = len(qs)
-            if ncerts == 0:
-                logger.warning('unable to find signing cert with ski=%s (%s)', obj.aki, obj.issuer)
-                return
-            else:
-                if ncerts > 1:
-                    # multiple matching certs, all of which are valid
-                    logger.warning('Found multiple certs matching ski=%s sn=%s', obj.aki, obj.issuer)
-                    for c in qs:
-                        logger.warning(c.repo.uri)
-                # just use the first match
-                inst.issuer = qs[0]
-
-        try:
-            # do object-specific tasks
-            dispatch[vs.file_class.__name__](obj, inst)
-        except:
-            logger.error('caught exception while processing rcynic_object:\n'
-                            'vs=' + repr(vs) + '\nobj=' + repr(obj))
-            # .show() writes to stdout
-            obj.show()
-            raise
-
-        logger.debug('object saved id=%s', id(inst))
-    else:
-        logger.debug('object is unchanged')
-
-
-@transaction.atomic
-def process_cache(root, xml_file):
-
-    last_uri = None
-    repo = None
-
-    logger.info('clearing validation statuses')
-    models.ValidationStatus.objects.all().delete()
-
-    logger.info('updating validation status')
-    for vs in rcynic_xml_iterator(root, xml_file):
-        if vs.uri != last_uri:
-            repo, created = models.RepositoryObject.objects.get_or_create(uri=vs.uri)
-            last_uri = vs.uri
-        save_status(repo, vs)
-
-    # garbage collection
-    # remove all objects which have no ValidationStatus references, which
-    # means they did not appear in the last XML output
-    logger.info('performing garbage collection')
-
-    # Delete all objects that have zero validation status elements.
-    models.RepositoryObject.objects.annotate(num_statuses=django.db.models.Count('statuses')).filter(num_statuses=0).delete()
-
-    # Delete all SignedObject instances that were not accepted.  There may
-    # exist rows for objects that were previously accepted.
-    # See https://trac.rpki.net/ticket/588#comment:30
-    #
-    # We have to do this here rather than in save_status() because the
-    # <validation_status/> elements are not guaranteed to be consecutive for a
-    # given URI.  see https://trac.rpki.net/ticket/625#comment:5
-    models.SignedObject.objects.exclude(repo__statuses__status=object_accepted).delete()
-
-    # ROAPrefixV* objects are M2M so they are not automatically deleted when
-    # their ROA object disappears
-    models.ROAPrefixV4.objects.annotate(num_roas=django.db.models.Count('roas')).filter(num_roas=0).delete()
-    models.ROAPrefixV6.objects.annotate(num_roas=django.db.models.Count('roas')).filter(num_roas=0).delete()
-    logger.info('done with garbage collection')
-
-
-@transaction.atomic
-def process_labels(xml_file):
-    logger.info('updating labels...')
-
-    for label, kind, desc in label_iterator(xml_file):
-        logger.debug('label=%s kind=%s desc=%s', label, kind, desc)
-        if kind:
-            q = models.ValidationLabel.objects.filter(label=label)
-            if not q:
-                obj = models.ValidationLabel(label=label)
-            else:
-                obj = q[0]
-
-            obj.kind = models.kinds_dict[kind]
-            obj.status = desc
-            obj.save()
-
-            LABEL_CACHE[label] = obj
-
-
-def fetch_published_objects():
-    """Query rpkid for all objects published by local users, and look up the
-    current validation status of each object.  The validation status is used
-    later to send alerts for objects which have transitioned to invalid.
-    """
-
-    logger.info('querying for published objects')
-
-    handles = [conf.handle for conf in Conf.objects.all()]
-    q_msg = Element(rpki.left_right.tag_msg, nsmap = rpki.left_right.nsmap,
-                    type = "query", version = rpki.left_right.version)
-    for h in handles:
-        SubElement(q_msg, rpki.left_right.tag_list_published_objects, action="list", tenant_handle=h, tag=h)
-    z = Zookeeper()
-    r_msg = z.call_rpkid(q_msg)
-    for r_pdu in r_msg:
-        if r_pdu.tag == rpki.left_right.tag_list_published_objects:
-            # Look up the object in the rcynic cache
-            qs = models.RepositoryObject.objects.filter(uri=r_pdu.get("uri"))
-            if qs:
-                # get the current validity state
-                valid = qs[0].statuses.filter(status=object_accepted).exists()
-                uris[r_pdu.get("uri")] = (r_pdu.get("tenant_handle"), valid, False, None)
-                logger.debug('adding %s', r_pdu.get("uri"))
-            else:
-                # this object is not in the cache.  it was either published
-                # recently, or disappared previously.  if it disappeared
-                # previously, it has already been alerted.  in either case, we
-                # omit the uri from the list since we are interested only in
-                # objects which were valid and are no longer valid
-                pass
-        elif r_pdu.tag == rpki.left_right.tag_report_error:
-            logging.error('rpkid reported an error: %s', r_pdu.get("error_code"))
-
-
-class Handle(object):
-    def __init__(self):
-        self.invalid = []
-        self.missing = []
-
-    def add_invalid(self, v):
-        self.invalid.append(v)
-
-    def add_missing(self, v):
-        self.missing.append(v)
-
-
-def notify_invalid():
-    """Send email alerts to the addresses registered in ghostbuster records for
-    any invalid objects that were published by users of this system.
-    """
-
-    logger.info('sending notifications for invalid objects')
-
-    # group invalid objects by user
-    notify = {}
-    for uri, v in uris.iteritems():
-        handle, old_status, new_status, obj = v
-
-        if obj is None:
-            # object went missing
-            n = notify.get(handle, Handle())
-            n.add_missing(uri)
-        # only select valid->invalid
-        elif old_status and not new_status:
-            n = notify.get(handle, Handle())
-            n.add_invalid(obj)
-
-    for handle, v in notify.iteritems():
-        conf = Conf.objects.get(handle)
-
-        msg = StringIO()
-        msg.write('This is an alert about problems with objects published by '
-                  'the resource handle %s.\n\n' % handle)
-
-        if v.invalid:
-            msg.write('The following objects were previously valid, but are '
-                      'now invalid:\n')
-
-            for o in v.invalid:
-                msg.write('\n')
-                msg.write(o.repo.uri)
-                msg.write('\n')
-                for s in o.statuses.all():
-                    msg.write('\t')
-                    msg.write(s.status.label)
-                    msg.write(': ')
-                    msg.write(s.status.status)
-                    msg.write('\n')
-
-        if v.missing:
-            msg.write('The following objects were previously valid but are no '
-                      'longer in the cache:\n')
-
-            for o in v.missing:
-                msg.write(o)
-                msg.write('\n')
-
-        msg.write("""--
-You are receiving this email because your address is published in a Ghostbuster
-record, or is the default email address for this resource holder account on
-%s.""" % getfqdn())
-
-        from_email = 'root@' + getfqdn()
-        subj = 'invalid RPKI object alert for resource handle %s' % conf.handle
-        conf.send_alert(subj, msg.getvalue(), from_email, severity=Alert.ERROR)
-
-
-def import_rcynic_xml(root=default_root, logfile=default_logfile):
-    """Load the contents of rcynic.xml into the rpki.gui.cacheview database."""
-
-    global object_accepted
-
-    start = time.time()
-    process_labels(logfile)
-    object_accepted = LABEL_CACHE['OBJECT_ACCEPTED']
-    fetch_published_objects()
-    process_cache(root, logfile)
-    notify_invalid()
-
-    rpki.gui.app.timestamp.update('rcynic_import')
-
-    stop = time.time()
-    logger.info('elapsed time %d seconds.', (stop - start))