Merge branches/tk161 to trunk.

svn path=/trunk/; revision=4415

9 files changed, 511 insertions, 749 deletions

diff --git a/rpkid/portal-gui/scripts/adduser.py b/rpkid/portal-gui/scripts/adduser.py
deleted file mode 100644
index b06c6bc4..00000000
--- a/rpkid/portal-gui/scripts/adduser.py
+++ /dev/null
@@ -1,86 +0,0 @@
-# $Id$
-#
-# Copyright (C) 2010, 2011  SPARTA, Inc. dba Cobham Analytic Solutions
-#
-# Permission to use, copy, modify, and distribute this software for any
-# purpose with or without fee is hereby granted, provided that the above
-# copyright notice and this permission notice appear in all copies.
-#
-# THE SOFTWARE IS PROVIDED "AS IS" AND SPARTA DISCLAIMS ALL WARRANTIES WITH
-# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-# AND FITNESS.  IN NO EVENT SHALL SPARTA BE LIABLE FOR ANY SPECIAL, DIRECT,
-# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
-# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-# PERFORMANCE OF THIS SOFTWARE.
-#
-#
-# Helper script to quickly set up a new portal-gui user/handle.  This script
-# is designed to be safe to run multiple times for the same user.
-#
-# DO NOT EDIT!  This script is automatically generated from adduser.py
-
-from django.contrib.auth.models import User
-from rpki.gui.app import settings
-from rpki.gui.app.models import Conf
-
-import getpass
-import pwd
-
-# The username that apache runs as.  This is required so that we can chown
-# the csv files that the portal-gui needs to write.
-web_uid = pwd.getpwnam(settings.WEB_USER)[2]
-
-if __name__ == '__main__':
-    if len(sys.argv) < 3:
-	print >>sys.stderr, 'usage: adduser <username> <user\'s email> <host handle>'
-	sys.exit(1)
-
-    if os.getuid() != 0:
-        print >>sys.stderr, 'error: this script must be run as root so it can set file permissions.'
-        sys.exit(1)
-
-    username = sys.argv[1]
-    email = sys.argv[2]
-    host = sys.argv[3]
-    print 'username=', username, 'email=', email, 'host=', host
-
-    user_set = User.objects.filter(username=username)
-    if user_set:
-	print >>sys.stderr, 'user already exists'
-	user = user_set[0]
-    else:
-	print >>sys.stderr, 'creating user'
-        password = getpass.getpass()
-	user = User.objects.create_user(username, email, password)
-
-    conf_set = Conf.objects.filter(handle=username)
-    if conf_set:
-	conf = conf_set[0]
-    else:
-	print >>sys.stderr, 'creating conf'
-	conf = Conf.objects.create(handle=username)
-
-    # always try to add the user as owner just in case the Conf object was
-    # created previously by the "list_resources" script
-    conf.owner.add(user)
-
-    if host != username:
-        host_set = Conf.objects.filter(handle=host)
-        if not host_set:
-            print >>sys.stderr, 'error: Conf object for host %s does not exist!' % host
-            sys.exit(1)
-
-        conf.host = host_set[0]
-    else:
-        print >>sys.stderr, '%s is self-hosted' % username
-    conf.save()
-    
-    myrpki_dir = '%s/%s' % (settings.CONFDIR, username)
-    print 'myrpki_dir=', myrpki_dir
-    if not os.path.exists(myrpki_dir):
-	print 'creating ', myrpki_dir
-	os.mkdir(myrpki_dir)
-    os.chown(myrpki_dir, web_uid, -1)
-
-# vim:sw=4 ts=8
diff --git a/rpkid/portal-gui/scripts/dumpdata.py b/rpkid/portal-gui/scripts/dumpdata.py
new file mode 100755
index 00000000..dcf23666
--- /dev/null
+++ b/rpkid/portal-gui/scripts/dumpdata.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# $Id$
+#
+# Copyright (C) 2012  SPARTA, Inc. a Parsons Company
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND SPARTA DISCLAIMS ALL WARRANTIES WITH
+# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS.  IN NO EVENT SHALL SPARTA BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+#
+# This is a helper script which will dump the rpki.gui.app models from
+# the old sqlite3 database, forcing the output order to the primary key in
+# order to avoid forward references for the AddressRange table.
+
+from django.conf import settings
+settings.configure(DEBUG=True,
+        DATABASES={
+            'default': {
+                'ENGINE': 'django.db.backends.sqlite3',
+                'NAME': '/usr/local/var/rpki/gui.db',
+                }
+            })
+
+from django.core import serializers
+import django.db.models
+
+from rpki.gui.app import models
+from django.contrib.auth import models as auth_models
+
+data = []
+for v in (auth_models.User, models.Conf, models.Parent, models.Child, models.AddressRange, models.Asn, models.ResourceCert, models.Roa, models.RoaRequest, models.Ghostbuster):
+    data.extend(list(v.objects.all().order_by('id')))
+
+print serializers.serialize('json', data, use_natural_keys=True)
+
+# vim:sw=4 ts=8 expandtab
diff --git a/rpkid/portal-gui/scripts/list_resources.py b/rpkid/portal-gui/scripts/list_resources.py
deleted file mode 100644
index 13864705..00000000
--- a/rpkid/portal-gui/scripts/list_resources.py
+++ /dev/null
@@ -1,200 +0,0 @@
-# $Id$
-#
-# Copyright (C) 2010, 2011  SPARTA, Inc. dba Cobham Analytic Solutions
-#
-# Permission to use, copy, modify, and distribute this software for any
-# purpose with or without fee is hereby granted, provided that the above
-# copyright notice and this permission notice appear in all copies.
-#
-# THE SOFTWARE IS PROVIDED "AS IS" AND SPARTA DISCLAIMS ALL WARRANTIES WITH
-# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-# AND FITNESS.  IN NO EVENT SHALL SPARTA BE LIABLE FOR ANY SPECIAL, DIRECT,
-# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
-# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-# PERFORMANCE OF THIS SOFTWARE.
-#
-#
-# This script is reponsible for talking to rpkid and populating the
-# portal-gui's sqlite database.  It asks rpkid for the list of received
-# resources, and the handles of any children.
-#
-# This script should be run in the directory containing the rpki.conf
-# for the handle that is self-hosting rpkid.
-#
-# Exit values:
-# 0     success, no errors
-# 1     fatal error
-# 2     usage error
-# 3     did not receive all <list_received_resources/> responses, try again
-#       later
-
-import sys
-
-import os
-os.environ['DJANGO_SETTINGS_MODULE'] = 'rpki.gui.settings'
-
-import getopt
-from datetime import datetime, timedelta
-from os.path import basename
-
-import rpki.config, rpki.left_right, rpki.resource_set
-from rpki.gui.app import models, glue
-
-verbose = False
-version = '$Id$'
-
-def query_rpkid():
-    """
-    Fetch our received resources from the local rpkid using the rpki.conf
-    in the current directory.
-    """
-    cfg = rpki.config.parser(section='myrpki')
-    call_rpkid = glue.build_rpkid_caller(cfg, verbose)
-
-    if verbose:
-        print 'retrieving the list of <self/> handles served by this rpkid'
-    rpkid_reply = call_rpkid(rpki.left_right.self_elt.make_pdu(action="list"))
-
-    # retrieve info about each handle
-    pdus = []
-    handles = []
-    for h in rpkid_reply:
-        assert isinstance(h, rpki.left_right.self_elt)
-        if verbose:
-            print 'adding handle %s to query' % (h.self_handle,)
-        # keep a list of the handles served by rpkid so that we may check that
-        # all expected responses are received.
-        handles.append(h.self_handle)
-        pdus.extend(
-            [rpki.left_right.child_elt.make_pdu(action="list", self_handle=h.self_handle),
-             rpki.left_right.list_received_resources_elt.make_pdu(self_handle=h.self_handle)
-             #rpki.left_right.parent_elt.make_pdu(action="list", tag="parents", self_handle=handle),
-             #rpki.left_right.list_roa_requests_elt.make_pdu(tag='roas', self_handle=handle),
-            ])
-
-    if verbose:
-        print 'querying for children and resources'
-    return handles, call_rpkid(*pdus)
-
-def usage(rc):
-    print 'usage: %s [ -hvV ] [ --help ] [ --verbose ] [ --version ]' % basename(sys.argv[0],)
-    sys.exit(rc)
-
-try:
-    opts, args = getopt.getopt(sys.argv[1:], 'hvV', [ 'help', 'verbose', 'version'])
-except getopt.GetoptError, err:
-    print str(err)
-    usage(2)
-
-for o,a in opts:
-    if o in ('-h', '--help'):
-        usage(0)
-    elif o in ('-v', '--verbose'):
-        verbose = True
-    elif o in ('-V', '--version'):
-        print basename(sys.argv[0]), version
-        sys.exit(0)
-
-handles, pdus = query_rpkid()
-seen = set() # which handles we got <list_received_resources/> responses
-for pdu in pdus:
-    conf_set = models.Conf.objects.filter(handle=pdu.self_handle)
-    if conf_set.count():
-        conf = conf_set[0]
-    else:
-        if verbose:
-            print 'creating new conf for %s' % (pdu.self_handle,)
-        conf = models.Conf.objects.create(handle=pdu.self_handle)
-
-    #if isinstance(pdu, rpki.left_right.parent_elt):
-#       print x.parent_handle, x.sia_base, x.sender_name, x.recipient_name, \
-#           x.peer_contact_uri
-    if isinstance(pdu, rpki.left_right.child_elt):
-        # have we seen this child before?
-        child_set = conf.children.filter(handle=pdu.child_handle)
-        if not child_set:
-            if verbose:
-                print 'creating new child %s' % (pdu.child_handle,)
-            # default to 1 year.  no easy way to query irdb for the
-            # current value.
-            valid_until = datetime.now() + timedelta(days=365)
-            child = models.Child(conf=conf, handle=pdu.child_handle,
-                                 valid_until=valid_until)
-            child.save()
-    #elif isinstance(x, rpki.left_right.list_roa_requests_elt):
-    #    print x.asn, x.ipv4, x.ipv6
-    elif isinstance(pdu, rpki.left_right.list_received_resources_elt):
-        # keep track of handles we got replies for
-        seen.add(pdu.self_handle)
-        # have we seen this parent before?
-        parent_set = conf.parents.filter(handle=pdu.parent_handle)
-        if not parent_set:
-            if verbose:
-                print 'creating new parent %s' % (pdu.parent_handle,)
-            parent = models.Parent(conf=conf, handle=pdu.parent_handle)
-            parent.save()
-        else:
-            parent = parent_set[0]
-
-        not_before = datetime.strptime(pdu.notBefore, "%Y-%m-%dT%H:%M:%SZ")
-        not_after = datetime.strptime(pdu.notAfter, "%Y-%m-%dT%H:%M:%SZ")
-
-        # have we seen this resource cert before?
-        cert_set = parent.resources.filter(uri=pdu.uri)
-        if cert_set.count() == 0:
-            cert = models.ResourceCert(uri=pdu.uri, parent=parent,
-                    not_before=not_before, not_after=not_after)
-        else:
-            cert = cert_set[0]
-            # update timestamps since it could have been modified
-            cert.not_before = not_before
-            cert.not_after = not_after
-        cert.save()
-
-        for asn in rpki.resource_set.resource_set_as(pdu.asn):
-            # see if this resource is already part of the cert
-            if cert.asn.filter(lo=asn.min, hi=asn.max).count() == 0:
-                # ensure this range wasn't seen from another of our parents
-                for v in models.Asn.objects.filter(lo=asn.min, hi=asn.max):
-                    # determine if resource is delegated from another parent
-                    if v.from_cert.filter(parent__in=conf.parents.all()).count():
-                        cert.asn.add(v)
-                        break
-                else:
-                    if verbose:
-                        print 'adding AS %s' % (asn,)
-                    cert.asn.create(lo=asn.min, hi=asn.max)
-                cert.save()
-
-        # IPv4/6 - not separated in the django db
-        def add_missing_address(addr_set):
-           for ip in addr_set:
-               lo=str(ip.min)
-               hi=str(ip.max)
-               if cert.address_range.filter(lo=lo, hi=hi).count() == 0:
-                   # ensure that this range wasn't previously seen from another of our parents
-                   for v in models.AddressRange.objects.filter(lo=lo, hi=hi):
-                       # determine if this resource is delegated from another parent as well
-                       if v.from_cert.filter(parent__in=conf.parents.all()).count():
-                           cert.address_range.add(v)
-                           break
-                   else:
-                       if verbose:
-                           print 'adding address range %s' % (ip,)
-                       cert.address_range.create(lo=lo, hi=hi)
-                   cert.save()
-
-        add_missing_address(rpki.resource_set.resource_set_ipv4(pdu.ipv4))
-        add_missing_address(rpki.resource_set.resource_set_ipv6(pdu.ipv6))
-
-# verify that we got responses for all expected handles
-for h in handles:
-    if h not in seen:
-        if verbose:
-            print 'warning: did not receive response for handle %s' % (h,)
-        sys.exit(3)
-
-sys.exit(0) # success
-
-# vim:sw=4 expandtab ts=4
diff --git a/rpkid/portal-gui/scripts/load_csv.py b/rpkid/portal-gui/scripts/load_csv.py
deleted file mode 100755
index 9d4fc1ac..00000000
--- a/rpkid/portal-gui/scripts/load_csv.py
+++ /dev/null
@@ -1,143 +0,0 @@
-# $Id$
-#
-# Copyright (C) 2010, 2011  SPARTA, Inc. dba Cobham Analytic Solutions
-#
-# Permission to use, copy, modify, and distribute this software for any
-# purpose with or without fee is hereby granted, provided that the above
-# copyright notice and this permission notice appear in all copies.
-#
-# THE SOFTWARE IS PROVIDED "AS IS" AND SPARTA DISCLAIMS ALL WARRANTIES WITH
-# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-# AND FITNESS.  IN NO EVENT SHALL SPARTA BE LIABLE FOR ANY SPECIAL, DIRECT,
-# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
-# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-# PERFORMANCE OF THIS SOFTWARE.
-#
-#
-# Helper script to load existing data from csv into the Django DB.
-# Primarly useful for the initial load, as the GUI does not sync changes
-# made directly to the csv files back into the database.
-#
-# This script should be run from the directory containing the rpki.conf
-# for the handle you are loading data
-#
-
-import csv
-import socket # for socket.error
-
-import rpki.resource_set, rpki.ipaddrs
-from rpki.myrpki import csv_reader
-from rpki.gui.app import models
-from rpki.gui.app.views import add_roa_requests
-
-cfg = rpki.config.parser(section='myrpki')
-handle = cfg.get('handle')
-asn_csv = cfg.get('asn_csv')
-prefix_csv = cfg.get('prefix_csv')
-roa_csv = cfg.get('roa_csv')
-
-print 'processing csv files for resource handle', handle
-
-conf = models.Conf.objects.get(handle=handle)
-
-class RangeError(Exception):
-    """
-    Problem with ASN range or address range.
-    """
-
-# every parent has a favorite
-def best_child(address_range, parent, parent_range):
-    '''Return the child address range that is the closest match, or
-    returns the arguments if no children.'''
-    if address_range == parent_range:
-        return (parent, parent_range)
-    for q in list(parent.children.all()): # force strict evaluation
-        t = q.as_resource_range()
-        if t.min <= address_range.min and t.max >= address_range.max:
-            return best_child(address_range, q, t)
-        # check for overlap
-        if t.min <= address_range.min <= t.max or t.min <= address_range.max <= t.max:
-            raise RangeError, \
-                    'can not handle overlapping ranges: %s and %s' % (address_range, t)
-    return parent, parent_range
-
-def get_or_create_prefix(address_range):
-    '''Returns a AddressRange object for the resource_range_ip specified
-    as an argument.  If no match is found, a new AddressRange object is
-    created as a child of the best matching received resource.'''
-
-    # get all resources from our parents
-    prefix_set = models.AddressRange.objects.filter(
-            from_cert__parent__in=conf.parents.all())
-
-    # gross, since we store the address ranges as strings in the django
-    # db, we can't use the normal __lte and __gte filters, so we get to
-    # do it in python instead.
-    for prefix in prefix_set:
-        prefix_range = prefix.as_resource_range()
-        if (prefix_range.min <= address_range.min and
-                prefix_range.max >= address_range.max):
-            # there should only ever be a single matching prefix
-            break
-    else:
-        raise RangeError, '%s does not match any received address range.' % (
-                address_range,)
-
-    # find the best match among the children + grandchildren
-    prefix, prefix_range = best_child(address_range, prefix, prefix_range)
-
-    print 'best match for %s is %s' % (address_range, prefix)
-    if prefix_range.min != address_range.min or prefix_range.max != address_range.max:
-        # create suballocation
-        print 'creating new range' 
-        prefix = models.AddressRange.objects.create(lo=str(address_range.min),
-                hi=str(address_range.max), parent=prefix)
-    return prefix
-
-def get_or_create_asn(asn):
-    asn_set = models.Asn.objects.filter(lo__lte=asn.min, hi__gte=asn.max,
-            from_cert__parent__in=conf.parents.all())
-    if not asn_set:
-        raise RangeError, '%s does not match any received AS range' % (asn,)
-    best = best_child(asn, asn_set[0], asn_set[0].as_resource_range())[0]
-    print 'best match for %s is %s' % (asn, best)
-    if best.lo != asn.min or best.hi != asn.max:
-        best = models.Asn.objects.create(lo=asn.min, hi=asn.max, parent=best)
-    return best
-
-def do_asns():
-    print 'processing', asn_csv
-    for child_handle, asn in csv_reader(asn_csv, columns=2):
-        asn_range = rpki.resource_set.resource_range_as.parse_str(asn)
-        child = conf.children.get(handle=child_handle)
-        asn = get_or_create_asn(asn_range)
-        child.asn.add(asn)
-
-def do_prefixes():
-    print 'processing', prefix_csv
-    for child_handle, prefix in csv_reader(prefix_csv, columns=2):
-        child = conf.children.get(handle=child_handle)
-        try:
-            rs = rpki.resource_set.resource_range_ipv4.parse_str(prefix)
-        except ValueError, err:
-            rs = rpki.resource_set.resource_range_ipv6.parse_str(prefix)
-        obj = get_or_create_prefix(rs)
-        obj.allocated = child
-        obj.save()
-
-def do_roas():
-    print 'processing', roa_csv
-    for prefix, asn, group in csv_reader(roa_csv, columns=3):
-        try:
-            rs = rpki.resource_set.roa_prefix_ipv4.parse_str(prefix)
-        except ValueError, err:
-            rs = rpki.resource_set.roa_prefix_ipv6.parse_str(prefix)
-
-        print str(rs.min()), str(rs.max()), rs.max_prefixlen
-        obj = get_or_create_prefix(rs.to_resource_range())
-        add_roa_requests(conf, obj, [int(asn)], rs.max_prefixlen)
-
-do_asns()
-do_prefixes()
-do_roas()
diff --git a/rpkid/portal-gui/scripts/roa_check.py b/rpkid/portal-gui/scripts/roa_check.py
deleted file mode 100755
index c280d935..00000000
--- a/rpkid/portal-gui/scripts/roa_check.py
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/usr/bin/env python
-# $Id$
-#
-# Runs through all the published ROAs and updates the Django DB with the
-# current active status of each defined ROA.
-#
-
-import socket
-
-from rcynic_output_iterator import rcynic_xml_iterator, rcynic_roa
-from rpki.resource_set import resource_set_ipv4, resource_set_ipv6
-from rpki.resource_set import roa_prefix_set_ipv4, roa_prefix_set_ipv6
-from rpki.resource_set import resource_range_ipv4, resource_range_ipv6
-from rpki.ipaddrs import v4addr, v6addr
-
-from rpki.gui.app.models import Roa
-
-# build up a list of all the authenticated roa's using the asn as the key
-roaiter = rcynic_xml_iterator(
-        rcynic_root='/home/melkins/rcynic/rcynic-data/',
-        xml_file='/home/melkins/rcynic/rcynic.xml')
-
-# key is an ASN
-# each element is a tuple of (resource_set_ipv4, resource_set_ipv6)
-roaauth = {}
-
-for roa in roaiter:
-    if isinstance(roa, rcynic_roa):
-        k = roa.asID
-        if not roaauth.has_key(k):
-            v = [resource_set_ipv4(), resource_set_ipv6()]
-            roaauth[k] = v
-        else:
-            v = roaauth[k]
-        for pfx in roa.prefix_sets:
-            if isinstance(pfx, roa_prefix_set_ipv4):
-                v[0] = v[0].union(pfx.to_resource_set())
-            elif isinstance(pfx, roa_prefix_set_ipv6):
-                v[1] = v[1].union(pfx.to_resource_set())
-
-#for k, v in roaauth.iteritems():
-#    print 'asn %d : prefixes %s' % (k, ' '.join(map(str,v)))
-
-# run through all the ROA's in the GUI's database
-for roa in Roa.objects.all():
-    k = int(roa.asn)
-    valid = False
-    if roaauth.has_key(k):
-        # ensure that all prefixes listed in the roa are present
-        # we convert the list of prefixes into prefix sets and use the
-        # resource_set class to perform set comparisons
-        ipv4_set = resource_set_ipv4()
-        ipv6_set = resource_set_ipv6()
-        for pfx in roa.prefix.all():
-            # IP addresses are just stored as strings in the sqlite db
-            try:
-                ipv4_set.append(resource_range_ipv4(v4addr(str(pfx.lo)), v4addr(str(pfx.hi))))
-            except socket.error:
-                ipv6_set.append(resource_range_ipv6(v6addr(str(pfx.lo)), v6addr(str(pfx.hi))))
-        r = roaauth[k]
-        if ipv4_set.issubset(r[0]) and ipv6_set.issubset(r[1]):
-            valid = True
-    if valid:
-        if not roa.active:
-            roa.active = True
-            roa.save()
-    else:
-        print 'roa for asn %s is not valid' % (roa.asn, )
-        if roa.active:
-            roa.active = False
-            roa.save()
diff --git a/rpkid/portal-gui/scripts/rpkigui-import-routes.py b/rpkid/portal-gui/scripts/rpkigui-import-routes.py
new file mode 100644
index 00000000..9a3748d3
--- /dev/null
+++ b/rpkid/portal-gui/scripts/rpkigui-import-routes.py
@@ -0,0 +1,250 @@
+# Copyright (C) 2012  SPARTA, Inc. a Parsons Company
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND SPARTA DISCLAIMS ALL WARRANTIES WITH
+# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS.  IN NO EVENT SHALL SPARTA BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+
+__version__ = '$Id$'
+
+import itertools
+import _mysql_exceptions
+import optparse
+import os.path
+import re
+import sys
+import struct
+import subprocess
+import time
+import logging
+
+from django.db import transaction, connection
+
+from rpki.resource_set import resource_range_ipv4, resource_range_ipv6
+from rpki.exceptions import BadIPResource
+import rpki.gui.app.timestamp
+
+# globals
+BGPDUMP = 'bgpdump'
+logger = logging.getLogger(__name__)
+
+
+def parse_text(f):
+    ip_re = re.compile(r'^[0-9a-fA-F:.]+/\d{1,3}$')
+    last_prefix = None
+    cursor = connection.cursor()
+    range_class = resource_range_ipv4
+    table = 'routeview_routeorigin'
+    sql = "INSERT INTO %s_new SET asn=%%s, prefix_min=%%s, prefix_max=%%s" % table
+
+    try:
+        logger.info('Dropping existing staging table...')
+        cursor.execute('DROP TABLE IF EXISTS %s_new' % table)
+    except _mysql_exceptions.Warning:
+        pass
+
+    logger.info('Creating staging table...')
+    cursor.execute('CREATE TABLE %(table)s_new LIKE %(table)s' % {'table': table})
+
+    logger.info('Disabling autocommit...')
+    cursor.execute('SET autocommit=0')
+
+    logger.info('Adding rows to table...')
+    for row in itertools.islice(f, 5, None):
+        cols = row.split()
+
+        # index -1 is i/e/? for igp/egp
+        origin_as = cols[-2]
+        # FIXME: skip AS_SETs
+        if origin_as[0] == '{':
+            continue
+
+        prefix = cols[1]
+
+        # validate the prefix since the "sh ip bgp" output is sometimes
+        # corrupt by no space between the prefix and the next hop IP
+        # address.
+        net, bits = prefix.split('/')
+        if len(bits) > 2:
+            s = ['mask for %s looks fishy...' % prefix]
+            prefix = '%s/%s' % (net, bits[0:2])
+            s.append('assuming it should be %s' % prefix)
+            logger.warning(' '.join(s))
+
+        # the output may contain multiple paths to the same origin.
+        # if this is the same prefix as the last entry, we don't need
+        # to validate it again.
+        #
+        # prefixes are sorted, but the origin_as is not, so we keep a set to
+        # avoid duplicates, and insert into the db once we've seen all the
+        # origin_as values for a given prefix
+        if prefix != last_prefix:
+            # output routes for previous prefix
+            if last_prefix is not None:
+                try:
+                    rng = range_class.parse_str(last_prefix)
+                    rmin = long(rng.min)
+                    rmax = long(rng.max)
+                    cursor.executemany(sql, [(asn, rmin, rmax) for asn in asns])
+                except BadIPResource:
+                    logger.warning('skipping bad prefix: ' + last_prefix)
+
+            asns = set()
+            last_prefix = prefix
+
+        asns.add(int(origin_as))
+
+    logger.info('Committing...')
+    cursor.execute('COMMIT')
+
+    try:
+        logger.info('Dropping old table...')
+        cursor.execute('DROP TABLE IF EXISTS %s_old' % table)
+    except _mysql_exceptions.Warning:
+        pass
+
+    logger.info('Swapping staging table with live table...')
+    cursor.execute('RENAME TABLE %(table)s TO %(table)s_old, %(table)s_new TO %(table)s' % {'table': table})
+
+    transaction.commit_unless_managed()
+
+    logger.info('Updating timestamp metadata...')
+    rpki.gui.app.timestamp.update('bgp_v4_import')
+
+
+def parse_mrt(f):
+    # filter input through bgpdump
+    pipe = subprocess.Popen([BGPDUMP, '-m', '-v', '-'], stdin=f,
+                            stdout=subprocess.PIPE)
+
+    last_prefix = None
+    last_as = None
+    for e in pipe.stdout.readlines():
+        a = e.split('|')
+        prefix = a[5]
+        try:
+            origin_as = int(a[6].split()[-1])
+        except ValueError:
+            # skip AS_SETs
+            continue
+
+        if prefix != last_prefix:
+            last_prefix = prefix
+        elif last_as == origin_as:
+            continue
+        last_as = origin_as
+
+        asns = PREFIXES.get(prefix)
+        if not asns:
+            asns = set()
+            PREFIXES[prefix] = asns
+        asns.add(origin_as)
+
+    pipe.wait()
+    if pipe.returncode:
+        raise ProgException('bgpdump exited with code %d' % pipe.returncode)
+
+
+class ProgException(Exception):
+    pass
+
+
+class BadArgument(ProgException):
+    pass
+
+
+class UnknownInputType(ProgException):
+    pass
+
+
+class PipeFailed(ProgException):
+    pass
+
+
+if __name__ == '__main__':
+    start_time = time.time()
+
+    parser = optparse.OptionParser(usage='%prog [options] PATH',
+            description="""This tool is used to import the IPv4/6 BGP table dumps
+from routeviews.org into the RPKI Web Portal database.  If the
+input file is a bzip2 compressed file, it will be decompressed
+automatically.""")
+    parser.add_option('-t', '--type', dest='filetype', metavar='TYPE',
+                      help='Specify the input file type (auto, text, mrt) [Default: %default]')
+    parser.add_option('-l', '--level', dest='log_level', default='INFO',
+                      help='Set logging level [Default: %default]')
+    parser.add_option('-u', '--bunzip2', dest='bunzip', metavar='PROG',
+                      help='Specify bunzip2 program to use')
+    parser.add_option('-b', '--bgpdump', dest='bgpdump', metavar='PROG',
+                      help='Specify path to bgdump binary')
+    parser.set_defaults(debug=False, verbose=False, filetype='auto')
+    options, args = parser.parse_args()
+
+    v = getattr(logging, options.log_level.upper())
+    logger.setLevel(v)
+    logging.basicConfig()
+    logger.info('logging level set to ' + logging.getLevelName(v))
+
+    if options.bgpdump:
+        BGPDUMP = os.path.expanduser(options.bgpdump)
+
+    try:
+        if len(args) != 1:
+            raise BadArgument('no filename specified, or more than one filename specified')
+        filename = args[0]
+
+        if options.filetype == 'auto':
+            # try to determine input type from filename, based on the default
+            # filenames from archive.routeviews.org
+            bname = os.path.basename(filename)
+            if bname.startswith('oix-full-snapshot-latest'):
+                filetype = 'text'
+            elif bname.startswith('rib.'):
+                filetype = 'mrt'
+            else:
+                raise UnknownInputType('unable to automatically determine input file type')
+            logging.info('Detected import format as "%s"' % filetype)
+        else:
+            filetype = options.filetype
+
+        pipe = None
+        if filename.endswith('.bz2'):
+            bunzip = 'bunzip2' if not options.bunzip else os.path.expanduser(options.bunzip)
+            logging.info('Decompressing input file on the fly...')
+            pipe = subprocess.Popen([bunzip, '--stdout', filename],
+                                    stdout=subprocess.PIPE)
+            input_file = pipe.stdout
+        else:
+            input_file = open(filename)
+
+        try:
+            dispatch = {'text': parse_text, 'mrt': parse_mrt}
+            dispatch[filetype](input_file)
+        except KeyError:
+            raise UnknownInputType('"%s" is an unknown input file type' % filetype)
+
+        if pipe:
+            logging.debug('Waiting for child to exit...')
+            pipe.wait()
+            if pipe.returncode:
+                raise PipeFailed('Child exited code %d' % pipe.returncode)
+            pipe = None
+        else:
+            input_file.close()
+
+        logger.info('Elapsed time %d secs' % (time.time() - start_time))
+        rc = 0
+
+    except ProgException, e:
+        logger.exception(e)
+        rc = 1
+
+    logging.shutdown()
+    sys.exit(rc)
diff --git a/rpkid/portal-gui/scripts/rpkigui-rcynic.py b/rpkid/portal-gui/scripts/rpkigui-rcynic.py
index 3dc0d9bd..3205fc8d 100644
--- a/rpkid/portal-gui/scripts/rpkigui-rcynic.py
+++ b/rpkid/portal-gui/scripts/rpkigui-rcynic.py
@@ -1,5 +1,5 @@
-# $Id$
 # Copyright (C) 2011  SPARTA, Inc. dba Cobham Analytic Solutions
+# Copyright (C) 2012  SPARTA, Inc. a Parsons Company
 #
 # Permission to use, copy, modify, and distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
@@ -12,216 +12,212 @@
 # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
 # OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 # PERFORMANCE OF THIS SOFTWARE.
-#
+
+__version__ = '$Id$'
 
 default_logfile = '/var/rcynic/data/summary.xml'
 default_root = '/var/rcynic/data'
 
-import time, vobject
+import time
+import vobject
+import logging
+
+from django.db import transaction
+import django.db.models
+from django.core.exceptions import ObjectDoesNotExist
+
+import rpki
+import rpki.gui.app.timestamp
 from rpki.gui.cacheview import models
 from rpki.rcynic import rcynic_xml_iterator, label_iterator
 from rpki.sundial import datetime
-from django.db import transaction
-import django.db.models
 
-debug = False
-fam_map = { 'roa_prefix_set_ipv6': 6, 'roa_prefix_set_ipv4': 4 }
+logger = logging.getLogger(__name__)
 
-class rcynic_object(object):
 
-    def __call__(self, vs):
-        """
-        do initial processing on a rcynic_object instance.
+def rcynic_cert(cert, obj):
+    obj.sia = cert.sia_directory_uri
 
-        return value is a tuple: first element is a boolean value indicating whether
-        the object is changed/new since the last time we processed it.  second
-        element is the db instance.
-        """
-        if debug:
-            print 'processing %s at %s' % (vs.file_class.__name__, vs.uri)
+    # object must be saved for the related manager methods below to work
+    obj.save()
+    # resources can change when a cert is updated
+    obj.asns.clear()
+    obj.addresses.clear()
 
-        # rcynic will generation <validation_status/> elements for objects
-        # listed in the manifest but not found on disk
-        if os.path.exists(vs.filename):
-            q = self.model_class.objects.filter(uri=vs.uri)
+    for asr in cert.resources.asn:
+        logger.debug('processing %s' % asr)
+
+        attrs = {'min': asr.min, 'max': asr.max}
+        q = models.ASRange.objects.filter(**attrs)
+        if not q:
+            obj.asns.create(**attrs)
+        else:
+            obj.asns.add(q[0])
+
+    for cls, addr_obj, addrset in (models.AddressRange, obj.addresses, cert.resources.v4), (models.AddressRangeV6, obj.addresses_v6, cert.resources.v6):
+        for rng in addrset:
+            logger.debug('processing %s' % rng)
+
+            attrs = {'prefix_min': rng.min, 'prefix_max': rng.max}
+            q = cls.objects.filter(**attrs)
             if not q:
-                if debug:
-                    print 'creating new db instance'
-                inst = self.model_class(uri=vs.uri)
+                addr_obj.create(**attrs)
             else:
-                inst = q[0]
+                addr_obj.add(q[0])
+
+
+def rcynic_roa(roa, obj):
+    obj.asid = roa.asID
+    # object must be saved for the related manager methods below to work
+    obj.save()
+    obj.prefixes.clear()
+    obj.prefixes_v6.clear()
+    for pfxset in roa.prefix_sets:
+        if pfxset.__class__.__name__ == 'roa_prefix_set_ipv6':
+            roa_cls = models.ROAPrefixV6
+            prefix_obj = obj.prefixes_v6
+        else:
+            roa_cls = models.ROAPrefixV4
+            prefix_obj = obj.prefixes
+
+        for pfx in pfxset:
+            attrs = {'prefix_min': pfx.min(),
+                        'prefix_max': pfx.max(),
+                        'max_length': pfx.max_prefixlen}
+            q = roa_cls.objects.filter(**attrs)
+            if not q:
+                prefix_obj.create(**attrs)
+            else:
+                prefix_obj.add(q[0])
 
-            # determine if the object is changed/new
-            mtime = os.stat(vs.filename)[8]
-            if mtime != inst.mtime:
-                inst.mtime = mtime
-                obj = vs.obj # causes object to be lazily loaded
-                inst.not_before = obj.notBefore.to_sql()
-                inst.not_after = obj.notAfter.to_sql()
-                if debug:
-                    sys.stderr.write('name=%s ski=%s\n' % (obj.subject, obj.ski))
-                inst.name = obj.subject
-                inst.keyid = obj.ski
 
-                # look up signing cert
-                if obj.issuer == obj.subject:
-                    # self-signed cert (TA)
-                    inst.cert = inst
-                else:
-                    q = models.Cert.objects.filter(keyid=obj.aki, name=obj.issuer)
-                    if q:
-                        inst.issuer = q[0]
-                    else:
-                        sys.stderr.write('warning: unable to find signing cert with ski=%s (%s)\n' % (obj.aki, obj.issuer))
-                        return None
-
-                self.callback(obj, inst)
-            else:
-                if debug:
-                    print 'object is unchanged'
+def rcynic_gbr(gbr, obj):
+    vcard = vobject.readOne(gbr.vcard)
+    logger.debug(vcard.prettyPrint())
+    obj.full_name = vcard.fn.value if hasattr(vcard, 'fn') else None
+    obj.email_address = vcard.email.value if hasattr(vcard, 'email') else None
+    obj.telephone = vcard.tel.value if hasattr(vcard, 'tel') else None
+    obj.organization = vcard.org.value[0] if hasattr(vcard, 'org') else None
+
+LABEL_CACHE = {}
 
-            # save required to create new ValidationStatus object refering to it
-            inst.save()
-            inst.statuses.create(generation=models.generations_dict[vs.generation] if vs.generation else None,
-                    timestamp=datetime.fromXMLtime(vs.timestamp).to_sql(),
-                    status=models.ValidationLabel.objects.get(label=vs.status))
 
-            return inst
+def save_statuses(inst, statuses):
+    for vs in statuses:
+        timestamp = datetime.fromXMLtime(vs.timestamp).to_sql()
+
+        # cache validation labels
+        if vs.status in LABEL_CACHE:
+            status = LABEL_CACHE[vs.status]
         else:
-            if debug:
-                print 'ERROR - file is missing: %s' % vs.filename
+            status = models.ValidationLabel.objects.get(label=vs.status)
+            LABEL_CACHE[vs.status] = status
+
+        g = models.generations_dict[vs.generation] if vs.generation else None
 
-        return True
+        inst.statuses.create(generation=g, timestamp=timestamp, status=status)
 
-class rcynic_cert(rcynic_object):
-    model_class = models.Cert
+@transaction.commit_on_success
+def process_cache(root, xml_file):
+    dispatch = {
+      'rcynic_certificate': rcynic_cert,
+      'rcynic_roa': rcynic_roa,
+      'rcynic_ghostbuster': rcynic_gbr
+    }
+    model_class = {
+      'rcynic_certificate': models.Cert,
+      'rcynic_roa': models.ROA,
+      'rcynic_ghostbuster': models.Ghostbuster
+    }
 
-    def callback(self, cert, obj):
-        """
-        Process a RPKI resource certificate.
-        """
+    last_uri = None
+    statuses = []
 
-        obj.sia = cert.sia_directory_uri
-        obj.save()
+    logger.info('clearing validation statuses')
+    models.ValidationStatus.objects.all().delete()
 
-        # resources can change when a cert is updated
-        obj.asns.clear()
-        obj.addresses.clear()
+    logger.info('updating validation status')
+    for vs in rcynic_xml_iterator(root, xml_file):
+        if vs.uri != last_uri:
+            if statuses:
+                obj, created = models.RepositoryObject.objects.get_or_create(uri=last_uri)
+                save_statuses(obj, statuses)
 
-        for asr in cert.resources.asn:
-            if debug:
-                sys.stderr.write('processing %s\n' % asr)
+            statuses = []
+            last_uri = vs.uri
 
-            attrs = { 'min': asr.min, 'max': asr.max }
-            q = models.ASRange.objects.filter(**attrs)
+        statuses.append(vs)
+
+        if vs.status == 'object_accepted':
+            logger.debug('processing %s' % vs.filename)
+
+            cls = model_class[vs.file_class.__name__]
+            q = cls.objects.filter(repo__uri=vs.uri)
             if not q:
-                obj.asns.create(**attrs)
+                repo, created = models.RepositoryObject.objects.get_or_create(uri=vs.uri)
+                inst = cls(repo=repo)
             else:
-                obj.asns.add(q[0])
+                inst = q[0]
 
-        for family, addrset in (4, cert.resources.v4), (6, cert.resources.v6):
-            for rng in addrset:
-                if debug:
-                    sys.stderr.write('processing %s\n' % rng)
+            # determine if the object is changed/new
+            mtime = os.stat(vs.filename)[8]
+            if mtime != inst.mtime:
+                inst.mtime = mtime
+                try:
+                    obj = vs.obj  # causes object to be lazily loaded
+                except rpki.POW._der.DerError, e:
+                    logger.warning('Caught %s while processing %s: %s' % (type(e), vs.filename, e))
+                    continue
 
-                attrs = { 'family': family, 'min': str(rng.min), 'max': str(rng.max) }
-                q = models.AddressRange.objects.filter(**attrs)
-                if not q:
-                    obj.addresses.create(**attrs)
-                else:
-                    obj.addresses.add(q[0])
-
-        if debug:
-            print 'finished processing rescert at %s' % cert.uri
-
-class rcynic_roa(rcynic_object):
-    model_class = models.ROA
-
-    def callback(self, roa, obj):
-        obj.asid = roa.asID
-        obj.save()
-        obj.prefixes.clear()
-        for pfxset in roa.prefix_sets:
-            family = fam_map[pfxset.__class__.__name__]
-            for pfx in pfxset:
-                attrs = { 'family' : family,
-                          'prefix': str(pfx.prefix),
-                          'bits' : pfx.prefixlen,
-                          'max_length': pfx.max_prefixlen }
-                q = models.ROAPrefix.objects.filter(**attrs)
-                if not q:
-                    obj.prefixes.create(**attrs)
-                else:
-                    obj.prefixes.add(q[0])
+                inst.not_before = obj.notBefore.to_sql()
+                inst.not_after = obj.notAfter.to_sql()
+                inst.name = obj.subject
+                inst.keyid = obj.ski
 
-class rcynic_gbr(rcynic_object):
-    model_class = models.Ghostbuster
+                # look up signing cert
+                if obj.issuer == obj.subject:
+                    # self-signed cert (TA)
+                    assert(isinstance(inst, models.Cert))
+                    inst.issuer = inst
+                else:
+                    try:
+                        inst.issuer = models.Cert.objects.get(keyid=obj.aki, name=obj.issuer)
+                    except ObjectDoesNotExist:
+                        logger.warning('unable to find signing cert with ski=%s (%s)' % (obj.aki, obj.issuer))
+                        continue
 
-    def callback(self, gbr, obj):
-        vcard = vobject.readOne(gbr.vcard)
-        if debug:
-            vcard.prettyPrint()
-        obj.full_name = vcard.fn.value if hasattr(vcard, 'fn') else None
-        obj.email_address = vcard.email.value if hasattr(vcard, 'email') else None
-        obj.telephone = vcard.tel.value if hasattr(vcard, 'tel') else None
-        obj.organization = vcard.org.value[0] if hasattr(vcard, 'org') else None
+                # do object-specific tasks
+                dispatch[vs.file_class.__name__](obj, inst)
 
-def process_cache(root, xml_file):
-    start = time.time()
+                inst.save()  # don't require a save in the dispatch methods
+            else:
+                logger.debug('object is unchanged')
 
-    dispatch = {
-      'rcynic_certificate': rcynic_cert(),
-      'rcynic_roa'        : rcynic_roa(),
-      'rcynic_ghostbuster': rcynic_gbr()
-    }
+            # insert the saved validation statuses now that the object has been
+            # created.
+            save_statuses(inst.repo, statuses)
+            statuses = []
 
-    # remove all existing ValidationStatus_* entries
-    models.ValidationStatus_Cert.objects.all().delete()
-    models.ValidationStatus_ROA.objects.all().delete()
-    models.ValidationStatus_Ghostbuster.objects.all().delete()
-
-    # loop over all rcynic objects and dispatch based on the returned
-    # rcynic_object subclass
-    n = 1
-    defer = rcynic_xml_iterator(root, xml_file)
-    while defer:
-        if debug:
-            print 'starting iteration %d for deferred objects' % n
-            n = n + 1
-
-        elts = defer
-        defer = []
-        for vs in elts:
-            # need to defer processing this object, most likely because
-            # the <validation_status/> element for the signing cert hasn't
-            # been seen yet
-            if not dispatch[vs.file_class.__name__](vs):
-                defer.append(vs)
+    # process any left over statuses for an object that was not ultimately
+    # accepted
+    if statuses:
+        obj, created = models.RepositoryObject.objects.get_or_create(uri=last_uri)
+        save_statuses(obj, statuses)
 
     # garbage collection
     # remove all objects which have no ValidationStatus references, which
     # means they did not appear in the last XML output
-    if debug:
-        print 'performing garbage collection'
+    logger.info('performing garbage collection')
 
-    # trying to .delete() the querysets directly results in a "too many sql variables" exception
-    for qs in (models.Cert.objects.annotate(num_statuses=django.db.models.Count('statuses')).filter(num_statuses=0),
-            models.Ghostbuster.objects.annotate(num_statuses=django.db.models.Count('statuses')).filter(num_statuses=0),
-            models.ROA.objects.annotate(num_statuses=django.db.models.Count('statuses')).filter(num_statuses=0)):
-        for e in qs:
-            e.delete()
-
-    if debug:
-        stop = time.time()
-        sys.stdout.write('elapsed time %d seconds.\n' % (stop - start))
+    # Delete all objects that have zero validation status elements.
+    models.RepositoryObject.objects.annotate(num_statuses=django.db.models.Count('statuses')).filter(num_statuses=0).delete()
 
+@transaction.commit_on_success
 def process_labels(xml_file):
-    if debug:
-        sys.stderr.write('updating labels...\n')
+    logger.info('updating labels...')
 
     for label, kind, desc in label_iterator(xml_file):
-        if debug:
-            sys.stderr.write('label=%s kind=%s desc=%s\n' % (label, kind, desc))
+        logger.debug('label=%s kind=%s desc=%s' % (label, kind, desc))
         if kind:
             q = models.ValidationLabel.objects.filter(label=label)
             if not q:
@@ -233,24 +229,33 @@ def process_labels(xml_file):
             obj.status = desc
             obj.save()
 
+
 if __name__ == '__main__':
     import optparse
 
     parser = optparse.OptionParser()
-    parser.add_option("-d", "--debug", action="store_true",
-            help="enable debugging message")
+    parser.add_option("-l", "--level", dest="log_level", default='INFO',
+                      help="specify the logging level [default: %default]")
     parser.add_option("-f", "--file", dest="logfile",
-            help="specify the rcynic XML file to parse [default: %default]",
-            default=default_logfile)
+                      help="specify the rcynic XML file to parse [default: %default]",
+                      default=default_logfile)
     parser.add_option("-r", "--root",
-            help="specify the chroot directory for the rcynic jail [default: %default]",
-            metavar="DIR", default=default_root)
+                      help="specify the chroot directory for the rcynic jail [default: %default]",
+                      metavar="DIR", default=default_root)
     options, args = parser.parse_args(sys.argv)
-    if options.debug:
-        debug = True
 
-    with transaction.commit_on_success():
-        process_labels(options.logfile)
-        process_cache(options.root, options.logfile)
+    v = getattr(logging, options.log_level.upper())
+    logger.setLevel(v)
+    logging.basicConfig()
+    logger.info('log level set to %s' % logging.getLevelName(v))
+
+    start = time.time()
+    process_labels(options.logfile)
+    process_cache(options.root, options.logfile)
+
+    rpki.gui.app.timestamp.update('rcynic_import')
+
+    stop = time.time()
+    logger.info('elapsed time %d seconds.' % (stop - start))
 
-# vim:sw=4 ts=8
+    logging.shutdown()
diff --git a/rpkid/portal-gui/scripts/rpkigui-reset-demo.py b/rpkid/portal-gui/scripts/rpkigui-reset-demo.py
new file mode 100644
index 00000000..acfddabd
--- /dev/null
+++ b/rpkid/portal-gui/scripts/rpkigui-reset-demo.py
@@ -0,0 +1,34 @@
+# Copyright (C) 2012  SPARTA, Inc. a Parsons Company
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND SPARTA DISCLAIMS ALL WARRANTIES WITH
+# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS.  IN NO EVENT SHALL SPARTA BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+
+"""
+This script is used to reset all of the labuser* accounts on demo.rpki.net back
+to a state suitable for a new workshop.  It removes all ROAs and Ghostbuster
+issued by the labuser accounts.
+
+"""
+
+__version__ = '$Id$'
+
+from rpki.irdb.models import ROARequest, GhostbusterRequest, ResourceHolderCA
+from rpki.gui.app.glue import list_received_resources
+
+for n in xrange(1, 33):
+    username = 'labuser%02d' % n
+    print 'removing objects for ' + username
+    for cls in (ROARequest, GhostbusterRequest):
+        cls.objects.filter(issuer__handle=username).delete()
+    print '... updating resource certificate cache'
+    conf = ResourceHolderCA.objects.get(handle=username)
+    list_received_resources(sys.stdout, conf)
diff --git a/rpkid/portal-gui/scripts/rpkigui-response.py b/rpkid/portal-gui/scripts/rpkigui-response.py
deleted file mode 100755
index 9b150c51..00000000
--- a/rpkid/portal-gui/scripts/rpkigui-response.py
+++ /dev/null
@@ -1,70 +0,0 @@
-# $Id$
-# Copyright (C) 2011  SPARTA, Inc. dba Cobham Analytic Solutions
-#
-# Permission to use, copy, modify, and distribute this software for any
-# purpose with or without fee is hereby granted, provided that the above
-# copyright notice and this permission notice appear in all copies.
-#
-# THE SOFTWARE IS PROVIDED "AS IS" AND SPARTA DISCLAIMS ALL WARRANTIES WITH
-# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-# AND FITNESS.  IN NO EVENT SHALL SPARTA BE LIABLE FOR ANY SPECIAL, DIRECT,
-# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
-# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-# PERFORMANCE OF THIS SOFTWARE.
-#
-#
-# Helper script for use on the server side when using rpkidemo.
-# Takes a xml result from either configure_parent or
-# configure_publication_client and places it in the portal gui
-# outbox with the appropriate rfc822 header fields.
-
-import os
-os.environ['DJANGO_SETTINGS_MODULE'] = 'rpki.gui.settings'
-
-import sys
-import pwd
-import email.message, email.utils, mailbox
-from django.conf import settings
-
-if len(sys.argv) < 4:
-    sys.stderr.write("""usage: rpkigui-response <target-handle> <response-type> <xml-response-file>
-
-<target-handle>        the handle for the rpkidemo user to which this
-                       response should be sent
-
-<response-type>        'parent' for a configure_child response, or
-                       'repository' for a configure_publication_client
-                       response
-
-<xml-response-file>    the file containing the xml response for a
-                       configure_child or configure_publication_client
-                       command
-""")
-
-    sys.exit(0)
-
-class InvalidResponseType(Exception):
-    """
-    Invalid response type.
-    """
-
-request_type = sys.argv[2]
-if not request_type in ('parent', 'repository'):
-    raise InvalidResponseType, 'invalid response type: %s' % request_type
-
-# make sure apache process can manipulate the outbox!
-os.setuid(pwd.getpwnam(settings.WEB_USER)[2])
-
-msg = email.message.Message()
-msg['X-rpki-self-handle'] = sys.argv[1]
-msg['X-rpki-type'] = request_type
-msg['Date'] = email.utils.formatdate()
-msg['Message-ID'] = email.utils.make_msgid()
-msg.set_type('application/x-rpki-setup')
-msg.set_payload(open(sys.argv[3]).read())
-
-box = mailbox.Maildir(settings.OUTBOX)
-box.add(msg)
-
-# vim:sw=4 ts=8 expandtab