portal-gui no longer invokes the myrpki.py command line tool.

It now updates the IRDB directly, and notifies rpkid to immediately
perform an update, rather than waiting for the next cron job to cycle.

added a 'valid_until' field to the Child model, to store the end of
the RPKI resource certificate validity period.

change load_csv.py and list_resources.py to use the code for locating
rpki.conf in rpki.config rather than duplicating it locally.

fixed bug in AddressRange model in function as_resource_range() where
IPv4 addresses were returned as IPv6.


svn path=/portal-gui/templates/myrpki/child_view.html; revision=3694

5 files changed, 132 insertions, 131 deletions

diff --git a/rpkid/rpki/gui/app/glue.py b/rpkid/rpki/gui/app/glue.py
index 0a5e9af3..a00eff50 100644
--- a/rpkid/rpki/gui/app/glue.py
+++ b/rpkid/rpki/gui/app/glue.py
@@ -19,121 +19,140 @@ PERFORMANCE OF THIS SOFTWARE.
 
 from __future__ import with_statement
 
-import os
-import os.path
-import csv
-import stat
-import sys
+import os, os.path, csv, stat, sys
 
 from django.db.models import F
 
-import rpki
-import rpki.config
-
+import rpki, rpki.async, rpki.http, rpki.x509, rpki.left_right
+from rpki.myrpki import CA, IRDB, csv_writer
 from rpki.gui import settings
 from rpki.gui.app import models
 
-def conf(handle):
-    return settings.CONFDIR + '/' + handle
-
-#def form_to_conf(data):
-#    """Write out a rpki.conf based on the given form data."""
-#    handle = data['handle']
-#    confdir = settings.MYRPKI_DATA_DIR + '/' + handle
-#    if os.path.exists(confdir):
-#        raise RuntimeError, '%s: directory already exists!' % (confdir, )
-#    os.makedirs(confdir)
-#    template = open(settings.MYRPKI_DATA_DIR + '/examples/rpki.conf', 'r').read()
-#    # stuff the appropriate output directory into the dict
-#    data['MYRPKI_DATA_DIR'] = confdir
-#    with open(confdir + '/rpki.conf', 'w') as conf:
-#        print >>conf, template % data
-#    invoke_rpki(handle, ['initialize'])
-
-def invoke_rpki(log, handle, args):
-    """Invoke the myrpki cli for the specified configuration."""
-    myrpki_dir = conf(handle)
-    config = myrpki_dir + '/rpki.conf'
-    # default rpki.conf uses relative paths, so chdir() to the repo first
-    cmd = 'cd %s && %s %s' % (myrpki_dir, settings.MYRPKI, ' '.join(['--config=' + config] + args))
-    print >>log, 'invoking', cmd
-    os.system(cmd)
+def conf(*handle):
+    """
+    Return the absolute pathname to the configuration directory for
+    the given resource handle.  If additional arguments are given, they
+    are taken to mean files/subdirectories.
+    """
+    argv = [ settings.CONFDIR ]
+    argv.extend(handle)
+    return os.path.join(*argv)
 
 def read_file_from_handle(handle, fname):
     """read a filename relative to the directory for the given resource handle.  returns
     a tuple of (content, mtime)"""
-    with open(conf(handle) + '/' + fname, 'r') as fp:
+    with open(conf(handle, fname), 'r') as fp:
         data = fp.read()
         mtime = os.fstat(fp.fileno())[stat.ST_MTIME]
     return data, mtime
 
-#def read_identity(handle):
-#    fname = settings.MYRPKI_DATA_DIR + '/' + handle + '/entitydb/identity.xml'
-#    with open(fname, 'r') as fp:
-#        data = fp.read()
-#    return data
 read_identity = lambda h: read_file_from_handle(h, 'entitydb/identity.xml')[0]
 
-def read_child_response(handle, child):
-    fname = '%s/entitydb/children/%s.xml' % (conf(handle), child)
-    with open(fname, 'r') as fp:
-        data = fp.read()
-    return data
-
-# FIXME - remove this once rpki.myrpki.csv_writer is an object with a
-# .file field
-def csv_writer(f):
-  return csv.writer(f, dialect = csv.get_dialect("excel-tab"))
-
 def output_asns(path, handle):
     '''Write out csv file containing asns delegated to my children.'''
     qs = models.Asn.objects.filter(lo=F('hi'), allocated__in=handle.children.all())
-    with open(path, 'w') as f:
-        w = csv_writer(f)
-        w.writerows([asn.allocated.handle, asn.lo] for asn in qs)
+    w = csv_writer(path)
+    w.writerows([asn.allocated.handle, asn.lo] for asn in qs)
+    w.close()
 
 def output_prefixes(path, handle):
     '''Write out csv file containing prefixes delegated to my children.'''
     qs = models.AddressRange.objects.filter(allocated__in=handle.children.all())
-    with open(path, 'w') as f:
-        w = csv_writer(f)
-        w.writerows([p.allocated.handle, p.as_resource_range()] for p in qs)
+    w = csv_writer(path)
+    w.writerows([p.allocated.handle, p.as_resource_range()] for p in qs)
+    w.close()
 
 def output_roas(path, handle):
     '''Write out csv file containing my roas.'''
     qs = models.RoaRequest.objects.filter(roa__in=handle.roas.all())
-    with open(path, 'w') as f:
-        w = csv_writer(f)
-        w.writerows([req.as_roa_prefix(), req.roa.asn,
-            '%s-group-%d' % (handle.handle, req.roa.pk)] for req in qs)
-
-def configure_daemons(log, handle):
-    args = ['configure_daemons']
-    for hosted in handle.hosting.all():
-        args.append(conf(hosted.handle) + '/myrpki.xml')
-    invoke_rpki(log, handle.handle, args)
+    w = csv_writer(path)
+    w.writerows([req.as_roa_prefix(), req.roa.asn,
+                '%s-group-%d' % (handle.handle, req.roa.pk)] for req in qs)
+    w.close()
+
+def qualify_path(pfx, fname):
+    """Ensure 'path' is an absolute filename."""
+    return fname if fname.startswith('/') else os.path.join(pfx, fname)
+
+def build_rpkid_caller(cfg, verbose=False):
+    """
+    Returns a function suitable for calling rpkid using the
+    configuration information specified in the rpki.config.parser
+    object.
+    """
+    bpki_servers_dir = cfg.get("bpki_servers_directory")
+    if not bpki_servers_dir.startswith('/'):
+        bpki_servers_dir = conf(cfg.get('handle'), bpki_servers_dir)
+
+    bpki_servers = CA(cfg.filename, bpki_servers_dir)
+    rpkid_base = "http://%s:%s/" % (cfg.get("rpkid_server_host"), cfg.get("rpkid_server_port"))
+
+    return rpki.async.sync_wrapper(rpki.http.caller(
+        proto       = rpki.left_right,
+        client_key  = rpki.x509.RSA(PEM_file = bpki_servers.dir + "/irbe.key"),
+        client_cert = rpki.x509.X509(PEM_file = bpki_servers.dir + "/irbe.cer"),
+        server_ta   = rpki.x509.X509(PEM_file = bpki_servers.cer),
+        server_cert = rpki.x509.X509(PEM_file = bpki_servers.dir + "/rpkid.cer"),
+        url         = rpkid_base + "left-right",
+        debug       = verbose))
 
 def configure_resources(log, handle):
-    '''Write out the csv files and invoke the myrpki.py command line tool.'''
-    # chdir to the repo dir since the default rpki.conf uses relative
-    # pathnames..
-    os.chdir(conf(handle.handle))
-    cfg = rpki.config.parser('rpki.conf', 'myrpki')
-    output_asns(cfg.get('asn_csv'), handle)
-    output_prefixes(cfg.get('prefix_csv'), handle)
-    output_roas(cfg.get('roa_csv'), handle)
-    run_rpkidemo = cfg.getboolean('run_rpkidemo', False)
-    if not run_rpkidemo:
-        run_rpkid = cfg.getboolean('run_rpkid')
-        if run_rpkid:
-            configure_daemons(log, handle)
-        else:
-            invoke_rpki(log, handle.handle, ['configure_resources'])
-
-            # send the myrpki.xml to the rpkid hosting me
-            configure_daemons(log, handle.host)
-
-            # process the response
-            invoke_rpki(log, handle.handle, ['configure_resources'])
+    """
+    This function should be called when resources for this resource
+    holder have changed.  It updates IRDB and notifies rpkid to
+    immediately process the changes, rather than waiting for the cron
+    job to run.
+
+    For backwards compatability (and backups), it also writes the csv
+    files for use with the myrpki.py command line script.
+    """
+
+    path = conf(handle.handle)
+    cfg = rpki.config.parser(os.path.join(path, 'rpki.conf'), 'myrpki')
+
+    output_asns(qualify_path(path, cfg.get('asn_csv')), handle)
+    output_prefixes(qualify_path(path, cfg.get('prefix_csv')), handle)
+    output_roas(qualify_path(path, cfg.get('roa_csv')), handle)
+
+    roa_requests = []
+    for roa in handle.roas.all():
+        v4 = rpki.resource_set.roa_prefix_set_ipv4()
+        v6 = rpki.resource_set.roa_prefix_set_ipv6()
+        for req in roa.from_roa_request.all():
+            pfx = req.as_roa_prefix()
+            if isinstance(pfx, rpki.resource_set.roa_prefix_ipv4):
+                v4.append(pfx)
+            else:
+                v6.append(pfx)
+        roa_requests.append((roa.asn, v4, v6))
+
+    children = []
+    for child in handle.children.all():
+        asns = rpki.resource_set.resource_set_as(a.as_resource_range() for a in child.asns.all())
+
+        v4 = rpki.resource_set.resource_set_ipv4()
+        v6 = rpki.resource_set.resource_set_ipv6()
+        for pfx in child.address_range.all():
+            rng = pfx.as_resource_range()
+            if isinstance(rng, rpki.resource_set.resource_range_ipv4):
+                v4.append(rng)
+            else:
+                v6.append(rng)
+            
+        # convert from datetime.datetime to rpki.sundial.datetime
+        valid_until = rpki.sundial.datetime.fromdatetime(child.valid_until)
+        children.append((child.handle, asns, v4, v6, valid_until))
+
+    irdb = IRDB(cfg)
+    irdb.update(handle, roa_requests, children)
+    irdb.close()
+
+    # for hosted handles, get the config for the rpkid host
+    if handle.host:
+        cfg = rpki.config.parser(conf(handle.host.handle, 'rpki.conf'), 'myrpki')
+
+    # contact rpkid to request immediate update
+    call_rpkid = build_rpkid_caller(cfg)
+    call_rpkid(rpki.left_right.self_elt.make_pdu(action='set', self_handle=handle.handle, run_now=True))
 
 # vim:sw=4 ts=8 expandtab
diff --git a/rpkid/rpki/gui/app/models.py b/rpkid/rpki/gui/app/models.py
index 78327b8f..3b161a59 100644
--- a/rpkid/rpki/gui/app/models.py
+++ b/rpkid/rpki/gui/app/models.py
@@ -90,7 +90,6 @@ class AddressRange(models.Model):
         try:
             self.as_resource_range().prefixlen()
         except rpki.exceptions.MustBePrefix, err:
-            print err
             return False
         return True
 
@@ -106,7 +105,7 @@ class RoaRequest(models.Model):
     def as_roa_prefix(self):
         '''Convert to a rpki.resouce_set.roa_prefix subclass.'''
         r = self.prefix.as_resource_range()
-        if isinstance(r, rpki.resource_set.resource_set_ipv4):
+        if isinstance(r, rpki.resource_set.resource_range_ipv4):
             return rpki.resource_set.roa_prefix_ipv4(r.min, r.prefixlen(),
                     self.max_length)
         else:
@@ -152,6 +151,7 @@ class Asn(models.Model):
 class Child(models.Model):
     conf = models.ForeignKey(Conf, related_name='children')
     handle = HandleField() # parent's name for child
+    valid_until = models.DateTimeField()
 
     def __unicode__(self):
 	return u"%s's child %s" % (self.conf, self.handle)
diff --git a/rpkid/rpki/gui/app/views.py b/rpkid/rpki/gui/app/views.py
index 77360d7d..db1cff3f 100644
--- a/rpkid/rpki/gui/app/views.py
+++ b/rpkid/rpki/gui/app/views.py
@@ -492,9 +492,13 @@ def myrpki_xml(request, self_handle):
         with open(fname, 'w') as myrpki_xml :
             myrpki_xml.write(request.POST['content'])
 
-        glue.configure_daemons(log, conf.host)
+        # FIXME: used to run configure_daemons here, but it takes too
+        # long with many hosted handles.  rpkidemo still needs a way
+        # to do initial bpki setup with rpkid!
 
-    return serve_file(self_handle, 'myrpki.xml', 'application/xml')
+        return http.HttpResponse('<p>success</p>')
+    else:
+        return serve_file(self_handle, 'myrpki.xml', 'application/xml')
 
 def login(request):
     """
diff --git a/rpkid/rpki/gui/scripts/list_resources.py b/rpkid/rpki/gui/scripts/list_resources.py
index 2b547e45..13864705 100644
--- a/rpkid/rpki/gui/scripts/list_resources.py
+++ b/rpkid/rpki/gui/scripts/list_resources.py
@@ -34,45 +34,23 @@ import sys
 import os
 os.environ['DJANGO_SETTINGS_MODULE'] = 'rpki.gui.settings'
 
-from datetime import datetime
 import getopt
+from datetime import datetime, timedelta
 from os.path import basename
 
-from rpki.myrpki import CA
-import rpki.config
-import rpki.x509
-import rpki.http
-import rpki.async
-import rpki.left_right
-import rpki.resource_set
-
-from rpki.gui.app import models
+import rpki.config, rpki.left_right, rpki.resource_set
+from rpki.gui.app import models, glue
 
 verbose = False
 version = '$Id$'
 
 def query_rpkid():
-    """Fetch our received resources from the local rpkid using the rpki.conf
-    in the current directory."""
-    cfg_file = os.getenv("RPKI_CONF", "rpki.conf")
-    cfg = rpki.config.parser(cfg_file, "myrpki")
-    bpki_servers = CA(cfg_file, cfg.get("bpki_servers_directory"))
-    rpkid_base = "http://%s:%s/" % (cfg.get("rpkid_server_host"), cfg.get("rpkid_server_port"))
-
-    if verbose:
-        print 'current directory is', os.getcwd()
-        print 'cfg_file=', cfg_file
-        print 'bpki_servers=', bpki_servers.dir
-        print 'rpkid_base=', rpkid_base
-
-    call_rpkid = rpki.async.sync_wrapper(rpki.http.caller(
-        proto       = rpki.left_right,
-        client_key  = rpki.x509.RSA(PEM_file = bpki_servers.dir + "/irbe.key"),
-        client_cert = rpki.x509.X509(PEM_file = bpki_servers.dir + "/irbe.cer"),
-        server_ta   = rpki.x509.X509(PEM_file = bpki_servers.cer),
-        server_cert = rpki.x509.X509(PEM_file = bpki_servers.dir + "/rpkid.cer"),
-        url         = rpkid_base + "left-right",
-        debug = verbose))
+    """
+    Fetch our received resources from the local rpkid using the rpki.conf
+    in the current directory.
+    """
+    cfg = rpki.config.parser(section='myrpki')
+    call_rpkid = glue.build_rpkid_caller(cfg, verbose)
 
     if verbose:
         print 'retrieving the list of <self/> handles served by this rpkid'
@@ -138,7 +116,11 @@ for pdu in pdus:
         if not child_set:
             if verbose:
                 print 'creating new child %s' % (pdu.child_handle,)
-            child = models.Child(conf=conf, handle=pdu.child_handle)
+            # default to 1 year.  no easy way to query irdb for the
+            # current value.
+            valid_until = datetime.now() + timedelta(days=365)
+            child = models.Child(conf=conf, handle=pdu.child_handle,
+                                 valid_until=valid_until)
             child.save()
     #elif isinstance(x, rpki.left_right.list_roa_requests_elt):
     #    print x.asn, x.ipv4, x.ipv6
diff --git a/rpkid/rpki/gui/scripts/load_csv.py b/rpkid/rpki/gui/scripts/load_csv.py
index 0ef49cce..ebb995b6 100755
--- a/rpkid/rpki/gui/scripts/load_csv.py
+++ b/rpkid/rpki/gui/scripts/load_csv.py
@@ -1,6 +1,6 @@
 # $Id$
 #
-# Copyright (C) 2010  SPARTA, Inc. dba Cobham Analytic Solutions
+# Copyright (C) 2010, 2011  SPARTA, Inc. dba Cobham Analytic Solutions
 #
 # Permission to use, copy, modify, and distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
@@ -23,22 +23,18 @@
 # for the handle you are loading data
 #
 
-import sys, os
+import os
 os.environ['DJANGO_SETTINGS_MODULE'] = 'rpki.gui.settings'
 
 import csv
 import socket # for socket.error
 
-import rpki
-import rpki.resource_set
-import rpki.ipaddrs
+import rpki.resource_set, rpki.ipaddrs
 from rpki.myrpki import csv_reader
-
 from rpki.gui.app import models
 from rpki.gui.app.views import add_roa_requests
 
-cfg_file = os.getenv("RPKI_CONF", "rpki.conf")
-cfg = rpki.config.parser(cfg_file, "myrpki")
+cfg = rpki.config.parser(section='myrpki')
 handle = cfg.get('handle')
 asn_csv = cfg.get('asn_csv')
 prefix_csv = cfg.get('prefix_csv')