Checkpoint. Add synchronize_prefixes and synchronize_asns commands.

svn path=/branches/tk100/; revision=4125

3 files changed, 170 insertions, 276 deletions

diff --git a/rpkid/rpki/irdb/models.py b/rpkid/rpki/irdb/models.py
index dbbc61a7..2cc46737 100644
--- a/rpkid/rpki/irdb/models.py
+++ b/rpkid/rpki/irdb/models.py
@@ -179,6 +179,9 @@ class CertificateManager(django.db.models.Manager):
 class Identity(django.db.models.Model):
   handle = HandleField(unique = True)
 
+  def __unicode__(self):
+    return self.handle
+
 class CA(django.db.models.Model):
   identity = django.db.models.ForeignKey(Identity)
   purpose = EnumField(choices = ("resources", "servers"))
@@ -290,6 +293,8 @@ class CrossCertification(Certificate):
       is_ca        = True,
       pathLenConstraint = 0)
 
+  def __unicode__(self):
+    return self.handle
 
 class Revocation(django.db.models.Model):
   issuer = django.db.models.ForeignKey(CA, related_name = "revocations")
@@ -334,6 +339,9 @@ class BSC(Certificate):
       interval     = self.default_interval,
       is_ca        = False)
 
+  def __unicode__(self):
+    return self.handle
+
 class Child(CrossCertification):
   issuer = django.db.models.ForeignKey(CA, related_name = "children")
   name = django.db.models.TextField(null = True, blank = True)
diff --git a/rpkid/rpki/rpkic.py b/rpkid/rpki/rpkic.py
index 43809d80..de1bb2cf 100644
--- a/rpkid/rpki/rpkic.py
+++ b/rpkid/rpki/rpkic.py
@@ -1,40 +1,19 @@
 """
-This (oversized) module used to be an (oversized) program.
-Refactoring in progress, some doc still needs updating.
-
-
-This program is now the merger of three different tools: the old
-myrpki.py script, the old myirbe.py script, and the newer setup.py CLI
-tool.  As such, it is still in need of some cleanup, but the need to
-provide a saner user interface is more urgent than internal code
-prettiness at the moment.  In the long run, 90% of the code in this
-file probably ought to move to well-designed library modules.
-
-Overall goal here is to build up the configuration necessary to run
-rpkid and friends, by reading a config file, a collection of .CSV
-files, and the results of a few out-of-band XML setup messages
-exchanged with one's parents, children, and so forth.
-
-The config file is in an OpenSSL-compatible format, the CSV files are
-simple tab-delimited text.  The XML files are all generated by this
-program, either the local instance or an instance being run by another
-player in the system; the mechanism used to exchange these setup
-messages is outside the scope of this program, feel free to use
-PGP-signed mail, a web interface (not provided), USB stick, carrier
-pigeons, whatever works.
-
-With one exception, the commands in this program avoid using any
-third-party Python code other than the rpki libraries themselves; with
-the same one exception, all OpenSSL work is done with the OpenSSL
-command line tool (the one built as a side effect of building rcynic
-will do, if your platform has no system copy or the system copy is too
-old).  This is all done in an attempt to make the code more portable,
-so one can run most of the RPKI back end software on a laptop or
-whatever.  The one exception is the configure_daemons command, which
-must, of necessity, use the same communication libraries as the
-daemons with which it is conversing.  So that one command will not
-work if the correct Python modules are not available.
+This is a command line configuration and control tool for rpkid et al.
 
+Type "help" on the prompt, or run the program with the --help option for an
+overview of the available commands; type "help foo" for (more) detailed help
+on the "foo" command.
+
+
+This program is a rewrite of the old myrpki program, replacing ten
+zillion XML and X.509 disk files and subprocess calls to the OpenSSL
+command line tool with SQL data and direct calls to the rpki.POW
+library.  This version abandons all pretense that this program might
+somehow work without rpki.POW, lxml, and Django installed, but since
+those packages are required for rpkid anyway, this seems like a small
+price to pay for major simplification of the code and better
+integration with the Django-based GUI interface.
 
 $Id$
 
@@ -53,6 +32,20 @@ OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 PERFORMANCE OF THIS SOFTWARE.
 """
 
+# NB: As of this writing, I'm trying really hard to avoid having this
+# program depend on a Django settings.py file.  This may prove to be a
+# waste of time in the long run, but for for now, this means that one
+# has to be careful about exactly how and when one imports Django
+# modules, or anything that imports Django modules.  Bottom line is
+# that we don't import such modules until we need them.
+
+
+# We need context managers for transactions.  Well, unless we're
+# willing to have this program depend on a Django settings.py file so
+# that we can use decorators, which I'm not, at the moment.
+
+from __future__ import with_statement
+
 import subprocess, csv, re, os, getopt, sys, base64, time, glob, copy, warnings
 import rpki.config, rpki.cli, rpki.sundial, rpki.log, rpki.oids
 import rpki.http, rpki.resource_set, rpki.relaxng, rpki.exceptions
@@ -72,47 +65,15 @@ namespace      = "http://www.hactrn.net/uris/rpki/myrpki/"
 version        = "2"
 namespaceQName = "{" + namespace + "}"
 
-## @var allow_incomplete
-# Whether to include incomplete entries when rendering to XML.
-
-allow_incomplete = False
-
-## @var whine
-# Whether to whine about incomplete entries while rendering to XML.
-
-whine = True
-
 # A whole lot of exceptions
 
-class BadCommandSyntax(Exception):
-  """
-  Bad command line syntax.
-  """
-
-class BadPrefixSyntax(Exception):
-  """
-  Bad prefix syntax.
-  """
-
-class CouldntTalkToDaemon(Exception):
-  """
-  Couldn't talk to daemon.
-  """
-
-class BadXMLMessage(Exception):
-  """
-  Bad XML message.
-  """
-
-class PastExpiration(Exception):
-  """
-  Expiration date has already passed.
-  """
+class BadCommandSyntax(Exception):      "Bad command line syntax."
+class BadPrefixSyntax(Exception):       "Bad prefix syntax."
+class CouldntTalkToDaemon(Exception):   "Couldn't talk to daemon."
+class BadXMLMessage(Exception):         "Bad XML message."
+class PastExpiration(Exception):        "Expiration date has already passed."
+class CantRunRootd(Exception):          "Can't run rootd."
 
-class CantRunRootd(Exception):
-  """
-  Can't run rootd.
-  """
 
 
 def B64Element(e, tag, obj, **kwargs):
@@ -130,100 +91,6 @@ def B64Element(e, tag, obj, **kwargs):
 
 
 
-class CA(object):
-  """
-  Representation of one certification authority.
-
-  This used to be a big complicated wrapper around the awfulness of
-  running the OpenSSL command line tool in a subprocess.
-
-  With the new Django-model-based IRDB, this is mostly just a wrapper
-  around the IRDB objects and their associated crypto methods.
-
-  For the moment we keep the BPKI directories, because we need some
-  place to write cert and key files for the daemons.  Not yet sure
-  quite where this is going in the long run.
-  """
-
-  def __init__(self, dir, identity, purpose):
-    self.dir     = dir
-    self.cer     = dir + "/ca.cer"
-    self.crl     = dir + "/ca.crl"
-    self.identiy = identity
-    self.purpose = purpose
-    self.ca      = None
-
-  def setup(self, ca_name):
-    """
-    Set up this CA.  I no longer remember why this is not part of
-    __init__().  Maybe it will come back to me
-    """
-
-    self.ca = rpki.irdb.CA.objects.get_or_certify(
-      identity = self.identity,
-      purpose = self.purpose)[0]
-
-    f = open(self.cer, "w")
-    f.write(self.ca.certificate.get_PEM())
-    f.close()
-
-    f = open(self.crl, "w")
-    f.write(self.ca.latest_crl.get_PEM())
-    f.close()
-
-  def ee(self, purpose):
-    """
-    Issue an end-enity certificate.
-    """
-
-    return rpki.irdb.EECertificate.objects.get_or_certify(
-      issuer = self.ca,
-      purpose = purpose)[0]
-
-  def cms_xml_sign(self, elt):
-    """
-    Sign an XML object with CMS, return Base64 text.
-    """
-
-    ee = self.ee("referral")
-    return rpki.x509.SignedReferral().wrap(
-      msg = elt,
-      keypair = ee.private_key,
-      certs = ee.certificate,
-      crls = self.ca.latest_crl)
-
-  def cms_xml_verify(self, b64, ca):
-    """
-    Attempt to verify and extract XML from a Base64-encoded signed CMS
-    object.
-
-    ca is a rpki.x509.X509 CA certificate which we expect to be the
-    issuer of the EE certificate bundled with the CMS; this CA
-    certificate must previously have been cross-certified under our
-    trust anchor.
-    """
-
-    return rpki.x509.SignedReferral(Base64 = b64).unwrap(
-      ta = (ca, self.ca.certificate))
-
-  def bsc(self, handle, pkcs10):
-    """
-    Issue BSC certificate, if we have a PKCS #10 request for it.
-
-    Returns IRDB BSC object, or None if we don't have one and can't
-    make one because we don't have the PKCS #10 yet.
-    """
-
-    if pkcs10 is None:
-      return None
-
-    return rpki.irdb.BSC.objects.get_or_certify(
-      issuer = self.ca,
-      handle = handle,
-      pkcs10 = rpki.x509.PKCS10(Base64 = pkcs10))[0]
-
-
-
 def etree_write(e, filename, verbose = False, msg = None):
   """
   Write out an etree to a file, safely.
@@ -329,10 +196,10 @@ class main(rpki.cli.Cmd):
     self.run_pubd  = self.cfg.getboolean("run_pubd")
     self.run_rootd = self.cfg.getboolean("run_rootd")
 
-    from django.conf import settings
-
     irdbd_section = "irdbd"
 
+    from django.conf import settings
+
     settings.configure(
       DATABASES = { "default" : {
         "ENGINE"   : "django.db.backends.mysql",
@@ -863,72 +730,85 @@ class main(rpki.cli.Cmd):
 
     return self.renew_children_common(arg, True)
 
-
-
 
-  def configure_resources_main(self, msg = None):
+  def do_synchronize_prefixes(self, arg):
     """
-    Main program of old myrpki.py script.  This remains separate
-    because it's called from more than one place.
+    Synchronize IRDB against prefixes.csv.
     """
 
-    roa_csv_file                  = self.cfg.get("roa_csv")
-    prefix_csv_file               = self.cfg.get("prefix_csv")
-    asn_csv_file                  = self.cfg.get("asn_csv")
+    argv = arg.split()
 
-    # This probably should become an argument instead of (or in
-    # addition to a default from?) a config file option.
-    xml_filename                  = self.cfg.get("xml_filename")
+    if len(argv) != 1:
+      raise BadCommandSyntax("Need to specify prefixes.csv filename")
 
-    try:
-      e = etree_read(xml_filename)
-      bsc = self.bpki_resources.bsc(e.findtext("bpki_bsc_pkcs10"))
-      service_uri = e.get("service_uri")
-    except IOError:
-      bsc = None
-      service_uri = None
+    grouped4 = {}
+    grouped6 = {}
+
+    for handle, prefix in csv_reader(argv[0], columns = 2):
+      grouped = grouped6 if ":" in prefix else grouped4
+      if handle not in grouped:
+        grouped[handle] = []
+      grouped[handle].append(prefix)
 
-    e = Element("myrpki", handle = self.handle)
+    import django.db.transaction
 
-    if service_uri:
-      e.set("service_uri", service_uri)
+    with django.db.transaction.commit_on_success():
 
-    roa_requests.from_csv(roa_csv_file).xml(e)
+      primary_keys = []
 
-    children.from_entitydb(
-      prefix_csv_file = prefix_csv_file,
-      asn_csv_file = asn_csv_file,
-      fxcert = self.bpki_resources.fxcert,
-      entitydb = self.entitydb).xml(e)
+      for version, grouped, rset in ((4, grouped4, rpki.resource_set.resource_set_ipv4),
+                                     (6, grouped6, rpki.resource_set.resource_set_ipv6)):
+        for handle, prefixes in grouped.iteritems():
+          child = self.resource_ca.children.get(handle = handle)
+          for prefix in rset(",".join(prefixes)):
+            obj = rpki.irdb.ChildNet.objects.get_or_create(
+              child    = child,
+              start_ip = str(prefix.min),
+              end_ip   = str(prefix.max),
+              version  = version)[0]
+            primary_keys.append(obj.pk)
 
-    parents.from_entitydb(
-      fxcert = self.bpki_resources.fxcert,
-      entitydb = self.entitydb).xml(e)
+      q = rpki.irdb.ChildNet.objects
+      q = q.filter(child__issuer__exact = self.resource_ca)
+      q = q.exclude(pk__in = primary_keys)
+      q.delete()
 
-    repositories.from_entitydb(
-      fxcert = self.bpki_resources.fxcert,
-      entitydb = self.entitydb).xml(e)
+  def do_synchronize_asns(self, arg):
+    """
+    Synchronize IRDB against asns.csv.
+    """
 
-    B64Element(e, "bpki_ca_certificate", self.bpki_resources.cer)
-    B64Element(e, "bpki_crl",            self.bpki_resources.crl)
+    argv = arg.split()
 
-    if bsc is not None:
-      B64Element(e, "bpki_bsc_certificate", bsc.certificate)
-      B64Element(e, "bpki_bsc_pkcs10",      bsc.pkcs10)
+    if len(argv) != 1:
+      raise BadCommandSyntax("Need to specify asns.csv filename")
 
-    etree_write(e, xml_filename, msg = msg)
+    grouped = {}
 
+    for handle, asn in csv_reader(argv[0], columns = 2):
+      if handle not in grouped:
+        grouped[handle] = []
+      grouped[handle].append(asn)
 
-  def do_configure_resources(self, arg):
-    """
-    Read CSV files and all the descriptions of parents and children
-    that we've built up, package the result up as a single XML file to
-    be shipped to a hosting rpkid.
-    """
+    import django.db.transaction
 
-    if arg:
-      raise BadCommandSyntax, "Unexpected argument %r" % arg
-    self.configure_resources_main(msg = "Send this file to the rpkid operator who is hosting you")
+    with django.db.transaction.commit_on_success():
+
+      primary_keys = []
+
+      for handle, asns in grouped.iteritems():
+        child = self.resource_ca.children.get(handle = handle)
+        for asn in rpki.resource_set.resource_set_as(",".join(asns)):
+          obj = rpki.irdb.ChildASN.objects.get_or_create(
+            child    = child,
+            start_as = str(asn.min),
+            end_as   = str(asn.max))[0]
+          primary_keys.append(obj.pk)
+
+      q = rpki.irdb.ChildASN.objects
+      q = q.filter(child__issuer__exact = self.resource_ca)
+      q = q.exclude(pk__in = primary_keys)
+      q.delete()
 
 
 
diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py
index d0a080e9..1fae02c4 100644
--- a/scripts/convert-from-entitydb-to-sql.py
+++ b/scripts/convert-from-entitydb-to-sql.py
@@ -29,9 +29,10 @@ from lxml.etree import ElementTree
 if os.getlogin() != "sra":
   sys.exit("I //said// this was a work in progress")
 
-cfg_file = "rpki.conf"
-entitydb = "entitydb"
-bpki     = "bpki"
+cfg_file        = "rpki.conf"
+entitydb        = "entitydb"
+bpki            = "bpki"
+copy_csv_data   = True
 
 opts, argv = getopt.getopt(sys.argv[1:], "c:h?", ["config=", "help"])
 for o, a in opts:
@@ -252,24 +253,26 @@ for filename in glob.iglob(os.path.join(entitydb, "children", "*.xml")):
     certificate = xcert,
     issuer = resource_ca)[0]
 
-  cur.execute("""
-              SELECT start_as, end_as FROM registrant_asn WHERE registrant_id = %s
-              """, (registrant_id,))
-  for start_as, end_as in cur.fetchall():
-    rpki.irdb.ChildASN.objects.get_or_create(
-      start_as = start_as,
-      end_as = end_as,
-      child = child)
-
-  cur.execute("""
-              SELECT start_ip, end_ip, version FROM registrant_net WHERE registrant_id = %s
-              """, (registrant_id,))
-  for start_ip, end_ip, version in cur.fetchall():
-    rpki.irdb.ChildNet.objects.get_or_create(
-      start_ip = start_ip,
-      end_ip = end_ip,
-      version = version,
-      child = child)
+  if copy_csv_data:
+
+    cur.execute("""
+                SELECT start_as, end_as FROM registrant_asn WHERE registrant_id = %s
+                """, (registrant_id,))
+    for start_as, end_as in cur.fetchall():
+      rpki.irdb.ChildASN.objects.get_or_create(
+        start_as = start_as,
+        end_as = end_as,
+        child = child)
+
+    cur.execute("""
+                SELECT start_ip, end_ip, version FROM registrant_net WHERE registrant_id = %s
+                """, (registrant_id,))
+    for start_ip, end_ip, version in cur.fetchall():
+      rpki.irdb.ChildNet.objects.get_or_create(
+        start_ip = start_ip,
+        end_ip = end_ip,
+        version = version,
+        child = child)
 
 # Scrape parent data out of the entitydb.
 
@@ -309,15 +312,16 @@ for filename in glob.iglob(os.path.join(entitydb, "parents", "*.xml")):
   # While we have the parent object in hand, load any Ghostbuster
   # entries specific to this parent.
 
-  cur.execute("""
-              SELECT vcard FROM ghostbuster_request
-              WHERE self_handle = %s AND parent_handle = %s
-              """, (self_handle, parent_handle))
-  for row in cur.fetchall():
-    rpki.irdb.GhostbusterRequest.objects.get_or_create(
-      identity = identity,
-      parent = parent,
-      vcard = row[0])
+  if copy_csv_data:
+    cur.execute("""
+                SELECT vcard FROM ghostbuster_request
+                WHERE self_handle = %s AND parent_handle = %s
+                """, (self_handle, parent_handle))
+    for row in cur.fetchall():
+      rpki.irdb.GhostbusterRequest.objects.get_or_create(
+        identity = identity,
+        parent = parent,
+        vcard = row[0])
 
 # Scrape repository data out of the entitydb.
 
@@ -370,37 +374,39 @@ for filename in glob.iglob(os.path.join(entitydb, "pubclients", "*.xml")):
     certificate = xcert,
     issuer = server_ca)
 
-# Copy over any ROA requests
+if copy_csv_data:
+
+  # Copy over any ROA requests
 
-cur.execute("""
-            SELECT roa_request_id, asn FROM roa_request
-            WHERE roa_request_handle = %s
-            """, (self_handle,))
-for roa_request_id, asn in cur.fetchall():
-  roa_request = rpki.irdb.ROARequest.objects.get_or_create(identity = identity, asn = asn)[0]
   cur.execute("""
-              SELECT prefix, prefixlen, max_prefixlen, version FROM roa_request_prefix
-              WHERE roa_request_id = %s
-              """, (roa_request_id,))
-  for prefix, prefixlen, max_prefixlen, version in cur.fetchall():
-    rpki.irdb.ROARequestPrefix.objects.get_or_create(
-      roa_request = roa_request,
-      version = version,
-      prefix = prefix,
-      prefixlen = prefixlen,
-      max_prefixlen = max_prefixlen)
-
-# Copy over any non-parent-specific Ghostbuster requests.
-
-cur.execute("""
-            SELECT vcard FROM ghostbuster_request
-            WHERE self_handle = %s AND parent_handle IS NULL
-            """, (self_handle,))
-for row in cur.fetchall():
-  rpki.irdb.GhostbusterRequest.objects.get_or_create(
-    identity = identity,
-    parent = None,
-    vcard = row[0])
+              SELECT roa_request_id, asn FROM roa_request
+              WHERE roa_request_handle = %s
+              """, (self_handle,))
+  for roa_request_id, asn in cur.fetchall():
+    roa_request = rpki.irdb.ROARequest.objects.get_or_create(identity = identity, asn = asn)[0]
+    cur.execute("""
+                SELECT prefix, prefixlen, max_prefixlen, version FROM roa_request_prefix
+                WHERE roa_request_id = %s
+                """, (roa_request_id,))
+    for prefix, prefixlen, max_prefixlen, version in cur.fetchall():
+      rpki.irdb.ROARequestPrefix.objects.get_or_create(
+        roa_request = roa_request,
+        version = version,
+        prefix = prefix,
+        prefixlen = prefixlen,
+        max_prefixlen = max_prefixlen)
+
+  # Copy over any non-parent-specific Ghostbuster requests.
+
+  cur.execute("""
+              SELECT vcard FROM ghostbuster_request
+              WHERE self_handle = %s AND parent_handle IS NULL
+              """, (self_handle,))
+  for row in cur.fetchall():
+    rpki.irdb.GhostbusterRequest.objects.get_or_create(
+      identity = identity,
+      parent = None,
+      vcard = row[0])
 
 # List cross certifications we didn't use.