Rework models slightly. Now parsing most of the BPKI stuff.

svn path=/branches/tk100/; revision=4109

7 files changed, 564 insertions, 49 deletions

diff --git a/buildtools/make-relaxng.py b/buildtools/make-relaxng.py
index 62decbae..0058ade5 100644
--- a/buildtools/make-relaxng.py
+++ b/buildtools/make-relaxng.py
@@ -3,7 +3,7 @@ Script to generate rpki/relaxng.py.
 
 $Id$
 
-Copyright (C) 2009  Internet Systems Consortium ("ISC")
+Copyright (C) 2009-2011  Internet Systems Consortium ("ISC")
 
 Permission to use, copy, modify, and distribute this software for any
 purpose with or without fee is hereby granted, provided that the above
@@ -32,7 +32,7 @@ OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 PERFORMANCE OF THIS SOFTWARE.
 """
 
-schemas = ("left_right", "up_down", "publication")
+import sys
 
 format_1 = """\
 # Automatically generated, do not edit.
@@ -46,9 +46,15 @@ format_2 = """\
 %(name)s = lxml.etree.RelaxNG(lxml.etree.fromstring('''%(rng)s'''))
 """
 
+def filename_to_symbol(s):
+  for suffix in (".rng", "-schema"):
+    if s.endswith(suffix):
+      s = s[:-len(suffix)]
+  return s.replace("-", "_")
+
 print format_1
 
-for name in schemas:
+for filename in sys.argv[1:]:
   print format_2 % {
-    "name" : name,
-    "rng"  : open(name.replace("_", "-") + "-schema.rng").read() }
+    "name" : filename_to_symbol(filename),
+    "rng"  : open(filename).read() }
diff --git a/rpkid/Makefile.in b/rpkid/Makefile.in
index 1ce31177..67a6cbe4 100644
--- a/rpkid/Makefile.in
+++ b/rpkid/Makefile.in
@@ -64,8 +64,10 @@ rpm deb:: all
 deb::
 	cd dist; for i in *.rpm; do case $$i in *.src.rpm) :;; *) (set -x; fakeroot alien -v $$i);; esac; done
 
-rpki/relaxng.py: ${abs_top_srcdir}/buildtools/make-relaxng.py left-right-schema.rng up-down-schema.rng publication-schema.rng
-	${PYTHON} ${abs_top_srcdir}/buildtools/make-relaxng.py >$@.tmp
+RNGS = left-right-schema.rng up-down-schema.rng publication-schema.rng myrpki.rng
+
+rpki/relaxng.py: ${abs_top_srcdir}/buildtools/make-relaxng.py ${RNGS}
+	${PYTHON} ${abs_top_srcdir}/buildtools/make-relaxng.py ${RNGS} >$@.tmp
 	mv $@.tmp $@
 
 left-right-schema.rng: left-right-schema.rnc
diff --git a/rpkid/myrpki.rnc b/rpkid/myrpki.rnc
index 5b8aa450..8acb16cf 100644
--- a/rpkid/myrpki.rnc
+++ b/rpkid/myrpki.rnc
@@ -2,10 +2,15 @@
 #
 # RelaxNG Schema for MyRPKI XML messages.
 #
+# This message protocol is on its way out, as we're in the process of
+# moving on from the user interface model that produced it, but even
+# after we finish replacing it we'll still need the schema for a while
+# to validate old messages when upgrading.
+#
 # libxml2 (including xmllint) only groks the XML syntax of RelaxNG, so
 # run the compact syntax through trang to get XML syntax.
 #
-# Copyright (C) 2009-2010  Internet Systems Consortium ("ISC")
+# Copyright (C) 2009-2011  Internet Systems Consortium ("ISC")
 #
 # Permission to use, copy, modify, and distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/rpkid/myrpki.rng b/rpkid/myrpki.rng
index a86d51a6..5f59e114 100644
--- a/rpkid/myrpki.rng
+++ b/rpkid/myrpki.rng
@@ -4,10 +4,15 @@
   
   RelaxNG Schema for MyRPKI XML messages.
   
+  This message protocol is on its way out, as we're in the process of
+  moving on from the user interface model that produced it, but even
+  after we finish replacing it we'll still need the schema for a while
+  to validate old messages when upgrading.
+  
   libxml2 (including xmllint) only groks the XML syntax of RelaxNG, so
   run the compact syntax through trang to get XML syntax.
   
-  Copyright (C) 2009-2010  Internet Systems Consortium ("ISC")
+  Copyright (C) 2009-2011  Internet Systems Consortium ("ISC")
   
   Permission to use, copy, modify, and distribute this software for any
   purpose with or without fee is hereby granted, provided that the above
diff --git a/rpkid/rpki/irdb/models.py b/rpkid/rpki/irdb/models.py
index d2db4067..0821b5cd 100644
--- a/rpkid/rpki/irdb/models.py
+++ b/rpkid/rpki/irdb/models.py
@@ -38,7 +38,7 @@ class ChoiceMap(dict):
 
   @property
   def choices(self):
-    return [(y, x) for (x, y) in self.iteritems()]
+    return tuple((y, x) for (x, y) in self.iteritems())
 
 class HandleField(django.db.models.CharField):
   """
@@ -74,42 +74,47 @@ ip_version_map = { "IPv4" : 4, "IPv6" : 6 }
 ## @var ip_version_choices
 # Choice argument for fields implementing IP version numbers.
 #
-ip_version_choices = [(y, x) for (x, y) in ip_version_map.iteritems()]
+ip_version_choices = tuple((y, x) for (x, y) in ip_version_map.iteritems())
 
 ###
 
 class Identity(django.db.models.Model):
   handle = HandleField()
 
-class BPKICertificate(django.db.models.Model):
-  certificate = BinaryField()
+class CA(django.db.models.Model):
   identity = django.db.models.ForeignKey(Identity, related_name = "bpki_certificates")
-
-class BPKIKey(BPKICertificate):
-  purpose_map = ChoiceMap("resource_ta", "server_ta", "rpkid", "pubd", "irdbd", "irbe")
+  purpose_map = ChoiceMap("resources", "servers")
   purpose = django.db.models.PositiveSmallIntegerField(choices = purpose_map.choices)
+  certificate = BinaryField()
   private_key = BinaryField()
+  next_serial = django.db.models.BigIntegerField(default = 1)
+  next_crl_number = django.db.models.BigIntegerField(default = 1)
+  last_crl_update = django.db.models.DateTimeField()
+  next_crl_update = django.db.models.DateTimeField()
 
-class BSC(BPKICertificate):
-  pkcs10 = BinaryField()
-
-class BPKICRL(django.db.models.Model):
-  serial = django.db.models.BigIntegerField()
-  thisupdate = django.db.models.DateTimeField()
-  nextupdate = django.db.models.DateTimeField()
-  issuer = django.db.models.OneToOneField(BPKICertificate, related_name = "crl")
+class Certificate(django.db.models.Model):
+  issuer = django.db.models.ForeignKey(CA, related_name = "certificates")
+  certificate = BinaryField()
 
-class BPKIRevocation(django.db.models.Model):
+class Revocation(django.db.models.Model):
+  issuer = django.db.models.ForeignKey(CA, related_name = "revocations")
   serial = django.db.models.BigIntegerField()
   revoked = django.db.models.DateTimeField()
   expires = django.db.models.DateTimeField()
-  crl = django.db.models.ForeignKey(BPKICRL, related_name = "revocations")
 
-class Child(BPKICertificate):
+class EECertificate(Certificate):
+  purpose_map = ChoiceMap("rpkid", "pubd", "irdbd", "irbe", "rootd")
+  purpose = django.db.models.PositiveSmallIntegerField(choices = purpose_map.choices)
+  private_key = BinaryField()
+
+class BSC(Certificate):
+  pkcs10 = BinaryField()
+
+class Child(Certificate):
   handle = HandleField()
-  name = django.db.models.TextField(blank = True)
+  name = django.db.models.TextField(null = True, blank = True)
   valid_until = django.db.models.DateTimeField()
-  bpki_ta = BinaryField()
+  ta = BinaryField()
 
 class ChildASN(django.db.models.Model):
   start_as = django.db.models.BigIntegerField()
@@ -124,11 +129,11 @@ class ChildNet(django.db.models.Model):
   version = django.db.models.PositiveSmallIntegerField(choices = ip_version_choices)
   child = django.db.models.ForeignKey(Child, related_name = "address_ranges")
 
-class Parent(BPKICertificate):
+class Parent(Certificate):
   handle = HandleField()
   parent_handle = HandleField()
   child_handle  = HandleField()
-  bpki_ta = BinaryField()
+  ta = BinaryField()
   service_uri = django.db.models.CharField(max_length = 255)
   repository_type_map = ChoiceMap("none", "offer", "referral")
   repository_type = django.db.models.PositiveSmallIntegerField(choices = repository_type_map.choices)
@@ -151,7 +156,7 @@ class GhostbusterRequest(django.db.models.Model):
   parent = django.db.models.ForeignKey(Parent,     related_name = "ghostbuster_requests", null = True)
   vcard = django.db.models.TextField()
 
-class Repository(BPKICertificate):
+class Repository(Certificate):
   handle = HandleField()
   client_handle = HandleField()
   bpki_ta = BinaryField()
@@ -159,6 +164,6 @@ class Repository(BPKICertificate):
   sia_base = django.db.models.TextField()
   parent = django.db.models.OneToOneField(Parent, related_name = "repository")
 
-class Client(BPKICertificate):
+class Client(Certificate):
   handle = HandleField()
   bpki_ta = BinaryField()
diff --git a/rpkid/rpki/relaxng.py b/rpkid/rpki/relaxng.py
index e1ea8f6b..24b3ab75 100644
--- a/rpkid/rpki/relaxng.py
+++ b/rpkid/rpki/relaxng.py
@@ -1839,3 +1839,382 @@ publication = lxml.etree.RelaxNG(lxml.etree.fromstring('''<?xml version="1.0" en
 -->
 '''))
 
+## @var myrpki
+## Parsed RelaxNG myrpki schema
+myrpki = lxml.etree.RelaxNG(lxml.etree.fromstring('''<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  $Id: myrpki.rnc 3723 2011-03-14 20:43:16Z sra $
+  
+  RelaxNG Schema for MyRPKI XML messages.
+  
+  This message protocol is on its way out, as we're in the process of
+  moving on from the user interface model that produced it, but even
+  after we finish replacing it we'll still need the schema for a while
+  to validate old messages when upgrading.
+  
+  libxml2 (including xmllint) only groks the XML syntax of RelaxNG, so
+  run the compact syntax through trang to get XML syntax.
+  
+  Copyright (C) 2009-2011  Internet Systems Consortium ("ISC")
+  
+  Permission to use, copy, modify, and distribute this software for any
+  purpose with or without fee is hereby granted, provided that the above
+  copyright notice and this permission notice appear in all copies.
+  
+  THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+  REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+  AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+  LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+  OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+  PERFORMANCE OF THIS SOFTWARE.
+-->
+<grammar ns="http://www.hactrn.net/uris/rpki/myrpki/" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <define name="version">
+    <value>2</value>
+  </define>
+  <define name="base64">
+    <data type="base64Binary">
+      <param name="maxLength">512000</param>
+    </data>
+  </define>
+  <define name="object_handle">
+    <data type="string">
+      <param name="maxLength">255</param>
+      <param name="pattern">[\-_A-Za-z0-9]*</param>
+    </data>
+  </define>
+  <define name="pubd_handle">
+    <data type="string">
+      <param name="maxLength">255</param>
+      <param name="pattern">[\-_A-Za-z0-9/]*</param>
+    </data>
+  </define>
+  <define name="uri">
+    <data type="anyURI">
+      <param name="maxLength">4096</param>
+    </data>
+  </define>
+  <define name="asn">
+    <data type="positiveInteger"/>
+  </define>
+  <define name="asn_list">
+    <data type="string">
+      <param name="maxLength">512000</param>
+      <param name="pattern">[\-,0-9]*</param>
+    </data>
+  </define>
+  <define name="ipv4_list">
+    <data type="string">
+      <param name="maxLength">512000</param>
+      <param name="pattern">[\-,0-9/.]*</param>
+    </data>
+  </define>
+  <define name="ipv6_list">
+    <data type="string">
+      <param name="maxLength">512000</param>
+      <param name="pattern">[\-,0-9/:a-fA-F]*</param>
+    </data>
+  </define>
+  <define name="timestamp">
+    <data type="dateTime">
+      <param name="pattern">.*Z</param>
+    </data>
+  </define>
+  <!--
+    Message formate used between configure_resources and
+    configure_daemons.
+  -->
+  <start combine="choice">
+    <element name="myrpki">
+      <attribute name="version">
+        <ref name="version"/>
+      </attribute>
+      <attribute name="handle">
+        <ref name="object_handle"/>
+      </attribute>
+      <optional>
+        <attribute name="service_uri">
+          <ref name="uri"/>
+        </attribute>
+      </optional>
+      <zeroOrMore>
+        <element name="roa_request">
+          <attribute name="asn">
+            <ref name="asn"/>
+          </attribute>
+          <attribute name="v4">
+            <ref name="ipv4_list"/>
+          </attribute>
+          <attribute name="v6">
+            <ref name="ipv6_list"/>
+          </attribute>
+        </element>
+      </zeroOrMore>
+      <zeroOrMore>
+        <element name="child">
+          <attribute name="handle">
+            <ref name="object_handle"/>
+          </attribute>
+          <attribute name="valid_until">
+            <ref name="timestamp"/>
+          </attribute>
+          <optional>
+            <attribute name="asns">
+              <ref name="asn_list"/>
+            </attribute>
+          </optional>
+          <optional>
+            <attribute name="v4">
+              <ref name="ipv4_list"/>
+            </attribute>
+          </optional>
+          <optional>
+            <attribute name="v6">
+              <ref name="ipv6_list"/>
+            </attribute>
+          </optional>
+          <optional>
+            <element name="bpki_certificate">
+              <ref name="base64"/>
+            </element>
+          </optional>
+        </element>
+      </zeroOrMore>
+      <zeroOrMore>
+        <element name="parent">
+          <attribute name="handle">
+            <ref name="object_handle"/>
+          </attribute>
+          <optional>
+            <attribute name="service_uri">
+              <ref name="uri"/>
+            </attribute>
+          </optional>
+          <optional>
+            <attribute name="myhandle">
+              <ref name="object_handle"/>
+            </attribute>
+          </optional>
+          <optional>
+            <attribute name="sia_base">
+              <ref name="uri"/>
+            </attribute>
+          </optional>
+          <optional>
+            <element name="bpki_cms_certificate">
+              <ref name="base64"/>
+            </element>
+          </optional>
+        </element>
+      </zeroOrMore>
+      <zeroOrMore>
+        <element name="repository">
+          <attribute name="handle">
+            <ref name="object_handle"/>
+          </attribute>
+          <optional>
+            <attribute name="service_uri">
+              <ref name="uri"/>
+            </attribute>
+          </optional>
+          <optional>
+            <element name="bpki_certificate">
+              <ref name="base64"/>
+            </element>
+          </optional>
+        </element>
+      </zeroOrMore>
+      <optional>
+        <element name="bpki_ca_certificate">
+          <ref name="base64"/>
+        </element>
+      </optional>
+      <optional>
+        <element name="bpki_crl">
+          <ref name="base64"/>
+        </element>
+      </optional>
+      <optional>
+        <element name="bpki_bsc_certificate">
+          <ref name="base64"/>
+        </element>
+      </optional>
+      <optional>
+        <element name="bpki_bsc_pkcs10">
+          <ref name="base64"/>
+        </element>
+      </optional>
+    </element>
+  </start>
+  <!-- Format of an identity.xml file. -->
+  <start combine="choice">
+    <element name="identity">
+      <attribute name="version">
+        <ref name="version"/>
+      </attribute>
+      <attribute name="handle">
+        <ref name="object_handle"/>
+      </attribute>
+      <element name="bpki_ta">
+        <ref name="base64"/>
+      </element>
+    </element>
+  </start>
+  <!--
+    Format of <authorization/> element used in referrals.  The Base64
+    text is a <referral/> (q. v.) element signed with CMS.
+  -->
+  <define name="authorization">
+    <element name="authorization">
+      <attribute name="referrer">
+        <ref name="pubd_handle"/>
+      </attribute>
+      <ref name="base64"/>
+    </element>
+  </define>
+  <!-- Format of <contact_info/> element used in referrals. -->
+  <define name="contact_info">
+    <element name="contact_info">
+      <optional>
+        <attribute name="uri">
+          <ref name="uri"/>
+        </attribute>
+      </optional>
+      <data type="string"/>
+    </element>
+  </define>
+  <!-- Variant payload portion of a <repository/> element. -->
+  <define name="repository_payload">
+    <choice>
+      <attribute name="type">
+        <value>none</value>
+      </attribute>
+      <attribute name="type">
+        <value>offer</value>
+      </attribute>
+      <group>
+        <attribute name="type">
+          <value>referral</value>
+        </attribute>
+        <ref name="authorization"/>
+        <ref name="contact_info"/>
+      </group>
+    </choice>
+  </define>
+  <!-- <parent/> element (response from configure_child). -->
+  <start combine="choice">
+    <element name="parent">
+      <attribute name="version">
+        <ref name="version"/>
+      </attribute>
+      <attribute name="valid_until">
+        <ref name="timestamp"/>
+      </attribute>
+      <optional>
+        <attribute name="service_uri">
+          <ref name="uri"/>
+        </attribute>
+      </optional>
+      <attribute name="child_handle">
+        <ref name="object_handle"/>
+      </attribute>
+      <attribute name="parent_handle">
+        <ref name="object_handle"/>
+      </attribute>
+      <element name="bpki_resource_ta">
+        <ref name="base64"/>
+      </element>
+      <element name="bpki_child_ta">
+        <ref name="base64"/>
+      </element>
+      <optional>
+        <element name="repository">
+          <ref name="repository_payload"/>
+        </element>
+      </optional>
+    </element>
+  </start>
+  <!--
+    <repository/> element, types offer and referral
+    (input to configure_publication_client).
+  -->
+  <start combine="choice">
+    <element name="repository">
+      <attribute name="version">
+        <ref name="version"/>
+      </attribute>
+      <attribute name="handle">
+        <ref name="object_handle"/>
+      </attribute>
+      <attribute name="parent_handle">
+        <ref name="object_handle"/>
+      </attribute>
+      <ref name="repository_payload"/>
+      <element name="bpki_client_ta">
+        <ref name="base64"/>
+      </element>
+    </element>
+  </start>
+  <!--
+    <repository/> element, confirmation type (output of
+    configure_publication_client).
+  -->
+  <start combine="choice">
+    <element name="repository">
+      <attribute name="version">
+        <ref name="version"/>
+      </attribute>
+      <attribute name="type">
+        <value>confirmed</value>
+      </attribute>
+      <attribute name="parent_handle">
+        <ref name="object_handle"/>
+      </attribute>
+      <attribute name="client_handle">
+        <ref name="pubd_handle"/>
+      </attribute>
+      <attribute name="service_uri">
+        <ref name="uri"/>
+      </attribute>
+      <attribute name="sia_base">
+        <ref name="uri"/>
+      </attribute>
+      <element name="bpki_server_ta">
+        <ref name="base64"/>
+      </element>
+      <element name="bpki_client_ta">
+        <ref name="base64"/>
+      </element>
+      <optional>
+        <ref name="authorization"/>
+      </optional>
+      <optional>
+        <ref name="contact_info"/>
+      </optional>
+    </element>
+  </start>
+  <!--
+    <referral/> element.  This is the entirety of a separate message
+    which is signed with CMS then included ase the Base64 content of an
+    <authorization/> element in the main message.
+  -->
+  <start combine="choice">
+    <element name="referral">
+      <attribute name="version">
+        <ref name="version"/>
+      </attribute>
+      <attribute name="authorized_sia_base">
+        <ref name="uri"/>
+      </attribute>
+      <ref name="base64"/>
+    </element>
+  </start>
+</grammar>
+<!--
+  Local Variables:
+  indent-tabs-mode: nil
+  End:
+-->
+'''))
+
diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py
index 09c5e973..b69e1ec2 100644
--- a/scripts/convert-from-entitydb-to-sql.py
+++ b/scripts/convert-from-entitydb-to-sql.py
@@ -21,8 +21,8 @@ OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 PERFORMANCE OF THIS SOFTWARE.
 """
 
-import sys, os, time, getopt
-import rpki.config
+import sys, os, time, getopt, glob, subprocess
+import rpki.config, rpki.x509, rpki.relaxng
 from rpki.mysql_import import MySQLdb
 from lxml.etree import ElementTree
 
@@ -31,24 +31,23 @@ if os.getlogin() != "sra":
 
 cfg_file = "rpki.conf"
 entitydb = "entitydb"
+bpki     = "bpki"
 
-opts, argv = getopt.getopt(sys.argv[1:], "c:e:h?", ["config=", "entitydb=", "help"])
+opts, argv = getopt.getopt(sys.argv[1:], "c:h?", ["config=", "help"])
 for o, a in opts:
   if o in ("-h", "--help", "-?"):
     print __doc__
     sys.exit(0)
   if o in ("-c", "--config"):
     cfg_file = a
-  elif o in ("-e", "--entitydb"):
-    entitydb = a
 if argv:
   sys.exit("Unexpected arguments %s" % argv)
 
-cfg = rpki.config.parser(cfg_file, "irdbd")
+cfg = rpki.config.parser(cfg_file)
 
-sql_database = cfg.get("sql-database")
-sql_username = cfg.get("sql-username")
-sql_password = cfg.get("sql-password")
+sql_database = cfg.get("sql-database", section = "irdbd")
+sql_username = cfg.get("sql-username", section = "irdbd")
+sql_password = cfg.get("sql-password", section = "irdbd")
 
 # Rename the old SQL tables, if they exist
 
@@ -80,7 +79,8 @@ settings.configure(
     "USER"     : sql_username,
     "PASSWORD" : sql_password,
     "HOST"     : "",
-    "PORT"     : "" }},
+    "PORT"     : "",
+    "OPTIONS"  : { "init_command": "SET storage_engine=INNODB" }}},
   INSTALLED_APPS = ("rpki.irdb",),
 )
 
@@ -93,8 +93,8 @@ import django.core.management
 django.core.management.call_command("syncdb", verbosity = 4, load_initial_data = False)
 
 # From here down will be an awful lot of messing about with XML and
-# X.509 data, extracting stuff from the old database and whacking it
-# into the new.  Still working out these bits.
+# X.509 data, extracting stuff from the old SQL database and whacking
+# it into the new.  Still working out these bits.
 
 xmlns = "{http://www.hactrn.net/uris/rpki/myrpki/}"
 
@@ -110,16 +110,129 @@ tag_parent           = xmlns + "parent"
 tag_repository       = xmlns + "repository"
 
 e = ElementTree(file = os.path.join(entitydb, "identity.xml")).getroot()
+rpki.relaxng.myrpki.assertValid(e)
 assert e.tag == tag_identity
 
 handle = e.get("handle")
-
-# Check handle against what's in rpki.conf?
+assert handle == cfg.get("handle", section = "myrpki")
 
 # Create identity if we haven't already
 
 identity = rpki.irdb.Identity.objects.get_or_create(handle = handle)[0]
 
+# Some BPKI utillity routines
+
+def read_openssl_serial(filename):
+  f = open(filename, "r")
+  text = f.read()
+  f.close()
+  return int(text.strip(), 16)
+
+def get_or_create_CA(purpose):
+  cer = rpki.x509.X509(Auto_file = os.path.join(bpki, purpose, "ca.cer"))
+  key = rpki.x509.RSA(Auto_file  = os.path.join(bpki, purpose, "ca.key"))
+  crl = rpki.x509.CRL(Auto_file  = os.path.join(bpki, purpose, "ca.crl"))
+  serial     = read_openssl_serial(os.path.join(bpki, purpose, "serial"))
+  crl_number = read_openssl_serial(os.path.join(bpki, purpose, "crl_number"))
+
+  return rpki.irdb.CA.objects.get_or_create(identity = identity,
+                                            purpose = rpki.irdb.CA.purpose_map[purpose],
+                                            certificate = cer.get_DER(),
+                                            private_key = key.get_DER(),
+                                            next_serial = serial,
+                                            next_crl_number = crl_number,
+                                            last_crl_update = crl.getThisUpdate().to_sql(),
+                                            next_crl_update = crl.getNextUpdate().to_sql())[0]
+
+def get_or_create_EECertificate(issuer, purpose):
+  cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "servers", purpose + ".cer"))
+  key = rpki.x509.RSA(Auto_file  = os.path.join(bpki, "servers", purpose + ".key"))
+  rpki.irdb.EECertificate.objects.get_or_create(
+    issuer      = issuer,
+    purpose     = rpki.irdb.EECertificate.purpose_map[purpose],
+    certificate = cer.get_DER(),
+    private_key = key.get_DER())
+
+# Load BPKI CA data
+
+resource_ca = get_or_create_CA("resources")
+
+# Load BPKI server EE certificates and keys
+
+run_flags = dict((i, cfg.getboolean(i, section = "myrpki"))
+                 for i in ("run_rpkid", "run_pubd", "run_rootd"))
+
+if any(run_flags.itervalues()):
+  server_ca = get_or_create_CA("servers")
+  get_or_create_EECertificate(server_ca, "irbe")
+  if run_flags["run_rpkid"]:
+    get_or_create_EECertificate(server_ca, "rpkid")
+    get_or_create_EECertificate(server_ca, "irdbd")
+  if run_flags["run_pubd"]:
+    get_or_create_EECertificate(server_ca, "pubd")
+  if run_flags["run_rootd"]:
+    get_or_create_EECertificate(server_ca, "rootd")
+else:
+  server_ca = None
+
+# Load BSC certificates and requests
+
+for fn in glob.iglob(os.path.join(bpki, "resources", "bsc.*.cer")):
+  cer = rpki.x509.X509(Auto_file = fn)
+  req = rpki.x509.X509(Auto_file = fn[:-4] + ".req")
+  rpki.irdb.BSC.objects.get_or_create(
+    issuer      = resource_ca,
+    certificate = cer.get_DER(),
+    pkcs10      = req.get_DER())
+
+
+def xcert_hash(cert):
+  """
+  Generate the filename hash that myrpki would have generated for a
+  cross-certification.  This is nasty, don't look.
+  """
+
+  cmd1 = ("openssl", "x509", "-noout", "-pubkey", "-subject")
+  cmd2 = ("openssl", "dgst", "-md5")
+
+  env = { "PATH" : os.environ["PATH"], "OPENSSL_CONF" : "/dev/null" }
+  p1 = subprocess.Popen(cmd1, env = env, stdin = subprocess.PIPE, stdout = subprocess.PIPE)
+  p2 = subprocess.Popen(cmd2, env = env, stdin = p1.stdout, stdout = subprocess.PIPE)
+  p1.stdin.write(cert.get_PEM())
+  p1.stdin.close()
+  hash = p2.stdout.read()
+  if p1.wait() != 0:
+    raise subprocess.CalledProcessError(returncode = p1.returncode, cmd = cmd1)
+  if p2.wait() != 0:
+    raise subprocess.CalledProcessError(returncode = p2.returncode, cmd = cmd2)
+
+  hash = "".join(hash.split())
+  if hash.startswith("(stdin)="):
+    hash =  hash[len("(stdin)="):]
+  return hash
+
+# Build a table of all the cross-certified BPKI certificates.
+
+xcerts = {}
+
+for filename in glob.iglob(os.path.join("bpki", "*", "xcert.*.cer")):
+  h = filename.split(".")[-2]
+
+  if not h in xcerts:
+    xcerts[h] = []
+  xcerts[h].append(filename)
+
+  # While we're at this, check to make sure that our reproduction of
+  # the hash algorithm is working correctly.
+  #
+  assert xcert_hash(rpki.x509.X509(Auto_file = filename)) == h
+
+
+
+# Somewhere around here I'm going to run out of things to do other
+# than scraping through the horrible entitydb XML.  Bother.
+
+
 # Copy over any ROA requests
 
 cur.execute("""
@@ -149,8 +262,8 @@ cur.execute("""
             WHERE self_handle = %s AND parent_handle IS NULL
             """, (handle,))
 for row in cur.fetchall():
-  rpki.irdb.GhostbusterRequest.objects.get_or_create(identity = identity, vcard = row[0],
-                                                     defaults = { "parent" : None })
+  rpki.irdb.GhostbusterRequest.objects.get_or_create(identity = identity, parent = None,
+                                                     vcard = row[0])
 
 cur.close()
 db.close()