From 61a8790f819698d446fad92678af6b73027f6bbf Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Fri, 9 Dec 2011 01:33:38 +0000 Subject: Rework models slightly. Now parsing most of the BPKI stuff. svn path=/branches/tk100/; revision=4109 --- buildtools/make-relaxng.py | 16 +- rpkid/Makefile.in | 6 +- rpkid/myrpki.rnc | 7 +- rpkid/myrpki.rng | 7 +- rpkid/rpki/irdb/models.py | 53 +++-- rpkid/rpki/relaxng.py | 379 ++++++++++++++++++++++++++++++++ scripts/convert-from-entitydb-to-sql.py | 145 ++++++++++-- 7 files changed, 564 insertions(+), 49 deletions(-) diff --git a/buildtools/make-relaxng.py b/buildtools/make-relaxng.py index 62decbae..0058ade5 100644 --- a/buildtools/make-relaxng.py +++ b/buildtools/make-relaxng.py @@ -3,7 +3,7 @@ Script to generate rpki/relaxng.py. $Id$ -Copyright (C) 2009 Internet Systems Consortium ("ISC") +Copyright (C) 2009-2011 Internet Systems Consortium ("ISC") Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above @@ -32,7 +32,7 @@ OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -schemas = ("left_right", "up_down", "publication") +import sys format_1 = """\ # Automatically generated, do not edit. @@ -46,9 +46,15 @@ format_2 = """\ %(name)s = lxml.etree.RelaxNG(lxml.etree.fromstring('''%(rng)s''')) """ +def filename_to_symbol(s): + for suffix in (".rng", "-schema"): + if s.endswith(suffix): + s = s[:-len(suffix)] + return s.replace("-", "_") + print format_1 -for name in schemas: +for filename in sys.argv[1:]: print format_2 % { - "name" : name, - "rng" : open(name.replace("_", "-") + "-schema.rng").read() } + "name" : filename_to_symbol(filename), + "rng" : open(filename).read() } diff --git a/rpkid/Makefile.in b/rpkid/Makefile.in index 1ce31177..67a6cbe4 100644 --- a/rpkid/Makefile.in +++ b/rpkid/Makefile.in @@ -64,8 +64,10 @@ rpm deb:: all deb:: cd dist; for i in *.rpm; do case $$i in *.src.rpm) :;; *) (set -x; fakeroot alien -v $$i);; esac; done -rpki/relaxng.py: ${abs_top_srcdir}/buildtools/make-relaxng.py left-right-schema.rng up-down-schema.rng publication-schema.rng - ${PYTHON} ${abs_top_srcdir}/buildtools/make-relaxng.py >$@.tmp +RNGS = left-right-schema.rng up-down-schema.rng publication-schema.rng myrpki.rng + +rpki/relaxng.py: ${abs_top_srcdir}/buildtools/make-relaxng.py ${RNGS} + ${PYTHON} ${abs_top_srcdir}/buildtools/make-relaxng.py ${RNGS} >$@.tmp mv $@.tmp $@ left-right-schema.rng: left-right-schema.rnc diff --git a/rpkid/myrpki.rnc b/rpkid/myrpki.rnc index 5b8aa450..8acb16cf 100644 --- a/rpkid/myrpki.rnc +++ b/rpkid/myrpki.rnc @@ -2,10 +2,15 @@ # # RelaxNG Schema for MyRPKI XML messages. # +# This message protocol is on its way out, as we're in the process of +# moving on from the user interface model that produced it, but even +# after we finish replacing it we'll still need the schema for a while +# to validate old messages when upgrading. +# # libxml2 (including xmllint) only groks the XML syntax of RelaxNG, so # run the compact syntax through trang to get XML syntax. # -# Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") +# Copyright (C) 2009-2011 Internet Systems Consortium ("ISC") # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above diff --git a/rpkid/myrpki.rng b/rpkid/myrpki.rng index a86d51a6..5f59e114 100644 --- a/rpkid/myrpki.rng +++ b/rpkid/myrpki.rng @@ -4,10 +4,15 @@ RelaxNG Schema for MyRPKI XML messages. + This message protocol is on its way out, as we're in the process of + moving on from the user interface model that produced it, but even + after we finish replacing it we'll still need the schema for a while + to validate old messages when upgrading. + libxml2 (including xmllint) only groks the XML syntax of RelaxNG, so run the compact syntax through trang to get XML syntax. - Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") + Copyright (C) 2009-2011 Internet Systems Consortium ("ISC") Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above diff --git a/rpkid/rpki/irdb/models.py b/rpkid/rpki/irdb/models.py index d2db4067..0821b5cd 100644 --- a/rpkid/rpki/irdb/models.py +++ b/rpkid/rpki/irdb/models.py @@ -38,7 +38,7 @@ class ChoiceMap(dict): @property def choices(self): - return [(y, x) for (x, y) in self.iteritems()] + return tuple((y, x) for (x, y) in self.iteritems()) class HandleField(django.db.models.CharField): """ @@ -74,42 +74,47 @@ ip_version_map = { "IPv4" : 4, "IPv6" : 6 } ## @var ip_version_choices # Choice argument for fields implementing IP version numbers. # -ip_version_choices = [(y, x) for (x, y) in ip_version_map.iteritems()] +ip_version_choices = tuple((y, x) for (x, y) in ip_version_map.iteritems()) ### class Identity(django.db.models.Model): handle = HandleField() -class BPKICertificate(django.db.models.Model): - certificate = BinaryField() +class CA(django.db.models.Model): identity = django.db.models.ForeignKey(Identity, related_name = "bpki_certificates") - -class BPKIKey(BPKICertificate): - purpose_map = ChoiceMap("resource_ta", "server_ta", "rpkid", "pubd", "irdbd", "irbe") + purpose_map = ChoiceMap("resources", "servers") purpose = django.db.models.PositiveSmallIntegerField(choices = purpose_map.choices) + certificate = BinaryField() private_key = BinaryField() + next_serial = django.db.models.BigIntegerField(default = 1) + next_crl_number = django.db.models.BigIntegerField(default = 1) + last_crl_update = django.db.models.DateTimeField() + next_crl_update = django.db.models.DateTimeField() -class BSC(BPKICertificate): - pkcs10 = BinaryField() - -class BPKICRL(django.db.models.Model): - serial = django.db.models.BigIntegerField() - thisupdate = django.db.models.DateTimeField() - nextupdate = django.db.models.DateTimeField() - issuer = django.db.models.OneToOneField(BPKICertificate, related_name = "crl") +class Certificate(django.db.models.Model): + issuer = django.db.models.ForeignKey(CA, related_name = "certificates") + certificate = BinaryField() -class BPKIRevocation(django.db.models.Model): +class Revocation(django.db.models.Model): + issuer = django.db.models.ForeignKey(CA, related_name = "revocations") serial = django.db.models.BigIntegerField() revoked = django.db.models.DateTimeField() expires = django.db.models.DateTimeField() - crl = django.db.models.ForeignKey(BPKICRL, related_name = "revocations") -class Child(BPKICertificate): +class EECertificate(Certificate): + purpose_map = ChoiceMap("rpkid", "pubd", "irdbd", "irbe", "rootd") + purpose = django.db.models.PositiveSmallIntegerField(choices = purpose_map.choices) + private_key = BinaryField() + +class BSC(Certificate): + pkcs10 = BinaryField() + +class Child(Certificate): handle = HandleField() - name = django.db.models.TextField(blank = True) + name = django.db.models.TextField(null = True, blank = True) valid_until = django.db.models.DateTimeField() - bpki_ta = BinaryField() + ta = BinaryField() class ChildASN(django.db.models.Model): start_as = django.db.models.BigIntegerField() @@ -124,11 +129,11 @@ class ChildNet(django.db.models.Model): version = django.db.models.PositiveSmallIntegerField(choices = ip_version_choices) child = django.db.models.ForeignKey(Child, related_name = "address_ranges") -class Parent(BPKICertificate): +class Parent(Certificate): handle = HandleField() parent_handle = HandleField() child_handle = HandleField() - bpki_ta = BinaryField() + ta = BinaryField() service_uri = django.db.models.CharField(max_length = 255) repository_type_map = ChoiceMap("none", "offer", "referral") repository_type = django.db.models.PositiveSmallIntegerField(choices = repository_type_map.choices) @@ -151,7 +156,7 @@ class GhostbusterRequest(django.db.models.Model): parent = django.db.models.ForeignKey(Parent, related_name = "ghostbuster_requests", null = True) vcard = django.db.models.TextField() -class Repository(BPKICertificate): +class Repository(Certificate): handle = HandleField() client_handle = HandleField() bpki_ta = BinaryField() @@ -159,6 +164,6 @@ class Repository(BPKICertificate): sia_base = django.db.models.TextField() parent = django.db.models.OneToOneField(Parent, related_name = "repository") -class Client(BPKICertificate): +class Client(Certificate): handle = HandleField() bpki_ta = BinaryField() diff --git a/rpkid/rpki/relaxng.py b/rpkid/rpki/relaxng.py index e1ea8f6b..24b3ab75 100644 --- a/rpkid/rpki/relaxng.py +++ b/rpkid/rpki/relaxng.py @@ -1839,3 +1839,382 @@ publication = lxml.etree.RelaxNG(lxml.etree.fromstring(''' ''')) +## @var myrpki +## Parsed RelaxNG myrpki schema +myrpki = lxml.etree.RelaxNG(lxml.etree.fromstring(''' + + + + 2 + + + + 512000 + + + + + 255 + [\-_A-Za-z0-9]* + + + + + 255 + [\-_A-Za-z0-9/]* + + + + + 4096 + + + + + + + + 512000 + [\-,0-9]* + + + + + 512000 + [\-,0-9/.]* + + + + + 512000 + [\-,0-9/:a-fA-F]* + + + + + .*Z + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + none + + + offer + + + + referral + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + confirmed + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +''')) + diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index 09c5e973..b69e1ec2 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -21,8 +21,8 @@ OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -import sys, os, time, getopt -import rpki.config +import sys, os, time, getopt, glob, subprocess +import rpki.config, rpki.x509, rpki.relaxng from rpki.mysql_import import MySQLdb from lxml.etree import ElementTree @@ -31,24 +31,23 @@ if os.getlogin() != "sra": cfg_file = "rpki.conf" entitydb = "entitydb" +bpki = "bpki" -opts, argv = getopt.getopt(sys.argv[1:], "c:e:h?", ["config=", "entitydb=", "help"]) +opts, argv = getopt.getopt(sys.argv[1:], "c:h?", ["config=", "help"]) for o, a in opts: if o in ("-h", "--help", "-?"): print __doc__ sys.exit(0) if o in ("-c", "--config"): cfg_file = a - elif o in ("-e", "--entitydb"): - entitydb = a if argv: sys.exit("Unexpected arguments %s" % argv) -cfg = rpki.config.parser(cfg_file, "irdbd") +cfg = rpki.config.parser(cfg_file) -sql_database = cfg.get("sql-database") -sql_username = cfg.get("sql-username") -sql_password = cfg.get("sql-password") +sql_database = cfg.get("sql-database", section = "irdbd") +sql_username = cfg.get("sql-username", section = "irdbd") +sql_password = cfg.get("sql-password", section = "irdbd") # Rename the old SQL tables, if they exist @@ -80,7 +79,8 @@ settings.configure( "USER" : sql_username, "PASSWORD" : sql_password, "HOST" : "", - "PORT" : "" }}, + "PORT" : "", + "OPTIONS" : { "init_command": "SET storage_engine=INNODB" }}}, INSTALLED_APPS = ("rpki.irdb",), ) @@ -93,8 +93,8 @@ import django.core.management django.core.management.call_command("syncdb", verbosity = 4, load_initial_data = False) # From here down will be an awful lot of messing about with XML and -# X.509 data, extracting stuff from the old database and whacking it -# into the new. Still working out these bits. +# X.509 data, extracting stuff from the old SQL database and whacking +# it into the new. Still working out these bits. xmlns = "{http://www.hactrn.net/uris/rpki/myrpki/}" @@ -110,16 +110,129 @@ tag_parent = xmlns + "parent" tag_repository = xmlns + "repository" e = ElementTree(file = os.path.join(entitydb, "identity.xml")).getroot() +rpki.relaxng.myrpki.assertValid(e) assert e.tag == tag_identity handle = e.get("handle") - -# Check handle against what's in rpki.conf? +assert handle == cfg.get("handle", section = "myrpki") # Create identity if we haven't already identity = rpki.irdb.Identity.objects.get_or_create(handle = handle)[0] +# Some BPKI utillity routines + +def read_openssl_serial(filename): + f = open(filename, "r") + text = f.read() + f.close() + return int(text.strip(), 16) + +def get_or_create_CA(purpose): + cer = rpki.x509.X509(Auto_file = os.path.join(bpki, purpose, "ca.cer")) + key = rpki.x509.RSA(Auto_file = os.path.join(bpki, purpose, "ca.key")) + crl = rpki.x509.CRL(Auto_file = os.path.join(bpki, purpose, "ca.crl")) + serial = read_openssl_serial(os.path.join(bpki, purpose, "serial")) + crl_number = read_openssl_serial(os.path.join(bpki, purpose, "crl_number")) + + return rpki.irdb.CA.objects.get_or_create(identity = identity, + purpose = rpki.irdb.CA.purpose_map[purpose], + certificate = cer.get_DER(), + private_key = key.get_DER(), + next_serial = serial, + next_crl_number = crl_number, + last_crl_update = crl.getThisUpdate().to_sql(), + next_crl_update = crl.getNextUpdate().to_sql())[0] + +def get_or_create_EECertificate(issuer, purpose): + cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "servers", purpose + ".cer")) + key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "servers", purpose + ".key")) + rpki.irdb.EECertificate.objects.get_or_create( + issuer = issuer, + purpose = rpki.irdb.EECertificate.purpose_map[purpose], + certificate = cer.get_DER(), + private_key = key.get_DER()) + +# Load BPKI CA data + +resource_ca = get_or_create_CA("resources") + +# Load BPKI server EE certificates and keys + +run_flags = dict((i, cfg.getboolean(i, section = "myrpki")) + for i in ("run_rpkid", "run_pubd", "run_rootd")) + +if any(run_flags.itervalues()): + server_ca = get_or_create_CA("servers") + get_or_create_EECertificate(server_ca, "irbe") + if run_flags["run_rpkid"]: + get_or_create_EECertificate(server_ca, "rpkid") + get_or_create_EECertificate(server_ca, "irdbd") + if run_flags["run_pubd"]: + get_or_create_EECertificate(server_ca, "pubd") + if run_flags["run_rootd"]: + get_or_create_EECertificate(server_ca, "rootd") +else: + server_ca = None + +# Load BSC certificates and requests + +for fn in glob.iglob(os.path.join(bpki, "resources", "bsc.*.cer")): + cer = rpki.x509.X509(Auto_file = fn) + req = rpki.x509.X509(Auto_file = fn[:-4] + ".req") + rpki.irdb.BSC.objects.get_or_create( + issuer = resource_ca, + certificate = cer.get_DER(), + pkcs10 = req.get_DER()) + + +def xcert_hash(cert): + """ + Generate the filename hash that myrpki would have generated for a + cross-certification. This is nasty, don't look. + """ + + cmd1 = ("openssl", "x509", "-noout", "-pubkey", "-subject") + cmd2 = ("openssl", "dgst", "-md5") + + env = { "PATH" : os.environ["PATH"], "OPENSSL_CONF" : "/dev/null" } + p1 = subprocess.Popen(cmd1, env = env, stdin = subprocess.PIPE, stdout = subprocess.PIPE) + p2 = subprocess.Popen(cmd2, env = env, stdin = p1.stdout, stdout = subprocess.PIPE) + p1.stdin.write(cert.get_PEM()) + p1.stdin.close() + hash = p2.stdout.read() + if p1.wait() != 0: + raise subprocess.CalledProcessError(returncode = p1.returncode, cmd = cmd1) + if p2.wait() != 0: + raise subprocess.CalledProcessError(returncode = p2.returncode, cmd = cmd2) + + hash = "".join(hash.split()) + if hash.startswith("(stdin)="): + hash = hash[len("(stdin)="):] + return hash + +# Build a table of all the cross-certified BPKI certificates. + +xcerts = {} + +for filename in glob.iglob(os.path.join("bpki", "*", "xcert.*.cer")): + h = filename.split(".")[-2] + + if not h in xcerts: + xcerts[h] = [] + xcerts[h].append(filename) + + # While we're at this, check to make sure that our reproduction of + # the hash algorithm is working correctly. + # + assert xcert_hash(rpki.x509.X509(Auto_file = filename)) == h + + + +# Somewhere around here I'm going to run out of things to do other +# than scraping through the horrible entitydb XML. Bother. + + # Copy over any ROA requests cur.execute(""" @@ -149,8 +262,8 @@ cur.execute(""" WHERE self_handle = %s AND parent_handle IS NULL """, (handle,)) for row in cur.fetchall(): - rpki.irdb.GhostbusterRequest.objects.get_or_create(identity = identity, vcard = row[0], - defaults = { "parent" : None }) + rpki.irdb.GhostbusterRequest.objects.get_or_create(identity = identity, parent = None, + vcard = row[0]) cur.close() db.close() -- cgit v1.2.3