diff options
33 files changed, 3951 insertions, 476 deletions
diff --git a/buildtools/make-relaxng.py b/buildtools/make-relaxng.py index 62decbae..0058ade5 100644 --- a/buildtools/make-relaxng.py +++ b/buildtools/make-relaxng.py @@ -3,7 +3,7 @@ Script to generate rpki/relaxng.py. $Id$ -Copyright (C) 2009 Internet Systems Consortium ("ISC") +Copyright (C) 2009-2011 Internet Systems Consortium ("ISC") Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above @@ -32,7 +32,7 @@ OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -schemas = ("left_right", "up_down", "publication") +import sys format_1 = """\ # Automatically generated, do not edit. @@ -46,9 +46,15 @@ format_2 = """\ %(name)s = lxml.etree.RelaxNG(lxml.etree.fromstring('''%(rng)s''')) """ +def filename_to_symbol(s): + for suffix in (".rng", "-schema"): + if s.endswith(suffix): + s = s[:-len(suffix)] + return s.replace("-", "_") + print format_1 -for name in schemas: +for filename in sys.argv[1:]: print format_2 % { - "name" : name, - "rng" : open(name.replace("_", "-") + "-schema.rng").read() } + "name" : filename_to_symbol(filename), + "rng" : open(filename).read() } diff --git a/buildtools/make-sql-schemas.py b/buildtools/make-sql-schemas.py index 700d2b9c..3ecde014 100644 --- a/buildtools/make-sql-schemas.py +++ b/buildtools/make-sql-schemas.py @@ -32,7 +32,7 @@ OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -schemas = ("rpkid", "irdbd", "pubd") +schemas = ("rpkid", "pubd") format_1 = """\ # Automatically generated, do not edit. diff --git a/rpkid/Makefile.in b/rpkid/Makefile.in index 36016fef..ff035f17 100644 --- a/rpkid/Makefile.in +++ b/rpkid/Makefile.in @@ -42,7 +42,7 @@ SETUP_PY = \ POW_SO = rpki/POW/_POW.so SCRIPTS = rpki-sql-backup rpki-sql-setup rpki-start-servers irbe_cli irdbd myrpki \ - pubd rootd rpkid portal-gui/scripts/rpkigui-load-csv \ + pubd rootd rpkic rpkid portal-gui/scripts/rpkigui-load-csv \ portal-gui/scripts/rpkigui-add-user portal-gui/scripts/rpkigui-response \ portal-gui/scripts/rpkigui-rcynic @@ -64,8 +64,10 @@ rpm deb:: all deb:: cd dist; for i in *.rpm; do case $$i in *.src.rpm) :;; *) (set -x; fakeroot alien -v $$i);; esac; done -rpki/relaxng.py: ${abs_top_srcdir}/buildtools/make-relaxng.py left-right-schema.rng up-down-schema.rng publication-schema.rng - ${PYTHON} ${abs_top_srcdir}/buildtools/make-relaxng.py >$@.tmp +RNGS = left-right-schema.rng up-down-schema.rng publication-schema.rng myrpki.rng + +rpki/relaxng.py: ${abs_top_srcdir}/buildtools/make-relaxng.py ${RNGS} + ${PYTHON} ${abs_top_srcdir}/buildtools/make-relaxng.py ${RNGS} >$@.tmp mv $@.tmp $@ left-right-schema.rng: left-right-schema.rnc @@ -80,7 +82,7 @@ publication-schema.rng: publication-schema.rnc myrpki.rng: myrpki.rnc trang myrpki.rnc myrpki.rng -rpki/sql_schemas.py: ${abs_top_srcdir}/buildtools/make-sql-schemas.py rpkid.sql irdbd.sql pubd.sql +rpki/sql_schemas.py: ${abs_top_srcdir}/buildtools/make-sql-schemas.py rpkid.sql pubd.sql ${PYTHON} ${abs_top_srcdir}/buildtools/make-sql-schemas.py >$@.tmp mv $@.tmp $@ @@ -121,7 +123,7 @@ tags: Makefile find . -type f \( -name '*.py' -o -name '*.sql' -o -name '*.rnc' -o -name '*.py.in' \) ! -name relaxng.py ! -name sql_schemas.py ! -name __doc__.py | etags - lint: - pylint --rcfile ${abs_top_srcdir}/buildtools/pylint.rc rpki/[a-z]*.py *d.py rpki-*.py myrpki.py irbe_cli.py tests/*.py + pylint --rcfile ${abs_top_srcdir}/buildtools/pylint.rc rpki/[a-z]*.py *d.py rpki-*.py myrpki.py rpkic.py irbe_cli.py tests/*.py # Documentation @@ -232,6 +234,9 @@ pubd: pubd.py rootd: rootd.py ${COMPILE_PYTHON} +rpkic: rpkic.py + ${COMPILE_PYTHON} + rpkid: rpkid.py ${COMPILE_PYTHON} diff --git a/rpkid/examples/rpki.conf b/rpkid/examples/rpki.conf index 41553c5a..53216b97 100644 --- a/rpkid/examples/rpki.conf +++ b/rpkid/examples/rpki.conf @@ -308,7 +308,7 @@ rpki-root-cert-uri = rsync://${myrpki::publication_rsync_server}/${myrpki: # Private key corresponding to rootd's root RPKI certificate -rpki-root-key = ${myrpki::bpki_servers_directory}/ca.key +rpki-root-key = ${myrpki::bpki_servers_directory}/root.key # Filename (as opposed to rsync URI) of rootd's root RPKI certificate diff --git a/rpkid/myrpki.rnc b/rpkid/myrpki.rnc index 5b8aa450..8acb16cf 100644 --- a/rpkid/myrpki.rnc +++ b/rpkid/myrpki.rnc @@ -2,10 +2,15 @@ # # RelaxNG Schema for MyRPKI XML messages. # +# This message protocol is on its way out, as we're in the process of +# moving on from the user interface model that produced it, but even +# after we finish replacing it we'll still need the schema for a while +# to validate old messages when upgrading. +# # libxml2 (including xmllint) only groks the XML syntax of RelaxNG, so # run the compact syntax through trang to get XML syntax. # -# Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") +# Copyright (C) 2009-2011 Internet Systems Consortium ("ISC") # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above diff --git a/rpkid/myrpki.rng b/rpkid/myrpki.rng index a86d51a6..5f59e114 100644 --- a/rpkid/myrpki.rng +++ b/rpkid/myrpki.rng @@ -4,10 +4,15 @@ RelaxNG Schema for MyRPKI XML messages. + This message protocol is on its way out, as we're in the process of + moving on from the user interface model that produced it, but even + after we finish replacing it we'll still need the schema for a while + to validate old messages when upgrading. + libxml2 (including xmllint) only groks the XML syntax of RelaxNG, so run the compact syntax through trang to get XML syntax. - Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") + Copyright (C) 2009-2011 Internet Systems Consortium ("ISC") Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above diff --git a/rpkid/rpki/csv_utils.py b/rpkid/rpki/csv_utils.py new file mode 100644 index 00000000..f7eed414 --- /dev/null +++ b/rpkid/rpki/csv_utils.py @@ -0,0 +1,100 @@ +""" +CSV utilities, moved here from myrpki.py. + +$Id$ + +Copyright (C) 2009--2011 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import csv +import os + +class BadCSVSyntax(Exception): + """ + Bad CSV syntax. + """ + +class csv_reader(object): + """ + Reader for tab-delimited text that's (slightly) friendlier than the + stock Python csv module (which isn't intended for direct use by + humans anyway, and neither was this package originally, but that + seems to be the way that it has evolved...). + + Columns parameter specifies how many columns users of the reader + expect to see; lines with fewer columns will be padded with None + values. + + Original API design for this class courtesy of Warren Kumari, but + don't blame him if you don't like what I did with his ideas. + """ + + def __init__(self, filename, columns = None, min_columns = None, comment_characters = "#;"): + assert columns is None or isinstance(columns, int) + assert min_columns is None or isinstance(min_columns, int) + if columns is not None and min_columns is None: + min_columns = columns + self.filename = filename + self.columns = columns + self.min_columns = min_columns + self.comment_characters = comment_characters + self.file = open(filename, "r") + + def __iter__(self): + line_number = 0 + for line in self.file: + line_number += 1 + line = line.strip() + if not line or line[0] in self.comment_characters: + continue + fields = line.split() + if self.min_columns is not None and len(fields) < self.min_columns: + raise BadCSVSyntax, "%s:%d: Not enough columns in line %r" % (self.filename, line_number, line) + if self.columns is not None and len(fields) > self.columns: + raise BadCSVSyntax, "%s:%d: Too many columns in line %r" % (self.filename, line_number, line) + if self.columns is not None and len(fields) < self.columns: + fields += tuple(None for i in xrange(self.columns - len(fields))) + yield fields + +class csv_writer(object): + """ + Writer object for tab delimited text. We just use the stock CSV + module in excel-tab mode for this. + + If "renmwo" is set (default), the file will be written to + a temporary name and renamed to the real filename after closing. + """ + + def __init__(self, filename, renmwo = True): + self.filename = filename + self.renmwo = "%s.~renmwo%d~" % (filename, os.getpid()) if renmwo else filename + self.file = open(self.renmwo, "w") + self.writer = csv.writer(self.file, dialect = csv.get_dialect("excel-tab")) + + def close(self): + """ + Close this writer. + """ + if self.file is not None: + self.file.close() + self.file = None + if self.filename != self.renmwo: + os.rename(self.renmwo, self.filename) + + def __getattr__(self, attr): + """ + Fake inheritance from whatever object csv.writer deigns to give us. + """ + return getattr(self.writer, attr) diff --git a/rpkid/rpki/http.py b/rpkid/rpki/http.py index d8afd44c..7d7e81ba 100644 --- a/rpkid/rpki/http.py +++ b/rpkid/rpki/http.py @@ -534,7 +534,7 @@ class http_server(http_stream): raise except Exception, e: rpki.log.traceback() - self.send_error(500, "Unhandled exception %s" % e) + self.send_error(500, reason = "Unhandled exception %s: %s" % (e.__class__.__name__, e)) else: self.send_error(code = error[0], reason = error[1]) diff --git a/rpkid/rpki/ipaddrs.py b/rpkid/rpki/ipaddrs.py index 531bcbb9..a192f92b 100644 --- a/rpkid/rpki/ipaddrs.py +++ b/rpkid/rpki/ipaddrs.py @@ -57,6 +57,8 @@ class v4addr(long): """ Construct a v4addr object. """ + if isinstance(x, unicode): + x = x.encode("ascii") if isinstance(x, str): return cls.from_bytes(socket.inet_pton(socket.AF_INET, ".".join(str(int(i)) for i in x.split(".")))) else: @@ -94,6 +96,8 @@ class v6addr(long): """ Construct a v6addr object. """ + if isinstance(x, unicode): + x = x.encode("ascii") if isinstance(x, str): return cls.from_bytes(socket.inet_pton(socket.AF_INET6, x)) else: diff --git a/rpkid/rpki/irdb/__init__.py b/rpkid/rpki/irdb/__init__.py new file mode 100644 index 00000000..3eb6fab7 --- /dev/null +++ b/rpkid/rpki/irdb/__init__.py @@ -0,0 +1,23 @@ +""" +Django really wants its models packaged as a models module within a +Python package, so humor it. + +$Id$ + +Copyright (C) 2011 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +from rpki.irdb.models import * +from rpki.irdb.zookeeper import Zookeeper diff --git a/rpkid/rpki/irdb/models.py b/rpkid/rpki/irdb/models.py new file mode 100644 index 00000000..1add3593 --- /dev/null +++ b/rpkid/rpki/irdb/models.py @@ -0,0 +1,525 @@ +""" +IR Database, Django-style. + +This is the back-end code's interface to the database. It's intended +to be usable by command line programs and other scripts, not just +Django GUI code, so be careful. + +$Id$ + +Copyright (C) 2011 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import django.db.models +import rpki.x509 +import rpki.sundial +import socket + +## @var ip_version_choices +# Choice argument for fields implementing IP version numbers. + +ip_version_choices = ((4, "IPv4"), (6, "IPv6")) + +### + +# Field types + +class HandleField(django.db.models.CharField): + """ + A handle field type. + """ + + description = 'A "handle" in one of the RPKI protocols' + + def __init__(self, *args, **kwargs): + kwargs["max_length"] = 120 + django.db.models.CharField.__init__(self, *args, **kwargs) + +class EnumField(django.db.models.PositiveSmallIntegerField): + """ + An enumeration type that uses strings in Python and small integers + in SQL. + """ + + description = "An enumeration type" + + __metaclass__ = django.db.models.SubfieldBase + + def __init__(self, *args, **kwargs): + if isinstance(kwargs["choices"], (tuple, list)) and isinstance(kwargs["choices"][0], str): + kwargs["choices"] = tuple(enumerate(kwargs["choices"], 1)) + django.db.models.PositiveSmallIntegerField.__init__(self, *args, **kwargs) + self.enum_i2s = dict(self.flatchoices) + self.enum_s2i = dict((v, k) for k, v in self.flatchoices) + + def to_python(self, value): + return self.enum_i2s.get(value, value) + + def get_prep_value(self, value): + return self.enum_s2i.get(value, value) + +class SundialField(django.db.models.DateTimeField): + """ + A field type for our customized datetime objects. + """ + + description = "A datetime type using our customized datetime objects" + + def to_python(self, value): + return rpki.sundial.datetime.fromdatetime( + django.db.models.DateTimeField.to_python(self, value)) + + def get_prep_value(self, value): + if isinstance(value, rpki.sundial.datetime): + return value.to_sql() + else: + return value + +### + +# Kludge to work around Django 1.2 problem. +# +# This should be a simple abstract base class DERField which we then +# subclass with trivial customization for specific kinds of DER +# objects. Sadly, subclassing of user defined field classes doesn't +# work in Django 1.2 with the django.db.models.SubfieldBase metaclass, +# so instead we fake it by defining methods externally and defining +# each concrete class as a direct subclass of django.db.models.Field. +# +# The bug has been fixed in Django 1.3, so we can revert this to the +# obvious form once we're ready to require Django 1.3 or later. The +# fix may have been backported to the 1.2 branch, but trying to test +# for it is likely more work than just working around it. +# +# See https://code.djangoproject.com/ticket/10728 for details. + +def DERField_init(self, *args, **kwargs): + kwargs["serialize"] = False + kwargs["blank"] = True + kwargs["default"] = None + django.db.models.Field.__init__(self, *args, **kwargs) + +def DERField_db_type(self, connection): + if connection.settings_dict['ENGINE'] == "django.db.backends.posgresql": + return "bytea" + else: + return "BLOB" + +def DERField_to_python(self, value): + assert value is None or isinstance(value, (self.rpki_type, str)) + if isinstance(value, str): + return self.rpki_type(DER = value) + else: + return value + +def DERField_get_prep_value(self, value): + assert value is None or isinstance(value, (self.rpki_type, str)) + if isinstance(value, self.rpki_type): + return value.get_DER() + else: + return value + +def DERField(cls): + cls.__init__ = DERField_init + cls.db_type = DERField_db_type + cls.to_python = DERField_to_python + cls.get_prep_value = DERField_get_prep_value + return cls + +@DERField +class CertificateField(django.db.models.Field): + __metaclass__ = django.db.models.SubfieldBase + description = "X.509 certificate" + rpki_type = rpki.x509.X509 + +@DERField +class RSAKeyField(django.db.models.Field): + __metaclass__ = django.db.models.SubfieldBase + description = "RSA keypair" + rpki_type = rpki.x509.RSA + +@DERField +class CRLField(django.db.models.Field): + __metaclass__ = django.db.models.SubfieldBase + description = "Certificate Revocation List" + rpki_type = rpki.x509.CRL + +@DERField +class PKCS10Field(django.db.models.Field): + __metaclass__ = django.db.models.SubfieldBase + description = "PKCS #10 certificate request" + rpki_type = rpki.x509.PKCS10 + +@DERField +class SignedReferralField(django.db.models.Field): + __metaclass__ = django.db.models.SubfieldBase + description = "CMS signed object containing XML" + rpki_type = rpki.x509.SignedReferral + +### + +# Custom managers + +class CertificateManager(django.db.models.Manager): + + def get_or_certify(self, **kwargs): + """ + Sort of like .get_or_create(), but for models containing + certificates which need to be generated based on other fields. + + Takes keyword arguments like .get(), checks for existing object. + If none, creates a new one; if found an existing object but some + of the non-key fields don't match, updates the existing object. + Runs certification method for new or updated objects. Returns a + tuple consisting of the object and a boolean indicating whether + anything has changed. + """ + + changed = False + + try: + obj = self.get(**self._get_or_certify_keys(kwargs)) + + except self.model.DoesNotExist: + obj = self.model(**kwargs) + changed = True + + else: + for k in kwargs: + if getattr(obj, k) != kwargs[k]: + setattr(obj, k, kwargs[k]) + changed = True + + if changed: + obj.avow() + obj.save() + + return obj, changed + + def _get_or_certify_keys(self, kwargs): + assert len(self.model._meta.unique_together) == 1 + return dict((k, kwargs[k]) for k in self.model._meta.unique_together[0]) + +class ResourceHolderCAManager(CertificateManager): + def _get_or_certify_keys(self, kwargs): + return { "handle" : kwargs["handle"] } + +class ServerCAManager(CertificateManager): + def _get_or_certify_keys(self, kwargs): + return { "pk" : 1 } + +class ResourceHolderEEManager(CertificateManager): + def _get_or_certify_keys(self, kwargs): + return { "issuer" : kwargs["issuer"] } + +### + +class CA(django.db.models.Model): + certificate = CertificateField() + private_key = RSAKeyField() + latest_crl = CRLField() + + # Might want to bring these into line with what rpkid does. Current + # variables here were chosen to map easily to what OpenSSL command + # line tool was keeping on disk. + + next_serial = django.db.models.BigIntegerField(default = 1) + next_crl_number = django.db.models.BigIntegerField(default = 1) + last_crl_update = SundialField() + next_crl_update = SundialField() + + # These should come from somewhere, but I don't yet know where + ca_certificate_lifetime = rpki.sundial.timedelta(days = 3652) + crl_interval = rpki.sundial.timedelta(days = 1) + + class Meta: + abstract = True + + def avow(self): + if self.private_key is None: + self.private_key = rpki.x509.RSA.generate() + now = rpki.sundial.now() + notAfter = now + self.ca_certificate_lifetime + self.certificate = rpki.x509.X509.bpki_self_certify( + keypair = self.private_key, + subject_name = self.subject_name, + serial = self.next_serial, + now = now, + notAfter = notAfter) + self.next_serial += 1 + self.generate_crl() + return self.certificate + + def certify(self, subject_name, subject_key, validity_interval, is_ca, pathLenConstraint = None): + now = rpki.sundial.now() + notAfter = now + validity_interval + result = self.certificate.bpki_certify( + keypair = self.private_key, + subject_name = subject_name, + subject_key = subject_key, + serial = self.next_serial, + now = now, + notAfter = notAfter, + is_ca = is_ca, + pathLenConstraint = pathLenConstraint) + self.next_serial += 1 + return result + + def revoke(self, cert): + Revocations.objects.create( + issuer = self, + revoked = rpki.sundial.now(), + serial = cert.certificate.getSerial(), + expires = cert.certificate.getNotAfter() + self.crl_interval) + cert.delete() + self.generate_crl() + + def generate_crl(self): + now = rpki.sundial.now() + self.revocations.filter(expires__lt = now).delete() + revoked = [(r.serial, rpki.sundial.datetime.fromdatetime(r.revoked).toASN1tuple(), ()) + for r in self.revocations.all()] + self.latest_crl = rpki.x509.CRL.generate( + keypair = self.private_key, + issuer = self.certificate, + serial = self.next_crl_number, + thisUpdate = now, + nextUpdate = now + self.crl_interval, + revokedCertificates = revoked) + self.last_crl_update = now + self.next_crl_update = now + self.crl_interval + self.next_crl_number += 1 + +class ServerCA(CA): + objects = ServerCAManager() + + def __unicode__(self): + return "" + + @property + def subject_name(self): + return rpki.x509.X501DN("%s BPKI server CA" % socket.gethostname()) + +class ResourceHolderCA(CA): + handle = HandleField(unique = True) + objects = ResourceHolderCAManager() + + def __unicode__(self): + return self.handle + + @property + def subject_name(self): + return rpki.x509.X501DN("%s BPKI resource CA" % self.handle) + +class Certificate(django.db.models.Model): + + certificate = CertificateField() + objects = CertificateManager() + + default_interval = rpki.sundial.timedelta(days = 60) + + class Meta: + abstract = True + unique_together = ("issuer", "handle") + + def revoke(self): + self.issuer.revoke(self) + +class CrossCertification(Certificate): + handle = HandleField() + ta = CertificateField() + + class Meta: + abstract = True + + def avow(self): + self.certificate = self.issuer.certify( + subject_name = self.ta.getSubject(), + subject_key = self.ta.getPublicKey(), + validity_interval = self.default_interval, + is_ca = True, + pathLenConstraint = 0) + + def __unicode__(self): + return self.handle + +class HostedCA(Certificate): + issuer = django.db.models.ForeignKey(ServerCA) + hosted = django.db.models.OneToOneField(ResourceHolderCA, related_name = "hosted_by") + + def avow(self): + self.certificate = self.issuer.certify( + subject_name = self.hosted.certificate.getSubject(), + subject_key = self.hosted.certificate.getPublicKey(), + validity_interval = self.default_interval, + is_ca = True, + pathLenConstraint = 1) + + class Meta: + unique_together = ("issuer", "hosted") + + def __unicode__(self): + return self.hosted_ca.handle + +class Revocation(django.db.models.Model): + serial = django.db.models.BigIntegerField() + revoked = SundialField() + expires = SundialField() + + class Meta: + abstract = True + unique_together = ("issuer", "serial") + +class ServerRevocation(Revocation): + issuer = django.db.models.ForeignKey(ServerCA, related_name = "revocations") + +class ResourceHolderRevocation(Revocation): + issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "revocations") + +class EECertificate(Certificate): + private_key = RSAKeyField() + + class Meta: + abstract = True + + def avow(self): + if self.private_key is None: + self.private_key = rpki.x509.RSA.generate() + self.certificate = self.issuer.certify( + subject_name = self.subject_name, + subject_key = self.private_key.get_RSApublic(), + validity_interval = self.default_interval, + is_ca = False) + +class ServerEE(EECertificate): + issuer = django.db.models.ForeignKey(ServerCA, related_name = "ee_certificates") + purpose = EnumField(choices = ("rpkid", "pubd", "irdbd", "irbe")) + + class Meta: + unique_together = ("issuer", "purpose") + + @property + def subject_name(self): + return rpki.x509.X501DN("%s BPKI %s EE" % (socket.gethostname(), self.get_purpose_display())) + +class Referral(EECertificate): + issuer = django.db.models.OneToOneField(ResourceHolderCA, related_name = "referral_certificate") + objects = ResourceHolderEEManager() + + @property + def subject_name(self): + return rpki.x509.X501DN("%s BPKI Referral EE" % self.issuer.handle) + +class Turtle(django.db.models.Model): + service_uri = django.db.models.CharField(max_length = 255) + +class Rootd(EECertificate, Turtle): + issuer = django.db.models.OneToOneField(ResourceHolderCA, related_name = "rootd") + objects = ResourceHolderEEManager() + + @property + def subject_name(self): + return rpki.x509.X501DN("%s BPKI rootd EE" % self.issuer.handle) + +class BSC(Certificate): + issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "bscs") + handle = HandleField() + pkcs10 = PKCS10Field() + + def avow(self): + self.certificate = self.issuer.certify( + subject_name = self.pkcs10.getSubject(), + subject_key = self.pkcs10.getPublicKey(), + validity_interval = self.default_interval, + is_ca = False) + + def __unicode__(self): + return self.handle + +class Child(CrossCertification): + issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "children") + name = django.db.models.TextField(null = True, blank = True) + valid_until = SundialField() + + # This shouldn't be necessary + class Meta: + unique_together = ("issuer", "handle") + +class ChildASN(django.db.models.Model): + child = django.db.models.ForeignKey(Child, related_name = "asns") + start_as = django.db.models.BigIntegerField() + end_as = django.db.models.BigIntegerField() + + class Meta: + unique_together = ("child", "start_as", "end_as") + +class ChildNet(django.db.models.Model): + child = django.db.models.ForeignKey(Child, related_name = "address_ranges") + start_ip = django.db.models.CharField(max_length = 40) + end_ip = django.db.models.CharField(max_length = 40) + version = EnumField(choices = ip_version_choices) + + class Meta: + unique_together = ("child", "start_ip", "end_ip", "version") + +class Parent(CrossCertification, Turtle): + issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "parents") + parent_handle = HandleField() + child_handle = HandleField() + repository_type = EnumField(choices = ("none", "offer", "referral")) + referrer = HandleField(null = True, blank = True) + referral_authorization = SignedReferralField(null = True, blank = True) + + # This shouldn't be necessary + class Meta: + unique_together = ("issuer", "handle") + +class ROARequest(django.db.models.Model): + issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "roa_requests") + asn = django.db.models.BigIntegerField() + +class ROARequestPrefix(django.db.models.Model): + roa_request = django.db.models.ForeignKey(ROARequest, related_name = "prefixes") + version = EnumField(choices = ip_version_choices) + prefix = django.db.models.CharField(max_length = 40) + prefixlen = django.db.models.PositiveSmallIntegerField() + max_prefixlen = django.db.models.PositiveSmallIntegerField() + + class Meta: + unique_together = ("roa_request", "version", "prefix", "prefixlen", "max_prefixlen") + +class GhostbusterRequest(django.db.models.Model): + issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "ghostbuster_requests") + parent = django.db.models.ForeignKey(Parent, related_name = "ghostbuster_requests", null = True) + vcard = django.db.models.TextField() + +class Repository(CrossCertification): + issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "repositories") + client_handle = HandleField() + service_uri = django.db.models.CharField(max_length = 255) + sia_base = django.db.models.TextField() + turtle = django.db.models.OneToOneField(Turtle, related_name = "repository") + + # This shouldn't be necessary + class Meta: + unique_together = ("issuer", "handle") + +class Client(CrossCertification): + issuer = django.db.models.ForeignKey(ServerCA, related_name = "clients") + sia_base = django.db.models.TextField() + + # This shouldn't be necessary + class Meta: + unique_together = ("issuer", "handle") diff --git a/rpkid/rpki/irdb/zookeeper.py b/rpkid/rpki/irdb/zookeeper.py new file mode 100644 index 00000000..ad078862 --- /dev/null +++ b/rpkid/rpki/irdb/zookeeper.py @@ -0,0 +1,1113 @@ +""" +Management code for the IRDB. + +$Id$ + +Copyright (C) 2009--2012 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import subprocess, csv, re, os, getopt, sys, base64, time, glob, copy, warnings +import rpki.config, rpki.cli, rpki.sundial, rpki.log, rpki.oids +import rpki.http, rpki.resource_set, rpki.relaxng, rpki.exceptions +import rpki.left_right, rpki.x509, rpki.async, rpki.irdb +import django.db.transaction + +from lxml.etree import (Element, SubElement, ElementTree, + fromstring as ElementFromString, + tostring as ElementToString) + +from rpki.csv_utils import (csv_reader, csv_writer, BadCSVSyntax) + + + +# XML namespace and protocol version for OOB setup protocol. The name +# is historical and may change before we propose this as the basis for +# a standard. + +myrpki_namespace = "http://www.hactrn.net/uris/rpki/myrpki/" +myrpki_version = "2" +myrpki_namespaceQName = "{" + myrpki_namespace + "}" + +myrpki_section = "myrpki" +irdbd_section = "irdbd" + +# A whole lot of exceptions + +class MissingHandle(Exception): "Missing handle" +class CouldntTalkToDaemon(Exception): "Couldn't talk to daemon." +class BadXMLMessage(Exception): "Bad XML message." +class PastExpiration(Exception): "Expiration date has already passed." +class CantRunRootd(Exception): "Can't run rootd." + + + +def B64Element(e, tag, obj, **kwargs): + """ + Create an XML element containing Base64 encoded data taken from a + DER object. + """ + + if e is None: + se = Element(tag, **kwargs) + else: + se = SubElement(e, tag, **kwargs) + if e is not None and e.text is None: + e.text = "\n" + se.text = "\n" + obj.get_Base64() + se.tail = "\n" + return se + +class PEM_writer(object): + """ + Write PEM files to disk, keeping track of which ones we've already + written and setting the file mode appropriately. + """ + + def __init__(self, verbose = False): + self.wrote = set() + self.verbose = verbose + + def __call__(self, filename, obj): + filename = os.path.realpath(filename) + if filename in self.wrote: + return + tempname = filename + if not filename.startswith("/dev/"): + tempname += ".%s.tmp" % os.getpid() + mode = 0400 if filename.endswith(".key") else 0444 + if self.verbose: + print "Writing", filename + f = os.fdopen(os.open(tempname, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode), "w") + f.write(obj.get_PEM()) + f.close() + if tempname != filename: + os.rename(tempname, filename) + self.wrote.add(filename) + + + + +def etree_read(filename): + """ + Read an etree from a file, verifying then stripping XML namespace + cruft. + """ + + e = ElementTree(file = filename).getroot() + rpki.relaxng.myrpki.assertValid(e) + for i in e.getiterator(): + if i.tag.startswith(myrpki_namespaceQName): + i.tag = i.tag[len(myrpki_namespaceQName):] + else: + raise BadXMLMessage, "XML tag %r is not in namespace %r" % (i.tag, myrpki_namespace) + return e + + +class etree_wrapper(object): + """ + Wrapper for ETree objects so we can return them as function results + without requiring the caller to understand much about them. + + """ + + def __init__(self, e, msg = None): + self.msg = msg + e = copy.deepcopy(e) + e.set("version", myrpki_version) + for i in e.getiterator(): + if i.tag[0] != "{": + i.tag = myrpki_namespaceQName + i.tag + assert i.tag.startswith(myrpki_namespaceQName) + rpki.relaxng.myrpki.assertValid(e) + self.etree = e + + def __str__(self): + return ElementToString(self.etree) + + def save(self, filename, blather = False): + filename = os.path.realpath(filename) + tempname = filename + if not filename.startswith("/dev/"): + tempname += ".%s.tmp" % os.getpid() + ElementTree(self.etree).write(tempname) + if tempname != filename: + os.rename(tempname, filename) + if blather: + print "Wrote", filename + if self.msg is not None: + print self.msg + + + +class Zookeeper(object): + + ## @var show_xml + # Whether to show XML for debugging + + show_xml = False + + def __init__(self, cfg = None, handle = None): + + if cfg is None: + cfg = rpki.config.parser() + + if handle is None: + handle = cfg.get("handle", section = myrpki_section) + + self.cfg = cfg + + self.run_rpkid = cfg.getboolean("run_rpkid", section = myrpki_section) + self.run_pubd = cfg.getboolean("run_pubd", section = myrpki_section) + self.run_rootd = cfg.getboolean("run_rootd", section = myrpki_section) + + if self.run_rootd and (not self.run_pubd or not self.run_rpkid): + raise CantRunRootd, "Can't run rootd unless also running rpkid and pubd" + + self.default_repository = cfg.get("default_repository", "", section = myrpki_section) + self.pubd_contact_info = cfg.get("pubd_contact_info", "", section = myrpki_section) + + self.rsync_module = cfg.get("publication_rsync_module", section = myrpki_section) + self.rsync_server = cfg.get("publication_rsync_server", section = myrpki_section) + + self.reset_identity(handle) + + + def reset_identity(self, handle): + """ + Select handle of current resource holding entity. + """ + + if handle is None: + raise MissingHandle + self.handle= handle + + + @property + def resource_ca(self): + """ + Get ResourceHolderCA object associated with current handle. + """ + + assert self.handle is not None + try: + return rpki.irdb.ResourceHolderCA.objects.get(handle = self.handle) + except rpki.irdb.ResourceHolderCA.DoesNotExist: + return None + + + @property + def server_ca(self): + """ + Get ServerCA object. + """ + + try: + return rpki.irdb.ServerCA.objects.get() + except rpki.irdb.ServerCA.DoesNotExist: + return None + + + @django.db.transaction.commit_on_success + def initialize(self): + """ + Initialize an RPKI installation. Reads the configuration file, + creates the BPKI and EntityDB directories, generates the initial + BPKI certificates, and creates an XML file describing the + resource-holding aspect of this RPKI installation. + """ + + resource_ca, created = rpki.irdb.ResourceHolderCA.objects.get_or_certify(handle = self.handle) + + if self.run_rpkid or self.run_pubd: + server_ca, created = rpki.irdb.ServerCA.objects.get_or_certify() + rpki.irdb.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "irbe") + + if self.run_rpkid: + rpki.irdb.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "rpkid") + rpki.irdb.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "irdbd") + + if self.run_pubd: + rpki.irdb.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "pubd") + + e = Element("identity", handle = self.handle) + B64Element(e, "bpki_ta", resource_ca.certificate) + return etree_wrapper(e, msg = 'This is the "identity" file you will need to send to your parent') + + @django.db.transaction.commit_on_success + def configure_rootd(self): + + assert self.run_rpkid and self.run_pubd and self.run_rootd + + rpki.irdb.Rootd.objects.get_or_certify( + issuer = self.resource_ca, + service_uri = "http://localhost:%s/" % self.cfg.get("rootd_server_port")) + + # The following assumes we'll set up the respository manually. + # Not sure this is a reasonable assumption, particularly if we + # ever fix rootd to use the publication protocol. + + try: + self.resource_ca.repositories.get(handle = self.handle) + return None + + except rpki.irdb.Repository.DoesNotExist: + e = Element("repository", type = "offer", handle = self.handle, parent_handle = self.handle) + B64Element(e, "bpki_client_ta", self.resource_ca.certificate) + return etree_wrapper(e, msg = 'This is the "repository offer" file for you to use if you want to publish in your own repository') + + + def write_bpki_files(self): + """ + Write out BPKI certificate, key, and CRL files for daemons that + need them. + """ + + writer = PEM_writer() + + if self.run_rpkid: + rpkid = self.server_ca.ee_certificates.get(purpose = "rpkid") + writer(self.cfg.get("bpki-ta", section = "rpkid"), self.server_ca.certificate) + writer(self.cfg.get("rpkid-key", section = "rpkid"), rpkid.private_key) + writer(self.cfg.get("rpkid-cert", section = "rpkid"), rpkid.certificate) + writer(self.cfg.get("irdb-cert", section = "rpkid"), + self.server_ca.ee_certificates.get(purpose = "irdbd").certificate) + writer(self.cfg.get("irbe-cert", section = "rpkid"), + self.server_ca.ee_certificates.get(purpose = "irbe").certificate) + + if self.run_pubd: + pubd = self.server_ca.ee_certificates.get(purpose = "pubd") + writer(self.cfg.get("bpki-ta", section = "pubd"), self.server_ca.certificate) + writer(self.cfg.get("pubd-key", section = "pubd"), pubd.private_key) + writer(self.cfg.get("pubd-cert", section = "pubd"), pubd.certificate) + writer(self.cfg.get("irbe-cert", section = "pubd"), + self.server_ca.ee_certificates.get(purpose = "irbe").certificate) + + if self.run_rootd: + rootd = rpki.irdb.ResourceHolderCA.objects.get(handle = self.cfg.get("handle", section = "myrpki")).rootd + writer(self.cfg.get("bpki-ta", section = "rootd"), self.server_ca.certificate) + writer(self.cfg.get("rootd-bpki-crl", section = "rootd"), self.server_ca.latest_crl) + writer(self.cfg.get("rootd-bpki-key", section = "rootd"), rootd.private_key) + writer(self.cfg.get("rootd-bpki-cert", section = "rootd"), rootd.certificate) + writer(self.cfg.get("child-bpki-cert", section = "rootd"), rootd.issuer.certificate) + + + @django.db.transaction.commit_on_success + def update_bpki(self): + """ + Update BPKI certificates. Assumes an existing RPKI installation. + + Basic plan here is to reissue all BPKI certificates we can, right + now. In the long run we might want to be more clever about only + touching ones that need maintenance, but this will do for a start. + + We also reissue CRLs for all CAs. + + Most likely this should be run under cron. + """ + + for model in (rpki.irdb.ServerCA, + rpki.irdb.ResourceHolderCA, + rpki.irdb.ServerEE, + rpki.irdb.Referral, + rpki.irdb.Rootd, + rpki.irdb.HostedCA, + rpki.irdb.BSC, + rpki.irdb.Child, + rpki.irdb.Parent, + rpki.irdb.Client, + rpki.irdb.Repository): + for obj in model.objects.all(): + print "Regenerating certificate", obj.certificate.getSubject() + obj.avow() + + print "Regenerating Server CRL" + self.server_ca.generate_crl() + + for ca in rpki.irdb.ResourceHolderCA.objects.all(): + print "Regenerating CRL for", ca.handle + ca.generate_crl() + + + @django.db.transaction.commit_on_success + def configure_child(self, filename, child_handle = None): + """ + Configure a new child of this RPKI entity, given the child's XML + identity file as an input. Extracts the child's data from the + XML, cross-certifies the child's resource-holding BPKI + certificate, and generates an XML file describing the relationship + between the child and this parent, including this parent's BPKI + data and up-down protocol service URI. + """ + + c = etree_read(filename) + + if child_handle is None: + child_handle = c.get("handle") + + service_uri = "http://%s:%s/up-down/%s/%s" % (self.cfg.get("rpkid_server_host"), + self.cfg.get("rpkid_server_port"), + self.handle, child_handle) + + valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 365) + + print "Child calls itself %r, we call it %r" % (c.get("handle"), child_handle) + + child, created = rpki.irdb.Child.objects.get_or_certify( + issuer = self.resource_ca, + handle = child_handle, + ta = rpki.x509.X509(Base64 = c.findtext("bpki_ta")), + valid_until = valid_until) + + e = Element("parent", parent_handle = self.handle, child_handle = child_handle, + service_uri = service_uri, valid_until = str(valid_until)) + B64Element(e, "bpki_resource_ta", self.resource_ca.certificate) + B64Element(e, "bpki_child_ta", child.ta) + + try: + if self.default_repository: + repo = self.resource_ca.repositories.get(handle = self.default_repository) + else: + repo = self.resource_ca.repositories.get() + except rpki.irdb.Repository.DoesNotExist: + repo = None + + if repo is None: + print "Couldn't find any usable repositories, not giving referral" + + elif repo.handle == self.handle: + SubElement(e, "repository", type = "offer") + + else: + proposed_sia_base = repo.sia_base + child_handle + "/" + referral_cert, created = rpki.irdb.Referral.objects.get_or_certify(issuer = self.resource_ca) + auth = rpki.x509.SignedReferral() + auth.set_content(B64Element(None, myrpki_namespaceQName + "referral", child.ta, + version = myrpki_version, + authorized_sia_base = proposed_sia_base)) + auth.schema_check() + auth.sign(referral_cert.private_key, referral_cert.certificate, self.resource_ca.latest_crl) + + r = SubElement(e, "repository", type = "referral") + B64Element(r, "authorization", auth, referrer = repo.client_handle) + SubElement(r, "contact_info") + + return etree_wrapper(e, msg = "Send this file back to the child you just configured"), child_handle + + + @django.db.transaction.commit_on_success + def delete_child(self, child_handle): + """ + Delete a child of this RPKI entity. + """ + + assert child_handle is not None + try: + self.resource_ca.children.get(handle = child_handle).delete() + except rpki.irdb.Child.DoesNotExist: + print "No such child \"%s\"" % arg + + + @django.db.transaction.commit_on_success + def configure_parent(self, filename, parent_handle = None): + """ + Configure a new parent of this RPKI entity, given the output of + the parent's configure_child command as input. Reads the parent's + response XML, extracts the parent's BPKI and service URI + information, cross-certifies the parent's BPKI data into this + entity's BPKI, and checks for offers or referrals of publication + service. If a publication offer or referral is present, we + generate a request-for-service message to that repository, in case + the user wants to avail herself of the referral or offer. + """ + + p = etree_read(filename) + + if parent_handle is None: + parent_handle = p.get("parent_handle") + + r = p.find("repository") + + repository_type = "none" + referrer = None + referral_authorization = None + + if r is not None: + repository_type = r.get("type") + + if repository_type == "referral": + a = r.find("authorization") + referrer = a.get("referrer") + referral_authorization = rpki.x509.SignedReferral(Base64 = a.text) + + print "Parent calls itself %r, we call it %r" % (p.get("parent_handle"), parent_handle) + print "Parent calls us %r" % p.get("child_handle") + + rpki.irdb.Parent.objects.get_or_certify( + issuer = self.resource_ca, + handle = parent_handle, + child_handle = p.get("child_handle"), + parent_handle = p.get("parent_handle"), + service_uri = p.get("service_uri"), + ta = rpki.x509.X509(Base64 = p.findtext("bpki_resource_ta")), + repository_type = repository_type, + referrer = referrer, + referral_authorization = referral_authorization) + + if repository_type == "none": + r = Element("repository", type = "none") + r.set("handle", self.handle) + r.set("parent_handle", parent_handle) + B64Element(r, "bpki_client_ta", self.resource_ca.certificate) + return etree_wrapper(r, msg = "This is the file to send to the repository operator"), parent_handle + + + @django.db.transaction.commit_on_success + def delete_parent(self, parent_handle): + """ + Delete a parent of this RPKI entity. + """ + + assert parent_handle is not None + try: + self.resource_ca.parents.get(handle = parent_handle).delete() + except rpki.irdb.Parent.DoesNotExist: + print "No such parent \"%s\"" % arg + + + @django.db.transaction.commit_on_success + def configure_publication_client(self, filename, sia_base = None): + """ + Configure publication server to know about a new client, given the + client's request-for-service message as input. Reads the client's + request for service, cross-certifies the client's BPKI data, and + generates a response message containing the repository's BPKI data + and service URI. + """ + + client = etree_read(filename) + + client_ta = rpki.x509.X509(Base64 = client.findtext("bpki_client_ta")) + + if sia_base is None and client.get("handle") == self.handle and self.resource_ca.certificate == client_ta: + print "This looks like self-hosted publication" + sia_base = "rsync://%s/%s/%s/" % (self.rsync_server, self.rsync_module, self.handle) + + if sia_base is None and client.get("type") == "referral": + print "This looks like a referral, checking" + try: + auth = client.find("authorization") + referrer = self.server_ca.clients.get(handle = auth.get("referrer")) + referral_cms = rpki.x509.SignedReferral(Base64 = auth.text) + referral_xml = referral_cms.unwrap(ta = (referrer.certificate, self.server_ca.certificate)) + if rpki.x509.X509(Base64 = referral_xml.text) != client_ta: + raise BadXMLMessage, "Referral trust anchor does not match" + sia_base = referral_xml.get("authorized_sia_base") + except rpki.irdb.Client.DoesNotExist: + print "We have no record of the client (%s) alleged to have made this referral" % auth.get("referrer") + + if sia_base is None and client.get("type") == "offer" and client.get("parent_handle") == self.handle: + print "This looks like an offer, client claims to be our child, checking" + try: + child = self.resource_ca.children.get(ta = client_ta) + except rpki.irdb.Child.DoesNotExist: + print "Can't find a child matching this client" + else: + sia_base = "rsync://%s/%s/%s/%s/" % (self.rsync_server, self.rsync_module, + self.handle, client.get("handle")) + + # If we still haven't figured out what to do with this client, it + # gets a top-level tree of its own, no attempt at nesting. + + if sia_base is None: + print "Don't know where to nest this client, defaulting to top-level" + sia_base = "rsync://%s/%s/%s/" % (self.rsync_server, self.rsync_module, client.get("handle")) + + if not sia_base.startswith("rsync://"): + raise BadXMLMessage, "Malformed sia_base parameter %r, should start with 'rsync://'" % sia_base + + client_handle = "/".join(sia_base.rstrip("/").split("/")[4:]) + + parent_handle = client.get("parent_handle") + + print "Client calls itself %r, we call it %r" % (client.get("handle"), client_handle) + print "Client says its parent handle is %r" % parent_handle + + rpki.irdb.Client.objects.get_or_certify( + issuer = self.server_ca, + handle = client_handle, + ta = client_ta, + sia_base = sia_base) + + e = Element("repository", type = "confirmed", + client_handle = client_handle, + parent_handle = parent_handle, + sia_base = sia_base, + service_uri = "http://%s:%s/client/%s" % (self.cfg.get("pubd_server_host"), + self.cfg.get("pubd_server_port"), + client_handle)) + + B64Element(e, "bpki_server_ta", self.server_ca.certificate) + B64Element(e, "bpki_client_ta", client_ta) + SubElement(e, "contact_info").text = self.pubd_contact_info + return (etree_wrapper(e, msg = "Send this file back to the publication client you just configured"), + client_handle) + + + @django.db.transaction.commit_on_success + def delete_publication_client(self, client_handle): + """ + Delete a publication client of this RPKI entity. + """ + + assert client_handle is not None + try: + self.resource_ca.clients.get(handle = client_handle).delete() + except rpki.irdb.Client.DoesNotExist: + print "No such client \"%s\"" % arg + + + @django.db.transaction.commit_on_success + def configure_repository(self, filename, parent_handle = None): + """ + Configure a publication repository for this RPKI entity, given the + repository's response to our request-for-service message as input. + Reads the repository's response, extracts and cross-certifies the + BPKI data and service URI, and links the repository data with the + corresponding parent data in our local database. + """ + + r = etree_read(filename) + + if parent_handle is None: + parent_handle = r.get("parent_handle") + + print "Repository calls us %r" % (r.get("client_handle")) + print "Repository response associated with parent_handle %r" % parent_handle + + try: + if parent_handle == self.handle: + turtle = self.resource_ca.rootd + else: + turtle = self.resource_ca.parents.get(handle = parent_handle) + + except (rpki.irdb.Parent.DoesNotExist, rpki.irdb.Rootd.DoesNotExist): + print "Could not find parent %r in our database" % parent_handle + + else: + rpki.irdb.Repository.objects.get_or_certify( + issuer = self.resource_ca, + handle = parent_handle, + client_handle = r.get("client_handle"), + service_uri = r.get("service_uri"), + sia_base = r.get("sia_base"), + ta = rpki.x509.X509(Base64 = r.findtext("bpki_server_ta")), + turtle = turtle) + + + @django.db.transaction.commit_on_success + def delete_repository(self, repository_handle): + """ + Delete a repository of this RPKI entity. + """ + + assert repository_handle is not None + try: + self.resource_ca.repositories.get(handle = arg).delete() + except rpki.irdb.Repository.DoesNotExist: + print "No such repository \"%s\"" % arg + + + @django.db.transaction.commit_on_success + def renew_children(self, child_handle, valid_until = None): + """ + Update validity period for one child entity or, if child_handle is + None, for all child entities. + """ + + if child_handle is None: + children = self.resource_ca.children + else: + children = self.resource_ca.children.filter(handle = child_handle) + + if valid_until is None: + valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 365) + else: + valid_until = rpki.sundial.fromXMLtime(valid_until) + if valid_until < rpki.sundial.now(): + raise PastExpiration, "Specified new expiration time %s has passed" % valid_until + + print "New validity date", valid_until + + for child in children: + child.valid_until = valid_until + child.save() + + + @django.db.transaction.commit_on_success + def load_prefixes(self, filename): + """ + Whack IRDB to match prefixes.csv. + """ + + grouped4 = {} + grouped6 = {} + + for handle, prefix in csv_reader(filename, columns = 2): + grouped = grouped6 if ":" in prefix else grouped4 + if handle not in grouped: + grouped[handle] = [] + grouped[handle].append(prefix) + + primary_keys = [] + + for version, grouped, rset in ((4, grouped4, rpki.resource_set.resource_set_ipv4), + (6, grouped6, rpki.resource_set.resource_set_ipv6)): + for handle, prefixes in grouped.iteritems(): + child = self.resource_ca.children.get(handle = handle) + for prefix in rset(",".join(prefixes)): + obj, created = rpki.irdb.ChildNet.objects.get_or_create( + child = child, + start_ip = str(prefix.min), + end_ip = str(prefix.max), + version = version) + primary_keys.append(obj.pk) + + q = rpki.irdb.ChildNet.objects + q = q.filter(child__issuer__exact = self.resource_ca) + q = q.exclude(pk__in = primary_keys) + q.delete() + + + @django.db.transaction.commit_on_success + def load_asns(self, filename): + """ + Whack IRDB to match asns.csv. + """ + + grouped = {} + + for handle, asn in csv_reader(filename, columns = 2): + if handle not in grouped: + grouped[handle] = [] + grouped[handle].append(asn) + + primary_keys = [] + + for handle, asns in grouped.iteritems(): + child = self.resource_ca.children.get(handle = handle) + for asn in rpki.resource_set.resource_set_as(",".join(asns)): + obj, created = rpki.irdb.ChildASN.objects.get_or_create( + child = child, + start_as = str(asn.min), + end_as = str(asn.max)) + primary_keys.append(obj.pk) + + q = rpki.irdb.ChildASN.objects + q = q.filter(child__issuer__exact = self.resource_ca) + q = q.exclude(pk__in = primary_keys) + q.delete() + + + @django.db.transaction.commit_on_success + def load_roa_requests(self, filename): + """ + Whack IRDB to match roa.csv. + """ + + grouped = {} + + # format: p/n-m asn group + for pnm, asn, group in csv_reader(filename, columns = 3): + key = (asn, group) + if key not in grouped: + grouped[key] = [] + grouped[key].append(pnm) + + # Deleting and recreating all the ROA requests is inefficient, + # but rpkid's current representation of ROA requests is wrong + # (see #32), so it's not worth a lot of effort here as we're + # just going to have to rewrite this soon anyway. + + self.resource_ca.roa_requests.all().delete() + + for key, pnms in grouped.iteritems(): + asn, group = key + + roa_request = self.resource_ca.roa_requests.create(asn = asn) + + for pnm in pnms: + if ":" in pnm: + p = rpki.resource_set.roa_prefix_ipv6.parse_str(pnm) + v = 6 + else: + p = rpki.resource_set.roa_prefix_ipv4.parse_str(pnm) + v = 4 + roa_request.prefixes.create( + version = v, + prefix = str(p.prefix), + prefixlen = int(p.prefixlen), + max_prefixlen = int(p.max_prefixlen)) + + + def call_rpkid(self, *pdus): + """ + Issue a call to rpkid, return result. + + Implementation is a little silly, constructs a wrapper object, + invokes it once, then throws it away. Hard to do better without + rewriting a bit of the HTTP code, as we want to be sure we're + using the current BPKI certificate and key objects. + """ + + url = "http://%s:%s/left-right" % ( + self.cfg.get("rpkid_server_host"), self.cfg.get("rpkid_server_port")) + + rpkid = self.server_ca.ee_certificates.get(purpose = "rpkid") + irbe = self.server_ca.ee_certificates.get(purpose = "irbe") + + call_rpkid = rpki.async.sync_wrapper(rpki.http.caller( + proto = rpki.left_right, + client_key = irbe.private_key, + client_cert = irbe.certificate, + server_ta = self.server_ca.certificate, + server_cert = rpkid.certificate, + url = url, + debug = self.show_xml)) + + return call_rpkid(*pdus) + + + def call_pubd(self, *pdus): + """ + Issue a call to pubd, return result. + + Implementation is a little silly, constructs a wrapper object, + invokes it once, then throws it away. Hard to do better without + rewriting a bit of the HTTP code, as we want to be sure we're + using the current BPKI certificate and key objects. + """ + + url = "http://%s:%s/control" % ( + self.cfg.get("pubd_server_host"), self.cfg.get("pubd_server_port")) + + pubd = self.server_ca.ee_certificates.get(purpose = "pubd") + irbe = self.server_ca.ee_certificates.get(purpose = "irbe") + + call_pubd = rpki.async.sync_wrapper(rpki.http.caller( + proto = rpki.publication, + client_key = irbe.private_key, + client_cert = irbe.certificate, + server_ta = self.server_ca.certificate, + server_cert = pubd.certificate, + url = url, + debug = self.show_xml)) + + return call_pubd(*pdus) + + + @django.db.transaction.commit_on_success + def synchronize(self, *handles_to_poke): + """ + Configure RPKI daemons with the data built up by the other + commands in this program. Most commands which modify the IRDB + should call this when they're done. + + Any arguments given are handles to be sent to rpkid at the end of + the synchronization run with a <self run_now="yes"/> operation. + """ + + # We can use a single BSC for everything -- except BSC key + # rollovers. Drive off that bridge when we get to it. + + bsc_handle = "bsc" + + # Default values for CRL parameters are low, for testing. Not + # quite as low as they once were, too much expired CRL whining. + + self_crl_interval = self.cfg.getint("self_crl_interval", 2 * 60 * 60) + self_regen_margin = self.cfg.getint("self_regen_margin", self_crl_interval / 4) + + # Make sure that pubd's BPKI CRL is up to date. + + if self.run_pubd: + self.call_pubd(rpki.publication.config_elt.make_pdu( + action = "set", + bpki_crl = self.server_ca.latest_crl)) + + for ca in rpki.irdb.ResourceHolderCA.objects.all(): + + # See what rpkid and pubd already have on file for this entity. + + if self.run_pubd: + pubd_reply = self.call_pubd(rpki.publication.client_elt.make_pdu(action = "list")) + client_pdus = dict((x.client_handle, x) for x in pubd_reply if isinstance(x, rpki.publication.client_elt)) + + rpkid_reply = self.call_rpkid( + rpki.left_right.self_elt.make_pdu( action = "get", tag = "self", self_handle = ca.handle), + rpki.left_right.bsc_elt.make_pdu( action = "list", tag = "bsc", self_handle = ca.handle), + rpki.left_right.repository_elt.make_pdu(action = "list", tag = "repository", self_handle = ca.handle), + rpki.left_right.parent_elt.make_pdu( action = "list", tag = "parent", self_handle = ca.handle), + rpki.left_right.child_elt.make_pdu( action = "list", tag = "child", self_handle = ca.handle)) + + self_pdu = rpkid_reply[0] + bsc_pdus = dict((x.bsc_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.bsc_elt)) + repository_pdus = dict((x.repository_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.repository_elt)) + parent_pdus = dict((x.parent_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.parent_elt)) + child_pdus = dict((x.child_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.child_elt)) + + pubd_query = [] + rpkid_query = [] + + self_cert, created = rpki.irdb.HostedCA.objects.get_or_certify( + issuer = self.server_ca, + hosted = ca) + + # There should be exactly one <self/> object per hosted entity, by definition + + if (isinstance(self_pdu, rpki.left_right.report_error_elt) or + self_pdu.crl_interval != self_crl_interval or + self_pdu.regen_margin != self_regen_margin or + self_pdu.bpki_cert != self_cert.certificate): + rpkid_query.append(rpki.left_right.self_elt.make_pdu( + action = "create" if isinstance(self_pdu, rpki.left_right.report_error_elt) else "set", + tag = "self", + self_handle = ca.handle, + bpki_cert = ca.certificate, + crl_interval = self_crl_interval, + regen_margin = self_regen_margin)) + + # In general we only need one <bsc/> per <self/>. BSC objects + # are a little unusual in that the keypair and PKCS #10 + # subelement is generated by rpkid, so complete setup requires + # two round trips. + + bsc_pdu = bsc_pdus.pop(bsc_handle, None) + + if bsc_pdu is None: + rpkid_query.append(rpki.left_right.bsc_elt.make_pdu( + action = "create", + tag = "bsc", + self_handle = ca.handle, + bsc_handle = bsc_handle, + generate_keypair = "yes")) + + elif bsc_pdu.pkcs10_request is None: + rpkid_query.append(rpki.left_right.bsc_elt.make_pdu( + action = "set", + tag = "bsc", + self_handle = ca.handle, + bsc_handle = bsc_handle, + generate_keypair = "yes")) + + rpkid_query.extend(rpki.left_right.bsc_elt.make_pdu( + action = "destroy", self_handle = ca.handle, bsc_handle = b) for b in bsc_pdus) + + # If we've already got actions queued up, run them now, so we + # can finish setting up the BSC before anything tries to use it. + + if rpkid_query: + rpkid_query.append(rpki.left_right.bsc_elt.make_pdu(action = "list", tag = "bsc", self_handle = ca.handle)) + rpkid_reply = self.call_rpkid(*rpkid_query) + bsc_pdus = dict((x.bsc_handle, x) + for x in rpkid_reply + if isinstance(x, rpki.left_right.bsc_elt) and x.action == "list") + bsc_pdu = bsc_pdus.pop(bsc_handle, None) + for r in rpkid_reply: + if isinstance(r, rpki.left_right.report_error_elt): + print "rpkid reported failure:", r.error_code + if r.error_text: + print r.error_text + if any(isinstance(r, rpki.left_right.report_error_elt) for r in rpkid_reply): + raise CouldntTalkToDaemon + + rpkid_query = [] + + assert bsc_pdu.pkcs10_request is not None + + bsc, created = rpki.irdb.BSC.objects.get_or_certify( + issuer = ca, + handle = bsc_handle, + pkcs10 = bsc_pdu.pkcs10_request) + + if bsc_pdu.signing_cert != bsc.certificate or bsc_pdu.signing_cert_crl != ca.latest_crl: + rpkid_query.append(rpki.left_right.bsc_elt.make_pdu( + action = "set", + tag = "bsc", + self_handle = ca.handle, + bsc_handle = bsc_handle, + signing_cert = bsc.certificate, + signing_cert_crl = ca.latest_crl)) + + # At present we need one <repository/> per <parent/>, not because + # rpkid requires that, but because pubd does. pubd probably should + # be fixed to support a single client allowed to update multiple + # trees, but for the moment the easiest way forward is just to + # enforce a 1:1 mapping between <parent/> and <repository/> objects + + for repository in ca.repositories.all(): + + repository_pdu = repository_pdus.pop(repository.handle, None) + + if (repository_pdu is None or + repository_pdu.bsc_handle != bsc_handle or + repository_pdu.peer_contact_uri != repository.service_uri or + repository_pdu.bpki_cert != repository.certificate): + rpkid_query.append(rpki.left_right.repository_elt.make_pdu( + action = "create" if repository_pdu is None else "set", + tag = repository.handle, + self_handle = ca.handle, + repository_handle = repository.handle, + bsc_handle = bsc_handle, + peer_contact_uri = repository.service_uri, + bpki_cert = repository.certificate)) + + rpkid_query.extend(rpki.left_right.repository_elt.make_pdu( + action = "destroy", self_handle = ca.handle, repository_handle = r) for r in repository_pdus) + + # <parent/> setup code currently assumes 1:1 mapping between + # <repository/> and <parent/>, and further assumes that the handles + # for an associated pair are the identical (that is: + # parent.repository_handle == parent.parent_handle). + + for parent in ca.parents.all(): + + parent_pdu = parent_pdus.pop(parent.handle, None) + + if (parent_pdu is None or + parent_pdu.bsc_handle != bsc_handle or + parent_pdu.repository_handle != parent.handle or + parent_pdu.peer_contact_uri != parent.service_uri or + parent_pdu.sia_base != parent.repository.sia_base or + parent_pdu.sender_name != parent.child_handle or + parent_pdu.recipient_name != parent.parent_handle or + parent_pdu.bpki_cms_cert != parent.certificate): + rpkid_query.append(rpki.left_right.parent_elt.make_pdu( + action = "create" if parent_pdu is None else "set", + tag = parent.handle, + self_handle = ca.handle, + parent_handle = parent.handle, + bsc_handle = bsc_handle, + repository_handle = parent.handle, + peer_contact_uri = parent.service_uri, + sia_base = parent.repository.sia_base, + sender_name = parent.child_handle, + recipient_name = parent.parent_handle, + bpki_cms_cert = parent.certificate)) + + try: + + parent_pdu = parent_pdus.pop(ca.handle, None) + + if (parent_pdu is None or + parent_pdu.bsc_handle != bsc_handle or + parent_pdu.repository_handle != ca.handle or + parent_pdu.peer_contact_uri != ca.rootd.service_uri or + parent_pdu.sia_base != ca.rootd.repository.sia_base or + parent_pdu.sender_name != ca.handle or + parent_pdu.recipient_name != ca.handle or + parent_pdu.bpki_cms_cert != ca.rootd.certificate): + rpkid_query.append(rpki.left_right.parent_elt.make_pdu( + action = "create" if parent_pdu is None else "set", + tag = ca.handle, + self_handle = ca.handle, + parent_handle = ca.handle, + bsc_handle = bsc_handle, + repository_handle = ca.handle, + peer_contact_uri = ca.rootd.service_uri, + sia_base = ca.rootd.repository.sia_base, + sender_name = ca.handle, + recipient_name = ca.handle, + bpki_cms_cert = ca.rootd.certificate)) + + except rpki.irdb.Rootd.DoesNotExist: + pass + + rpkid_query.extend(rpki.left_right.parent_elt.make_pdu( + action = "destroy", self_handle = ca.handle, parent_handle = p) for p in parent_pdus) + + # Children are simpler than parents, because they call us, so no URL + # to construct and figuring out what certificate to use is their + # problem, not ours. + + for child in ca.children.all(): + + child_pdu = child_pdus.pop(child.handle, None) + + if (child_pdu is None or + child_pdu.bsc_handle != bsc_handle or + child_pdu.bpki_cert != child.certificate): + rpkid_query.append(rpki.left_right.child_elt.make_pdu( + action = "create" if child_pdu is None else "set", + tag = child.handle, + self_handle = ca.handle, + child_handle = child.handle, + bsc_handle = bsc_handle, + bpki_cert = child.certificate)) + + rpkid_query.extend(rpki.left_right.child_elt.make_pdu( + action = "destroy", self_handle = ca.handle, child_handle = c) for c in child_pdus) + + # Publication setup. + + # Um, why are we doing this per resource holder? + + if self.run_pubd: + + for client in self.server_ca.clients.all(): + + client_pdu = client_pdus.pop(client.handle, None) + + if (client_pdu is None or + client_pdu.base_uri != client.sia_base or + client_pdu.bpki_cert != client.certificate): + pubd_query.append(rpki.publication.client_elt.make_pdu( + action = "create" if client_pdu is None else "set", + client_handle = client.handle, + bpki_cert = client.certificate, + base_uri = client.sia_base)) + + pubd_query.extend(rpki.publication.client_elt.make_pdu( + action = "destroy", client_handle = p) for p in client_pdus) + + # Poke rpkid to run immediately for any requested handles. + + rpkid_query.extend(rpki.left_right.self_elt.make_pdu( + action = "set", self_handle = h, run_now = "yes") for h in handles_to_poke) + + # If we changed anything, ship updates off to daemons + + if rpkid_query: + rpkid_reply = self.call_rpkid(*rpkid_query) + bsc_pdus = dict((x.bsc_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.bsc_elt)) + if bsc_handle in bsc_pdus and bsc_pdus[bsc_handle].pkcs10_request: + bsc_req = bsc_pdus[bsc_handle].pkcs10_request + for r in rpkid_reply: + if isinstance(r, rpki.left_right.report_error_elt): + print "rpkid reported failure:", r.error_code + if r.error_text: + print r.error_text + if any(isinstance(r, rpki.left_right.report_error_elt) for r in rpkid_reply): + raise CouldntTalkToDaemon + + if pubd_query: + assert self.run_pubd + pubd_reply = self.call_pubd(*pubd_query) + for r in pubd_reply: + if isinstance(r, rpki.publication.report_error_elt): + print "pubd reported failure:", r.error_code + if r.error_text: + print r.error_text + if any(isinstance(r, rpki.publication.report_error_elt) for r in pubd_reply): + raise CouldntTalkToDaemon diff --git a/rpkid/rpki/irdbd.py b/rpkid/rpki/irdbd.py index c2e01287..60e122c7 100644 --- a/rpkid/rpki/irdbd.py +++ b/rpkid/rpki/irdbd.py @@ -5,7 +5,7 @@ Usage: python irdbd.py [ { -c | --config } configfile ] [ { -h | --help } ] $Id$ -Copyright (C) 2009--2011 Internet Systems Consortium ("ISC") +Copyright (C) 2009--2012 Internet Systems Consortium ("ISC") Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above @@ -38,160 +38,92 @@ import sys, os, time, getopt, urlparse, warnings import rpki.http, rpki.config, rpki.resource_set, rpki.relaxng import rpki.exceptions, rpki.left_right, rpki.log, rpki.x509 -from rpki.mysql_import import MySQLdb - class main(object): - def handle_list_resources(self, q_pdu, r_msg): - + child = rpki.irdb.Child.objects.get(issuer__handle__exact = q_pdu.self_handle, handle = q_pdu.child_handle) r_pdu = rpki.left_right.list_resources_elt() r_pdu.tag = q_pdu.tag r_pdu.self_handle = q_pdu.self_handle r_pdu.child_handle = q_pdu.child_handle - - self.cur.execute( - "SELECT registrant_id, valid_until FROM registrant WHERE registry_handle = %s AND registrant_handle = %s", - (q_pdu.self_handle, q_pdu.child_handle)) - - if self.cur.rowcount != 1: - raise rpki.exceptions.NotInDatabase, \ - "This query should have produced a single exact match, something's messed up (rowcount = %d, self_handle = %s, child_handle = %s)" \ - % (self.cur.rowcount, q_pdu.self_handle, q_pdu.child_handle) - - registrant_id, valid_until = self.cur.fetchone() - - r_pdu.valid_until = valid_until.strftime("%Y-%m-%dT%H:%M:%SZ") - - r_pdu.asn = rpki.resource_set.resource_set_as.from_sql( - self.cur, - "SELECT start_as, end_as FROM registrant_asn WHERE registrant_id = %s", - (registrant_id,)) - - r_pdu.ipv4 = rpki.resource_set.resource_set_ipv4.from_sql( - self.cur, - "SELECT start_ip, end_ip FROM registrant_net WHERE registrant_id = %s AND version = 4", - (registrant_id,)) - - r_pdu.ipv6 = rpki.resource_set.resource_set_ipv6.from_sql( - self.cur, - "SELECT start_ip, end_ip FROM registrant_net WHERE registrant_id = %s AND version = 6", - (registrant_id,)) - + r_pdu.valid_until = child.valid_until.strftime("%Y-%m-%dT%H:%M:%SZ") + r_pdu.asn = rpki.resource_set.resource_set_as.from_django( + (a.start_as, a.end_as) for a in child.asns.all()) + r_pdu.ipv4 = rpki.resource_set.resource_set_ipv4.from_django( + (a.start_ip, a.end_ip) for a in child.address_ranges.filter(version = 4)) + r_pdu.ipv6 = rpki.resource_set.resource_set_ipv6.from_django( + (a.start_ip, a.end_ip) for a in child.address_ranges.filter(version = 6)) r_msg.append(r_pdu) - def handle_list_roa_requests(self, q_pdu, r_msg): - - self.cur.execute( - "SELECT roa_request_id, asn FROM roa_request WHERE roa_request_handle = %s", - (q_pdu.self_handle,)) - - for roa_request_id, asn in self.cur.fetchall(): - + for request in rpki.irdb.ROARequest.objects.filter(issuer__handle__exact = q_pdu.self_handle): r_pdu = rpki.left_right.list_roa_requests_elt() r_pdu.tag = q_pdu.tag r_pdu.self_handle = q_pdu.self_handle - r_pdu.asn = asn - - r_pdu.ipv4 = rpki.resource_set.roa_prefix_set_ipv4.from_sql( - self.cur, - "SELECT prefix, prefixlen, max_prefixlen FROM roa_request_prefix WHERE roa_request_id = %s AND version = 4", - (roa_request_id,)) - - r_pdu.ipv6 = rpki.resource_set.roa_prefix_set_ipv6.from_sql( - self.cur, - "SELECT prefix, prefixlen, max_prefixlen FROM roa_request_prefix WHERE roa_request_id = %s AND version = 6", - (roa_request_id,)) - + r_pdu.asn = request.asn + r_pdu.ipv4 = rpki.resource_set.roa_prefix_set_ipv4.from_django( + (p.prefix, p.prefixlen, p.max_prefixlen) for p in request.prefixes.filter(version = 4)) + r_pdu.ipv6 = rpki.resource_set.roa_prefix_set_ipv6.from_django( + (p.prefix, p.prefixlen, p.max_prefixlen) for p in request.prefixes.filter(version = 6)) r_msg.append(r_pdu) - def handle_list_ghostbuster_requests(self, q_pdu, r_msg): - - self.cur.execute( - "SELECT vcard FROM ghostbuster_request WHERE self_handle = %s AND parent_handle = %s", - (q_pdu.self_handle, q_pdu.parent_handle)) - - vcards = [result[0] for result in self.cur.fetchall()] - - if not vcards: - - self.cur.execute( - "SELECT vcard FROM ghostbuster_request WHERE self_handle = %s AND parent_handle IS NULL", - (q_pdu.self_handle,)) - - vcards = [result[0] for result in self.cur.fetchall()] - - for vcard in vcards: + ghostbusters = rpki.irdb.GhostbusterRequest.objects.filter( + issuer__handle__exact = q_pdu.self_handle, + parent__handle__exact = q_pdu.parent_handle) + if ghostbusters.count() == 0: + ghostbusters = rpki.irdb.GhostbusterRequest.objects.filter( + issuer__handle__exact = q_pdu.self_handle, + parent = None) + for ghostbuster in ghostbusters: r_pdu = rpki.left_right.list_ghostbuster_requests_elt() r_pdu.tag = q_pdu.tag r_pdu.self_handle = q_pdu.self_handle r_pdu.parent_handle = q_pdu.parent_handle - r_pdu.vcard = vcard + r_pdu.vcard = ghostbuster.vcard r_msg.append(r_pdu) - - handle_dispatch = { - rpki.left_right.list_resources_elt : handle_list_resources, - rpki.left_right.list_roa_requests_elt : handle_list_roa_requests, - rpki.left_right.list_ghostbuster_requests_elt : handle_list_ghostbuster_requests} - - def handler(self, query, path, cb): try: - - self.db.ping(True) - + q_pdu = None r_msg = rpki.left_right.msg.reply() - + self.start_new_transaction() + serverCA = rpki.irdb.ServerCA.objects.get() + rpkid = serverCA.ee_certificates.get(purpose = "rpkid") try: - - q_msg = rpki.left_right.cms_msg(DER = query).unwrap((self.bpki_ta, self.rpkid_cert)) - + q_msg = rpki.left_right.cms_msg(DER = query).unwrap((serverCA.certificate, rpkid.certificate)) if not isinstance(q_msg, rpki.left_right.msg) or not q_msg.is_query(): - raise rpki.exceptions.BadQuery, "Unexpected %r PDU" % q_msg - + raise rpki.exceptions.BadQuery("Unexpected %r PDU" % q_msg) for q_pdu in q_msg: - - try: - - try: - h = self.handle_dispatch[type(q_pdu)] - except KeyError: - raise rpki.exceptions.BadQuery, "Unexpected %r PDU" % q_pdu - else: - h(self, q_pdu, r_msg) - - except (rpki.async.ExitNow, SystemExit): - raise - - except Exception, e: - rpki.log.traceback() - r_msg.append(rpki.left_right.report_error_elt.from_exception(e, q_pdu.self_handle, q_pdu.tag)) - + self.dispatch(q_pdu, r_msg) except (rpki.async.ExitNow, SystemExit): raise - except Exception, e: rpki.log.traceback() - r_msg.append(rpki.left_right.report_error_elt.from_exception(e)) - - cb(200, body = rpki.left_right.cms_msg().wrap(r_msg, self.irdbd_key, self.irdbd_cert)) - + if q_pdu is None: + r_msg.append(rpki.left_right.report_error_elt.from_exception(e)) + else: + r_msg.append(rpki.left_right.report_error_elt.from_exception(e, q_pdu.self_handle, q_pdu.tag)) + irdbd = serverCA.ee_certificates.get(purpose = "irdbd") + cb(200, body = rpki.left_right.cms_msg().wrap(r_msg, irdbd.private_key, irdbd.certificate)) except (rpki.async.ExitNow, SystemExit): raise - except Exception, e: rpki.log.traceback() - - # We only get here in cases where we couldn't or wouldn't generate - # <report_error/>, so just return HTTP failure. - cb(500, reason = "Unhandled exception %s: %s" % (e.__class__.__name__, e)) + def dispatch(self, q_pdu, r_msg): + try: + handler = self.dispatch_vector[type(q_pdu)] + except KeyError: + raise rpki.exceptions.BadQuery("Unexpected %r PDU" % q_pdu) + else: + handler(q_pdu, r_msg) + + def __init__(self, **kwargs): - def __init__(self): + global rpki + from django.conf import settings os.environ["TZ"] = "UTC" time.tzset() @@ -208,31 +140,60 @@ class main(object): elif o in ("-d", "--debug"): rpki.log.use_syslog = False if argv: - raise rpki.exceptions.CommandParseFailure, "Unexpected arguments %s" % argv + raise rpki.exceptions.CommandParseFailure("Unexpected arguments %s" % argv) rpki.log.init("irdbd") - self.cfg = rpki.config.parser(cfg_file, "irdbd") + cfg = rpki.config.parser(cfg_file, "irdbd") - startup_msg = self.cfg.get("startup-message", "") + startup_msg = cfg.get("startup-message", "") if startup_msg: rpki.log.info(startup_msg) - self.cfg.set_global_flags() - - self.db = MySQLdb.connect(user = self.cfg.get("sql-username"), - db = self.cfg.get("sql-database"), - passwd = self.cfg.get("sql-password")) - - self.cur = self.db.cursor() - self.db.autocommit(True) - - self.bpki_ta = rpki.x509.X509(Auto_update = self.cfg.get("bpki-ta")) - self.rpkid_cert = rpki.x509.X509(Auto_update = self.cfg.get("rpkid-cert")) - self.irdbd_cert = rpki.x509.X509(Auto_update = self.cfg.get("irdbd-cert")) - self.irdbd_key = rpki.x509.RSA( Auto_update = self.cfg.get("irdbd-key")) - - u = urlparse.urlparse(self.cfg.get("http-url")) + cfg.set_global_flags() + + settings.configure( + DEBUG = True, + DATABASES = { + "default" : { + "ENGINE" : "django.db.backends.mysql", + "NAME" : cfg.get("sql-database"), + "USER" : cfg.get("sql-username"), + "PASSWORD" : cfg.get("sql-password"), + "HOST" : "", + "PORT" : "" }}, + INSTALLED_APPS = ("rpki.irdb",),) + + import rpki.irdb + + # Entirely too much fun with read-only access to transactional databases. + # + # http://stackoverflow.com/questions/3346124/how-do-i-force-django-to-ignore-any-caches-and-reload-data + # http://devblog.resolversystems.com/?p=439 + # http://groups.google.com/group/django-users/browse_thread/thread/e25cec400598c06d + # http://stackoverflow.com/questions/1028671/python-mysqldb-update-query-fails + # http://dev.mysql.com/doc/refman/5.0/en/set-transaction.html + # + # It turns out that MySQL is doing us a favor with this weird + # transactional behavior on read, because without it there's a + # race condition if multiple updates are committed to the IRDB + # while we're in the middle of processing a query. Note that + # proper transaction management by the committers doesn't protect + # us, this is a transactional problem on read. So we need to use + # explicit transaction management. Since irdbd is a read-only + # consumer of IRDB data, this means we need to commit an empty + # transaction at the beginning of processing each query, to reset + # the transaction isolation snapshot. + + import django.db.transaction + self.start_new_transaction = django.db.transaction.commit_manually(django.db.transaction.commit) + + self.dispatch_vector = { + rpki.left_right.list_resources_elt : self.handle_list_resources, + rpki.left_right.list_roa_requests_elt : self.handle_list_roa_requests, + rpki.left_right.list_ghostbuster_requests_elt : self.handle_list_ghostbuster_requests } + + u = urlparse.urlparse(cfg.get("http-url")) assert u.scheme in ("", "http") and \ u.username is None and \ @@ -241,6 +202,7 @@ class main(object): u.query == "" and \ u.fragment == "" - rpki.http.server(host = u.hostname or "localhost", - port = u.port or 443, - handlers = ((u.path, self.handler),)) + rpki.http.server( + host = u.hostname or "localhost", + port = u.port or 443, + handlers = ((u.path, self.handler),)) diff --git a/rpkid/rpki/left_right.py b/rpkid/rpki/left_right.py index 72412cbe..ac480ff0 100644 --- a/rpkid/rpki/left_right.py +++ b/rpkid/rpki/left_right.py @@ -394,7 +394,7 @@ class self_elt(data_elt): def list_failed(e): rpki.log.traceback() - rpki.log.warn("Couldn't get resource class list from parent %r, skipping: %s" % (parent, e)) + rpki.log.warn("Couldn't get resource class list from parent %r, skipping: %s (%r)" % (parent, e, e)) parent_iterator() rpki.up_down.list_pdu.query(parent, got_list, list_failed) diff --git a/rpkid/rpki/myrpki.py b/rpkid/rpki/myrpki.py index 2fa2f8cb..ec36371c 100644 --- a/rpkid/rpki/myrpki.py +++ b/rpkid/rpki/myrpki.py @@ -793,9 +793,17 @@ class CA(object): Write PEM certificate to file, then cross-certify. """ fn = os.path.join(self.dir, filename or "temp.%s.cer" % os.getpid()) + der = base64.b64decode(b64) + if True: + try: + text = self.run_openssl("x509", "-inform", "DER", "-noout", + "-issuer", "-subject", stdin = der) + except: + text = "" + print "fxcert():", self.dir, filename, text try: self.run_openssl("x509", "-inform", "DER", "-out", fn, - stdin = base64.b64decode(b64)) + stdin = der) return self.xcert(fn, path_restriction) finally: if not filename and os.path.exists(fn): diff --git a/rpkid/rpki/old_irdbd.py b/rpkid/rpki/old_irdbd.py new file mode 100644 index 00000000..c63ce9e2 --- /dev/null +++ b/rpkid/rpki/old_irdbd.py @@ -0,0 +1,249 @@ +""" +IR database daemon. + +Usage: python irdbd.py [ { -c | --config } configfile ] [ { -h | --help } ] + +This is the old (pre-Django) version of irdbd, still used by smoketest +and perhaps still useful as a minimal example. + +$Id$ + +Copyright (C) 2009--2012 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + +Portions copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import sys, os, time, getopt, urlparse, warnings +import rpki.http, rpki.config, rpki.resource_set, rpki.relaxng +import rpki.exceptions, rpki.left_right, rpki.log, rpki.x509 + +from rpki.mysql_import import MySQLdb + +class main(object): + + + def handle_list_resources(self, q_pdu, r_msg): + + r_pdu = rpki.left_right.list_resources_elt() + r_pdu.tag = q_pdu.tag + r_pdu.self_handle = q_pdu.self_handle + r_pdu.child_handle = q_pdu.child_handle + + self.cur.execute( + "SELECT registrant_id, valid_until FROM registrant WHERE registry_handle = %s AND registrant_handle = %s", + (q_pdu.self_handle, q_pdu.child_handle)) + + if self.cur.rowcount != 1: + raise rpki.exceptions.NotInDatabase, \ + "This query should have produced a single exact match, something's messed up (rowcount = %d, self_handle = %s, child_handle = %s)" \ + % (self.cur.rowcount, q_pdu.self_handle, q_pdu.child_handle) + + registrant_id, valid_until = self.cur.fetchone() + + r_pdu.valid_until = valid_until.strftime("%Y-%m-%dT%H:%M:%SZ") + + r_pdu.asn = rpki.resource_set.resource_set_as.from_sql( + self.cur, + "SELECT start_as, end_as FROM registrant_asn WHERE registrant_id = %s", + (registrant_id,)) + + r_pdu.ipv4 = rpki.resource_set.resource_set_ipv4.from_sql( + self.cur, + "SELECT start_ip, end_ip FROM registrant_net WHERE registrant_id = %s AND version = 4", + (registrant_id,)) + + r_pdu.ipv6 = rpki.resource_set.resource_set_ipv6.from_sql( + self.cur, + "SELECT start_ip, end_ip FROM registrant_net WHERE registrant_id = %s AND version = 6", + (registrant_id,)) + + r_msg.append(r_pdu) + + + def handle_list_roa_requests(self, q_pdu, r_msg): + + self.cur.execute( + "SELECT roa_request_id, asn FROM roa_request WHERE roa_request_handle = %s", + (q_pdu.self_handle,)) + + for roa_request_id, asn in self.cur.fetchall(): + + r_pdu = rpki.left_right.list_roa_requests_elt() + r_pdu.tag = q_pdu.tag + r_pdu.self_handle = q_pdu.self_handle + r_pdu.asn = asn + + r_pdu.ipv4 = rpki.resource_set.roa_prefix_set_ipv4.from_sql( + self.cur, + "SELECT prefix, prefixlen, max_prefixlen FROM roa_request_prefix WHERE roa_request_id = %s AND version = 4", + (roa_request_id,)) + + r_pdu.ipv6 = rpki.resource_set.roa_prefix_set_ipv6.from_sql( + self.cur, + "SELECT prefix, prefixlen, max_prefixlen FROM roa_request_prefix WHERE roa_request_id = %s AND version = 6", + (roa_request_id,)) + + r_msg.append(r_pdu) + + + def handle_list_ghostbuster_requests(self, q_pdu, r_msg): + + self.cur.execute( + "SELECT vcard FROM ghostbuster_request WHERE self_handle = %s AND parent_handle = %s", + (q_pdu.self_handle, q_pdu.parent_handle)) + + vcards = [result[0] for result in self.cur.fetchall()] + + if not vcards: + + self.cur.execute( + "SELECT vcard FROM ghostbuster_request WHERE self_handle = %s AND parent_handle IS NULL", + (q_pdu.self_handle,)) + + vcards = [result[0] for result in self.cur.fetchall()] + + for vcard in vcards: + r_pdu = rpki.left_right.list_ghostbuster_requests_elt() + r_pdu.tag = q_pdu.tag + r_pdu.self_handle = q_pdu.self_handle + r_pdu.parent_handle = q_pdu.parent_handle + r_pdu.vcard = vcard + r_msg.append(r_pdu) + + + handle_dispatch = { + rpki.left_right.list_resources_elt : handle_list_resources, + rpki.left_right.list_roa_requests_elt : handle_list_roa_requests, + rpki.left_right.list_ghostbuster_requests_elt : handle_list_ghostbuster_requests} + + + def handler(self, query, path, cb): + try: + + self.db.ping(True) + + r_msg = rpki.left_right.msg.reply() + + try: + + q_msg = rpki.left_right.cms_msg(DER = query).unwrap((self.bpki_ta, self.rpkid_cert)) + + if not isinstance(q_msg, rpki.left_right.msg) or not q_msg.is_query(): + raise rpki.exceptions.BadQuery, "Unexpected %r PDU" % q_msg + + for q_pdu in q_msg: + + try: + + try: + h = self.handle_dispatch[type(q_pdu)] + except KeyError: + raise rpki.exceptions.BadQuery, "Unexpected %r PDU" % q_pdu + else: + h(self, q_pdu, r_msg) + + except (rpki.async.ExitNow, SystemExit): + raise + + except Exception, e: + rpki.log.traceback() + r_msg.append(rpki.left_right.report_error_elt.from_exception(e, q_pdu.self_handle, q_pdu.tag)) + + except (rpki.async.ExitNow, SystemExit): + raise + + except Exception, e: + rpki.log.traceback() + r_msg.append(rpki.left_right.report_error_elt.from_exception(e)) + + cb(200, body = rpki.left_right.cms_msg().wrap(r_msg, self.irdbd_key, self.irdbd_cert)) + + except (rpki.async.ExitNow, SystemExit): + raise + + except Exception, e: + rpki.log.traceback() + + # We only get here in cases where we couldn't or wouldn't generate + # <report_error/>, so just return HTTP failure. + + cb(500, reason = "Unhandled exception %s: %s" % (e.__class__.__name__, e)) + + + def __init__(self): + + os.environ["TZ"] = "UTC" + time.tzset() + + cfg_file = None + + opts, argv = getopt.getopt(sys.argv[1:], "c:dh?", ["config=", "debug", "help"]) + for o, a in opts: + if o in ("-h", "--help", "-?"): + print __doc__ + sys.exit(0) + if o in ("-c", "--config"): + cfg_file = a + elif o in ("-d", "--debug"): + rpki.log.use_syslog = False + if argv: + raise rpki.exceptions.CommandParseFailure, "Unexpected arguments %s" % argv + + rpki.log.init("irdbd") + + self.cfg = rpki.config.parser(cfg_file, "irdbd") + + startup_msg = self.cfg.get("startup-message", "") + if startup_msg: + rpki.log.info(startup_msg) + + self.cfg.set_global_flags() + + self.db = MySQLdb.connect(user = self.cfg.get("sql-username"), + db = self.cfg.get("sql-database"), + passwd = self.cfg.get("sql-password")) + + self.cur = self.db.cursor() + self.db.autocommit(True) + + self.bpki_ta = rpki.x509.X509(Auto_update = self.cfg.get("bpki-ta")) + self.rpkid_cert = rpki.x509.X509(Auto_update = self.cfg.get("rpkid-cert")) + self.irdbd_cert = rpki.x509.X509(Auto_update = self.cfg.get("irdbd-cert")) + self.irdbd_key = rpki.x509.RSA( Auto_update = self.cfg.get("irdbd-key")) + + u = urlparse.urlparse(self.cfg.get("http-url")) + + assert u.scheme in ("", "http") and \ + u.username is None and \ + u.password is None and \ + u.params == "" and \ + u.query == "" and \ + u.fragment == "" + + rpki.http.server(host = u.hostname or "localhost", + port = u.port or 443, + handlers = ((u.path, self.handler),)) diff --git a/rpkid/rpki/pubd.py b/rpkid/rpki/pubd.py index bde1260e..6968780d 100644 --- a/rpkid/rpki/pubd.py +++ b/rpkid/rpki/pubd.py @@ -134,7 +134,7 @@ class main(object): raise except Exception, e: rpki.log.traceback() - cb(500, reason = "Unhandled exception %s" % e) + cb(500, reason = "Unhandled exception %s: %s" % (e.__class__.__name__, e)) client_url_regexp = re.compile("/client/([-A-Z0-9_/]+)$", re.I) diff --git a/rpkid/rpki/relaxng.py b/rpkid/rpki/relaxng.py index e1ea8f6b..24b3ab75 100644 --- a/rpkid/rpki/relaxng.py +++ b/rpkid/rpki/relaxng.py @@ -1839,3 +1839,382 @@ publication = lxml.etree.RelaxNG(lxml.etree.fromstring('''<?xml version="1.0" en --> ''')) +## @var myrpki +## Parsed RelaxNG myrpki schema +myrpki = lxml.etree.RelaxNG(lxml.etree.fromstring('''<?xml version="1.0" encoding="UTF-8"?> +<!-- + $Id: myrpki.rnc 3723 2011-03-14 20:43:16Z sra $ + + RelaxNG Schema for MyRPKI XML messages. + + This message protocol is on its way out, as we're in the process of + moving on from the user interface model that produced it, but even + after we finish replacing it we'll still need the schema for a while + to validate old messages when upgrading. + + libxml2 (including xmllint) only groks the XML syntax of RelaxNG, so + run the compact syntax through trang to get XML syntax. + + Copyright (C) 2009-2011 Internet Systems Consortium ("ISC") + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH + REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, + INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + PERFORMANCE OF THIS SOFTWARE. +--> +<grammar ns="http://www.hactrn.net/uris/rpki/myrpki/" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes"> + <define name="version"> + <value>2</value> + </define> + <define name="base64"> + <data type="base64Binary"> + <param name="maxLength">512000</param> + </data> + </define> + <define name="object_handle"> + <data type="string"> + <param name="maxLength">255</param> + <param name="pattern">[\-_A-Za-z0-9]*</param> + </data> + </define> + <define name="pubd_handle"> + <data type="string"> + <param name="maxLength">255</param> + <param name="pattern">[\-_A-Za-z0-9/]*</param> + </data> + </define> + <define name="uri"> + <data type="anyURI"> + <param name="maxLength">4096</param> + </data> + </define> + <define name="asn"> + <data type="positiveInteger"/> + </define> + <define name="asn_list"> + <data type="string"> + <param name="maxLength">512000</param> + <param name="pattern">[\-,0-9]*</param> + </data> + </define> + <define name="ipv4_list"> + <data type="string"> + <param name="maxLength">512000</param> + <param name="pattern">[\-,0-9/.]*</param> + </data> + </define> + <define name="ipv6_list"> + <data type="string"> + <param name="maxLength">512000</param> + <param name="pattern">[\-,0-9/:a-fA-F]*</param> + </data> + </define> + <define name="timestamp"> + <data type="dateTime"> + <param name="pattern">.*Z</param> + </data> + </define> + <!-- + Message formate used between configure_resources and + configure_daemons. + --> + <start combine="choice"> + <element name="myrpki"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <optional> + <attribute name="service_uri"> + <ref name="uri"/> + </attribute> + </optional> + <zeroOrMore> + <element name="roa_request"> + <attribute name="asn"> + <ref name="asn"/> + </attribute> + <attribute name="v4"> + <ref name="ipv4_list"/> + </attribute> + <attribute name="v6"> + <ref name="ipv6_list"/> + </attribute> + </element> + </zeroOrMore> + <zeroOrMore> + <element name="child"> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <attribute name="valid_until"> + <ref name="timestamp"/> + </attribute> + <optional> + <attribute name="asns"> + <ref name="asn_list"/> + </attribute> + </optional> + <optional> + <attribute name="v4"> + <ref name="ipv4_list"/> + </attribute> + </optional> + <optional> + <attribute name="v6"> + <ref name="ipv6_list"/> + </attribute> + </optional> + <optional> + <element name="bpki_certificate"> + <ref name="base64"/> + </element> + </optional> + </element> + </zeroOrMore> + <zeroOrMore> + <element name="parent"> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <optional> + <attribute name="service_uri"> + <ref name="uri"/> + </attribute> + </optional> + <optional> + <attribute name="myhandle"> + <ref name="object_handle"/> + </attribute> + </optional> + <optional> + <attribute name="sia_base"> + <ref name="uri"/> + </attribute> + </optional> + <optional> + <element name="bpki_cms_certificate"> + <ref name="base64"/> + </element> + </optional> + </element> + </zeroOrMore> + <zeroOrMore> + <element name="repository"> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <optional> + <attribute name="service_uri"> + <ref name="uri"/> + </attribute> + </optional> + <optional> + <element name="bpki_certificate"> + <ref name="base64"/> + </element> + </optional> + </element> + </zeroOrMore> + <optional> + <element name="bpki_ca_certificate"> + <ref name="base64"/> + </element> + </optional> + <optional> + <element name="bpki_crl"> + <ref name="base64"/> + </element> + </optional> + <optional> + <element name="bpki_bsc_certificate"> + <ref name="base64"/> + </element> + </optional> + <optional> + <element name="bpki_bsc_pkcs10"> + <ref name="base64"/> + </element> + </optional> + </element> + </start> + <!-- Format of an identity.xml file. --> + <start combine="choice"> + <element name="identity"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <element name="bpki_ta"> + <ref name="base64"/> + </element> + </element> + </start> + <!-- + Format of <authorization/> element used in referrals. The Base64 + text is a <referral/> (q. v.) element signed with CMS. + --> + <define name="authorization"> + <element name="authorization"> + <attribute name="referrer"> + <ref name="pubd_handle"/> + </attribute> + <ref name="base64"/> + </element> + </define> + <!-- Format of <contact_info/> element used in referrals. --> + <define name="contact_info"> + <element name="contact_info"> + <optional> + <attribute name="uri"> + <ref name="uri"/> + </attribute> + </optional> + <data type="string"/> + </element> + </define> + <!-- Variant payload portion of a <repository/> element. --> + <define name="repository_payload"> + <choice> + <attribute name="type"> + <value>none</value> + </attribute> + <attribute name="type"> + <value>offer</value> + </attribute> + <group> + <attribute name="type"> + <value>referral</value> + </attribute> + <ref name="authorization"/> + <ref name="contact_info"/> + </group> + </choice> + </define> + <!-- <parent/> element (response from configure_child). --> + <start combine="choice"> + <element name="parent"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="valid_until"> + <ref name="timestamp"/> + </attribute> + <optional> + <attribute name="service_uri"> + <ref name="uri"/> + </attribute> + </optional> + <attribute name="child_handle"> + <ref name="object_handle"/> + </attribute> + <attribute name="parent_handle"> + <ref name="object_handle"/> + </attribute> + <element name="bpki_resource_ta"> + <ref name="base64"/> + </element> + <element name="bpki_child_ta"> + <ref name="base64"/> + </element> + <optional> + <element name="repository"> + <ref name="repository_payload"/> + </element> + </optional> + </element> + </start> + <!-- + <repository/> element, types offer and referral + (input to configure_publication_client). + --> + <start combine="choice"> + <element name="repository"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <attribute name="parent_handle"> + <ref name="object_handle"/> + </attribute> + <ref name="repository_payload"/> + <element name="bpki_client_ta"> + <ref name="base64"/> + </element> + </element> + </start> + <!-- + <repository/> element, confirmation type (output of + configure_publication_client). + --> + <start combine="choice"> + <element name="repository"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="type"> + <value>confirmed</value> + </attribute> + <attribute name="parent_handle"> + <ref name="object_handle"/> + </attribute> + <attribute name="client_handle"> + <ref name="pubd_handle"/> + </attribute> + <attribute name="service_uri"> + <ref name="uri"/> + </attribute> + <attribute name="sia_base"> + <ref name="uri"/> + </attribute> + <element name="bpki_server_ta"> + <ref name="base64"/> + </element> + <element name="bpki_client_ta"> + <ref name="base64"/> + </element> + <optional> + <ref name="authorization"/> + </optional> + <optional> + <ref name="contact_info"/> + </optional> + </element> + </start> + <!-- + <referral/> element. This is the entirety of a separate message + which is signed with CMS then included ase the Base64 content of an + <authorization/> element in the main message. + --> + <start combine="choice"> + <element name="referral"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="authorized_sia_base"> + <ref name="uri"/> + </attribute> + <ref name="base64"/> + </element> + </start> +</grammar> +<!-- + Local Variables: + indent-tabs-mode: nil + End: +--> +''')) + diff --git a/rpkid/rpki/resource_set.py b/rpkid/rpki/resource_set.py index 2fd10756..06b2ebc7 100644 --- a/rpkid/rpki/resource_set.py +++ b/rpkid/rpki/resource_set.py @@ -500,6 +500,18 @@ class resource_set(list): for (b, e) in sql.fetchall()]) @classmethod + def from_django(cls, iterable): + """ + Create resource set from a Django query. + + iterable is something which returns (min, max) pairs. + """ + + return cls(ini = [cls.range_type(cls.range_type.datum_type(b), + cls.range_type.datum_type(e)) + for (b, e) in iterable]) + + @classmethod def parse_str(cls, s): """ Parse resource set from text string (eg, XML attributes). This is @@ -983,6 +995,19 @@ class roa_prefix_set(list): return cls([cls.prefix_type(cls.prefix_type.range_type.datum_type(x), int(y), int(z)) for (x, y, z) in sql.fetchall()]) + @classmethod + def from_django(cls, iterable): + """ + Create ROA prefix set from a Django query. + + iterable is something which returns (prefix, prefixlen, + max_prefixlen) triples. + """ + + return cls([cls.prefix_type(cls.prefix_type.range_type.datum_type(x), int(y), int(z)) + for (x, y, z) in iterable]) + + def to_roa_tuple(self): """ Convert ROA prefix set into tuple format used by ROA ASN.1 diff --git a/rpkid/rpki/rootd.py b/rpkid/rpki/rootd.py index b289c3e8..7c569f7b 100644 --- a/rpkid/rpki/rootd.py +++ b/rpkid/rpki/rootd.py @@ -301,7 +301,7 @@ class main(object): self.rpki_root_dir = self.cfg.get("rpki-root-dir") self.rpki_base_uri = self.cfg.get("rpki-base-uri", "rsync://" + self.rpki_class_name + ".invalid/") - self.rpki_root_key = rpki.x509.RSA( Auto_file = self.cfg.get("rpki-root-key")) + self.rpki_root_key = rpki.x509.RSA(Auto_update = self.cfg.get("rpki-root-key")) self.rpki_root_cert_file = self.cfg.get("rpki-root-cert") self.rpki_root_cert_uri = self.cfg.get("rpki-root-cert-uri", self.rpki_base_uri + "Root.cer") diff --git a/rpkid/rpki/rpkic.py b/rpkid/rpki/rpkic.py new file mode 100644 index 00000000..96431114 --- /dev/null +++ b/rpkid/rpki/rpkic.py @@ -0,0 +1,455 @@ +""" +This is a command line configuration and control tool for rpkid et al. + +Type "help" on the prompt, or run the program with the --help option for an +overview of the available commands; type "help foo" for (more) detailed help +on the "foo" command. + + +This program is a rewrite of the old myrpki program, replacing ten +zillion XML and X.509 disk files and subprocess calls to the OpenSSL +command line tool with SQL data and direct calls to the rpki.POW +library. This version abandons all pretense that this program might +somehow work without rpki.POW, lxml, and Django installed, but since +those packages are required for rpkid anyway, this seems like a small +price to pay for major simplification of the code and better +integration with the Django-based GUI interface. + +$Id$ + +Copyright (C) 2009--2011 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +# NB: As of this writing, I'm trying really hard to avoid having this +# program depend on a Django settings.py file. This may prove to be a +# waste of time in the long run, but for for now, this means that one +# has to be careful about exactly how and when one imports Django +# modules, or anything that imports Django modules. Bottom line is +# that we don't import such modules until we need them. + + +# We need context managers for transactions. Well, unless we're +# willing to have this program depend on a Django settings.py file so +# that we can use decorators, which I'm not, at the moment. + +from __future__ import with_statement + +import csv, re, os, getopt, sys, base64, time, glob, copy, warnings +import rpki.config, rpki.cli, rpki.sundial, rpki.log, rpki.oids +import rpki.http, rpki.resource_set, rpki.relaxng, rpki.exceptions +import rpki.left_right, rpki.x509, rpki.async + +class BadCommandSyntax(Exception): "Bad command line syntax." +class BadPrefixSyntax(Exception): "Bad prefix syntax." +class CouldntTalkToDaemon(Exception): "Couldn't talk to daemon." +class BadXMLMessage(Exception): "Bad XML message." +class PastExpiration(Exception): "Expiration date has already passed." +class CantRunRootd(Exception): "Can't run rootd." + +class main(rpki.cli.Cmd): + + prompt = "rpkic> " + + completedefault = rpki.cli.Cmd.filename_complete + + def __init__(self): + os.environ["TZ"] = "UTC" + time.tzset() + + rpki.log.use_syslog = False + + cfg_file = None + handle = None + + opts, argv = getopt.getopt(sys.argv[1:], "c:hi:?", ["config=", "help", "identity="]) + for o, a in opts: + if o in ("-c", "--config"): + cfg_file = a + elif o in ("-h", "--help", "-?"): + argv = ["help"] + elif o in ("-i", "--identity"): + handle = a + + if not argv or argv[0] != "help": + rpki.log.init("rpkic") + self.read_config(cfg_file, handle) + + rpki.cli.Cmd.__init__(self, argv) + + def read_config(self, cfg_file, handle): + global rpki + + cfg = rpki.config.parser(cfg_file, "myrpki") + cfg.set_global_flags() + + from django.conf import settings + + settings.configure( + DATABASES = { "default" : { + "ENGINE" : "django.db.backends.mysql", + "NAME" : cfg.get("sql-database", section = "irdbd"), + "USER" : cfg.get("sql-username", section = "irdbd"), + "PASSWORD" : cfg.get("sql-password", section = "irdbd"), + "HOST" : "", + "PORT" : "", + "OPTIONS" : { "init_command": "SET storage_engine=INNODB" }}}, + INSTALLED_APPS = ("rpki.irdb",), + ) + + import rpki.irdb + + import django.core.management + django.core.management.call_command("syncdb", verbosity = 0, load_initial_data = False) + + self.zoo = rpki.irdb.Zookeeper(cfg = cfg, handle = handle) + + def help_overview(self): + """ + Show program __doc__ string. Perhaps there's some clever way to + do this using the textwrap module, but for now something simple + and crude will suffice. + """ + + for line in __doc__.splitlines(True): + self.stdout.write(" " * 4 + line) + self.stdout.write("\n") + + def irdb_handle_complete(self, klass, text, line, begidx, endidx): + return [obj.handle for obj in klass.objects.all() if obj.handle and obj.handle.startswith(text)] + + def do_select_identity(self, arg): + """ + Select an identity handle for use with later commands. + """ + + argv = arg.split() + if len(argv) != 1: + raise BadCommandSyntax("This command expexcts one argument, not %r" % arg) + self.zoo.reset_identity(argv[0]) + + def complete_select_identity(self, *args): + return self.irdb_handle_complete(rpki.irdb.ResourceHolderCA, *args) + + + def do_initialize(self, arg): + """ + Initialize an RPKI installation. This command reads the + configuration file, creates the BPKI and EntityDB directories, + generates the initial BPKI certificates, and creates an XML file + describing the resource-holding aspect of this RPKI installation. + """ + + if arg: + raise BadCommandSyntax, "This command takes no arguments" + + r = self.zoo.initialize() + r.save("%s.identity.xml" % self.zoo.handle, not self.zoo.run_pubd) + + if self.zoo.run_rootd and self.zoo.handle == self.zoo.cfg.get("handle"): + r = self.zoo.configure_rootd() + if r is not None: + r.save("%s.%s.repository-request.xml" % (self.zoo.handle, self.zoo.handle), True) + + self.zoo.write_bpki_files() + + + def do_update_bpki(self, arg): + """ + Update BPKI certificates. Assumes an existing RPKI installation. + + Basic plan here is to reissue all BPKI certificates we can, right + now. In the long run we might want to be more clever about only + touching ones that need maintenance, but this will do for a start. + + We also reissue CRLs for all CAs. + + Most likely this should be run under cron. + """ + + self.zoo.update_bpki() + + + def do_configure_child(self, arg): + """ + Configure a new child of this RPKI entity, given the child's XML + identity file as an input. This command extracts the child's data + from the XML, cross-certifies the child's resource-holding BPKI + certificate, and generates an XML file describing the relationship + between the child and this parent, including this parent's BPKI + data and up-down protocol service URI. + """ + + child_handle = None + + opts, argv = getopt.getopt(arg.split(), "", ["child_handle="]) + for o, a in opts: + if o == "--child_handle": + child_handle = a + + if len(argv) != 1: + raise BadCommandSyntax, "Need to specify filename for child.xml" + + r, child_handle = self.zoo.configure_child(argv[0], child_handle) + r.save("%s.%s.parent-response.xml" % (self.zoo.handle, child_handle), True) + + + def do_delete_child(self, arg): + """ + Delete a child of this RPKI entity. + """ + + try: + self.zoo.delete_child(arg) + except rpki.irdb.Child.DoesNotExist: + print "No such child \"%s\"" % arg + + def complete_delete_child(self, *args): + return self.irdb_handle_complete(rpki.irdb.Child, *args) + + + def do_configure_parent(self, arg): + """ + Configure a new parent of this RPKI entity, given the output of + the parent's configure_child command as input. This command reads + the parent's response XML, extracts the parent's BPKI and service + URI information, cross-certifies the parent's BPKI data into this + entity's BPKI, and checks for offers or referrals of publication + service. If a publication offer or referral is present, we + generate a request-for-service message to that repository, in case + the user wants to avail herself of the referral or offer. + """ + + parent_handle = None + + opts, argv = getopt.getopt(arg.split(), "", ["parent_handle="]) + for o, a in opts: + if o == "--parent_handle": + parent_handle = a + + if len(argv) != 1: + raise BadCommandSyntax, "Need to specify filename for parent.xml on command line" + + r, parent_handle = self.zoo.configure_parent(argv[0], parent_handle) + r.save("%s.%s.repository-request.xml" % (self.zoo.handle, parent_handle), True) + + + def do_delete_parent(self, arg): + """ + Delete a parent of this RPKI entity. + """ + + try: + self.zoo.delete_parent(arg) + except rpki.irdb.Parent.DoesNotExist: + print "No such parent \"%s\"" % arg + + def complete_delete_parent(self, *args): + return self.irdb_handle_complete(rpki.irdb.Parent, *args) + + + def do_configure_publication_client(self, arg): + """ + Configure publication server to know about a new client, given the + client's request-for-service message as input. This command reads + the client's request for service, cross-certifies the client's + BPKI data, and generates a response message containing the + repository's BPKI data and service URI. + """ + + sia_base = None + + opts, argv = getopt.getopt(arg.split(), "", ["sia_base="]) + for o, a in opts: + if o == "--sia_base": + sia_base = a + + if len(argv) != 1: + raise BadCommandSyntax, "Need to specify filename for client.xml" + + r, client_handle = self.zoo.configure_publication_client(argv[0], sia_base) + r.save("%s.repository-response.xml" % client_handle.replace("/", "."), True) + + + def do_delete_publication_client(self, arg): + """ + Delete a publication client of this RPKI entity. + """ + + try: + self.zoo.delete_publication_client(arg).delete() + except rpki.irdb.Client.DoesNotExist: + print "No such client \"%s\"" % arg + + def complete_delete_publication_client(self, *args): + return self.irdb_handle_complete(rpki.irdb.Client, *args) + + + def do_configure_repository(self, arg): + """ + Configure a publication repository for this RPKI entity, given the + repository's response to our request-for-service message as input. + This command reads the repository's response, extracts and + cross-certifies the BPKI data and service URI, and links the + repository data with the corresponding parent data in our local + database. + """ + + parent_handle = None + + opts, argv = getopt.getopt(arg.split(), "", ["parent_handle="]) + for o, a in opts: + if o == "--parent_handle": + parent_handle = a + + if len(argv) != 1: + raise BadCommandSyntax, "Need to specify filename for repository.xml on command line" + + self.zoo.configure_repository(argv[0], parent_handle) + + def do_delete_repository(self, arg): + """ + Delete a repository of this RPKI entity. + + This should check that the XML file it's deleting really is a + repository, but doesn't, yet. + """ + + try: + self.zoo.delete_repository(arg) + except rpki.irdb.Repository.DoesNotExist: + print "No such repository \"%s\"" % arg + + def complete_delete_repository(self, *args): + return self.irdb_handle_complete(rpki.irdb.Repository, *args) + + + def do_renew_child(self, arg): + """ + Update validity period for one child entity. + """ + + valid_until = None + + opts, argv = getopt.getopt(arg.split(), "", ["valid_until"]) + for o, a in opts: + if o == "--valid_until": + valid_until = a + + if len(argv) != 1: + raise BadCommandSyntax, "Need to specify child handle" + + self.zoo.renew_children(argv[0], valid_until) + + def complete_renew_child(self, *args): + return self.irdb_handle_complete(rpki.irdb.Child, *args) + + + def do_renew_all_children(self, arg): + """ + Update validity period for all child entities. + """ + + valid_until = None + + opts, argv = getopt.getopt(arg.split(), "", ["valid_until"]) + for o, a in opts: + if o == "--valid_until": + valid_until = a + + if len(argv) != 0: + raise BadCommandSyntax, "Unexpected arguments" + + self.zoo.renew_children(None, valid_until) + + + def do_load_prefixes(self, arg): + """ + Load prefixes into IRDB from CSV file. + """ + + argv = arg.split() + + if len(argv) != 1: + raise BadCommandSyntax("Need to specify prefixes.csv filename") + + self.zoo.load_prefixes(argv[0]) + + + def do_show_child_resources(self, arg): + """ + Show resources assigned to children. + """ + + if arg.strip(): + raise BadCommandSyntax("This command takes no arguments") + + for child in self.resource_ca.children.all(): + + asn = rpki.resource_set.resource_set_as.from_django( + (a.start_as, a.end_as) for a in child.asns.all()) + ipv4 = rpki.resource_set.resource_set_ipv4.from_django( + (a.start_ip, a.end_ip) for a in child.address_ranges.filter(version = 4)) + ipv6 = rpki.resource_set.resource_set_ipv6.from_django( + (a.start_ip, a.end_ip) for a in child.address_ranges.filter(version = 6)) + + print "Child:", child.handle + if asn: + print " ASN:", asn + if ipv4: + print " IPv4:", ipv4 + if ipv6: + print " IPv6:", ipv6 + + + def do_load_asns(self, arg): + """ + Load ASNs into IRDB from CSV file. + """ + + argv = arg.split() + + if len(argv) != 1: + raise BadCommandSyntax("Need to specify asns.csv filename") + + self.zoo.load_asns(argv[0]) + + + def do_load_roa_requests(self, arg): + """ + Load ROA requests into IRDB from CSV file. + """ + + argv = arg.split() + + if len(argv) != 1: + raise BadCommandSyntax("Need to specify roa.csv filename") + + self.zoo.load_roa_requests(argv[0]) + + + + + def do_synchronize(self, arg): + """ + Whack daemons to match IRDB. + + This command may be replaced by implicit synchronization embedded + in of other commands, haven't decided yet. + """ + + if arg: + raise BadCommandSyntax("Unexpected argument(s): %r" % arg) + + self.zoo.synchronize() diff --git a/rpkid/rpki/rpkid.py b/rpkid/rpki/rpkid.py index 75624a3c..2c185f18 100644 --- a/rpkid/rpki/rpkid.py +++ b/rpkid/rpki/rpkid.py @@ -242,7 +242,7 @@ class main(object): raise except Exception, e: rpki.log.traceback() - cb(500, reason = "Unhandled exception %s" % e) + cb(500, reason = "Unhandled exception %s: %s" % (e.__class__.__name__, e)) up_down_url_regexp = re.compile("/up-down/([-A-Z0-9_]+)/([-A-Z0-9_]+)$", re.I) diff --git a/rpkid/rpki/sql_schemas.py b/rpkid/rpki/sql_schemas.py index 154ab5c1..e7c65299 100644 --- a/rpkid/rpki/sql_schemas.py +++ b/rpkid/rpki/sql_schemas.py @@ -239,115 +239,6 @@ CREATE TABLE ghostbuster ( -- End: ''' -## @var irdbd -## SQL schema irdbd -irdbd = '''-- $Id: irdbd.sql 3730 2011-03-21 12:42:43Z sra $ - --- Copyright (C) 2009--2011 Internet Systems Consortium ("ISC") --- --- Permission to use, copy, modify, and distribute this software for any --- purpose with or without fee is hereby granted, provided that the above --- copyright notice and this permission notice appear in all copies. --- --- THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH --- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY --- AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, --- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM --- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE --- OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR --- PERFORMANCE OF THIS SOFTWARE. - --- Copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") --- --- Permission to use, copy, modify, and distribute this software for any --- purpose with or without fee is hereby granted, provided that the above --- copyright notice and this permission notice appear in all copies. --- --- THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH --- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY --- AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, --- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM --- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE --- OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR --- PERFORMANCE OF THIS SOFTWARE. - --- SQL objects needed by irdbd.py. You only need this if you're using --- irdbd.py as your IRDB; if you have a "real" backend you can do --- anything you like so long as you implement the relevant portion of --- the left-right protocol. - --- DROP TABLE commands must be in correct (reverse dependency) order --- to satisfy FOREIGN KEY constraints. - -DROP TABLE IF EXISTS roa_request_prefix; -DROP TABLE IF EXISTS roa_request; -DROP TABLE IF EXISTS registrant_net; -DROP TABLE IF EXISTS registrant_asn; -DROP TABLE IF EXISTS registrant; -DROP TABLE IF EXISTS ghostbuster_request; - -CREATE TABLE registrant ( - registrant_id SERIAL NOT NULL, - registrant_handle VARCHAR(255) NOT NULL, - registrant_name TEXT, - registry_handle VARCHAR(255), - valid_until DATETIME NOT NULL, - PRIMARY KEY (registrant_id), - UNIQUE (registry_handle, registrant_handle) -) ENGINE=InnoDB; - -CREATE TABLE registrant_asn ( - registrant_asn_id SERIAL NOT NULL, - start_as BIGINT UNSIGNED NOT NULL, - end_as BIGINT UNSIGNED NOT NULL, - registrant_id BIGINT UNSIGNED NOT NULL, - PRIMARY KEY (registrant_asn_id), - CONSTRAINT registrant_asn_registrant_id - FOREIGN KEY (registrant_id) REFERENCES registrant (registrant_id) ON DELETE CASCADE -) ENGINE=InnoDB; - -CREATE TABLE registrant_net ( - registrant_net_id SERIAL NOT NULL, - start_ip VARCHAR(40) NOT NULL, - end_ip VARCHAR(40) NOT NULL, - version TINYINT UNSIGNED NOT NULL, - registrant_id BIGINT UNSIGNED NOT NULL, - PRIMARY KEY (registrant_net_id), - CONSTRAINT registrant_net_registrant_id - FOREIGN KEY (registrant_id) REFERENCES registrant (registrant_id) ON DELETE CASCADE -) ENGINE=InnoDB; - -CREATE TABLE roa_request ( - roa_request_id SERIAL NOT NULL, - roa_request_handle VARCHAR(255) NOT NULL, - asn BIGINT UNSIGNED NOT NULL, - PRIMARY KEY (roa_request_id) -) ENGINE=InnoDB; - -CREATE TABLE roa_request_prefix ( - prefix VARCHAR(40) NOT NULL, - prefixlen TINYINT UNSIGNED NOT NULL, - max_prefixlen TINYINT UNSIGNED NOT NULL, - version TINYINT UNSIGNED NOT NULL, - roa_request_id BIGINT UNSIGNED NOT NULL, - PRIMARY KEY (roa_request_id, prefix, prefixlen, max_prefixlen), - CONSTRAINT roa_request_prefix_roa_request_id - FOREIGN KEY (roa_request_id) REFERENCES roa_request (roa_request_id) ON DELETE CASCADE -) ENGINE=InnoDB; - -CREATE TABLE ghostbuster_request ( - ghostbuster_request_id SERIAL NOT NULL, - self_handle VARCHAR(40) NOT NULL, - parent_handle VARCHAR(40), - vcard LONGBLOB NOT NULL, - PRIMARY KEY (ghostbuster_request_id) -) ENGINE=InnoDB; - --- Local Variables: --- indent-tabs-mode: nil --- End: -''' - ## @var pubd ## SQL schema pubd pubd = '''-- $Id: pubd.sql 3465 2010-10-07 00:59:39Z sra $ diff --git a/rpkid/rpki/x509.py b/rpkid/rpki/x509.py index 58129363..7bbb47bc 100644 --- a/rpkid/rpki/x509.py +++ b/rpkid/rpki/x509.py @@ -47,6 +47,7 @@ import rpki.POW, rpki.POW.pkix, base64, lxml.etree, os, subprocess, sys import email.mime.application, email.utils, mailbox, time import rpki.exceptions, rpki.resource_set, rpki.oids, rpki.sundial import rpki.manifest, rpki.roa, rpki.log, rpki.async, rpki.ghostbuster +import rpki.relaxng def base64_with_linebreaks(der): """ @@ -120,6 +121,74 @@ def _find_xia_uri(extension, name): return location[1] return None +class X501DN(object): + """ + Class to hold an X.501 Distinguished Name. + + This is nothing like a complete implementation, just enough for our + purposes. POW has one interface to this, POW.pkix has another. In + terms of completeness in the Python representation, the POW.pkix + representation is much closer to right, but the whole thing is a + horrible mess. + + See RFC 5280 4.1.2.4 for the ASN.1 details. In brief: + + - A DN is a SEQUENCE of RDNs. + + - A RDN is a set of AttributeAndValues; in practice, multi-value + RDNs are rare, so an RDN is almost always a set with a single + element. + + - An AttributeAndValue is an OID and a value, where a whole bunch + of things including both syntax and semantics of the value are + determined by the OID. + + - The value is some kind of ASN.1 string; there are far too many + encoding options options, most of which are either strongly + discouraged or outright forbidden by the PKIX profile, but which + persist for historical reasons. The only ones PKIX actually + likes are PrintableString and UTF8String, but there are nuances + and special cases where some of the others are required. + + The RPKI profile further restricts DNs to a single mandatory + CommonName attribute with a single optional SerialNumber attribute + (not to be confused with the certificate serial number). + + BPKI certificates should (we hope) follow the general PKIX guideline + but the ones we construct ourselves are likely to be relatively + simple. + + The main purpose of this class is to hide as much as possible of + this mess from code that has to work with these wretched things. + """ + + def __init__(self, ini = None, **kwargs): + assert ini is None or not kwargs + if len(kwargs) == 1 and "CN" in kwargs: + ini = kwargs.pop("CN") + if isinstance(ini, (str, unicode)): + self.dn = (((rpki.oids.name2oid["commonName"], ("printableString", ini)),),) + elif isinstance(ini, tuple): + self.dn = ini + elif kwargs: + raise NotImplementedError("Sorry, I haven't implemented keyword arguments yet") + elif ini is not None: + raise TypeError("Don't know how to interpret %r as an X.501 DN" % (ini,), ini) + + def __str__(self): + return "".join("/" + "+".join("%s=%s" % (rpki.oids.oid2name[a[0]], a[1][1]) + for a in rdn) + for rdn in self.dn) + + def __cmp__(self, other): + return cmp(self.dn, other.dn) + + def get_POWpkix(self): + return self.dn + + def get_POW(self): + raise NotImplementedError("Sorry, I haven't written the conversion to POW format yet") + class DER_object(object): """ Virtual class to hold a generic DER object. @@ -259,6 +328,8 @@ class DER_object(object): return -1 elif other is None: return 1 + elif isinstance(other, str): + return cmp(self.get_DER(), other) else: return cmp(self.get_DER(), other.get_DER()) @@ -456,13 +527,13 @@ class X509(DER_object): """ Get the issuer of this certificate. """ - return "".join("/%s=%s" % rdn for rdn in self.get_POW().getIssuer()) + return X501DN(self.get_POWpkix().getIssuer()) def getSubject(self): """ Get the subject of this certificate. """ - return "".join("/%s=%s" % rdn for rdn in self.get_POW().getSubject()) + return X501DN(self.get_POWpkix().getSubject()) def getNotBefore(self): """ @@ -497,11 +568,60 @@ class X509(DER_object): def issue(self, keypair, subject_key, serial, sia, aia, crldp, notAfter, cn = None, resources = None, is_ca = True): """ - Issue a certificate. + Issue an RPKI certificate. + """ + + assert aia is not None and crldp is not None + + return self._issue( + keypair = keypair, + subject_key = subject_key, + serial = serial, + sia = sia, + aia = aia, + crldp = crldp, + notAfter = notAfter, + cn = cn, + resources = resources, + is_ca = is_ca, + aki = self.get_SKI(), + issuer_name = self.get_POWpkix().getSubject()) + + + @classmethod + def self_certify(cls, keypair, subject_key, serial, sia, notAfter, + cn = None, resources = None): + """ + Generate a self-certified RPKI certificate. + """ + + ski = subject_key.get_SKI() + if cn is None: + cn = "".join(("%02X" % ord(i) for i in ski)) + + return cls._issue( + keypair = keypair, + subject_key = subject_key, + serial = serial, + sia = sia, + aia = None, + crldp = None, + notAfter = notAfter, + cn = cn, + resources = resources, + is_ca = True, + aki = ski, + issuer_name = (((rpki.oids.name2oid["commonName"], ("printableString", cn)),),)) + + + @staticmethod + def _issue(keypair, subject_key, serial, sia, aia, crldp, notAfter, + cn, resources, is_ca, aki, issuer_name): + """ + Common code to issue an RPKI certificate. """ now = rpki.sundial.now() - aki = self.get_SKI() ski = subject_key.get_SKI() if cn is None: @@ -512,7 +632,7 @@ class X509(DER_object): cert = rpki.POW.pkix.Certificate() cert.setVersion(2) cert.setSerial(serial) - cert.setIssuer(self.get_POWpkix().getSubject()) + cert.setIssuer(issuer_name) cert.setSubject((((rpki.oids.name2oid["commonName"], ("printableString", cn)),),)) cert.setNotBefore(now.toASN1tuple()) cert.setNotAfter(notAfter.toASN1tuple()) @@ -520,10 +640,15 @@ class X509(DER_object): exts = [ ["subjectKeyIdentifier", False, ski], ["authorityKeyIdentifier", False, (aki, (), None)], - ["cRLDistributionPoints", False, ((("fullName", (("uri", crldp),)), None, ()),)], - ["authorityInfoAccess", False, ((rpki.oids.name2oid["id-ad-caIssuers"], ("uri", aia)),)], ["certificatePolicies", True, ((rpki.oids.name2oid["id-cp-ipAddr-asNumber"], ()),)] ] + + if crldp is not None: + exts.append(["cRLDistributionPoints", False, ((("fullName", (("uri", crldp),)), None, ()),)]) + + if aia is not None: + exts.append(["authorityInfoAccess", False, ((rpki.oids.name2oid["id-ad-caIssuers"], ("uri", aia)),)]) + if is_ca: exts.append(["basicConstraints", True, (1, None)]) exts.append(["keyUsage", True, (0, 0, 0, 0, 0, 1, 1)]) @@ -555,33 +680,96 @@ class X509(DER_object): return X509(POWpkix = cert) - def cross_certify(self, keypair, source_cert, serial, notAfter, now = None, pathLenConstraint = 0): + def bpki_cross_certify(self, keypair, source_cert, serial, notAfter, + now = None, pathLenConstraint = 0): """ - Issue a certificate with values taking from an existing certificate. - This is used to construct some kinds oF BPKI certificates. + Issue a BPKI certificate with values taking from an existing certificate. + """ + return self.bpki_certify( + keypair = keypair, + subject_name = source_cert.getSubject(), + subject_key = source_cert.getPublicKey(), + serial = serial, + notAfter = notAfter, + now = now, + pathLenConstraint = pathLenConstraint, + is_ca = True) + + @classmethod + def bpki_self_certify(cls, keypair, subject_name, serial, notAfter, + now = None, pathLenConstraint = None): + """ + Issue a self-signed BPKI CA certificate. + """ + return cls._bpki_certify( + keypair = keypair, + issuer_name = subject_name, + subject_name = subject_name, + subject_key = keypair.get_RSApublic(), + serial = serial, + now = now, + notAfter = notAfter, + pathLenConstraint = pathLenConstraint, + is_ca = True) + + def bpki_certify(self, keypair, subject_name, subject_key, serial, notAfter, is_ca, + now = None, pathLenConstraint = None): + """ + Issue a normal BPKI certificate. + """ + assert keypair.get_RSApublic() == self.getPublicKey() + return self._bpki_certify( + keypair = keypair, + issuer_name = self.getSubject(), + subject_name = subject_name, + subject_key = subject_key, + serial = serial, + now = now, + notAfter = notAfter, + pathLenConstraint = pathLenConstraint, + is_ca = is_ca) + + @classmethod + def _bpki_certify(cls, keypair, issuer_name, subject_name, subject_key, + serial, now, notAfter, pathLenConstraint, is_ca): + """ + Issue a BPKI certificate. This internal method does the real + work, after one of the wrapper methods has extracted the relevant + fields. """ if now is None: now = rpki.sundial.now() - assert isinstance(pathLenConstraint, int) and pathLenConstraint >= 0 + issuer_key = keypair.get_RSApublic() + + assert (issuer_key == subject_key) == (issuer_name == subject_name) + assert is_ca or issuer_name != subject_name + assert is_ca or pathLenConstraint is None + assert pathLenConstraint is None or (isinstance(pathLenConstraint, (int, long)) and + pathLenConstraint >= 0) + + extensions = [ + (rpki.oids.name2oid["subjectKeyIdentifier" ], False, subject_key.get_SKI())] + if issuer_key != subject_key: + extensions.append( + (rpki.oids.name2oid["authorityKeyIdentifier"], False, (issuer_key.get_SKI(), (), None))) + if is_ca: + extensions.append( + (rpki.oids.name2oid["basicConstraints" ], True, (1, pathLenConstraint))) cert = rpki.POW.pkix.Certificate() cert.setVersion(2) cert.setSerial(serial) - cert.setIssuer(self.get_POWpkix().getSubject()) - cert.setSubject(source_cert.get_POWpkix().getSubject()) + cert.setIssuer(issuer_name.get_POWpkix()) + cert.setSubject(subject_name.get_POWpkix()) cert.setNotBefore(now.toASN1tuple()) cert.setNotAfter(notAfter.toASN1tuple()) - cert.tbs.subjectPublicKeyInfo.set( - source_cert.get_POWpkix().tbs.subjectPublicKeyInfo.get()) - cert.setExtensions(( - (rpki.oids.name2oid["subjectKeyIdentifier" ], False, source_cert.get_SKI()), - (rpki.oids.name2oid["authorityKeyIdentifier"], False, (self.get_SKI(), (), None)), - (rpki.oids.name2oid["basicConstraints" ], True, (1, 0)))) + cert.tbs.subjectPublicKeyInfo.fromString(subject_key.get_DER()) + cert.setExtensions(extensions) cert.sign(keypair.get_POW(), rpki.POW.SHA256_DIGEST) - return X509(POWpkix = cert) + return cls(POWpkix = cert) @classmethod def normalize_chain(cls, chain): @@ -628,6 +816,12 @@ class PKCS10(DER_object): self.POWpkix = req return self.POWpkix + def getSubject(self): + """ + Extract the subject name from this certification request. + """ + return X501DN(self.get_POWpkix().certificationRequestInfo.subject.get()) + def getPublicKey(self): """ Extract the public key from this certification request. @@ -955,7 +1149,7 @@ class CMS_object(DER_object): if certs and (len(certs) > 1 or certs[0].getSubject() != trusted_ee.getSubject() or certs[0].getPublicKey() != trusted_ee.getPublicKey()): raise rpki.exceptions.UnexpectedCMSCerts # , certs if crls: - raise rpki.exceptions.UnexpectedCMSCRLs # , crls + rpki.log.warn("Ignoring unexpected CMS CRL%s from trusted peer" % ("" if len(crls) == 1 else "s")) else: if not certs: raise rpki.exceptions.MissingCMSEEcert # , certs @@ -1246,7 +1440,10 @@ class XML_CMS_object(CMS_object): Wrap an XML PDU in CMS and return its DER encoding. """ rpki.log.trace() - self.set_content(msg.toXML()) + if self.saxify is None: + self.set_content(msg) + else: + self.set_content(msg.toXML()) self.schema_check() self.sign(keypair, certs, crls) if self.dump_outbound_cms: @@ -1261,7 +1458,22 @@ class XML_CMS_object(CMS_object): self.dump_inbound_cms.dump(self) self.verify(ta) self.schema_check() - return self.saxify(self.get_content()) + if self.saxify is None: + return self.get_content() + else: + return self.saxify(self.get_content()) + + ## @var saxify + # SAX handler hook. Subclasses can set this to a SAX handler, in + # which case .unwrap() will call it and return the result. + # Otherwise, .unwrap() just returns a verified element tree. + + saxify = None + +class SignedReferral(XML_CMS_object): + encoding = "us-ascii" + schema = rpki.relaxng.myrpki + saxify = None class Ghostbuster(CMS_object): """ @@ -1357,7 +1569,7 @@ class CRL(DER_object): """ Get issuer value of this CRL. """ - return "".join("/%s=%s" % rdn for rdn in self.get_POW().getIssuer()) + return X501DN(self.get_POWpkix().getIssuer()) @classmethod def generate(cls, keypair, issuer, serial, thisUpdate, nextUpdate, revokedCertificates, version = 1, digestType = "sha256WithRSAEncryption"): diff --git a/rpkid/rpkic.py b/rpkid/rpkic.py new file mode 100644 index 00000000..6ef3a67b --- /dev/null +++ b/rpkid/rpkic.py @@ -0,0 +1,21 @@ +""" +$Id$ + +Copyright (C) 2010-2011 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +if __name__ == "__main__": + import rpki.rpkic + rpki.rpkic.main() diff --git a/rpkid/tests/old_irdbd.py b/rpkid/tests/old_irdbd.py new file mode 100644 index 00000000..3fa84b80 --- /dev/null +++ b/rpkid/tests/old_irdbd.py @@ -0,0 +1,21 @@ +""" +$Id$ + +Copyright (C) 2010-2012 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +if __name__ == "__main__": + import rpki.old_irdbd + rpki.old_irdbd.main() diff --git a/rpkid/irdbd.sql b/rpkid/tests/old_irdbd.sql index bf324cd8..bf324cd8 100644 --- a/rpkid/irdbd.sql +++ b/rpkid/tests/old_irdbd.sql diff --git a/rpkid/tests/smoketest.py b/rpkid/tests/smoketest.py index 8145a0eb..7dfc584c 100644 --- a/rpkid/tests/smoketest.py +++ b/rpkid/tests/smoketest.py @@ -124,8 +124,8 @@ pubd_name = cfg.get("pubd_name", "pubd") prog_python = cfg.get("prog_python", sys.executable) prog_rpkid = cfg.get("prog_rpkid", "../../rpkid.py") -prog_irdbd = cfg.get("prog_irdbd", "../../irdbd.py") -prog_poke = cfg.get("prog_poke", "../../testpoke.py") +prog_irdbd = cfg.get("prog_irdbd", "../old_irdbd.py") +prog_poke = cfg.get("prog_poke", "../testpoke.py") prog_rootd = cfg.get("prog_rootd", "../../rootd.py") prog_pubd = cfg.get("prog_pubd", "../../pubd.py") prog_rsyncd = cfg.get("prog_rsyncd", "rsync") @@ -135,7 +135,7 @@ prog_openssl = cfg.get("prog_openssl", "../../../openssl/openssl/apps/openss rcynic_stats = cfg.get("rcynic_stats", "echo ; ../../../rcynic/show.sh %s.xml ; echo" % rcynic_name) rpki_sql_file = cfg.get("rpki_sql_file", "../rpkid.sql") -irdb_sql_file = cfg.get("irdb_sql_file", "../irdbd.sql") +irdb_sql_file = cfg.get("irdb_sql_file", "old_irdbd.sql") pub_sql_file = cfg.get("pub_sql_file", "../pubd.sql") startup_delay = int(cfg.get("startup_delay", "10")) @@ -868,7 +868,12 @@ class allocation(object): except IOError: serial = 1 - x = parent.cross_certify(keypair, child, serial, notAfter, now) + x = parent.bpki_cross_certify( + keypair = keypair, + source_cert = child, + serial = serial, + notAfter = notAfter, + now = now) f = open(serial_file, "w") f.write("%02x\n" % (serial + 1)) @@ -1264,16 +1269,12 @@ def mangle_sql(filename): """ Mangle an SQL file into a sequence of SQL statements. """ - - # There is no pretty way to do this. Just shut your eyes, it'll be - # over soon. - + words = [] f = open(filename) - statements = " ".join(" ".join(word for word in line.expandtabs().split(" ") if word) - for line in [line.strip(" \t\n") for line in f.readlines()] - if line and not line.startswith("--")).rstrip(";").split(";") + for line in f: + words.extend(line.partition("--")[0].split()) f.close() - return [stmt.strip() for stmt in statements] + return " ".join(words).strip(";").split(";") bpki_cert_fmt_1 = '''\ [ req ] diff --git a/rpkid/tests/sql-cleaner.py b/rpkid/tests/sql-cleaner.py index 5c772bc4..5db122e1 100644 --- a/rpkid/tests/sql-cleaner.py +++ b/rpkid/tests/sql-cleaner.py @@ -3,7 +3,7 @@ $Id$ -Copyright (C) 2009--2010 Internet Systems Consortium ("ISC") +Copyright (C) 2009--2011 Internet Systems Consortium ("ISC") Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above @@ -18,7 +18,8 @@ OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -import subprocess, rpki.config +import rpki.config, rpki.sql_schemas +from rpki.mysql_import import MySQLdb cfg = rpki.config.parser(None, "yamltest", allow_missing = True) @@ -26,8 +27,30 @@ for name in ("rpkid", "irdbd", "pubd"): username = cfg.get("%s_sql_username" % name, name[:4]) password = cfg.get("%s_sql_password" % name, "fnord") + + schema = [] + for line in getattr(rpki.sql_schemas, name, "").splitlines(): + schema.extend(line.partition("--")[0].split()) + schema = " ".join(schema).strip(";").split(";") + schema = [statement.strip() for statement in schema if statement and "DROP TABLE" not in statement] for i in xrange(12): - subprocess.check_call( - ("mysql", "-u", username, "-p" + password, "%s%d" % (name[:4], i)), - stdin = open("../%s.sql" % name)) + + database = "%s%d" % (name[:4], i) + + db = MySQLdb.connect(user = username, db = database, passwd = password) + cur = db.cursor() + + cur.execute("SHOW TABLES") + tables = [r[0] for r in cur.fetchall()] + + cur.execute("SET foreign_key_checks = 0") + for table in tables: + cur.execute("DROP TABLE %s" % table) + cur.execute("SET foreign_key_checks = 1") + + for statement in schema: + cur.execute(statement) + + cur.close() + db.close() diff --git a/rpkid/tests/yamltest-test-all.sh b/rpkid/tests/yamltest-test-all.sh index f6a05237..46f3c59e 100644 --- a/rpkid/tests/yamltest-test-all.sh +++ b/rpkid/tests/yamltest-test-all.sh @@ -1,7 +1,7 @@ #!/bin/sh - # $Id$ -# Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") +# Copyright (C) 2009-2012 Internet Systems Consortium ("ISC") # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above @@ -17,18 +17,18 @@ set -x -export TZ=UTC MYRPKI_RNG=$(pwd)/myrpki.rng +export TZ=UTC test -z "$STY" && exec screen -L sh $0 screen -X split screen -X focus -runtime=$((30 * 60)) +: ${runtime=900} for yaml in smoketest.*.yaml do - rm -rf test + rm -rf test rcynic-data python sql-cleaner.py screen python yamltest.py -p yamltest.pid $yaml now=$(date +%s) @@ -42,9 +42,13 @@ do date ../../rcynic/rcynic ../../rcynic/show.sh + ../../utils/scan_roas/scan_roas rcynic-data/authenticated date done - test -r yamltest.pid && kill -INT $(cat yamltest.pid) - sleep 30 + if test -r yamltest.pid + then + kill -INT $(cat yamltest.pid) + sleep 30 + fi make backup done diff --git a/rpkid/tests/yamltest.py b/rpkid/tests/yamltest.py index ecd00af2..290120f7 100644 --- a/rpkid/tests/yamltest.py +++ b/rpkid/tests/yamltest.py @@ -1,6 +1,6 @@ """ Test framework, using the same YAML test description format as -smoketest.py, but using the myrpki.py tool to do all the back-end +smoketest.py, but using the rpkic.py tool to do all the back-end work. Reads YAML file, generates .csv and .conf files, runs daemons and waits for one of them to exit. @@ -10,7 +10,7 @@ Still to do: - Implement smoketest.py-style delta actions, that is, modify the allocation database under control of the YAML file, dump out new - .csv files, and run myrpki.py again to feed resulting changes into + .csv files, and run rpkic.py again to feed resulting changes into running daemons. $Id$ @@ -46,7 +46,8 @@ PERFORMANCE OF THIS SOFTWARE. """ import subprocess, re, os, getopt, sys, yaml, signal, time -import rpki.resource_set, rpki.sundial, rpki.config, rpki.log, rpki.myrpki +import rpki.resource_set, rpki.sundial, rpki.config, rpki.log +import rpki.csv_utils, rpki.x509 # Nasty regular expressions for parsing config files. Sadly, while # the Python ConfigParser supports writing config files, it does so in @@ -67,11 +68,11 @@ this_dir = os.getcwd() test_dir = cleanpath(this_dir, "yamltest.dir") rpkid_dir = cleanpath(this_dir, "..") -prog_myrpki = cleanpath(rpkid_dir, "myrpki.py") -prog_rpkid = cleanpath(rpkid_dir, "rpkid.py") -prog_irdbd = cleanpath(rpkid_dir, "irdbd.py") -prog_pubd = cleanpath(rpkid_dir, "pubd.py") -prog_rootd = cleanpath(rpkid_dir, "rootd.py") +prog_rpkic = cleanpath(rpkid_dir, "rpkic") +prog_rpkid = cleanpath(rpkid_dir, "rpkid") +prog_irdbd = cleanpath(rpkid_dir, "irdbd") +prog_pubd = cleanpath(rpkid_dir, "pubd") +prog_rootd = cleanpath(rpkid_dir, "rootd") prog_openssl = cleanpath(this_dir, "../../openssl/openssl/apps/openssl") if not os.path.exists(prog_openssl): @@ -116,14 +117,14 @@ class allocation_db(list): def __init__(self, yaml): list.__init__(self) self.root = allocation(yaml, self) - assert self.root.is_root() + assert self.root.is_root if self.root.crl_interval is None: self.root.crl_interval = 24 * 60 * 60 if self.root.regen_margin is None: self.root.regen_margin = 24 * 60 * 60 for a in self: if a.sia_base is None: - if a.runs_pubd(): + if a.runs_pubd: base = "rsync://localhost:%d/rpki/" % a.rsync_port else: base = a.parent.sia_base @@ -134,14 +135,13 @@ class allocation_db(list): a.crl_interval = a.parent.crl_interval if a.regen_margin is None: a.regen_margin = a.parent.regen_margin - a.client_handle = "/".join(a.sia_base.rstrip("/").split("/")[3:]) self.root.closure() self.map = dict((a.name, a) for a in self) for a in self: - if a.is_hosted(): + if a.is_hosted: a.hosted_by = self.map[a.hosted_by] a.hosted_by.hosts.append(a) - assert not a.is_root() and not a.hosted_by.is_hosted() + assert not a.is_root and not a.hosted_by.is_hosted def dump(self): """ @@ -154,7 +154,7 @@ class allocation_db(list): class allocation(object): """ One entity in our allocation database. Every entity in the database - is assumed to hold resources, so needs at least myrpki services. + is assumed to hold resources, so needs at least rpkic services. Entities that don't have the hosted_by property run their own copies of rpkid, irdbd, and pubd, so they also need myirbe services. """ @@ -218,14 +218,14 @@ class allocation(object): self.base.v6 = self.base.v6.union(r.v6.to_resource_set()) self.hosted_by = yaml.get("hosted_by") self.hosts = [] - if not self.is_hosted(): + if not self.is_hosted: self.engine = self.allocate_engine() self.rpkid_port = self.allocate_port() self.irdbd_port = self.allocate_port() - if self.runs_pubd(): + if self.runs_pubd: self.pubd_port = self.allocate_port() self.rsync_port = self.allocate_port() - if self.is_root(): + if self.is_root: self.rootd_port = self.allocate_port() def closure(self): @@ -253,55 +253,56 @@ class allocation(object): if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) if self.parent: s += " Up: %s\n" % self.parent.name if self.sia_base: s += " SIA: %s\n" % self.sia_base - if self.is_hosted(): s += " Host: %s\n" % self.hosted_by.name + if self.is_hosted: s += " Host: %s\n" % self.hosted_by.name if self.hosts: s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts) for r in self.roa_requests: s += " ROA: %s\n" % r - if not self.is_hosted(): s += " IPort: %s\n" % self.irdbd_port - if self.runs_pubd(): s += " PPort: %s\n" % self.pubd_port - if not self.is_hosted(): s += " RPort: %s\n" % self.rpkid_port - if self.runs_pubd(): s += " SPort: %s\n" % self.rsync_port - if self.is_root(): s += " TPort: %s\n" % self.rootd_port + if not self.is_hosted: s += " IPort: %s\n" % self.irdbd_port + if self.runs_pubd: s += " PPort: %s\n" % self.pubd_port + if not self.is_hosted: s += " RPort: %s\n" % self.rpkid_port + if self.runs_pubd: s += " SPort: %s\n" % self.rsync_port + if self.is_root: s += " TPort: %s\n" % self.rootd_port return s + " Until: %s\n" % self.resources.valid_until + @property def is_root(self): """ Is this the root node? """ return self.parent is None + @property def is_hosted(self): """ Is this entity hosted? """ return self.hosted_by is not None + @property def runs_pubd(self): """ Does this entity run a pubd? """ - return self.is_root() or not (self.is_hosted() or only_one_pubd) + return self.is_root or not (self.is_hosted or only_one_pubd) def path(self, *names): """ Construct pathnames in this entity's test directory. """ - return cleanpath(test_dir, self.name, *names) + return cleanpath(test_dir, self.host.name, *names) def csvout(self, fn): """ - Open and log a CSV output file. We use delimiter and dialect - settings imported from the myrpki module, so that we automatically - write CSV files in the right format. + Open and log a CSV output file. """ path = self.path(fn) print "Writing", path - return rpki.myrpki.csv_writer(path) + return rpki.csv_utils.csv_writer(path) def up_down_url(self): """ Construct service URL for this node's parent. """ - parent_port = self.parent.hosted_by.rpkid_port if self.parent.is_hosted() else self.parent.rpkid_port + parent_port = self.parent.hosted_by.rpkid_port if self.parent.is_hosted else self.parent.rpkid_port return "http://localhost:%d/up-down/%s/%s" % (parent_port, self.parent.name, self.name) def dump_asns(self, fn): @@ -312,37 +313,7 @@ class allocation(object): for k in self.kids: f.writerows((k.name, a) for a in k.resources.asn) f.close() - - def dump_children(self, fn): - """ - Write children CSV file. - """ - f = self.csvout(fn) - f.writerows((k.name, k.resources.valid_until, k.path("bpki/resources/ca.cer")) - for k in self.kids) - f.close() - - def dump_parents(self, fn): - """ - Write parents CSV file. - """ - f = self.csvout(fn) - if self.is_root(): - f.writerow(("rootd", - "http://localhost:%d/" % self.rootd_port, - self.path("bpki/servers/ca.cer"), - self.path("bpki/servers/ca.cer"), - self.name, - self.sia_base)) - else: - parent_host = self.parent.hosted_by if self.parent.is_hosted() else self.parent - f.writerow((self.parent.name, - self.up_down_url(), - self.parent.path("bpki/resources/ca.cer"), - parent_host.path("bpki/servers/ca.cer"), - self.name, - self.sia_base)) - f.close() + self.run_rpkic("load_asns", fn) def dump_prefixes(self, fn): """ @@ -352,43 +323,45 @@ class allocation(object): for k in self.kids: f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) f.close() + self.run_rpkic("load_prefixes", fn) def dump_roas(self, fn): """ Write ROA CSV file. """ - group = self.name if self.is_root() else self.parent.name + group = self.name if self.is_root else self.parent.name f = self.csvout(fn) for r in self.roa_requests: f.writerows((p, r.asn, group) for p in (r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ())) f.close() + self.run_rpkic("load_roa_requests", fn) - def dump_clients(self, fn, db): - """ - Write pubclients CSV file. - """ - if self.runs_pubd(): - f = self.csvout(fn) - f.writerows((s.client_handle, s.path("bpki/resources/ca.cer"), s.sia_base) - for s in (db if only_one_pubd else [self] + self.kids)) - f.close() - - def find_pubd(self): + @property + def pubd(self): """ Walk up tree until we find somebody who runs pubd. """ s = self - path = [s] - while not s.runs_pubd(): + while not s.runs_pubd: s = s.parent - path.append(s) - return s, ".".join(i.name for i in reversed(path)) + return s - def find_host(self): + @property + def client_handle(self): """ - Figure out who hosts this entity. + Work out what pubd configure_publication_client will call us. """ + path = [] + s = self + while not s.runs_pubd: + path.append(s) + s = s.parent + path.append(s) + return ".".join(i.name for i in reversed(path)) + + @property + def host(self): return self.hosted_by or self def dump_conf(self, fn): @@ -396,12 +369,10 @@ class allocation(object): Write configuration file for OpenSSL and RPKI tools. """ - s, ignored = self.find_pubd() - r = { "handle" : self.name, - "run_rpkid" : str(not self.is_hosted()), - "run_pubd" : str(self.runs_pubd()), - "run_rootd" : str(self.is_root()), + "run_rpkid" : str(not self.is_hosted), + "run_pubd" : str(self.runs_pubd), + "run_rootd" : str(self.is_root), "openssl" : prog_openssl, "irdbd_sql_database" : "irdb%d" % self.engine, "irdbd_sql_username" : "irdb", @@ -415,8 +386,8 @@ class allocation(object): "pubd_sql_database" : "pubd%d" % self.engine, "pubd_sql_username" : "pubd", "pubd_server_host" : "localhost", - "pubd_server_port" : str(s.pubd_port), - "publication_rsync_server" : "localhost:%s" % s.rsync_port } + "pubd_server_port" : str(self.pubd.pubd_port), + "publication_rsync_server" : "localhost:%s" % self.pubd.rsync_port } r.update(config_overrides) @@ -442,7 +413,7 @@ class allocation(object): Write rsyncd configuration file. """ - if self.runs_pubd(): + if self.runs_pubd: f = open(self.path(fn), "w") print "Writing", f.name f.writelines(s + "\n" for s in @@ -457,40 +428,26 @@ class allocation(object): "comment = RPKI test")) f.close() - def run_configure_daemons(self): - """ - Run configure_daemons if this entity is not hosted by another engine. - """ - if self.is_hosted(): - print "%s is hosted, skipping configure_daemons" % self.name - else: - files = [h.path("myrpki.xml") for h in self.hosts] - self.run_myrpki("configure_daemons", *[f for f in files if os.path.exists(f)]) - - def run_configure_resources(self): + def run_rpkic(self, *args): """ - Run configure_resources for this entity. + Run rpkic for this entity. """ - self.run_myrpki("configure_resources") - - def run_myrpki(self, *args): - """ - Run myrpki.py for this entity. - """ - print 'Running "%s" for %s' % (" ".join(("myrpki",) + args), self.name) - subprocess.check_call((sys.executable, prog_myrpki) + args, cwd = self.path()) + cmd = (prog_rpkic, "-i", self.name, "-c", self.path("rpki.conf")) + args + print 'Running "%s"' % " ".join(cmd) + subprocess.check_call(cmd, cwd = self.host.path()) def run_python_daemon(self, prog): """ Start a Python daemon and return a subprocess.Popen object representing the running daemon. """ - basename = os.path.basename(prog) - p = subprocess.Popen((sys.executable, prog, "-d", "-c", self.path("rpki.conf")), + cmd = (prog, "-d", "-c", self.path("rpki.conf")) + log = os.path.splitext(os.path.basename(prog))[0] + ".log" + p = subprocess.Popen(cmd, cwd = self.path(), - stdout = open(self.path(os.path.splitext(basename)[0] + ".log"), "w"), + stdout = open(self.path(log), "w"), stderr = subprocess.STDOUT) - print "Running %s for %s: pid %d process %r" % (basename, self.name, p.pid, p) + print 'Running %s for %s: pid %d process %r' % (" ".join(cmd), self.name, p.pid, p) return p def run_rpkid(self): @@ -604,45 +561,61 @@ try: # Show what we loaded - db.dump() + #db.dump() # Set up each entity in our test for d in db: - os.makedirs(d.path()) - d.dump_asns("asns.csv") - d.dump_prefixes("prefixes.csv") - d.dump_roas("roas.csv") - d.dump_conf("rpki.conf") - d.dump_rsyncd("rsyncd.conf") - if False: - d.dump_children("children.csv") - d.dump_parents("parents.csv") - d.dump_clients("pubclients.csv", db) + if not d.is_hosted: + os.makedirs(d.path()) + os.makedirs(d.path("bpki/resources")) + os.makedirs(d.path("bpki/servers")) + d.dump_conf("rpki.conf") + if d.runs_pubd: + d.dump_rsyncd("rsyncd.conf") # Initialize BPKI and generate self-descriptor for each entity. for d in db: - d.run_myrpki("initialize") + d.run_rpkic("initialize") # Create publication directories. for d in db: - if d.is_root() or d.runs_pubd(): + if d.is_root or d.runs_pubd: os.makedirs(d.path("publication")) # Create RPKI root certificate. print "Creating rootd RPKI root certificate" - # Should use req -subj here to set subject name. Later. - db.root.run_openssl("x509", "-req", "-sha256", "-outform", "DER", - "-signkey", "bpki/servers/ca.key", - "-in", "bpki/servers/ca.req", - "-out", "publication/root.cer", - "-extfile", "rpki.conf", - "-extensions", "rootd_x509_extensions") + root_resources = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as("0-4294967295"), + v4 = rpki.resource_set.resource_set_ipv4("0.0.0.0/0"), + v6 = rpki.resource_set.resource_set_ipv6("::/0")) + root_key = rpki.x509.RSA.generate() + + root_uri = "rsync://localhost:%d/rpki/" % db.root.pubd.rsync_port + + root_sia = ((rpki.oids.name2oid["id-ad-caRepository"], ("uri", root_uri)), + (rpki.oids.name2oid["id-ad-rpkiManifest"], ("uri", root_uri + "root.mnf"))) + + root_cert = rpki.x509.X509.self_certify( + keypair = root_key, + subject_key = root_key.get_RSApublic(), + serial = 1, + sia = root_sia, + notAfter = rpki.sundial.now() + rpki.sundial.timedelta(days = 365), + resources = root_resources) + + f = open(db.root.path("publication/root.cer"), "wb") + f.write(root_cert.get_DER()) + f.close() + + f = open(db.root.path("bpki/servers/root.key"), "wb") + f.write(root_key.get_DER()) + f.close() # From here on we need to pay attention to initialization order. We # used to do all the pre-configure_daemons stuff before running any @@ -659,62 +632,62 @@ try: print print "Configuring", d.name print - if d.is_root(): - d.run_myrpki("configure_publication_client", d.path("entitydb", "repositories", "%s.xml" % d.name)) + if d.is_root: + assert not d.is_hosted + d.run_rpkic("configure_publication_client", + d.path("%s.%s.repository-request.xml" % (d.name, d.name))) print - d.run_myrpki("configure_repository", d.path("entitydb", "pubclients", "%s.xml" % d.name)) + d.run_rpkic("configure_repository", + d.path("%s.repository-response.xml" % d.client_handle)) print else: - d.parent.run_myrpki("configure_child", d.path("entitydb", "identity.xml")) + d.parent.run_rpkic("configure_child", d.path("%s.identity.xml" % d.name)) print - d.run_myrpki("configure_parent", d.parent.path("entitydb", "children", "%s.xml" % d.name)) + d.run_rpkic("configure_parent", + d.parent.path("%s.%s.parent-response.xml" % (d.parent.name, d.name))) print - publisher, path = d.find_pubd() - publisher.run_myrpki("configure_publication_client", d.path("entitydb", "repositories", "%s.xml" % d.parent.name)) + d.pubd.run_rpkic("configure_publication_client", + d.path("%s.%s.repository-request.xml" % (d.name, d.parent.name))) print - d.run_myrpki("configure_repository", publisher.path("entitydb", "pubclients", "%s.xml" % path)) + d.run_rpkic("configure_repository", + d.pubd.path("%s.repository-response.xml" % d.client_handle)) print - parent_host = d.parent.find_host() - if d.parent is not parent_host: - d.parent.run_configure_resources() - print - parent_host.run_configure_daemons() + d.parent.run_rpkic("synchronize") print - if publisher is not parent_host: - publisher.run_configure_daemons() + if d.pubd is not d.parent.host: + d.pubd.run_rpkic("synchronize") print print "Running daemons for", d.name - if d.is_root(): + if d.is_root: progs.append(d.run_rootd()) - if not d.is_hosted(): + if not d.is_hosted: progs.append(d.run_irdbd()) progs.append(d.run_rpkid()) - if d.runs_pubd(): + if d.runs_pubd: progs.append(d.run_pubd()) progs.append(d.run_rsyncd()) - if d.is_root() or not d.is_hosted() or d.runs_pubd(): + if d.is_root or not d.is_hosted or d.runs_pubd: print "Giving", d.name, "daemons time to start up" time.sleep(20) print assert all(p.poll() is None for p in progs) - # Run configure_daemons to set up IRDB and RPKI objects. Need to - # run a second time to push BSC certs out to rpkid. Nothing - # should happen on the third pass. Oops, when hosting we need to - # run configure_resources between passes, since only the hosted - # entity can issue the BSC, etc. + # In theory we now only need to synchronize the new entity once. + d.run_rpkic("synchronize") + # Run through list again, to be sure we catch hosted cases. + # In theory this is no longer necessary. + if False: for i in xrange(3): - d.run_configure_resources() - d.find_host().run_configure_daemons() + for d in db: + d.run_rpkic("synchronize") - # Run through list again, to be sure we catch hosted cases - - for i in xrange(3): - for d in db: - d.run_configure_resources() - d.run_configure_daemons() + # Load all the CSV files + for d in db: + d.dump_asns("%s.asns.csv" % d.name) + d.dump_prefixes("%s.prefixes.csv" % d.name) + d.dump_roas("%s.roas.csv" % d.name) print "Done initializing daemons" diff --git a/rtr-origin/rtr-origin.py b/rtr-origin/rtr-origin.py index 246e4120..3b6ec145 100755 --- a/rtr-origin/rtr-origin.py +++ b/rtr-origin/rtr-origin.py @@ -1240,6 +1240,25 @@ class client_channel(pdu_channel): proc = subprocess.Popen(argv, stdin = s[0], stdout = s[0], close_fds = True), killsig = signal.SIGINT) + @classmethod + def tls(cls, host, port): + """ + Set up TLS connection and start listening for first PDU. + + NB: This uses OpenSSL's "s_client" command, which does not + check server certificates properly, so this is not suitable for + production use. Fixing this would be a trivial change, it just + requires using a client program which does check certificates + properly (eg, gnutls-cli, or stunnel's client mode if that works + for such purposes this week). + """ + args = ("openssl", "s_client", "-tls1", "-quiet", "-connect", "%s:%s" % (host, port)) + blather("[Running: %s]" % " ".join(args)) + s = socket.socketpair() + return cls(sock = s[1], + proc = subprocess.Popen(args, stdin = s[0], stdout = s[0], close_fds = True), + killsig = signal.SIGKILL) + def deliver_pdu(self, pdu): """ Handle received PDU. @@ -1572,6 +1591,10 @@ def client_main(argv): direct (and completely insecure!) TCP connection to the server. The remaining arguments should be a hostname (or IP address) and a TCP port number. + + If the first argument is "tls", the client will attempt to open a TLS connection to the server. The + remaining arguments should be a hostname (or IP address) and a TCP + port number. """ blather("[Startup]") @@ -1583,6 +1606,8 @@ def client_main(argv): client = client_channel.ssh(*argv[1:]) elif argv[0] == "tcp" and len(argv) == 3: client = client_channel.tcp(*argv[1:]) + elif argv[0] == "tls" and len(argv) == 3: + client = client_channel.tls(*argv[1:]) else: sys.exit("Unexpected arguments: %r" % (argv,)) while True: diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py new file mode 100644 index 00000000..1b469261 --- /dev/null +++ b/scripts/convert-from-entitydb-to-sql.py @@ -0,0 +1,440 @@ +""" +Merge XML entitydb and OpenSSL command-line BPKI into SQL IRDB. + +This is a work in progress, don't use it unless you really know what +you're doing. + +$Id$ + +Copyright (C) 2011 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import sys, os, time, getopt, glob, subprocess, base64 +import rpki.config, rpki.x509, rpki.relaxng, rpki.sundial +from rpki.mysql_import import MySQLdb +from lxml.etree import ElementTree + +if os.getlogin() != "sra": + sys.exit("I //said// this was a work in progress") + +cfg_file = "rpki.conf" +entitydb = "entitydb" +bpki = "bpki" +copy_csv_data = True + +opts, argv = getopt.getopt(sys.argv[1:], "c:h?", ["config=", "help"]) +for o, a in opts: + if o in ("-h", "--help", "-?"): + print __doc__ + sys.exit(0) + if o in ("-c", "--config"): + cfg_file = a +if argv: + sys.exit("Unexpected arguments %s" % argv) + +cfg = rpki.config.parser(cfg_file) + +sql_database = cfg.get("sql-database", section = "irdbd") +sql_username = cfg.get("sql-username", section = "irdbd") +sql_password = cfg.get("sql-password", section = "irdbd") + +db = MySQLdb.connect(user = sql_username, db = sql_database, passwd = sql_password) +cur = db.cursor() + +# Configure the Django model system + +from django.conf import settings + +settings.configure( + DATABASES = { "default" : { + "ENGINE" : "django.db.backends.mysql", + "NAME" : sql_database, + "USER" : sql_username, + "PASSWORD" : sql_password, + "HOST" : "", + "PORT" : "", + "OPTIONS" : { "init_command": "SET storage_engine=INNODB" }}}, + INSTALLED_APPS = ("rpki.irdb",), +) + +import rpki.irdb + +# Create the model-based tables if they don't already exist + +import django.core.management + +django.core.management.call_command("syncdb", verbosity = 4, load_initial_data = False) + +# From here down will be an awful lot of messing about with XML and +# X.509 data, extracting stuff from the old SQL database and whacking +# it into the new. Still working out these bits. + +xmlns = "{http://www.hactrn.net/uris/rpki/myrpki/}" + +tag_authorization = xmlns + "authorization" +tag_bpki_child_ta = xmlns + "bpki_child_ta" +tag_bpki_client_ta = xmlns + "bpki_client_ta" +tag_bpki_resource_ta = xmlns + "bpki_resource_ta" +tag_bpki_server_ta = xmlns + "bpki_server_ta" +tag_bpki_ta = xmlns + "bpki_ta" +tag_contact_info = xmlns + "contact_info" +tag_identity = xmlns + "identity" +tag_parent = xmlns + "parent" +tag_repository = xmlns + "repository" + +e = ElementTree(file = os.path.join(entitydb, "identity.xml")).getroot() +rpki.relaxng.myrpki.assertValid(e) +assert e.tag == tag_identity + +self_handle = e.get("handle") +assert self_handle == cfg.get("handle", section = "myrpki") + +# Some BPKI utillity routines + +def read_openssl_serial(filename): + f = open(filename, "r") + text = f.read() + f.close() + return int(text.strip(), 16) + +def get_or_create_ServerEE(issuer, purpose): + cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "servers", purpose + ".cer")) + key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "servers", purpose + ".key")) + rpki.irdb.ServerEE.objects.get_or_create( + issuer = issuer, + purpose = purpose, + certificate = cer, + private_key = key) + +# Load BPKI CAs and directly certified EEs + +cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "resources", "ca.cer")) +key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "resources", "ca.key")) +crl = rpki.x509.CRL(Auto_file = os.path.join(bpki, "resources", "ca.crl")) +serial = read_openssl_serial(os.path.join(bpki, "resources", "serial")) +crl_number = read_openssl_serial(os.path.join(bpki, "resources", "crl_number")) + +resource_ca = rpki.irdb.ResourceHolderCA.objects.get_or_create( + handle = self_handle, + certificate = cer, + private_key = key, + latest_crl = crl, + next_serial = serial, + next_crl_number = crl_number, + last_crl_update = crl.getThisUpdate().to_sql(), + next_crl_update = crl.getNextUpdate().to_sql())[0] + +if os.path.exists(os.path.join(bpki, "resources", "referral.cer")): + cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "resources", "referral.cer")) + key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "resources", "referral.key")) + rpki.irdb.Referral.objects.get_or_create( + issuer = resource_ca, + certificate = cer, + private_key = key) + +run_rpkid = cfg.getboolean("run_rpkid", section = "myrpki") +run_pubd = cfg.getboolean("run_pubd", section = "myrpki") +run_rootd = cfg.getboolean("run_rootd", section = "myrpki") + +if run_rpkid or run_pubd: + cer = rpki.x509.X509(Auto_file = os.path.join(bpki, "servers", "ca.cer")) + key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "servers", "ca.key")) + crl = rpki.x509.CRL(Auto_file = os.path.join(bpki, "servers", "ca.crl")) + serial = read_openssl_serial(os.path.join(bpki, "servers", "serial")) + crl_number = read_openssl_serial(os.path.join(bpki, "servers", "crl_number")) + server_ca = rpki.irdb.ServerCA.objects.get_or_create( + certificate = cer, + private_key = key, + latest_crl = crl, + next_serial = serial, + next_crl_number = crl_number, + last_crl_update = crl.getThisUpdate().to_sql(), + next_crl_update = crl.getNextUpdate().to_sql())[0] + get_or_create_ServerEE(server_ca, "irbe") + +else: + server_ca = None + +if run_rpkid: + get_or_create_ServerEE(server_ca, "rpkid") + get_or_create_ServerEE(server_ca, "irdbd") + +if run_pubd: + get_or_create_ServerEE(server_ca, "pubd") + +# Certification model for rootd has changed. We can reuse the old +# key, but we have to recertify under a different CA than previously. +# Yes, we're pulling a key from the servers BPKI tree and certifying +# it under the resource holder CA, that's part of the change. + +if run_rootd: + rpki.irdb.Rootd.objects.get_or_certify( + issuer = resource_ca, + service_uri = "http://localhost:%s/" % cfg.get("rootd_server_port", section = "myrpki"), + private_key = rpki.x509.RSA(Auto_file = os.path.join(bpki, "servers", "rootd.key"))) + +# Load BSC certificates and requests. Yes, this currently wires in +# exactly one BSC handle, "bsc". So does the old myrpki code. Ick. + +for fn in glob.iglob(os.path.join(bpki, "resources", "bsc.*.cer")): + rpki.irdb.BSC.objects.get_or_create( + issuer = resource_ca, + handle = "bsc", + certificate = rpki.x509.X509(Auto_file = fn), + pkcs10 = rpki.x509.PKCS10(Auto_file = fn[:-4] + ".req")) + +def xcert_hash(cert): + """ + Generate the filename hash that myrpki would have generated for a + cross-certification. This is nasty, don't look. + """ + + cmd1 = ("openssl", "x509", "-noout", "-pubkey", "-subject") + cmd2 = ("openssl", "dgst", "-md5") + + env = { "PATH" : os.environ["PATH"], "OPENSSL_CONF" : "/dev/null" } + p1 = subprocess.Popen(cmd1, env = env, stdin = subprocess.PIPE, stdout = subprocess.PIPE) + p2 = subprocess.Popen(cmd2, env = env, stdin = p1.stdout, stdout = subprocess.PIPE) + p1.stdin.write(cert.get_PEM()) + p1.stdin.close() + hash = p2.stdout.read() + if p1.wait() != 0: + raise subprocess.CalledProcessError(returncode = p1.returncode, cmd = cmd1) + if p2.wait() != 0: + raise subprocess.CalledProcessError(returncode = p2.returncode, cmd = cmd2) + + hash = "".join(hash.split()) + if hash.startswith("(stdin)="): + hash = hash[len("(stdin)="):] + return hash + +# Let's try keeping track of all the xcert filenames we use, so we can +# list the ones we didn't. + +xcert_filenames = set(glob.iglob(os.path.join(bpki, "*", "xcert.*.cer"))) + +# Scrape child data out of the entitydb. + +for filename in glob.iglob(os.path.join(entitydb, "children", "*.xml")): + child_handle = os.path.splitext(os.path.split(filename)[1])[0] + + e = ElementTree(file = filename).getroot() + rpki.relaxng.myrpki.assertValid(e) + assert e.tag == tag_parent + + ta = rpki.x509.X509(Base64 = e.findtext(tag_bpki_child_ta)) + xcfn = os.path.join(bpki, "resources", "xcert.%s.cer" % xcert_hash(ta)) + xcert_filenames.discard(xcfn) + xcert = rpki.x509.X509(Auto_file = xcfn) + + cur.execute(""" + SELECT registrant_id, valid_until FROM registrant + WHERE registry_handle = %s AND registrant_handle = %s + """, (self_handle, child_handle)) + assert cur.rowcount == 1 + registrant_id, valid_until = cur.fetchone() + + valid_until = rpki.sundial.datetime.fromdatetime(valid_until) + if valid_until != rpki.sundial.datetime.fromXMLtime(e.get("valid_until")): + print "WARNING: valid_until dates in XML and SQL do not match for child", child_handle + print " SQL:", str(valid_until) + print " XML:", str(rpki.sundial.datetime.fromXMLtime(e.get("valid_until"))) + print "Blundering onwards" + + child = rpki.irdb.Child.objects.get_or_create( + handle = child_handle, + valid_until = valid_until.to_sql(), + ta = ta, + certificate = xcert, + issuer = resource_ca)[0] + + if copy_csv_data: + + cur.execute(""" + SELECT start_as, end_as FROM registrant_asn WHERE registrant_id = %s + """, (registrant_id,)) + for start_as, end_as in cur.fetchall(): + rpki.irdb.ChildASN.objects.get_or_create( + start_as = start_as, + end_as = end_as, + child = child) + + cur.execute(""" + SELECT start_ip, end_ip, version FROM registrant_net WHERE registrant_id = %s + """, (registrant_id,)) + for start_ip, end_ip, version in cur.fetchall(): + rpki.irdb.ChildNet.objects.get_or_create( + start_ip = start_ip, + end_ip = end_ip, + version = version, + child = child) + +# Scrape parent data out of the entitydb. + +for filename in glob.iglob(os.path.join(entitydb, "parents", "*.xml")): + parent_handle = os.path.splitext(os.path.split(filename)[1])[0] + + e = ElementTree(file = filename).getroot() + rpki.relaxng.myrpki.assertValid(e) + assert e.tag == tag_parent + + if parent_handle == self_handle: + assert run_rootd + assert e.get("service_uri") == "http://localhost:%s/" % cfg.get("rootd_server_port", section = "myrpki") + continue + + ta = rpki.x509.X509(Base64 = e.findtext(tag_bpki_resource_ta)) + xcfn = os.path.join(bpki, "resources", "xcert.%s.cer" % xcert_hash(ta)) + xcert_filenames.discard(xcfn) + xcert = rpki.x509.X509(Auto_file = xcfn) + + r = e.find(tag_repository) + repository_type = r.get("type") + if repository_type == "referral": + a = r.find(tag_authorization) + referrer = a.get("referrer") + referral_authorization = base64.b64decode(a.text) + else: + referrer = None + referral_authorization = None + + parent = rpki.irdb.Parent.objects.get_or_create( + handle = parent_handle, + parent_handle = e.get("parent_handle"), + child_handle = e.get("child_handle"), + ta = ta, + certificate = xcert, + service_uri = e.get("service_uri"), + repository_type = repository_type, + referrer = referrer, + referral_authorization = referral_authorization, + issuer = resource_ca)[0] + + # While we have the parent object in hand, load any Ghostbuster + # entries specific to this parent. + + if copy_csv_data: + cur.execute(""" + SELECT vcard FROM ghostbuster_request + WHERE self_handle = %s AND parent_handle = %s + """, (self_handle, parent_handle)) + for row in cur.fetchall(): + rpki.irdb.GhostbusterRequest.objects.get_or_create( + issuer = resource_ca, + parent = parent, + vcard = row[0]) + +# Scrape repository data out of the entitydb. + +for filename in glob.iglob(os.path.join(entitydb, "repositories", "*.xml")): + repository_handle = os.path.splitext(os.path.split(filename)[1])[0] + + e = ElementTree(file = filename).getroot() + rpki.relaxng.myrpki.assertValid(e) + assert e.tag == tag_repository + + if e.get("type") != "confirmed": + continue + + ta = rpki.x509.X509(Base64 = e.findtext(tag_bpki_server_ta)) + xcfn = os.path.join(bpki, "resources", "xcert.%s.cer" % xcert_hash(ta)) + xcert_filenames.discard(xcfn) + xcert = rpki.x509.X509(Auto_file = xcfn) + + parent_handle = e.get("parent_handle") + if parent_handle == self_handle: + turtle = resource_ca.rootd + else: + turtle = rpki.irdb.Parent.objects.get(handle = parent_handle) + + rpki.irdb.Repository.objects.get_or_create( + handle = repository_handle, + client_handle = e.get("client_handle"), + ta = ta, + certificate = xcert, + service_uri = e.get("service_uri"), + sia_base = e.get("sia_base"), + turtle = turtle, + issuer = resource_ca) + +# Scrape client data out of the entitydb. + +for filename in glob.iglob(os.path.join(entitydb, "pubclients", "*.xml")): + client_handle = os.path.splitext(os.path.split(filename)[1])[0].replace(".", "/") + + e = ElementTree(file = filename).getroot() + rpki.relaxng.myrpki.assertValid(e) + assert e.tag == tag_repository + + assert e.get("type") == "confirmed" + + ta = rpki.x509.X509(Base64 = e.findtext(tag_bpki_client_ta)) + xcfn = os.path.join(bpki, "servers", "xcert.%s.cer" % xcert_hash(ta)) + xcert_filenames.discard(xcfn) + xcert = rpki.x509.X509(Auto_file = xcfn) + + rpki.irdb.Client.objects.get_or_create( + handle = client_handle, + ta = ta, + certificate = xcert, + issuer = server_ca, + sia_base = e.get("sia_base")) + +if copy_csv_data: + + # Copy over any ROA requests + + cur.execute(""" + SELECT roa_request_id, asn FROM roa_request + WHERE roa_request_handle = %s + """, (self_handle,)) + for roa_request_id, asn in cur.fetchall(): + roa_request = rpki.irdb.ROARequest.objects.get_or_create(issuer = resource_ca, asn = asn)[0] + cur.execute(""" + SELECT prefix, prefixlen, max_prefixlen, version FROM roa_request_prefix + WHERE roa_request_id = %s + """, (roa_request_id,)) + for prefix, prefixlen, max_prefixlen, version in cur.fetchall(): + rpki.irdb.ROARequestPrefix.objects.get_or_create( + roa_request = roa_request, + version = version, + prefix = prefix, + prefixlen = prefixlen, + max_prefixlen = max_prefixlen) + + # Copy over any non-parent-specific Ghostbuster requests. + + cur.execute(""" + SELECT vcard FROM ghostbuster_request + WHERE self_handle = %s AND parent_handle IS NULL + """, (self_handle,)) + for row in cur.fetchall(): + rpki.irdb.GhostbusterRequest.objects.get_or_create( + issuer = resource_ca, + parent = None, + vcard = row[0]) + +# List cross certifications we didn't use. + +if False: + for filename in sorted(xcert_filenames): + cer = rpki.x509.X509(Auto_file = filename) + #print "Unused cross-certificate:", filename, cer.getSubject() + print "Unused cross-certificate:", filename, cer.get_POW().pprint() + +# Done! + +cur.close() +db.close() |