diff options
| -rw-r--r-- | buildtools/make-sql-schemas.py | 2 | ||||
| -rw-r--r-- | rpkid/Makefile.in | 2 | ||||
| -rw-r--r-- | rpkid/examples/rpki.conf | 2 | ||||
| -rw-r--r-- | rpkid/rpki/irdb/models.py | 16 | ||||
| -rw-r--r-- | rpkid/rpki/irdbd.py | 202 | ||||
| -rw-r--r-- | rpkid/rpki/old_irdbd.py | 249 | ||||
| -rw-r--r-- | rpkid/rpki/resource_set.py | 25 | ||||
| -rw-r--r-- | rpkid/rpki/rpkic.py | 195 | ||||
| -rw-r--r-- | rpkid/rpki/sql_schemas.py | 109 | ||||
| -rw-r--r-- | rpkid/tests/old_irdbd.py | 21 | ||||
| -rw-r--r-- | rpkid/tests/old_irdbd.sql (renamed from rpkid/irdbd.sql) | 0 | ||||
| -rw-r--r-- | rpkid/tests/smoketest.py | 6 | ||||
| -rw-r--r-- | rpkid/tests/sql-cleaner.py | 4 | ||||
| -rw-r--r-- | rpkid/tests/yamltest.py | 265 | ||||
| -rw-r--r-- | scripts/convert-from-entitydb-to-sql.py | 6 |
15 files changed, 680 insertions, 424 deletions
diff --git a/buildtools/make-sql-schemas.py b/buildtools/make-sql-schemas.py index 700d2b9c..3ecde014 100644 --- a/buildtools/make-sql-schemas.py +++ b/buildtools/make-sql-schemas.py @@ -32,7 +32,7 @@ OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -schemas = ("rpkid", "irdbd", "pubd") +schemas = ("rpkid", "pubd") format_1 = """\ # Automatically generated, do not edit. diff --git a/rpkid/Makefile.in b/rpkid/Makefile.in index aba6872b..db47b0da 100644 --- a/rpkid/Makefile.in +++ b/rpkid/Makefile.in @@ -82,7 +82,7 @@ publication-schema.rng: publication-schema.rnc myrpki.rng: myrpki.rnc trang myrpki.rnc myrpki.rng -rpki/sql_schemas.py: ${abs_top_srcdir}/buildtools/make-sql-schemas.py rpkid.sql irdbd.sql pubd.sql +rpki/sql_schemas.py: ${abs_top_srcdir}/buildtools/make-sql-schemas.py rpkid.sql pubd.sql ${PYTHON} ${abs_top_srcdir}/buildtools/make-sql-schemas.py >$@.tmp mv $@.tmp $@ diff --git a/rpkid/examples/rpki.conf b/rpkid/examples/rpki.conf index 06b853a9..17e37b7b 100644 --- a/rpkid/examples/rpki.conf +++ b/rpkid/examples/rpki.conf @@ -308,7 +308,7 @@ rpki-root-cert-uri = rsync://${myrpki::publication_rsync_server}/${myrpki: # Private key corresponding to rootd's root RPKI certificate -rpki-root-key = ${myrpki::bpki_servers_directory}/ca.key +rpki-root-key = ${myrpki::bpki_servers_directory}/root.key # Filename (as opposed to rsync URI) of rootd's root RPKI certificate diff --git a/rpkid/rpki/irdb/models.py b/rpkid/rpki/irdb/models.py index 071e0918..1add3593 100644 --- a/rpkid/rpki/irdb/models.py +++ b/rpkid/rpki/irdb/models.py @@ -209,6 +209,7 @@ class CertificateManager(django.db.models.Manager): return obj, changed def _get_or_certify_keys(self, kwargs): + assert len(self.model._meta.unique_together) == 1 return dict((k, kwargs[k]) for k in self.model._meta.unique_together[0]) class ResourceHolderCAManager(CertificateManager): @@ -452,6 +453,10 @@ class Child(CrossCertification): name = django.db.models.TextField(null = True, blank = True) valid_until = SundialField() + # This shouldn't be necessary + class Meta: + unique_together = ("issuer", "handle") + class ChildASN(django.db.models.Model): child = django.db.models.ForeignKey(Child, related_name = "asns") start_as = django.db.models.BigIntegerField() @@ -477,6 +482,10 @@ class Parent(CrossCertification, Turtle): referrer = HandleField(null = True, blank = True) referral_authorization = SignedReferralField(null = True, blank = True) + # This shouldn't be necessary + class Meta: + unique_together = ("issuer", "handle") + class ROARequest(django.db.models.Model): issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "roa_requests") asn = django.db.models.BigIntegerField() @@ -503,7 +512,14 @@ class Repository(CrossCertification): sia_base = django.db.models.TextField() turtle = django.db.models.OneToOneField(Turtle, related_name = "repository") + # This shouldn't be necessary + class Meta: + unique_together = ("issuer", "handle") + class Client(CrossCertification): issuer = django.db.models.ForeignKey(ServerCA, related_name = "clients") sia_base = django.db.models.TextField() + # This shouldn't be necessary + class Meta: + unique_together = ("issuer", "handle") diff --git a/rpkid/rpki/irdbd.py b/rpkid/rpki/irdbd.py index c2e01287..adda4be2 100644 --- a/rpkid/rpki/irdbd.py +++ b/rpkid/rpki/irdbd.py @@ -5,7 +5,7 @@ Usage: python irdbd.py [ { -c | --config } configfile ] [ { -h | --help } ] $Id$ -Copyright (C) 2009--2011 Internet Systems Consortium ("ISC") +Copyright (C) 2009--2012 Internet Systems Consortium ("ISC") Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above @@ -38,116 +38,106 @@ import sys, os, time, getopt, urlparse, warnings import rpki.http, rpki.config, rpki.resource_set, rpki.relaxng import rpki.exceptions, rpki.left_right, rpki.log, rpki.x509 -from rpki.mysql_import import MySQLdb +from django.conf import settings -class main(object): +os.environ["TZ"] = "UTC" +time.tzset() +cfg_file = None - def handle_list_resources(self, q_pdu, r_msg): +opts, argv = getopt.getopt(sys.argv[1:], "c:dh?", ["config=", "debug", "help"]) +for o, a in opts: + if o in ("-h", "--help", "-?"): + print __doc__ + sys.exit(0) + if o in ("-c", "--config"): + cfg_file = a + elif o in ("-d", "--debug"): + rpki.log.use_syslog = False +if argv: + raise rpki.exceptions.CommandParseFailure, "Unexpected arguments %s" % argv - r_pdu = rpki.left_right.list_resources_elt() - r_pdu.tag = q_pdu.tag - r_pdu.self_handle = q_pdu.self_handle - r_pdu.child_handle = q_pdu.child_handle +rpki.log.init("irdbd") + +cfg = rpki.config.parser(cfg_file, "irdbd") - self.cur.execute( - "SELECT registrant_id, valid_until FROM registrant WHERE registry_handle = %s AND registrant_handle = %s", - (q_pdu.self_handle, q_pdu.child_handle)) +startup_msg = cfg.get("startup-message", "") +if startup_msg: + rpki.log.info(startup_msg) - if self.cur.rowcount != 1: - raise rpki.exceptions.NotInDatabase, \ - "This query should have produced a single exact match, something's messed up (rowcount = %d, self_handle = %s, child_handle = %s)" \ - % (self.cur.rowcount, q_pdu.self_handle, q_pdu.child_handle) +cfg.set_global_flags() - registrant_id, valid_until = self.cur.fetchone() +settings.configure( + DEBUG = True, + DATABASES = { "default" : { + "ENGINE" : "django.db.backends.mysql", + "NAME" : cfg.get("sql-database"), + "USER" : cfg.get("sql-username"), + "PASSWORD" : cfg.get("sql-password"), + "HOST" : "", + "PORT" : ""}}, + INSTALLED_APPS = ("rpki.irdb",),) - r_pdu.valid_until = valid_until.strftime("%Y-%m-%dT%H:%M:%SZ") +import rpki.irdb - r_pdu.asn = rpki.resource_set.resource_set_as.from_sql( - self.cur, - "SELECT start_as, end_as FROM registrant_asn WHERE registrant_id = %s", - (registrant_id,)) - r_pdu.ipv4 = rpki.resource_set.resource_set_ipv4.from_sql( - self.cur, - "SELECT start_ip, end_ip FROM registrant_net WHERE registrant_id = %s AND version = 4", - (registrant_id,)) +class Main(object): - r_pdu.ipv6 = rpki.resource_set.resource_set_ipv6.from_sql( - self.cur, - "SELECT start_ip, end_ip FROM registrant_net WHERE registrant_id = %s AND version = 6", - (registrant_id,)) + def handle_list_resources(self, q_pdu, r_msg): + child = rpki.irdb.Child.objects.get(issuer__handle__exact = q_pdu.self_handle, handle = q_pdu.child_handle) + r_pdu = rpki.left_right.list_resources_elt() + r_pdu.tag = q_pdu.tag + r_pdu.self_handle = q_pdu.self_handle + r_pdu.child_handle = q_pdu.child_handle + r_pdu.valid_until = child.valid_until.strftime("%Y-%m-%dT%H:%M:%SZ") + r_pdu.asn = rpki.resource_set.resource_set_as.from_django( + (a.start_as, a.end_s) for a in child.asns.all()) + r_pdu.ipv4 = rpki.resource_set.resource_set_ipv4.from_django( + (a.start_ip, a.end_ip) for a in child.address_ranges.filter(version = 4)) + r_pdu.ipv6 = rpki.resource_set.resource_set_ipv6.from_django( + (a.start_ip, a.end_ip) for a in child.address_ranges.filter(version = 6)) r_msg.append(r_pdu) def handle_list_roa_requests(self, q_pdu, r_msg): - - self.cur.execute( - "SELECT roa_request_id, asn FROM roa_request WHERE roa_request_handle = %s", - (q_pdu.self_handle,)) - - for roa_request_id, asn in self.cur.fetchall(): - + for request in rpki.irdb.ROARequest.objects.filter(issuer__handle__exact = q_pdu.self_handle): r_pdu = rpki.left_right.list_roa_requests_elt() r_pdu.tag = q_pdu.tag r_pdu.self_handle = q_pdu.self_handle - r_pdu.asn = asn - - r_pdu.ipv4 = rpki.resource_set.roa_prefix_set_ipv4.from_sql( - self.cur, - "SELECT prefix, prefixlen, max_prefixlen FROM roa_request_prefix WHERE roa_request_id = %s AND version = 4", - (roa_request_id,)) - - r_pdu.ipv6 = rpki.resource_set.roa_prefix_set_ipv6.from_sql( - self.cur, - "SELECT prefix, prefixlen, max_prefixlen FROM roa_request_prefix WHERE roa_request_id = %s AND version = 6", - (roa_request_id,)) - + r_pdu.asn = request.asn + r_pdu.ipv4 = rpki.resource_set.roa_prefix_set_ipv4.from_django( + (p.prefix, p.prefixlen, p.max_prefixlen) for p in request.prefixes.filter(version = 4)) + r_pdu.ipv6 = rpki.resource_set.roa_prefix_set_ipv6.from_django( + (p.prefix, p.prefixlen, p.max_prefixlen) for p in request.prefixes.filter(version = 6)) r_msg.append(r_pdu) def handle_list_ghostbuster_requests(self, q_pdu, r_msg): - - self.cur.execute( - "SELECT vcard FROM ghostbuster_request WHERE self_handle = %s AND parent_handle = %s", - (q_pdu.self_handle, q_pdu.parent_handle)) - - vcards = [result[0] for result in self.cur.fetchall()] - - if not vcards: - - self.cur.execute( - "SELECT vcard FROM ghostbuster_request WHERE self_handle = %s AND parent_handle IS NULL", - (q_pdu.self_handle,)) - - vcards = [result[0] for result in self.cur.fetchall()] - - for vcard in vcards: + ghostbusters = rpki.irdb.GhostbusterRequest.objects.filter(issuer__handle__exact = q_pdu.self_handle, + parent__handle__exact = q_pdu.parent_handle) + if ghostbusters.count() == 0: + ghostbusters = rpki.irdb.GhostbusterRequest.objects.filter(issuer__handle__exact = q_pdu.self_handle, + parent = None) + for ghostbuster in ghostbusters: r_pdu = rpki.left_right.list_ghostbuster_requests_elt() r_pdu.tag = q_pdu.tag r_pdu.self_handle = q_pdu.self_handle r_pdu.parent_handle = q_pdu.parent_handle - r_pdu.vcard = vcard + r_pdu.vcard = ghostbuster.vcard r_msg.append(r_pdu) - handle_dispatch = { - rpki.left_right.list_resources_elt : handle_list_resources, - rpki.left_right.list_roa_requests_elt : handle_list_roa_requests, - rpki.left_right.list_ghostbuster_requests_elt : handle_list_ghostbuster_requests} - - def handler(self, query, path, cb): - try: - - self.db.ping(True) + try: r_msg = rpki.left_right.msg.reply() - try: + serverCA = rpki.irdb.ServerCA.objects.get() + rpkid = serverCA.ee_certificates.get(purpose = "rpkid") - q_msg = rpki.left_right.cms_msg(DER = query).unwrap((self.bpki_ta, self.rpkid_cert)) + try: + q_msg = rpki.left_right.cms_msg(DER = query).unwrap((serverCA.certificate, rpkid.certificate)) if not isinstance(q_msg, rpki.left_right.msg) or not q_msg.is_query(): raise rpki.exceptions.BadQuery, "Unexpected %r PDU" % q_msg @@ -177,7 +167,9 @@ class main(object): rpki.log.traceback() r_msg.append(rpki.left_right.report_error_elt.from_exception(e)) - cb(200, body = rpki.left_right.cms_msg().wrap(r_msg, self.irdbd_key, self.irdbd_cert)) + irdbd = serverCA.ee_certificates.get(purpose = "irdbd") + + cb(200, body = rpki.left_right.cms_msg().wrap(r_msg, irdbd.private_key, irdbd.certificate)) except (rpki.async.ExitNow, SystemExit): raise @@ -191,56 +183,28 @@ class main(object): cb(500, reason = "Unhandled exception %s: %s" % (e.__class__.__name__, e)) - def __init__(self): - - os.environ["TZ"] = "UTC" - time.tzset() - - cfg_file = None - - opts, argv = getopt.getopt(sys.argv[1:], "c:dh?", ["config=", "debug", "help"]) - for o, a in opts: - if o in ("-h", "--help", "-?"): - print __doc__ - sys.exit(0) - if o in ("-c", "--config"): - cfg_file = a - elif o in ("-d", "--debug"): - rpki.log.use_syslog = False - if argv: - raise rpki.exceptions.CommandParseFailure, "Unexpected arguments %s" % argv - - rpki.log.init("irdbd") - - self.cfg = rpki.config.parser(cfg_file, "irdbd") - - startup_msg = self.cfg.get("startup-message", "") - if startup_msg: - rpki.log.info(startup_msg) - - self.cfg.set_global_flags() - - self.db = MySQLdb.connect(user = self.cfg.get("sql-username"), - db = self.cfg.get("sql-database"), - passwd = self.cfg.get("sql-password")) + handle_dispatch = { + rpki.left_right.list_resources_elt : handle_list_resources, + rpki.left_right.list_roa_requests_elt : handle_list_roa_requests, + rpki.left_right.list_ghostbuster_requests_elt : handle_list_ghostbuster_requests} - self.cur = self.db.cursor() - self.db.autocommit(True) - self.bpki_ta = rpki.x509.X509(Auto_update = self.cfg.get("bpki-ta")) - self.rpkid_cert = rpki.x509.X509(Auto_update = self.cfg.get("rpkid-cert")) - self.irdbd_cert = rpki.x509.X509(Auto_update = self.cfg.get("irdbd-cert")) - self.irdbd_key = rpki.x509.RSA( Auto_update = self.cfg.get("irdbd-key")) + def __init__(self, **kwargs): + for k in kwargs: + setattr(self, k, kwargs[k]) - u = urlparse.urlparse(self.cfg.get("http-url")) + def __call__(self): + u = urlparse.urlparse(self.http_url) assert u.scheme in ("", "http") and \ u.username is None and \ u.password is None and \ u.params == "" and \ u.query == "" and \ u.fragment == "" + rpki.http.server(host = u.hostname or "localhost", + port = u.port or 443, + handlers = ((u.path, self.handler),)) + - rpki.http.server(host = u.hostname or "localhost", - port = u.port or 443, - handlers = ((u.path, self.handler),)) +main = Main(http_url = cfg.get("http-url")) diff --git a/rpkid/rpki/old_irdbd.py b/rpkid/rpki/old_irdbd.py new file mode 100644 index 00000000..c63ce9e2 --- /dev/null +++ b/rpkid/rpki/old_irdbd.py @@ -0,0 +1,249 @@ +""" +IR database daemon. + +Usage: python irdbd.py [ { -c | --config } configfile ] [ { -h | --help } ] + +This is the old (pre-Django) version of irdbd, still used by smoketest +and perhaps still useful as a minimal example. + +$Id$ + +Copyright (C) 2009--2012 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + +Portions copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import sys, os, time, getopt, urlparse, warnings +import rpki.http, rpki.config, rpki.resource_set, rpki.relaxng +import rpki.exceptions, rpki.left_right, rpki.log, rpki.x509 + +from rpki.mysql_import import MySQLdb + +class main(object): + + + def handle_list_resources(self, q_pdu, r_msg): + + r_pdu = rpki.left_right.list_resources_elt() + r_pdu.tag = q_pdu.tag + r_pdu.self_handle = q_pdu.self_handle + r_pdu.child_handle = q_pdu.child_handle + + self.cur.execute( + "SELECT registrant_id, valid_until FROM registrant WHERE registry_handle = %s AND registrant_handle = %s", + (q_pdu.self_handle, q_pdu.child_handle)) + + if self.cur.rowcount != 1: + raise rpki.exceptions.NotInDatabase, \ + "This query should have produced a single exact match, something's messed up (rowcount = %d, self_handle = %s, child_handle = %s)" \ + % (self.cur.rowcount, q_pdu.self_handle, q_pdu.child_handle) + + registrant_id, valid_until = self.cur.fetchone() + + r_pdu.valid_until = valid_until.strftime("%Y-%m-%dT%H:%M:%SZ") + + r_pdu.asn = rpki.resource_set.resource_set_as.from_sql( + self.cur, + "SELECT start_as, end_as FROM registrant_asn WHERE registrant_id = %s", + (registrant_id,)) + + r_pdu.ipv4 = rpki.resource_set.resource_set_ipv4.from_sql( + self.cur, + "SELECT start_ip, end_ip FROM registrant_net WHERE registrant_id = %s AND version = 4", + (registrant_id,)) + + r_pdu.ipv6 = rpki.resource_set.resource_set_ipv6.from_sql( + self.cur, + "SELECT start_ip, end_ip FROM registrant_net WHERE registrant_id = %s AND version = 6", + (registrant_id,)) + + r_msg.append(r_pdu) + + + def handle_list_roa_requests(self, q_pdu, r_msg): + + self.cur.execute( + "SELECT roa_request_id, asn FROM roa_request WHERE roa_request_handle = %s", + (q_pdu.self_handle,)) + + for roa_request_id, asn in self.cur.fetchall(): + + r_pdu = rpki.left_right.list_roa_requests_elt() + r_pdu.tag = q_pdu.tag + r_pdu.self_handle = q_pdu.self_handle + r_pdu.asn = asn + + r_pdu.ipv4 = rpki.resource_set.roa_prefix_set_ipv4.from_sql( + self.cur, + "SELECT prefix, prefixlen, max_prefixlen FROM roa_request_prefix WHERE roa_request_id = %s AND version = 4", + (roa_request_id,)) + + r_pdu.ipv6 = rpki.resource_set.roa_prefix_set_ipv6.from_sql( + self.cur, + "SELECT prefix, prefixlen, max_prefixlen FROM roa_request_prefix WHERE roa_request_id = %s AND version = 6", + (roa_request_id,)) + + r_msg.append(r_pdu) + + + def handle_list_ghostbuster_requests(self, q_pdu, r_msg): + + self.cur.execute( + "SELECT vcard FROM ghostbuster_request WHERE self_handle = %s AND parent_handle = %s", + (q_pdu.self_handle, q_pdu.parent_handle)) + + vcards = [result[0] for result in self.cur.fetchall()] + + if not vcards: + + self.cur.execute( + "SELECT vcard FROM ghostbuster_request WHERE self_handle = %s AND parent_handle IS NULL", + (q_pdu.self_handle,)) + + vcards = [result[0] for result in self.cur.fetchall()] + + for vcard in vcards: + r_pdu = rpki.left_right.list_ghostbuster_requests_elt() + r_pdu.tag = q_pdu.tag + r_pdu.self_handle = q_pdu.self_handle + r_pdu.parent_handle = q_pdu.parent_handle + r_pdu.vcard = vcard + r_msg.append(r_pdu) + + + handle_dispatch = { + rpki.left_right.list_resources_elt : handle_list_resources, + rpki.left_right.list_roa_requests_elt : handle_list_roa_requests, + rpki.left_right.list_ghostbuster_requests_elt : handle_list_ghostbuster_requests} + + + def handler(self, query, path, cb): + try: + + self.db.ping(True) + + r_msg = rpki.left_right.msg.reply() + + try: + + q_msg = rpki.left_right.cms_msg(DER = query).unwrap((self.bpki_ta, self.rpkid_cert)) + + if not isinstance(q_msg, rpki.left_right.msg) or not q_msg.is_query(): + raise rpki.exceptions.BadQuery, "Unexpected %r PDU" % q_msg + + for q_pdu in q_msg: + + try: + + try: + h = self.handle_dispatch[type(q_pdu)] + except KeyError: + raise rpki.exceptions.BadQuery, "Unexpected %r PDU" % q_pdu + else: + h(self, q_pdu, r_msg) + + except (rpki.async.ExitNow, SystemExit): + raise + + except Exception, e: + rpki.log.traceback() + r_msg.append(rpki.left_right.report_error_elt.from_exception(e, q_pdu.self_handle, q_pdu.tag)) + + except (rpki.async.ExitNow, SystemExit): + raise + + except Exception, e: + rpki.log.traceback() + r_msg.append(rpki.left_right.report_error_elt.from_exception(e)) + + cb(200, body = rpki.left_right.cms_msg().wrap(r_msg, self.irdbd_key, self.irdbd_cert)) + + except (rpki.async.ExitNow, SystemExit): + raise + + except Exception, e: + rpki.log.traceback() + + # We only get here in cases where we couldn't or wouldn't generate + # <report_error/>, so just return HTTP failure. + + cb(500, reason = "Unhandled exception %s: %s" % (e.__class__.__name__, e)) + + + def __init__(self): + + os.environ["TZ"] = "UTC" + time.tzset() + + cfg_file = None + + opts, argv = getopt.getopt(sys.argv[1:], "c:dh?", ["config=", "debug", "help"]) + for o, a in opts: + if o in ("-h", "--help", "-?"): + print __doc__ + sys.exit(0) + if o in ("-c", "--config"): + cfg_file = a + elif o in ("-d", "--debug"): + rpki.log.use_syslog = False + if argv: + raise rpki.exceptions.CommandParseFailure, "Unexpected arguments %s" % argv + + rpki.log.init("irdbd") + + self.cfg = rpki.config.parser(cfg_file, "irdbd") + + startup_msg = self.cfg.get("startup-message", "") + if startup_msg: + rpki.log.info(startup_msg) + + self.cfg.set_global_flags() + + self.db = MySQLdb.connect(user = self.cfg.get("sql-username"), + db = self.cfg.get("sql-database"), + passwd = self.cfg.get("sql-password")) + + self.cur = self.db.cursor() + self.db.autocommit(True) + + self.bpki_ta = rpki.x509.X509(Auto_update = self.cfg.get("bpki-ta")) + self.rpkid_cert = rpki.x509.X509(Auto_update = self.cfg.get("rpkid-cert")) + self.irdbd_cert = rpki.x509.X509(Auto_update = self.cfg.get("irdbd-cert")) + self.irdbd_key = rpki.x509.RSA( Auto_update = self.cfg.get("irdbd-key")) + + u = urlparse.urlparse(self.cfg.get("http-url")) + + assert u.scheme in ("", "http") and \ + u.username is None and \ + u.password is None and \ + u.params == "" and \ + u.query == "" and \ + u.fragment == "" + + rpki.http.server(host = u.hostname or "localhost", + port = u.port or 443, + handlers = ((u.path, self.handler),)) diff --git a/rpkid/rpki/resource_set.py b/rpkid/rpki/resource_set.py index 2fd10756..2be8a614 100644 --- a/rpkid/rpki/resource_set.py +++ b/rpkid/rpki/resource_set.py @@ -500,6 +500,18 @@ class resource_set(list): for (b, e) in sql.fetchall()]) @classmethod + def from_django(cls, iterable): + """ + Create resource set from a Django query. + + iterable is something which returns (min, max) pairs. + """ + + return cls(ini = [cls.range_type(cls.range_type.datum_type(b), + cls.range_type.datum_type(e)) + for (b, e) in iterable]) + + @classmethod def parse_str(cls, s): """ Parse resource set from text string (eg, XML attributes). This is @@ -983,6 +995,19 @@ class roa_prefix_set(list): return cls([cls.prefix_type(cls.prefix_type.range_type.datum_type(x), int(y), int(z)) for (x, y, z) in sql.fetchall()]) + @classmethod + def from_sql(cls, iterable): + """ + Create ROA prefix set from a Django query. + + iterable is something which returns (prefix, prefixlen, + max_prefixlen) triples. + """ + + return cls([cls.prefix_type(cls.prefix_type.range_type.datum_type(x), int(y), int(z)) + for (x, y, z) in iterable]) + + def to_roa_tuple(self): """ Convert ROA prefix set into tuple format used by ROA ASN.1 diff --git a/rpkid/rpki/rpkic.py b/rpkid/rpki/rpkic.py index a946ec65..c098eea5 100644 --- a/rpkid/rpki/rpkic.py +++ b/rpkid/rpki/rpkic.py @@ -82,13 +82,43 @@ def B64Element(e, tag, obj, **kwargs): DER object. """ - if e.text is None: + if e is None: + se = Element(tag, **kwargs) + else: + se = SubElement(e, tag, **kwargs) + if e is not None and e.text is None: e.text = "\n" - se = SubElement(e, tag, **kwargs) se.text = "\n" + obj.get_Base64() se.tail = "\n" return se +class PEM_writer(object): + """ + Write PEM files to disk, keeping track of which ones we've already + written and setting the file mode appropriately. + """ + + def __init__(self, verbose = False): + self.wrote = set() + self.verbose = verbose + + def __call__(self, filename, obj): + filename = os.path.realpath(filename) + if filename in self.wrote: + return + tempname = filename + if not filename.startswith("/dev/"): + tempname += ".%s.tmp" % os.getpid() + mode = 0400 if filename.endswith(".key") else 0444 + if self.verbose: + print "Writing", filename + f = os.fdopen(os.open(tempname, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode), "w") + f.write(obj.get_PEM()) + f.close() + if tempname != filename: + os.rename(tempname, filename) + self.wrote.add(filename) + def etree_write(e, filename, verbose = False, msg = None): @@ -199,12 +229,16 @@ class main(rpki.cli.Cmd): "USER" : self.cfg.get("sql-username", section = irdbd_section), "PASSWORD" : self.cfg.get("sql-password", section = irdbd_section), "HOST" : "", - "PORT" : "" }}, + "PORT" : "", + "OPTIONS" : { "init_command": "SET storage_engine=INNODB" }}}, INSTALLED_APPS = ("rpki.irdb",), ) import rpki.irdb + import django.core.management + django.core.management.call_command("syncdb", verbosity = 0, load_initial_data = False) + if self.run_rootd and (not self.run_pubd or not self.run_rpkid): raise CantRunRootd, "Can't run rootd unless also running rpkid and pubd" @@ -286,12 +320,14 @@ class main(rpki.cli.Cmd): # Build the identity.xml file. Need to check for existing file so we don't # overwrite? Worry about that later. + run_rootd = self.run_rootd and self.handle == self.cfg.get("handle") + e = Element("identity", handle = self.handle) B64Element(e, "bpki_ta", self.resource_ca.certificate) - etree_write(e, "identity.xml", - msg = None if self.run_rootd else 'This is the "identity" file you will need to send to your parent') + etree_write(e, "%s.identity.xml" % self.handle, + msg = None if run_rootd else 'This is the "identity" file you will need to send to your parent') - if self.run_rootd: + if run_rootd: assert self.run_rpkid and self.run_pubd rpki.irdb.Rootd.objects.get_or_certify( @@ -308,9 +344,47 @@ class main(rpki.cli.Cmd): except rpki.irdb.Repository.DoesNotExist: e = Element("repository", type = "offer", handle = self.handle, parent_handle = self.handle) B64Element(e, "bpki_client_ta", self.resource_ca.certificate) - etree_write(e, "rootd_repository_offer.xml", + etree_write(e, "%s.%s.repository-request.xml" % (self.handle, self.handle), msg = 'This is the "repository offer" file for you to use if you want to publish in your own repository') + # Not (yet) sure whether we should be calling this here, try it for now + self.write_bpki_files() + + + def write_bpki_files(self): + """ + Write out BPKI certificate, key, and CRL files for daemons that + need them. + """ + + writer = PEM_writer() + + if self.run_rpkid: + rpkid = self.server_ca.ee_certificates.get(purpose = "rpkid") + writer(self.cfg.get("bpki-ta", section = "rpkid"), self.server_ca.certificate) + writer(self.cfg.get("rpkid-key", section = "rpkid"), rpkid.private_key) + writer(self.cfg.get("rpkid-cert", section = "rpkid"), rpkid.certificate) + writer(self.cfg.get("irdb-cert", section = "rpkid"), + self.server_ca.ee_certificates.get(purpose = "irdbd").certificate) + writer(self.cfg.get("irbe-cert", section = "rpkid"), + self.server_ca.ee_certificates.get(purpose = "irbe").certificate) + + if self.run_pubd: + pubd = self.server_ca.ee_certificates.get(purpose = "pubd") + writer(self.cfg.get("bpki-ta", section = "pubd"), self.server_ca.certificate) + writer(self.cfg.get("pubd-key", section = "pubd"), pubd.private_key) + writer(self.cfg.get("pubd-cert", section = "pubd"), pubd.certificate) + writer(self.cfg.get("irbe-cert", section = "pubd"), + self.server_ca.ee_certificates.get(purpose = "irbe").certificate) + + if self.run_rootd: + rootd = rpki.irdb.ResourceHolderCA.objects.get(handle = self.cfg.get("handle", section = "myrpki")).rootd + writer(self.cfg.get("bpki-ta", section = "rootd"), self.server_ca.certificate) + writer(self.cfg.get("rootd-bpki-crl", section = "rootd"), self.server_ca.latest_crl) + writer(self.cfg.get("rootd-bpki-key", section = "rootd"), rootd.private_key) + writer(self.cfg.get("rootd-bpki-cert", section = "rootd"), rootd.certificate) + writer(self.cfg.get("child-bpki-cert", section = "rootd"), rootd.issuer.certificate) + def do_update_bpki(self, arg): """ @@ -347,6 +421,9 @@ class main(rpki.cli.Cmd): print "Regenerating CRL for", ca.handle ca.generate_crl() + self.write_bpki_files() + + def do_configure_child(self, arg): """ Configure a new child of this RPKI entity, given the child's XML @@ -380,42 +457,46 @@ class main(rpki.cli.Cmd): print "Child calls itself %r, we call it %r" % (c.get("handle"), child_handle) - rpki.irdb.Child.objects.get_or_certify( + child, created = rpki.irdb.Child.objects.get_or_certify( issuer = self.resource_ca, handle = child_handle, ta = rpki.x509.X509(Base64 = c.findtext("bpki_ta")), - valid_until = valid_until.toXMLtime()) + valid_until = valid_until) e = Element("parent", parent_handle = self.handle, child_handle = child_handle, - service_uri = service_uri, valid_until = valid_until) + service_uri = service_uri, valid_until = str(valid_until)) B64Element(e, "bpki_resource_ta", self.resource_ca.certificate) - SubElement(e, "bpki_child_ta").text = c.findtext("bpki_ta") + B64Element(e, "bpki_child_ta", child.ta) try: - repo = self.resource_ca.repositories.get(handle = self.default_repository) + if self.default_repository: + repo = self.resource_ca.repositories.get(handle = self.default_repository) + else: + repo = self.resource_ca.repositories.get() except rpki.irdb.Repository.DoesNotExist: - try: - repo = self.resource_ca.repositories[0] - except rpki.irdb.Repository.DoesNotExist: - repo = None + repo = None if repo is None: print "Couldn't find any usable repositories, not giving referral" - elif repo_handle == self.handle: + elif repo.handle == self.handle: SubElement(e, "repository", type = "offer") else: - proposed_sia_base = repo.get("sia_base") + child_handle + "/" - r = Element("referral", authorized_sia_base = proposed_sia_base) - r.text = c.findtext("bpki_ta") - auth = self.bpki_resources.cms_xml_sign(r) + proposed_sia_base = repo.sia_base + child_handle + "/" + referral_cert, created = rpki.irdb.Referral.objects.get_or_certify(issuer = self.resource_ca) + auth = rpki.x509.SignedReferral() + auth.set_content(B64Element(None, namespaceQName + "referral", child.ta, + version = version, + authorized_sia_base = proposed_sia_base)) + auth.schema_check() + auth.sign(referral_cert.private_key, referral_cert.certificate, self.resource_ca.latest_crl) r = SubElement(e, "repository", type = "referral") - SubElement(r, "authorization", referrer = repo.get("client_handle")).text = auth - SubElement(r, "contact_info").text = repo.findtext("contact_info") + B64Element(r, "authorization", auth, referrer = repo.client_handle) + SubElement(r, "contact_info") - etree_write(e, "parent-response-to-%s.xml" % child_handle, + etree_write(e, "%s.%s.parent-response.xml" % (self.handle, child_handle), msg = "Send this file back to the child you just configured") @@ -477,7 +558,7 @@ class main(rpki.cli.Cmd): print "Parent calls itself %r, we call it %r" % (p.get("parent_handle"), parent_handle) print "Parent calls us %r" % p.get("child_handle") - rpki.irdb.Parent.get_or_certify( + rpki.irdb.Parent.objects.get_or_certify( issuer = self.resource_ca, handle = parent_handle, child_handle = p.get("child_handle"), @@ -493,7 +574,7 @@ class main(rpki.cli.Cmd): r.set("handle", self.handle) r.set("parent_handle", parent_handle) B64Element(r, "bpki_client_ta", self.resource_ca.certificate) - etree_write(r, "repository-request-for-%s.xml" % parent_handle, + etree_write(r, "%s.%s.repository-request.xml" % (self.handle, parent_handle), msg = "This is the file to send to the repository operator") @@ -542,12 +623,12 @@ class main(rpki.cli.Cmd): print "This looks like a referral, checking" try: auth = client.find("authorization") - referrer = self.resource_ca.clients.get(handle = auth.get("referrer")) + referrer = self.server_ca.clients.get(handle = auth.get("referrer")) referral_cms = rpki.x509.SignedReferral(Base64 = auth.text) referral_xml = referral_cms.unwrap(ta = (referrer.certificate, self.server_ca.certificate)) if rpki.x509.X509(Base64 = referral_xml.text) != client_ta: raise BadXMLMessage, "Referral trust anchor does not match" - sia_base = referral.get("authorized_sia_base") + sia_base = referral_xml.get("authorized_sia_base") except rpki.irdb.Client.DoesNotExist: print "We have no record of the client (%s) alleged to have made this referral" % auth.get("referrer") @@ -578,7 +659,7 @@ class main(rpki.cli.Cmd): print "Client calls itself %r, we call it %r" % (client.get("handle"), client_handle) print "Client says its parent handle is %r" % parent_handle - rpki.irdb.Client.get_or_certify( + rpki.irdb.Client.objects.get_or_certify( issuer = self.server_ca, handle = client_handle, ta = client_ta, @@ -595,7 +676,7 @@ class main(rpki.cli.Cmd): B64Element(e, "bpki_server_ta", self.server_ca.certificate) B64Element(e, "bpki_client_ta", client_ta) SubElement(e, "contact_info").text = self.pubd_contact_info - etree_write(e, "repository-response-to-%s.xml" % client_handle.replace("/", "."), + etree_write(e, "%s.repository-response.xml" % client_handle.replace("/", "."), msg = "Send this file back to the publication client you just configured") @@ -651,7 +732,7 @@ class main(rpki.cli.Cmd): print "Could not find parent %r in our database" % parent_handle else: - rpki.irdb.Repository.get_or_certify( + rpki.irdb.Repository.objects.get_or_certify( issuer = self.resource_ca, handle = parent_handle, client_handle = r.get("client_handle"), @@ -809,6 +890,58 @@ class main(rpki.cli.Cmd): q.delete() + def do_synchronize_roa_requests(self, arg): + """ + Synchronize IRDB against roa.csv. + """ + + argv = arg.split() + + if len(argv) != 1: + raise BadCommandSyntax("Need to specify roa.csv filename") + + grouped = {} + + # format: p/n-m asn group + for pnm, asn, group in csv_reader(argv[0], columns = 3): + key = (asn, group) + if key not in grouped: + grouped[key] = [] + grouped[key].append(pnm) + + import django.db.transaction + + with django.db.transaction.commit_on_success(): + + # Deleting and recreating all the ROA requests is inefficient, + # but rpkid's current representation of ROA requests is wrong + # (see #32), so it's not worth a lot of effort here as we're + # just going to have to rewrite this soon anyway. + + self.resource_ca.roa_requests.all().delete() + + for key, pnms in grouped.iteritems(): + asn, group = key + + roa_request = rpki.irdb.ROARequest.objects.create( + issuer = self.resource_ca, + asn = asn) + + for pnm in pnms: + if ":" in pnm: + p = rpki.resource_set.roa_prefix_ipv6.parse_str(pnm) + v = 6 + else: + p = rpki.resource_set.roa_prefix_ipv4.parse_str(pnm) + v = 4 + rpki.irdb.ROARequestPrefix.objects.create( + roa_request = roa_request, + version = v, + prefix = str(p.prefix), + prefixlen = int(p.prefixlen), + max_prefixlen = int(p.max_prefixlen)) + + def do_synchronize(self, arg): """ Temporary testing hack (probably) to let me run .synchronize() diff --git a/rpkid/rpki/sql_schemas.py b/rpkid/rpki/sql_schemas.py index 154ab5c1..e7c65299 100644 --- a/rpkid/rpki/sql_schemas.py +++ b/rpkid/rpki/sql_schemas.py @@ -239,115 +239,6 @@ CREATE TABLE ghostbuster ( -- End: ''' -## @var irdbd -## SQL schema irdbd -irdbd = '''-- $Id: irdbd.sql 3730 2011-03-21 12:42:43Z sra $ - --- Copyright (C) 2009--2011 Internet Systems Consortium ("ISC") --- --- Permission to use, copy, modify, and distribute this software for any --- purpose with or without fee is hereby granted, provided that the above --- copyright notice and this permission notice appear in all copies. --- --- THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH --- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY --- AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, --- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM --- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE --- OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR --- PERFORMANCE OF THIS SOFTWARE. - --- Copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") --- --- Permission to use, copy, modify, and distribute this software for any --- purpose with or without fee is hereby granted, provided that the above --- copyright notice and this permission notice appear in all copies. --- --- THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH --- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY --- AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, --- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM --- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE --- OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR --- PERFORMANCE OF THIS SOFTWARE. - --- SQL objects needed by irdbd.py. You only need this if you're using --- irdbd.py as your IRDB; if you have a "real" backend you can do --- anything you like so long as you implement the relevant portion of --- the left-right protocol. - --- DROP TABLE commands must be in correct (reverse dependency) order --- to satisfy FOREIGN KEY constraints. - -DROP TABLE IF EXISTS roa_request_prefix; -DROP TABLE IF EXISTS roa_request; -DROP TABLE IF EXISTS registrant_net; -DROP TABLE IF EXISTS registrant_asn; -DROP TABLE IF EXISTS registrant; -DROP TABLE IF EXISTS ghostbuster_request; - -CREATE TABLE registrant ( - registrant_id SERIAL NOT NULL, - registrant_handle VARCHAR(255) NOT NULL, - registrant_name TEXT, - registry_handle VARCHAR(255), - valid_until DATETIME NOT NULL, - PRIMARY KEY (registrant_id), - UNIQUE (registry_handle, registrant_handle) -) ENGINE=InnoDB; - -CREATE TABLE registrant_asn ( - registrant_asn_id SERIAL NOT NULL, - start_as BIGINT UNSIGNED NOT NULL, - end_as BIGINT UNSIGNED NOT NULL, - registrant_id BIGINT UNSIGNED NOT NULL, - PRIMARY KEY (registrant_asn_id), - CONSTRAINT registrant_asn_registrant_id - FOREIGN KEY (registrant_id) REFERENCES registrant (registrant_id) ON DELETE CASCADE -) ENGINE=InnoDB; - -CREATE TABLE registrant_net ( - registrant_net_id SERIAL NOT NULL, - start_ip VARCHAR(40) NOT NULL, - end_ip VARCHAR(40) NOT NULL, - version TINYINT UNSIGNED NOT NULL, - registrant_id BIGINT UNSIGNED NOT NULL, - PRIMARY KEY (registrant_net_id), - CONSTRAINT registrant_net_registrant_id - FOREIGN KEY (registrant_id) REFERENCES registrant (registrant_id) ON DELETE CASCADE -) ENGINE=InnoDB; - -CREATE TABLE roa_request ( - roa_request_id SERIAL NOT NULL, - roa_request_handle VARCHAR(255) NOT NULL, - asn BIGINT UNSIGNED NOT NULL, - PRIMARY KEY (roa_request_id) -) ENGINE=InnoDB; - -CREATE TABLE roa_request_prefix ( - prefix VARCHAR(40) NOT NULL, - prefixlen TINYINT UNSIGNED NOT NULL, - max_prefixlen TINYINT UNSIGNED NOT NULL, - version TINYINT UNSIGNED NOT NULL, - roa_request_id BIGINT UNSIGNED NOT NULL, - PRIMARY KEY (roa_request_id, prefix, prefixlen, max_prefixlen), - CONSTRAINT roa_request_prefix_roa_request_id - FOREIGN KEY (roa_request_id) REFERENCES roa_request (roa_request_id) ON DELETE CASCADE -) ENGINE=InnoDB; - -CREATE TABLE ghostbuster_request ( - ghostbuster_request_id SERIAL NOT NULL, - self_handle VARCHAR(40) NOT NULL, - parent_handle VARCHAR(40), - vcard LONGBLOB NOT NULL, - PRIMARY KEY (ghostbuster_request_id) -) ENGINE=InnoDB; - --- Local Variables: --- indent-tabs-mode: nil --- End: -''' - ## @var pubd ## SQL schema pubd pubd = '''-- $Id: pubd.sql 3465 2010-10-07 00:59:39Z sra $ diff --git a/rpkid/tests/old_irdbd.py b/rpkid/tests/old_irdbd.py new file mode 100644 index 00000000..3fa84b80 --- /dev/null +++ b/rpkid/tests/old_irdbd.py @@ -0,0 +1,21 @@ +""" +$Id$ + +Copyright (C) 2010-2012 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +if __name__ == "__main__": + import rpki.old_irdbd + rpki.old_irdbd.main() diff --git a/rpkid/irdbd.sql b/rpkid/tests/old_irdbd.sql index bf324cd8..bf324cd8 100644 --- a/rpkid/irdbd.sql +++ b/rpkid/tests/old_irdbd.sql diff --git a/rpkid/tests/smoketest.py b/rpkid/tests/smoketest.py index 4c888f67..7dfc584c 100644 --- a/rpkid/tests/smoketest.py +++ b/rpkid/tests/smoketest.py @@ -124,8 +124,8 @@ pubd_name = cfg.get("pubd_name", "pubd") prog_python = cfg.get("prog_python", sys.executable) prog_rpkid = cfg.get("prog_rpkid", "../../rpkid.py") -prog_irdbd = cfg.get("prog_irdbd", "../../irdbd.py") -prog_poke = cfg.get("prog_poke", "../../testpoke.py") +prog_irdbd = cfg.get("prog_irdbd", "../old_irdbd.py") +prog_poke = cfg.get("prog_poke", "../testpoke.py") prog_rootd = cfg.get("prog_rootd", "../../rootd.py") prog_pubd = cfg.get("prog_pubd", "../../pubd.py") prog_rsyncd = cfg.get("prog_rsyncd", "rsync") @@ -135,7 +135,7 @@ prog_openssl = cfg.get("prog_openssl", "../../../openssl/openssl/apps/openss rcynic_stats = cfg.get("rcynic_stats", "echo ; ../../../rcynic/show.sh %s.xml ; echo" % rcynic_name) rpki_sql_file = cfg.get("rpki_sql_file", "../rpkid.sql") -irdb_sql_file = cfg.get("irdb_sql_file", "../irdbd.sql") +irdb_sql_file = cfg.get("irdb_sql_file", "old_irdbd.sql") pub_sql_file = cfg.get("pub_sql_file", "../pubd.sql") startup_delay = int(cfg.get("startup_delay", "10")) diff --git a/rpkid/tests/sql-cleaner.py b/rpkid/tests/sql-cleaner.py index 5d11781f..5db122e1 100644 --- a/rpkid/tests/sql-cleaner.py +++ b/rpkid/tests/sql-cleaner.py @@ -29,10 +29,10 @@ for name in ("rpkid", "irdbd", "pubd"): password = cfg.get("%s_sql_password" % name, "fnord") schema = [] - for line in getattr(rpki.sql_schemas, name).splitlines(): + for line in getattr(rpki.sql_schemas, name, "").splitlines(): schema.extend(line.partition("--")[0].split()) schema = " ".join(schema).strip(";").split(";") - schema = [statement.strip() for statement in schema if "DROP TABLE" not in statement] + schema = [statement.strip() for statement in schema if statement and "DROP TABLE" not in statement] for i in xrange(12): diff --git a/rpkid/tests/yamltest.py b/rpkid/tests/yamltest.py index 403076f1..600a0e62 100644 --- a/rpkid/tests/yamltest.py +++ b/rpkid/tests/yamltest.py @@ -46,7 +46,7 @@ PERFORMANCE OF THIS SOFTWARE. """ import subprocess, re, os, getopt, sys, yaml, signal, time -import rpki.resource_set, rpki.sundial, rpki.config, rpki.log, rpki.rpkic +import rpki.resource_set, rpki.sundial, rpki.config, rpki.log, rpki.csv_utils # Nasty regular expressions for parsing config files. Sadly, while # the Python ConfigParser supports writing config files, it does so in @@ -67,11 +67,11 @@ this_dir = os.getcwd() test_dir = cleanpath(this_dir, "yamltest.dir") rpkid_dir = cleanpath(this_dir, "..") -prog_rpkic = cleanpath(rpkid_dir, "rpkic.py") -prog_rpkid = cleanpath(rpkid_dir, "rpkid.py") -prog_irdbd = cleanpath(rpkid_dir, "irdbd.py") -prog_pubd = cleanpath(rpkid_dir, "pubd.py") -prog_rootd = cleanpath(rpkid_dir, "rootd.py") +prog_rpkic = cleanpath(rpkid_dir, "rpkic") +prog_rpkid = cleanpath(rpkid_dir, "rpkid") +prog_irdbd = cleanpath(rpkid_dir, "irdbd") +prog_pubd = cleanpath(rpkid_dir, "pubd") +prog_rootd = cleanpath(rpkid_dir, "rootd") prog_openssl = cleanpath(this_dir, "../../openssl/openssl/apps/openssl") if not os.path.exists(prog_openssl): @@ -116,14 +116,14 @@ class allocation_db(list): def __init__(self, yaml): list.__init__(self) self.root = allocation(yaml, self) - assert self.root.is_root() + assert self.root.is_root if self.root.crl_interval is None: self.root.crl_interval = 24 * 60 * 60 if self.root.regen_margin is None: self.root.regen_margin = 24 * 60 * 60 for a in self: if a.sia_base is None: - if a.runs_pubd(): + if a.runs_pubd: base = "rsync://localhost:%d/rpki/" % a.rsync_port else: base = a.parent.sia_base @@ -134,14 +134,13 @@ class allocation_db(list): a.crl_interval = a.parent.crl_interval if a.regen_margin is None: a.regen_margin = a.parent.regen_margin - a.client_handle = "/".join(a.sia_base.rstrip("/").split("/")[3:]) self.root.closure() self.map = dict((a.name, a) for a in self) for a in self: - if a.is_hosted(): + if a.is_hosted: a.hosted_by = self.map[a.hosted_by] a.hosted_by.hosts.append(a) - assert not a.is_root() and not a.hosted_by.is_hosted() + assert not a.is_root and not a.hosted_by.is_hosted def dump(self): """ @@ -218,14 +217,14 @@ class allocation(object): self.base.v6 = self.base.v6.union(r.v6.to_resource_set()) self.hosted_by = yaml.get("hosted_by") self.hosts = [] - if not self.is_hosted(): + if not self.is_hosted: self.engine = self.allocate_engine() self.rpkid_port = self.allocate_port() self.irdbd_port = self.allocate_port() - if self.runs_pubd(): + if self.runs_pubd: self.pubd_port = self.allocate_port() self.rsync_port = self.allocate_port() - if self.is_root(): + if self.is_root: self.rootd_port = self.allocate_port() def closure(self): @@ -253,55 +252,56 @@ class allocation(object): if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) if self.parent: s += " Up: %s\n" % self.parent.name if self.sia_base: s += " SIA: %s\n" % self.sia_base - if self.is_hosted(): s += " Host: %s\n" % self.hosted_by.name + if self.is_hosted: s += " Host: %s\n" % self.hosted_by.name if self.hosts: s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts) for r in self.roa_requests: s += " ROA: %s\n" % r - if not self.is_hosted(): s += " IPort: %s\n" % self.irdbd_port - if self.runs_pubd(): s += " PPort: %s\n" % self.pubd_port - if not self.is_hosted(): s += " RPort: %s\n" % self.rpkid_port - if self.runs_pubd(): s += " SPort: %s\n" % self.rsync_port - if self.is_root(): s += " TPort: %s\n" % self.rootd_port + if not self.is_hosted: s += " IPort: %s\n" % self.irdbd_port + if self.runs_pubd: s += " PPort: %s\n" % self.pubd_port + if not self.is_hosted: s += " RPort: %s\n" % self.rpkid_port + if self.runs_pubd: s += " SPort: %s\n" % self.rsync_port + if self.is_root: s += " TPort: %s\n" % self.rootd_port return s + " Until: %s\n" % self.resources.valid_until + @property def is_root(self): """ Is this the root node? """ return self.parent is None + @property def is_hosted(self): """ Is this entity hosted? """ return self.hosted_by is not None + @property def runs_pubd(self): """ Does this entity run a pubd? """ - return self.is_root() or not (self.is_hosted() or only_one_pubd) + return self.is_root or not (self.is_hosted or only_one_pubd) def path(self, *names): """ Construct pathnames in this entity's test directory. """ - return cleanpath(test_dir, self.name, *names) + return cleanpath(test_dir, self.host.name, *names) def csvout(self, fn): """ - Open and log a CSV output file. We use delimiter and dialect - settings imported from the rpkic module, so that we automatically - write CSV files in the right format. + Open and log a CSV output file. """ path = self.path(fn) print "Writing", path - return rpki.rpkic.csv_writer(path) + return rpki.csv_utils.csv_writer(path) def up_down_url(self): """ Construct service URL for this node's parent. """ - parent_port = self.parent.hosted_by.rpkid_port if self.parent.is_hosted() else self.parent.rpkid_port + parent_port = self.parent.hosted_by.rpkid_port if self.parent.is_hosted else self.parent.rpkid_port return "http://localhost:%d/up-down/%s/%s" % (parent_port, self.parent.name, self.name) def dump_asns(self, fn): @@ -312,37 +312,7 @@ class allocation(object): for k in self.kids: f.writerows((k.name, a) for a in k.resources.asn) f.close() - - def dump_children(self, fn): - """ - Write children CSV file. - """ - f = self.csvout(fn) - f.writerows((k.name, k.resources.valid_until, k.path("bpki/resources/ca.cer")) - for k in self.kids) - f.close() - - def dump_parents(self, fn): - """ - Write parents CSV file. - """ - f = self.csvout(fn) - if self.is_root(): - f.writerow(("rootd", - "http://localhost:%d/" % self.rootd_port, - self.path("bpki/servers/ca.cer"), - self.path("bpki/servers/ca.cer"), - self.name, - self.sia_base)) - else: - parent_host = self.parent.hosted_by if self.parent.is_hosted() else self.parent - f.writerow((self.parent.name, - self.up_down_url(), - self.parent.path("bpki/resources/ca.cer"), - parent_host.path("bpki/servers/ca.cer"), - self.name, - self.sia_base)) - f.close() + self.run_rpkic("synchronize_asns", fn) def dump_prefixes(self, fn): """ @@ -352,43 +322,45 @@ class allocation(object): for k in self.kids: f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) f.close() + self.run_rpkic("synchronize_prefixes", fn) def dump_roas(self, fn): """ Write ROA CSV file. """ - group = self.name if self.is_root() else self.parent.name + group = self.name if self.is_root else self.parent.name f = self.csvout(fn) for r in self.roa_requests: f.writerows((p, r.asn, group) for p in (r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ())) f.close() + self.run_rpkic("synchronize_roa_requests", fn) - def dump_clients(self, fn, db): - """ - Write pubclients CSV file. - """ - if self.runs_pubd(): - f = self.csvout(fn) - f.writerows((s.client_handle, s.path("bpki/resources/ca.cer"), s.sia_base) - for s in (db if only_one_pubd else [self] + self.kids)) - f.close() - - def find_pubd(self): + @property + def pubd(self): """ Walk up tree until we find somebody who runs pubd. """ s = self - path = [s] - while not s.runs_pubd(): + while not s.runs_pubd: s = s.parent - path.append(s) - return s, ".".join(i.name for i in reversed(path)) + return s - def find_host(self): + @property + def client_handle(self): """ - Figure out who hosts this entity. + Work out what pubd configure_publication_client will call us. """ + path = [] + s = self + while not s.runs_pubd: + path.append(s) + s = s.parent + path.append(s) + return ".".join(i.name for i in reversed(path)) + + @property + def host(self): return self.hosted_by or self def dump_conf(self, fn): @@ -396,12 +368,10 @@ class allocation(object): Write configuration file for OpenSSL and RPKI tools. """ - s, ignored = self.find_pubd() - r = { "handle" : self.name, - "run_rpkid" : str(not self.is_hosted()), - "run_pubd" : str(self.runs_pubd()), - "run_rootd" : str(self.is_root()), + "run_rpkid" : str(not self.is_hosted), + "run_pubd" : str(self.runs_pubd), + "run_rootd" : str(self.is_root), "openssl" : prog_openssl, "irdbd_sql_database" : "irdb%d" % self.engine, "irdbd_sql_username" : "irdb", @@ -415,8 +385,8 @@ class allocation(object): "pubd_sql_database" : "pubd%d" % self.engine, "pubd_sql_username" : "pubd", "pubd_server_host" : "localhost", - "pubd_server_port" : str(s.pubd_port), - "publication_rsync_server" : "localhost:%s" % s.rsync_port } + "pubd_server_port" : str(self.pubd.pubd_port), + "publication_rsync_server" : "localhost:%s" % self.pubd.rsync_port } r.update(config_overrides) @@ -442,7 +412,7 @@ class allocation(object): Write rsyncd configuration file. """ - if self.runs_pubd(): + if self.runs_pubd: f = open(self.path(fn), "w") print "Writing", f.name f.writelines(s + "\n" for s in @@ -457,40 +427,26 @@ class allocation(object): "comment = RPKI test")) f.close() - def run_configure_daemons(self): - """ - Run configure_daemons if this entity is not hosted by another engine. - """ - if self.is_hosted(): - print "%s is hosted, skipping configure_daemons" % self.name - else: - files = [h.path("myrpki.xml") for h in self.hosts] - self.run_rpkic("configure_daemons", *[f for f in files if os.path.exists(f)]) - - def run_configure_resources(self): - """ - Run configure_resources for this entity. - """ - self.run_rpkic("configure_resources") - def run_rpkic(self, *args): """ - Run rpkic.py for this entity. + Run rpkic for this entity. """ - print 'Running "%s" for %s' % (" ".join(("rpkic",) + args), self.name) - subprocess.check_call((sys.executable, prog_rpkic) + args, cwd = self.path()) + cmd = (prog_rpkic, "-i", self.name, "-c", self.path("rpki.conf")) + args + print 'Running "%s"' % " ".join(cmd) + subprocess.check_call(cmd, cwd = self.host.path()) def run_python_daemon(self, prog): """ Start a Python daemon and return a subprocess.Popen object representing the running daemon. """ - basename = os.path.basename(prog) - p = subprocess.Popen((sys.executable, prog, "-d", "-c", self.path("rpki.conf")), + cmd = (prog, "-d", "-c", self.path("rpki.conf")) + log = os.path.splitext(os.path.basename(prog))[0] + ".log" + p = subprocess.Popen(cmd, cwd = self.path(), - stdout = open(self.path(os.path.splitext(basename)[0] + ".log"), "w"), + stdout = open(self.path(log), "w"), stderr = subprocess.STDOUT) - print "Running %s for %s: pid %d process %r" % (basename, self.name, p.pid, p) + print 'Running %s for %s: pid %d process %r' % (" ".join(cmd), self.name, p.pid, p) return p def run_rpkid(self): @@ -604,21 +560,18 @@ try: # Show what we loaded - db.dump() + #db.dump() # Set up each entity in our test for d in db: - os.makedirs(d.path()) - d.dump_asns("asns.csv") - d.dump_prefixes("prefixes.csv") - d.dump_roas("roas.csv") - d.dump_conf("rpki.conf") - d.dump_rsyncd("rsyncd.conf") - if False: - d.dump_children("children.csv") - d.dump_parents("parents.csv") - d.dump_clients("pubclients.csv", db) + if not d.is_hosted: + os.makedirs(d.path()) + os.makedirs(d.path("bpki/resources")) + os.makedirs(d.path("bpki/servers")) + d.dump_conf("rpki.conf") + if d.runs_pubd: + d.dump_rsyncd("rsyncd.conf") # Initialize BPKI and generate self-descriptor for each entity. @@ -628,7 +581,7 @@ try: # Create publication directories. for d in db: - if d.is_root() or d.runs_pubd(): + if d.is_root or d.runs_pubd: os.makedirs(d.path("publication")) # Create RPKI root certificate. @@ -636,11 +589,11 @@ try: print "Creating rootd RPKI root certificate" # Should use req -subj here to set subject name. Later. - db.root.run_openssl("x509", "-req", "-sha256", "-outform", "DER", - "-signkey", "bpki/servers/ca.key", - "-in", "bpki/servers/ca.req", - "-out", "publication/root.cer", - "-extfile", "rpki.conf", + db.root.run_openssl("req", "-x509", "-sha256", "-outform", "DER", + "-newkey", "rsa:2048", + "-keyout", "bpki/servers/root.key", + "-out", "publication/root.cer", + "-config", "rpki.conf", "-extensions", "rootd_x509_extensions") @@ -659,62 +612,62 @@ try: print print "Configuring", d.name print - if d.is_root(): - d.run_rpkic("configure_publication_client", d.path("entitydb", "repositories", "%s.xml" % d.name)) + if d.is_root: + assert not d.is_hosted + d.run_rpkic("configure_publication_client", + d.path("%s.%s.repository-request.xml" % (d.name, d.name))) print - d.run_rpkic("configure_repository", d.path("entitydb", "pubclients", "%s.xml" % d.name)) + d.run_rpkic("configure_repository", + d.path("%s.repository-response.xml" % d.client_handle)) print else: - d.parent.run_rpkic("configure_child", d.path("entitydb", "identity.xml")) + d.parent.run_rpkic("configure_child", d.path("%s.identity.xml" % d.name)) print - d.run_rpkic("configure_parent", d.parent.path("entitydb", "children", "%s.xml" % d.name)) + d.run_rpkic("configure_parent", + d.parent.path("%s.%s.parent-response.xml" % (d.parent.name, d.name))) print - publisher, path = d.find_pubd() - publisher.run_rpkic("configure_publication_client", d.path("entitydb", "repositories", "%s.xml" % d.parent.name)) + d.pubd.run_rpkic("configure_publication_client", + d.path("%s.%s.repository-request.xml" % (d.name, d.parent.name))) print - d.run_rpkic("configure_repository", publisher.path("entitydb", "pubclients", "%s.xml" % path)) + d.run_rpkic("configure_repository", + d.pubd.path("%s.repository-response.xml" % d.client_handle)) print - parent_host = d.parent.find_host() - if d.parent is not parent_host: - d.parent.run_configure_resources() - print - parent_host.run_configure_daemons() + d.parent.run_rpkic("synchronize") print - if publisher is not parent_host: - publisher.run_configure_daemons() + if d.pubd is not d.parent.host: + d.pubd.run_rpkic("synchronize") print print "Running daemons for", d.name - if d.is_root(): + if d.is_root: progs.append(d.run_rootd()) - if not d.is_hosted(): + if not d.is_hosted: progs.append(d.run_irdbd()) progs.append(d.run_rpkid()) - if d.runs_pubd(): + if d.runs_pubd: progs.append(d.run_pubd()) progs.append(d.run_rsyncd()) - if d.is_root() or not d.is_hosted() or d.runs_pubd(): + if d.is_root or not d.is_hosted or d.runs_pubd: print "Giving", d.name, "daemons time to start up" time.sleep(20) print assert all(p.poll() is None for p in progs) - # Run configure_daemons to set up IRDB and RPKI objects. Need to - # run a second time to push BSC certs out to rpkid. Nothing - # should happen on the third pass. Oops, when hosting we need to - # run configure_resources between passes, since only the hosted - # entity can issue the BSC, etc. + # In theory we now only need to synchronize the new entity once. + d.run_rpkic("synchronize") + # Run through list again, to be sure we catch hosted cases. + # In theory this is no longer necessary. + if False: for i in xrange(3): - d.run_configure_resources() - d.find_host().run_configure_daemons() + for d in db: + d.run_rpkic("synchronize") - # Run through list again, to be sure we catch hosted cases - - for i in xrange(3): - for d in db: - d.run_configure_resources() - d.run_configure_daemons() + # Load all the CSV files + for d in db: + d.dump_asns("%s.asns.csv" % d.name) + d.dump_prefixes("%s.prefixes.csv" % d.name) + d.dump_roas("%s.roas.csv" % d.name) print "Done initializing daemons" diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index d96dd62d..1b469261 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -247,7 +247,11 @@ for filename in glob.iglob(os.path.join(entitydb, "children", "*.xml")): registrant_id, valid_until = cur.fetchone() valid_until = rpki.sundial.datetime.fromdatetime(valid_until) - assert valid_until == rpki.sundial.datetime.fromXMLtime(e.get("valid_until")) + if valid_until != rpki.sundial.datetime.fromXMLtime(e.get("valid_until")): + print "WARNING: valid_until dates in XML and SQL do not match for child", child_handle + print " SQL:", str(valid_until) + print " XML:", str(rpki.sundial.datetime.fromXMLtime(e.get("valid_until"))) + print "Blundering onwards" child = rpki.irdb.Child.objects.get_or_create( handle = child_handle, |