diff options
| -rwxr-xr-x | ca/tests/bgpsec-yaml.py | 71 | ||||
| -rw-r--r-- | ca/tests/smoketest.py | 30 | ||||
| -rwxr-xr-x | potpourri/rpki-rtr-replay | 139 | ||||
| -rw-r--r-- | rpki/exceptions.py | 10 | ||||
| -rw-r--r-- | rpki/pubd.py | 59 | ||||
| -rw-r--r-- | rpki/publication.py | 29 | ||||
| -rw-r--r-- | rpki/publication_control.py | 4 | ||||
| -rw-r--r-- | rpki/relaxng.py | 19 | ||||
| -rw-r--r-- | rpki/rtr/server.py | 4 | ||||
| -rw-r--r-- | rpki/sql_schemas.py | 2 | ||||
| -rw-r--r-- | schemas/relaxng/publication.rnc | 9 | ||||
| -rw-r--r-- | schemas/relaxng/publication.rng | 19 | ||||
| -rw-r--r-- | schemas/sql/pubd.sql | 2 |
13 files changed, 332 insertions, 65 deletions
diff --git a/ca/tests/bgpsec-yaml.py b/ca/tests/bgpsec-yaml.py index 32388056..1562f86e 100755 --- a/ca/tests/bgpsec-yaml.py +++ b/ca/tests/bgpsec-yaml.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # # $Id$ -# +# # Copyright (C) 2014 Dragon Research Labs ("DRL") # # Permission to use, copy, modify, and distribute this software for any @@ -28,26 +28,61 @@ import yaml root = "Root" -def kid(n): # pylint: disable=W0621 - name = "ISP-%03d" % n - ipv4 = "10.%d.0.0/16" % n - asn = n - router_id = n * 10000 +class Kid(object): + + def __init__(self, n): + self.name = "ISP-%03d" % n + self.ipv4 = "10.%d.0.0/16" % n + self.asn = n + self.router_id = n * 10000 + + @property + def declare(self): + return dict(name = self.name, + ipv4 = self.ipv4, + asn = self.asn, + hosted_by = root, + roa_request = [dict(asn = self.asn, ipv4 = self.ipv4)], + router_cert = [dict(asn = self.asn, router_id = self.router_id)]) + + @property + def del_routercert(self): + return dict(name = self.name, router_cert_del = [dict(asn = self.asn, router_id = self.router_id)]) + + @property + def add_routercert(self): + return dict(name = self.name, router_cert_add = [dict(asn = self.asn, router_id = self.router_id)]) + + +kids = [Kid(n + 1) for n in xrange(200)] - return dict(name = name, - ipv4 = ipv4, - asn = asn, - hosted_by = root, - roa_request = [dict(asn = asn, ipv4 = ipv4)], - router_cert = [dict(asn = asn, router_id = router_id)]) +shell_fmt = "shell set -x; ../../../rp/rpki-rtr/rpki-rtr cronjob rcynic-data/authenticated && tar %svf rpki-rtr.tar *.[ai]x*.v*" +shell_first = shell_fmt % "c" +shell_next = shell_fmt % "u" + +sleeper = "sleep 30" + +docs = [dict(name = root, + valid_for = "1y", + kids = [kid.declare for kid in kids])] + +docs.append([shell_first, + sleeper]) + +gym = kids[50:70] + +for kid in gym: + docs.append([shell_next, + kid.del_routercert, + sleeper]) + +for kid in gym: + docs.append([shell_next, + kid.add_routercert, + sleeper]) print '''\ # This configuration was generated by a script. Edit at your own risk. ''' -print yaml.dump(dict(name = root, - crl_interval = "1h", - regen_margin = "20m", - valid_for = "1y", - kids = [kid(n + 1) for n in xrange(200)])) - +print yaml.safe_dump_all(docs, default_flow_style = False, allow_unicode = False) diff --git a/ca/tests/smoketest.py b/ca/tests/smoketest.py index ccf27e12..53e65b9f 100644 --- a/ca/tests/smoketest.py +++ b/ca/tests/smoketest.py @@ -163,6 +163,8 @@ def main(): log_handler = lambda: logging.StreamHandler(sys.stdout))) logger.info("Starting") + rpki.http.http_client.timeout = rpki.sundial.timedelta(hours = 1) + pubd_process = None rootd_process = None rsyncd_process = None @@ -382,6 +384,9 @@ class router_cert(object): """ _ecparams = None + _keypair = None + _pkcs10 = None + _gski = None @classmethod def ecparams(cls): @@ -392,18 +397,33 @@ class router_cert(object): def __init__(self, asn, router_id): self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) self.router_id = router_id - self.keypair = rpki.x509.ECDSA.generate(self.ecparams()) - self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) - self.gski = self.pkcs10.gSKI() self.cn = "ROUTER-%08x" % self.asn[0].min self.sn = "%08x" % self.router_id self.eku = rpki.oids.id_kp_bgpsec_router + @property + def keypair(self): + if self._keypair is None: + self._keypair = rpki.x509.ECDSA.generate(self.ecparams()) + return self._keypair + + @property + def pkcs10(self): + if self._pkcs10 is None: + self._pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) + return self._pkcs10 + + @property + def gski(self): + if self._gski is None: + self._gski = self.pkcs10.gSKI() + return self._gski + def __eq__(self, other): - return self.asn == other.asn and self.sn == other.sn and self.gski == other.gski + return self.asn == other.asn and self.sn == other.sn def __hash__(self): - return tuple(self.asn).__hash__() + self.cn.__hash__() + self.sn.__hash__() + self.gski.__hash__() + return tuple(self.asn).__hash__() + self.cn.__hash__() + self.sn.__hash__() def __str__(self): return "%s: %s,%s: %s" % (self.asn, self.cn, self.sn, self.gski) diff --git a/potpourri/rpki-rtr-replay b/potpourri/rpki-rtr-replay new file mode 100755 index 00000000..6f8de99e --- /dev/null +++ b/potpourri/rpki-rtr-replay @@ -0,0 +1,139 @@ +#!/usr/bin/env python + +# $Id$ +# +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# Portions copyright (C) 2009-2013 Internet Systems Consortium ("ISC") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notices and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL +# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR +# ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL +# DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA +# OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +import asyncore +import bisect +import glob +import logging +import os +import shutil +import sqlite3 +import subprocess +import sys +import time + +import rpki.POW +import rpki.oids +import rpki.rtr.channels +import rpki.rtr.client +import rpki.rtr.generator +import rpki.rtr.pdus +import rpki.rtr.server + +from rpki.rtr.channels import Timestamp + + +class ReplayClock(object): + """ + Internal clock for replaying a set of rpki-rtr database files. + + This class replaces the normal on-disk serial number mechanism with + an in-memory version based on pre-computed data. + + DO NOT USE THIS IN PRODUCTION. + + You have been warned. + """ + + def __init__(self): + self.timestamps = dict((v, sorted(set(Timestamp(int(f.split(".")[0])) + for f in glob.iglob("*.ax.v%d" % v)))) + for v in rpki.rtr.pdus.PDU.version_map) + self.epoch = min(t[0] for t in self.timestamps.itervalues()) + self.offset = self.epoch - Timestamp.now() + self.nonce = rpki.rtr.generator.AXFRSet.new_nonce(0) + + def __nonzero__(self): + return sum(len(t) for t in self.timestamps.itervalues()) > 0 + + def now(self): + now = Timestamp.now(self.offset) + return now + + def read_current(self, version): + now = self.now() + if version is None: + return self.epoch, self.nonce + while len(self.timestamps[version]) > 1 and now >= self.timestamps[version][1]: + del self.timestamps[version][0] + return self.timestamps[version][0], self.nonce + + def siesta(self): + try: + when = min(t[1] for t in self.timestamps.itervalues() if len(t) > 1) + except ValueError: + return None + now = self.now() + if now < when: + return when - now + else: + return 1 + + +def server_main(args): + """ + Reply rpki-data from a historical database. + + This is a clone of server_main() which replaces the external serial + number updates triggered via the kickme channel by cronjob_main with + an internal clocking mechanism to replay historical test data. + + DO NOT USE THIS IN PRODUCTION. + + You have been warned. + """ + + logger = logging.LoggerAdapter(logging.root, dict(connection = rpki.rtr.server._hostport_tag())) + + logger.debug("[Starting]") + + if args.rpki_rtr_dir: + try: + os.chdir(args.rpki_rtr_dir) + except OSError, e: + sys.exit(e) + + # Yes, this really does replace a global function defined in another + # module with a bound method to our clock object. Fun stuff, huh? + + clock = ReplayClock() + rpki.rtr.server.read_current = clock.read_current + + try: + server = rpki.rtr.server.ServerChannel(logger = logger, refresh = args.refresh, retry = args.retry, expire = args.expire) + old_serial = server.get_serial() + logger.debug("[Starting at serial %d (%s)]", old_serial, old_serial) + while clock: + new_serial = server.get_serial() + if old_serial != new_serial: + logger.debug("[Serial bumped from %d (%s) to %d (%s)]", old_serial, old_serial, new_serial, new_serial) + server.notify(force = True) + old_serial = new_serial + asyncore.loop(timeout = clock.siesta(), count = 1) + except KeyboardInterrupt: + sys.exit(0) + + +# Splice our extensions into server +rpki.rtr.server.server_main = server_main + +# And run the program +import rpki.rtr.main +rpki.rtr.main.main() diff --git a/rpki/exceptions.py b/rpki/exceptions.py index 504c6f28..86c7fa27 100644 --- a/rpki/exceptions.py +++ b/rpki/exceptions.py @@ -288,6 +288,16 @@ class NoObjectAtURI(RPKI_Exception): No object published at specified URI. """ +class ExistingObjectAtURI(RPKI_Exception): + """ + An object has already been published at specified URI. + """ + +class DifferentObjectAtURI(RPKI_Exception): + """ + An object with a different hash exists at specified URI. + """ + class CMSContentNotSet(RPKI_Exception): """ Inner content of a CMS_object has not been set. If object is known diff --git a/rpki/pubd.py b/rpki/pubd.py index 14de1999..0ee4d38c 100644 --- a/rpki/pubd.py +++ b/rpki/pubd.py @@ -40,6 +40,8 @@ import rpki.publication import rpki.publication_control import rpki.daemonize +from lxml.etree import Element, SubElement, ElementTree, Comment + logger = logging.getLogger(__name__) class main(object): @@ -107,6 +109,9 @@ class main(object): self.publication_multimodule = self.cfg.getboolean("publication-multimodule", False) + self.rrdp_expiration_interval = rpki.sundial.timedelta.parse(self.cfg.get("rrdp-expiration-interval", "6h")) + self.rrdp_publication_base = self.cfg.get("rrdp-publication-base", "rrdp-publication/") + self.session = session_obj.fetch(self) rpki.http.server( @@ -187,11 +192,6 @@ class session_obj(rpki.sql.sql_persistent): "session_id", "uuid") - ## @var expiration_interval - # How long to wait after retiring a snapshot before purging it from the database. - - expiration_interval = rpki.sundial.timedelta(hours = 6) - def __repr__(self): return rpki.log.log_repr(self, self.uuid, self.serial) @@ -231,7 +231,7 @@ class session_obj(rpki.sql.sql_persistent): now = rpki.sundial.now() old_snapshot = self.current_snapshot if old_snapshot is not None: - old_snapshot.expires = now + self.expiration_interval + old_snapshot.expires = now + self.gctx.rrdp_expiration_interval old_snapshot.sql_store() new_snapshot.activated = now new_snapshot.sql_store() @@ -280,40 +280,27 @@ class snapshot_obj(rpki.sql.sql_persistent): Well, OK, only almost the right properties. auto-increment probably does not back up if we ROLLBACK, which could leave gaps - in the sequence. So may need to rework this. Ignore for now. + in the sequence. So may need to rework this, eg, to use a serial + field in the session object. Ignore the issue until we have the + rest of this working. """ return self.snapshot_id - def publish(self, client, obj, uri): - - # Still a bit confused as to what we should do here. The - # overwrite <publish/> with another <publish/> model doens't - # really match the IXFR model. Current proposal is an attribute - # on <publish/> to say that this is an overwrite, haven't - # implemented that yet. Would need to push knowledge of when - # we're overwriting all the way from rpkid code that decides to - # write each kind of object. In most cases it looks like we - # already know, a priori, might be a few corner cases. - - # Temporary kludge - if True: - try: - self.withdraw(client, uri) - except rpki.exceptions.NoObjectAtURI: - logger.debug("Withdrew %s", uri) - else: - logger.debug("No prior %s", uri) - + def publish(self, client, obj, uri, hash): + if hash is not None: + self.withdraw(client, uri, hash) + if object_obj.current_object_at_uri(client, self, uri) is not None: + raise rpki.exceptions.ExistingObjectAtURI("Object already published at %s" % uri) logger.debug("Publishing %s", uri) return object_obj.create(client, self, obj, uri) - def withdraw(self, client, uri): - obj = object_obj.sql_fetch_where1(self.gctx, - "session_id = %s AND client_id = %s AND withdrawn_snapshot_id IS NULL AND uri = %s", - (self.session_id, client.client_id, uri)) + def withdraw(self, client, uri, hash): + obj = object_obj.current_object_at_uri(client, self, uri) if obj is None: raise rpki.exceptions.NoObjectAtURI("No object published at %s" % uri) + if obj.hash != hash: + raise rpki.exceptions.DifferentObjectAtURI("Found different object at %s (%s, %s)" % (uri, obj.hash, hash)) logger.debug("Withdrawing %s", uri) obj.delete(self) @@ -354,8 +341,8 @@ class object_obj(rpki.sql.sql_persistent): self.gctx = snapshot.gctx self.uri = uri self.payload = obj - self.hash = rpki.x509.sha256(obj.get_Base64()) - logger.debug("Computed hash %s of %r", self.hash.encode("hex"), obj) + self.hash = rpki.x509.sha256(obj.get_Base64()).encode("hex") + logger.debug("Computed hash %s of %r", self.hash, obj) self.published_snapshot_id = snapshot.snapshot_id self.withdrawn_snapshot_id = None self.session_id = snapshot.session_id @@ -367,3 +354,9 @@ class object_obj(rpki.sql.sql_persistent): self.withdrawn_snapshot_id = snapshot.snapshot_id #self.sql_mark_dirty() self.sql_store() + + @classmethod + def current_object_at_uri(cls, client, snapshot, uri): + return cls.sql_fetch_where1(client.gctx, + "session_id = %s AND client_id = %s AND withdrawn_snapshot_id IS NULL AND uri = %s", + (snapshot.session_id, client.client_id, uri)) diff --git a/rpki/publication.py b/rpki/publication.py index 7b5abaf9..ec088a46 100644 --- a/rpki/publication.py +++ b/rpki/publication.py @@ -39,7 +39,6 @@ logger = logging.getLogger(__name__) class publication_namespace(object): - xmlns = "http://www.hactrn.net/uris/rpki/publication-spec/" nsmap = { None : xmlns } @@ -103,6 +102,9 @@ class base_publication_elt(rpki.xml_utils.base_elt, publication_namespace): class publish_elt(base_publication_elt): + """ + <publish/> element. + """ element_name = "publish" @@ -132,7 +134,7 @@ class publish_elt(base_publication_elt): """ logger.info("Publishing %s", self.payload.tracking_data(self.uri)) - snapshot.publish(self.client, self.payload, self.uri) + snapshot.publish(self.client, self.payload, self.uri, self.hash) filename = self.uri_to_filename() filename_tmp = filename + ".tmp" dirname = os.path.dirname(filename) @@ -144,6 +146,9 @@ class publish_elt(base_publication_elt): class withdraw_elt(base_publication_elt): + """ + <withdraw/> element. + """ element_name = "withdraw" @@ -153,7 +158,7 @@ class withdraw_elt(base_publication_elt): """ logger.info("Withdrawing %s", self.uri) - snapshot.withdraw(self.client, self.uri) + snapshot.withdraw(self.client, self.uri, self.hash) filename = self.uri_to_filename() try: os.remove(filename) @@ -173,6 +178,24 @@ class withdraw_elt(base_publication_elt): dirname = os.path.dirname(dirname) +class list_elt(base_publication_elt): + """ + <list/> element. + """ + + def serve_dispatch(self, r_msg, snapshot, cb, eb): + """ + Action dispatch handler. + """ + + for obj in self.client.published_objects: + r_pdu = self.__class__() + r_pdu.tag = self.tag + r_pdu.uri = obj.uri + r_pdu.hash = obj.hash + r_msg.append(r_pdu) + + class report_error_elt(rpki.xml_utils.text_elt, publication_namespace): """ <report_error/> element. diff --git a/rpki/publication_control.py b/rpki/publication_control.py index f65fa15d..f1cc5f2c 100644 --- a/rpki/publication_control.py +++ b/rpki/publication_control.py @@ -90,6 +90,10 @@ class client_elt(rpki.xml_utils.data_elt, rpki.sql.sql_persistent, publication_c def objects(self): return rpki.pubd.object_obj.sql_fetch_where(self.gctx, "client_id = %s", (self.client_id,)) + @property + def published_object(self): + return rpki.pubd.object_obj.sql_fetch_where(self.gctx, "client_id = %s AND withdrawn_snapshot_id IS NULL", (self.client_id,)) + def serve_post_save_hook(self, q_pdu, r_pdu, cb, eb): """ Extra server actions for client_elt. diff --git a/rpki/relaxng.py b/rpki/relaxng.py index 4e8e9242..93ac16fe 100644 --- a/rpki/relaxng.py +++ b/rpki/relaxng.py @@ -1837,12 +1837,14 @@ publication = lxml.etree.RelaxNG(lxml.etree.fromstring(r'''<?xml version="1.0" e <choice> <ref name="publish_query"/> <ref name="withdraw_query"/> + <ref name="list_query"/> </choice> </define> <define name="reply_elt"> <choice> <ref name="publish_reply"/> <ref name="withdraw_reply"/> + <ref name="list_reply"/> <ref name="report_error_reply"/> </choice> </define> @@ -1919,6 +1921,23 @@ publication = lxml.etree.RelaxNG(lxml.etree.fromstring(r'''<?xml version="1.0" e <ref name="uri"/> </element> </define> + <!-- <list/> element --> + <define name="list_query"> + <element name="list"> + <optional> + <ref name="tag"/> + </optional> + </element> + </define> + <define name="list_reply"> + <element name="list"> + <optional> + <ref name="tag"/> + </optional> + <ref name="uri"/> + <ref name="hash"/> + </element> + </define> <!-- <report_error/> element --> <define name="report_error_reply"> <element name="report_error"> diff --git a/rpki/rtr/server.py b/rpki/rtr/server.py index b3e4fd7c..1c7a5e78 100644 --- a/rpki/rtr/server.py +++ b/rpki/rtr/server.py @@ -324,7 +324,7 @@ class ServerChannel(rpki.rtr.channels.PDUChannel): old_serial = self.current_serial return old_serial != self.get_serial() - def notify(self, data = None): + def notify(self, data = None, force = False): """ Cronjob instance kicked us: check whether our serial number has changed, and send a notify message if so. @@ -335,7 +335,7 @@ class ServerChannel(rpki.rtr.channels.PDUChannel): whether we care about a particular change set or not. """ - if self.check_serial(): + if force or self.check_serial(): self.push_pdu(SerialNotifyPDU(version = self.version, serial = self.current_serial, nonce = self.current_nonce)) diff --git a/rpki/sql_schemas.py b/rpki/sql_schemas.py index 7c7079c0..b28c8231 100644 --- a/rpki/sql_schemas.py +++ b/rpki/sql_schemas.py @@ -309,7 +309,7 @@ CREATE TABLE snapshot ( CREATE TABLE object ( object_id SERIAL NOT NULL, uri VARCHAR(255) NOT NULL, - hash BINARY(32) NOT NULL, + hash CHAR(64) NOT NULL, payload LONGBLOB NOT NULL, published_snapshot_id BIGINT UNSIGNED, withdrawn_snapshot_id BIGINT UNSIGNED, diff --git a/schemas/relaxng/publication.rnc b/schemas/relaxng/publication.rnc index 8c129546..f3d1f94e 100644 --- a/schemas/relaxng/publication.rnc +++ b/schemas/relaxng/publication.rnc @@ -58,8 +58,8 @@ start |= element msg { # PDUs allowed in queries and replies. -query_elt = publish_query | withdraw_query -reply_elt = publish_reply | withdraw_reply | report_error_reply +query_elt = publish_query | withdraw_query | list_query +reply_elt = publish_reply | withdraw_reply | list_reply | report_error_reply # Tag attributes for bulk operations. @@ -91,6 +91,11 @@ publish_reply = element publish { tag?, uri } withdraw_query = element withdraw { tag?, uri, hash } withdraw_reply = element withdraw { tag?, uri } +# <list/> element + +list_query = element list { tag? } +list_reply = element list { tag?, uri, hash } + # <report_error/> element report_error_reply = element report_error { diff --git a/schemas/relaxng/publication.rng b/schemas/relaxng/publication.rng index e88db011..39d78c00 100644 --- a/schemas/relaxng/publication.rng +++ b/schemas/relaxng/publication.rng @@ -74,12 +74,14 @@ <choice> <ref name="publish_query"/> <ref name="withdraw_query"/> + <ref name="list_query"/> </choice> </define> <define name="reply_elt"> <choice> <ref name="publish_reply"/> <ref name="withdraw_reply"/> + <ref name="list_reply"/> <ref name="report_error_reply"/> </choice> </define> @@ -156,6 +158,23 @@ <ref name="uri"/> </element> </define> + <!-- <list/> element --> + <define name="list_query"> + <element name="list"> + <optional> + <ref name="tag"/> + </optional> + </element> + </define> + <define name="list_reply"> + <element name="list"> + <optional> + <ref name="tag"/> + </optional> + <ref name="uri"/> + <ref name="hash"/> + </element> + </define> <!-- <report_error/> element --> <define name="report_error_reply"> <element name="report_error"> diff --git a/schemas/sql/pubd.sql b/schemas/sql/pubd.sql index 34778491..210396d5 100644 --- a/schemas/sql/pubd.sql +++ b/schemas/sql/pubd.sql @@ -62,7 +62,7 @@ CREATE TABLE snapshot ( CREATE TABLE object ( object_id SERIAL NOT NULL, uri VARCHAR(255) NOT NULL, - hash BINARY(32) NOT NULL, + hash CHAR(64) NOT NULL, payload LONGBLOB NOT NULL, published_snapshot_id BIGINT UNSIGNED, withdrawn_snapshot_id BIGINT UNSIGNED, |