diff options
Diffstat (limited to 'rpkid.stable/testbed.py')
| -rw-r--r-- | rpkid.stable/testbed.py | 1142 |
1 files changed, 1142 insertions, 0 deletions
diff --git a/rpkid.stable/testbed.py b/rpkid.stable/testbed.py new file mode 100644 index 00000000..f3afa2d5 --- /dev/null +++ b/rpkid.stable/testbed.py @@ -0,0 +1,1142 @@ +""" +Test framework to configure and drive a collection of rpkid.py and +irdbd.py instances under control of a master script. + +Usage: python testbed.py [ { -c | --config } config_file ] + [ { -h | --help } ] + [ { -p | --profile } ] + [ { -y | --yaml } yaml_script ] + +Default config_file is testbed.conf, override with --config option. + +Default yaml_script is testbed.yaml, override with -yaml option. + +yaml_script is a YAML file describing the tests to be run, and is +intended to be implementation agnostic. + +config_file contains settings for various implementation-specific +things that don't belong in yaml_script. + +$Id$ + +Copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import os, yaml, MySQLdb, subprocess, signal, time, re, getopt, sys, lxml +import rpki.resource_set, rpki.sundial, rpki.x509, rpki.https +import rpki.log, rpki.left_right, rpki.config, rpki.publication + +os.environ["TZ"] = "UTC" +time.tzset() + +cfg_file = "testbed.conf" + +yaml_script = None +profile = False + +opts,argv = getopt.getopt(sys.argv[1:], "c:hpy:?", ["config=", "help", "profile", "yaml="]) +for o,a in opts: + if o in ("-h", "--help", "-?"): + print __doc__ + sys.exit(0) + elif o in ("-c", "--config"): + cfg_file = a + elif o in ("-p", "--profile"): + profile = True + elif o in ("-y", "--yaml"): + yaml_script = a +if argv: + print __doc__ + raise RuntimeError, "Unexpected arguments %s" % argv + +cfg = rpki.config.parser(cfg_file, "testbed") + +# Load the YAML script early, so we can report errors ASAP + +if yaml_script is None: + yaml_script = cfg.get("yaml_script", "testbed.yaml") +try: + yaml_script = [y for y in yaml.safe_load_all(open(yaml_script))] +except: + print __doc__ + raise + +# Define port allocator early, so we can use it while reading config + +def allocate_port(): + """Allocate a TCP port number.""" + global base_port + p = base_port + base_port += 1 + return p + +# Most filenames in the following are relative to the working directory. + +testbed_name = cfg.get("testbed_name", "testbed") +testbed_dir = cfg.get("testbed_dir", testbed_name + ".dir") + +irdb_db_pass = cfg.get("irdb_db_pass", "fnord") +rpki_db_pass = cfg.get("rpki_db_pass", "fnord") +pubd_db_pass = cfg.get("pubd_db_pass", "fnord") + +base_port = int(cfg.get("base_port", "4400")) + +rsyncd_port = allocate_port() +rootd_port = allocate_port() +pubd_port = allocate_port() + +rsyncd_module = cfg.get("rsyncd_module", testbed_name) +rootd_sia = cfg.get("rootd_sia", "rsync://localhost:%d/%s/" % (rsyncd_port, rsyncd_module)) + +rootd_name = cfg.get("rootd_name", "rootd") +rsyncd_name = cfg.get("rcynic_name", "rsyncd") +rcynic_name = cfg.get("rcynic_name", "rcynic") +pubd_name = cfg.get("pubd_name", "pubd") + +prog_python = cfg.get("prog_python", "python") +prog_rpkid = cfg.get("prog_rpkid", "../rpkid.py") +prog_irdbd = cfg.get("prog_irdbd", "../irdbd.py") +prog_poke = cfg.get("prog_poke", "../testpoke.py") +prog_rootd = cfg.get("prog_rootd", "../rootd.py") +prog_pubd = cfg.get("prog_pubd", "../pubd.py") +prog_openssl = cfg.get("prog_openssl", "../../openssl/openssl/apps/openssl") +prog_rsyncd = cfg.get("prog_rsyncd", "rsync") +prog_rcynic = cfg.get("prog_rcynic", "../../rcynic/rcynic") + +rcynic_stats = cfg.get("rcynic_stats", "xsltproc --param refresh 0 ../../rcynic/rcynic.xsl %s.xml | w3m -T text/html -dump" % rcynic_name) + +rpki_sql_file = cfg.get("rpki_sql_file", "rpkid.sql") +irdb_sql_file = cfg.get("irdb_sql_file", "irdbd.sql") +pub_sql_file = cfg.get("pub_sql_file", "pubd.sql") + +startup_delay = int(cfg.get("startup_delay", "10")) + +def main(): + """Main program, up front to make control logic more obvious.""" + + rpki.log.init(testbed_name) + rpki.log.info("Starting") + + signal.signal(signal.SIGALRM, wakeup) + + pubd_process = None + rootd_process = None + rsyncd_process = None + + rpki_sql = mangle_sql(rpki_sql_file) + irdb_sql = mangle_sql(irdb_sql_file) + pubd_sql = mangle_sql(pub_sql_file) + + rpki.log.info("Initializing test directory") + + # Connect to test directory, creating it if necessary + try: + os.chdir(testbed_dir) + except: + os.makedirs(testbed_dir) + os.chdir(testbed_dir) + + # Discard everything but keys, which take a while to generate + for root, dirs, files in os.walk(".", topdown = False): + for file in files: + if not file.endswith(".key"): + os.remove(os.path.join(root, file)) + for dir in dirs: + os.rmdir(os.path.join(root, dir)) + + rpki.log.info("Reading master YAML configuration") + y = yaml_script.pop(0) + + rpki.log.info("Constructing internal allocation database") + db = allocation_db(y) + + rpki.log.info("Constructing BPKI keys and certs for rootd") + setup_bpki_cert_chain(rootd_name, ee = ("RPKI",)) + + rpki.log.info("Constructing BPKI keys and certs for pubd") + setup_bpki_cert_chain(pubd_name, ee = ("PUBD", "IRBE")) + + for a in db: + a.setup_bpki_certs() + + setup_publication(pubd_sql) + setup_rootd(db.root.name, "SELF-1", y.get("rootd", {})) + setup_rsyncd() + setup_rcynic() + + for a in db.engines: + a.setup_conf_file() + a.setup_sql(rpki_sql, irdb_sql) + a.sync_sql() + + try: + + rpki.log.info("Starting rootd") + rootd_process = subprocess.Popen((prog_python, prog_rootd, "-c", rootd_name + ".conf")) + + rpki.log.info("Starting pubd") + pubd_process = subprocess.Popen((prog_python, prog_pubd, "-c", pubd_name + ".conf") + (("-p", pubd_name + ".prof") if profile else ())) + + rpki.log.info("Starting rsyncd") + rsyncd_process = subprocess.Popen((prog_rsyncd, "--daemon", "--no-detach", "--config", rsyncd_name + ".conf")) + + # Start rpkid and irdbd instances + for a in db.engines: + a.run_daemons() + + rpki.log.info("Sleeping %d seconds while daemons start up" % startup_delay) + time.sleep(startup_delay) + + # Create objects in RPKI engines + for a in db.engines: + a.create_rpki_objects() + + # Setup keys and certs and write YAML files for leaves + for a in db.leaves: + a.setup_yaml_leaf() + + # Set pubd's BPKI CRL + set_pubd_crl() + + # Loop until we run out of control YAML + while True: + + # This is probably where we should be updating expired BPKI + # objects, particular CRLs + + # Run cron in all RPKI instances + for a in db.engines: + a.run_cron() + + # Run all YAML clients + for a in db.leaves: + a.run_yaml() + + # Run rcynic to check results + run_rcynic() + + # If we've run out of deltas to apply, we're done + if not yaml_script: + rpki.log.info("No more deltas to apply, done") + break + + rpki.log.info("Applying deltas") + db.apply_delta(yaml_script.pop(0)) + + # Resync IRDBs + for a in db.engines: + a.sync_sql() + + # Clean up + + finally: + + try: + rpki.log.info("Shutting down") + for a in db.engines: + a.kill_daemons() + for p,n in ((rootd_process, "rootd"), (pubd_process, "pubd"), (rsyncd_process, "rsyncd")): + if p is not None: + rpki.log.info("Killing %s" % n) + os.kill(p.pid, signal.SIGTERM) + except Exception, data: + rpki.log.warn("Couldn't clean up daemons (%s), continuing" % data) + +def wakeup(signum, frame): + """Handler called when we receive a SIGALRM signal.""" + rpki.log.info("Wakeup call received, continuing") + +def cmd_sleep(interval = None): + """Set an alarm, then wait for it to go off.""" + if interval is None: + rpki.log.info("Pausing indefinitely, send a SIGALRM to wake me up") + else: + seconds = rpki.sundial.timedelta.parse(interval).convert_to_seconds() + rpki.log.info("Sleeping %s seconds" % seconds) + signal.alarm(seconds) + signal.pause() + +def cmd_shell(*cmd): + """Run a shell command.""" + cmd = " ".join(cmd) + status = subprocess.call(cmd, shell = True) + rpki.log.info("Shell command returned status %d" % status) + +def cmd_echo(*words): + """Echo some text to the log.""" + rpki.log.note(" ".join(words)) + +## @var cmds +# Dispatch table for commands embedded in delta sections + +cmds = { "sleep" : cmd_sleep, + "shell" : cmd_shell, + "echo" : cmd_echo } + +class route_origin(object): + """Representation for a route_origin object.""" + + def __init__(self, asn, ipv4, ipv6): + self.asn = asn + self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None + self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None + + def __eq__(self, other): + return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 + + def __hash__(self): + v4 = tuple(self.v4) if self.v4 is not None else None + v6 = tuple(self.v6) if self.v6 is not None else None + return self.asn.__hash__() + v4.__hash__() + v6.__hash__() + + def __str__(self): + if self.v4 and self.v6: s = str(self.v4) + "," + str(self.v6) + elif self.v4: s = str(self.v4) + else: s = str(self.v6) + return "%s: %s" % (self.asn, s) + + @classmethod + def parse(cls, yaml): + return cls(yaml.get("asn"), yaml.get("ipv4"), yaml.get("ipv6")) + +class allocation_db(list): + """Representation of all the entities and allocations in the test system. + Almost everything is generated out of this database. + """ + + def __init__(self, yaml): + """Initialize database from the (first) YAML document.""" + self.root = allocation(yaml, self) + assert self.root.is_root() + if self.root.crl_interval is None: + self.root.crl_interval = rpki.sundial.timedelta.parse(cfg.get("crl_interval", "1d")).convert_to_seconds() + if self.root.regen_margin is None: + self.root.regen_margin = rpki.sundial.timedelta.parse(cfg.get("regen_margin", "1d")).convert_to_seconds() + for a in self: + if a.sia_base is None: + a.sia_base = (rootd_sia if a.is_root() else a.parent.sia_base) + a.name + "/" + if a.base.valid_until is None: + a.base.valid_until = a.parent.base.valid_until + if a.crl_interval is None: + a.crl_interval = a.parent.crl_interval + if a.regen_margin is None: + a.regen_margin = a.parent.regen_margin + self.root.closure() + self.map = dict((a.name, a) for a in self) + self.engines = [a for a in self if not a.is_leaf()] + self.leaves = [a for a in self if a.is_leaf()] + for i, a in zip(range(len(self.engines)), self.engines): + a.set_engine_number(i) + + def apply_delta(self, delta): + """Apply a delta or run a command.""" + if delta is None: + return + for d in delta: + if isinstance(d, str): + c = d.split() + cmds[c[0]](*c[1:]) + else: + self.map[d["name"]].apply_delta(d) + self.root.closure() + + def dump(self): + """Print content of the database.""" + for a in self: + print a + +class allocation(object): + + parent = None + irdb_db_name = None + irdb_port = None + rpki_db_name = None + rpki_port = None + crl_interval = None + regen_margin = None + + def __init__(self, yaml, db, parent = None): + """Initialize one entity and insert it into the database.""" + db.append(self) + self.name = yaml["name"] + self.parent = parent + self.kids = [allocation(k, db, self) for k in yaml.get("kids", ())] + valid_until = None + if "valid_until" in yaml: + valid_until = rpki.sundial.datetime.fromdatetime(yaml.get("valid_until")) + if valid_until is None and "valid_for" in yaml: + valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(yaml["valid_for"]) + self.base = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as(yaml.get("asn")), + v4 = rpki.resource_set.resource_set_ipv4(yaml.get("ipv4")), + v6 = rpki.resource_set.resource_set_ipv6(yaml.get("ipv6")), + valid_until = valid_until) + self.sia_base = yaml.get("sia_base") + if "crl_interval" in yaml: + self.crl_interval = rpki.sundial.timedelta.parse(yaml["crl_interval"]).convert_to_seconds() + if "regen_margin" in yaml: + self.regen_margin = rpki.sundial.timedelta.parse(yaml["regen_margin"]).convert_to_seconds() + self.route_origins = set() + if "route_origin" in yaml: + for y in yaml.get("route_origin"): + self.route_origins.add(route_origin.parse(y)) + self.extra_conf = yaml.get("extra_conf", []) + + def closure(self): + """Compute the transitive resource closure.""" + resources = self.base + for kid in self.kids: + resources = resources.union(kid.closure()) + self.resources = resources + return resources + + def apply_delta(self, yaml): + """Apply deltas to this entity.""" + rpki.log.info("Applying delta: %s" % yaml) + for k,v in yaml.items(): + if k != "name": + getattr(self, "apply_" + k)(v) + + def apply_add_as(self, text): self.base.asn = self.base.asn.union(rpki.resource_set.resource_set_as(text)) + def apply_add_v4(self, text): self.base.v4 = self.base.v4.union(rpki.resource_set.resource_set_ipv4(text)) + def apply_add_v6(self, text): self.base.v6 = self.base.v6.union(rpki.resource_set.resource_set_ipv6(text)) + def apply_sub_as(self, text): self.base.asn = self.base.asn.difference(rpki.resource_set.resource_set_as(text)) + def apply_sub_v4(self, text): self.base.v4 = self.base.v4.difference(rpki.resource_set.resource_set_ipv4(text)) + def apply_sub_v6(self, text): self.base.v6 = self.base.v6.difference(rpki.resource_set.resource_set_ipv6(text)) + + def apply_valid_until(self, stamp): self.base.valid_until = rpki.sundial.datetime.fromdatetime(stamp) + def apply_valid_for(self, text): self.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(text) + def apply_valid_add(self, text): self.base.valid_until += rpki.sundial.timedelta.parse(text) + def apply_valid_sub(self, text): self.base.valid_until -= rpki.sundial.timedelta.parse(text) + + def apply_route_origin_add(self, yaml): + for y in yaml: + self.route_origins.add(route_origin.parse(y)) + + def apply_route_origin_del(self, yaml): + for y in yaml: + self.route_origins.remove(route_origin.parse(y)) + + def apply_rekey(self, target): + if self.is_leaf(): + raise RuntimeError, "Can't rekey YAML leaf %s, sorry" % self.name + elif target is None: + rpki.log.info("Rekeying <self/> %s" % self.name) + self.call_rpkid(rpki.left_right.self_elt.make_pdu(action = "set", self_id = self.self_id, rekey = "yes")) + else: + rpki.log.info("Rekeying <parent/> %s %s" % (self.name, target)) + self.call_rpkid(rpki.left_right.parent_elt.make_pdu(action = "set", self_id = self.self_id, parent_id = target, rekey = "yes")) + + def apply_revoke(self, target): + if self.is_leaf(): + rpki.log.info("Attempting to revoke YAML leaf %s" % self.name) + subprocess.check_call((prog_python, prog_poke, "-y", self.name + ".yaml", "-r", "revoke")) + elif target is None: + rpki.log.info("Revoking <self/> %s" % self.name) + self.call_rpkid(rpki.left_right.self_elt.make_pdu(action = "set", self_id = self.self_id, revoke = "yes")) + else: + rpki.log.info("Revoking <parent/> %s %s" % (self.name, target)) + self.call_rpkid(rpki.left_right.parent_elt.make_pdu(action = "set", self_id = self.self_id, parent_id = target, revoke = "yes")) + + def __str__(self): + s = self.name + "\n" + if self.resources.asn: s += " ASN: %s\n" % self.resources.asn + if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 + if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 + if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) + if self.parent: s += " Up: %s\n" % self.parent.name + if self.sia_base: s += " SIA: %s\n" % self.sia_base + return s + "Until: %s\n" % self.resources.valid_until + + def is_leaf(self): return not self.kids and not self.route_origins + def is_root(self): return self.parent is None + def is_twig(self): return not self.is_leaf() and not self.is_root() + + def set_engine_number(self, n): + """Set the engine number for this entity.""" + self.irdb_db_name = "irdb%d" % n + self.irdb_port = allocate_port() + self.rpki_db_name = "rpki%d" % n + self.rpki_port = allocate_port() + + def setup_bpki_certs(self): + """Create BPKI certificates for this entity.""" + rpki.log.info("Constructing BPKI keys and certs for %s" % self.name) + if self.is_leaf(): + setup_bpki_cert_chain(self.name, ee = ("RPKI",)) + else: + setup_bpki_cert_chain(self.name, ee = ("RPKI", "IRDB", "IRBE"), ca = ("SELF-1",)) + self.rpkid_ta = rpki.x509.X509(PEM_file = self.name + "-TA.cer") + self.irbe_key = rpki.x509.RSA( PEM_file = self.name + "-IRBE.key") + self.irbe_cert = rpki.x509.X509(PEM_file = self.name + "-IRBE.cer") + self.rpkid_cert = rpki.x509.X509(PEM_file = self.name + "-RPKI.cer") + + def setup_conf_file(self): + """Write config files for this entity.""" + rpki.log.info("Writing config files for %s" % self.name) + d = { "my_name" : self.name, + "testbed_name" : testbed_name, + "irdb_db_name" : self.irdb_db_name, + "irdb_db_pass" : irdb_db_pass, + "irdb_port" : self.irdb_port, + "rpki_db_name" : self.rpki_db_name, + "rpki_db_pass" : rpki_db_pass, + "rpki_port" : self.rpki_port } + f = open(self.name + ".conf", "w") + f.write(conf_fmt_1 % d) + for line in self.extra_conf: + f.write(line + "\n") + f.close() + + def setup_sql(self, rpki_sql, irdb_sql): + """Set up this entity's IRDB.""" + rpki.log.info("Setting up MySQL for %s" % self.name) + db = MySQLdb.connect(user = "rpki", db = self.rpki_db_name, passwd = rpki_db_pass) + cur = db.cursor() + for sql in rpki_sql: + cur.execute(sql) + db.close() + db = MySQLdb.connect(user = "irdb", db = self.irdb_db_name, passwd = irdb_db_pass) + cur = db.cursor() + for sql in irdb_sql: + cur.execute(sql) + for kid in self.kids: + cur.execute("INSERT registrant (IRBE_mapped_id, subject_name, valid_until) VALUES (%s, %s, %s)", (kid.name, kid.name, kid.resources.valid_until.to_sql())) + db.close() + + def sync_sql(self): + """Whack this entity's IRDB to match our master database. We do + this once during setup, then do it again every time we apply a + delta to this entity. + """ + rpki.log.info("Updating MySQL data for IRDB %s" % self.name) + db = MySQLdb.connect(user = "irdb", db = self.irdb_db_name, passwd = irdb_db_pass) + cur = db.cursor() + cur.execute("DELETE FROM asn") + cur.execute("DELETE FROM net") + for kid in self.kids: + cur.execute("SELECT registrant_id FROM registrant WHERE IRBE_mapped_id = %s", (kid.name,)) + registrant_id = cur.fetchone()[0] + for as_range in kid.resources.asn: + cur.execute("INSERT asn (start_as, end_as, registrant_id) VALUES (%s, %s, %s)", (as_range.min, as_range.max, registrant_id)) + for v4_range in kid.resources.v4: + cur.execute("INSERT net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 4, %s)", (v4_range.min, v4_range.max, registrant_id)) + for v6_range in kid.resources.v6: + cur.execute("INSERT net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 6, %s)", (v6_range.min, v6_range.max, registrant_id)) + cur.execute("UPDATE registrant SET valid_until = %s WHERE registrant_id = %s", (kid.resources.valid_until.to_sql(), registrant_id)) + db.close() + + def run_daemons(self): + """Run daemons for this entity.""" + rpki.log.info("Running daemons for %s" % self.name) + self.rpkid_process = subprocess.Popen((prog_python, prog_rpkid, "-c", self.name + ".conf") + (("-p", self.name + ".prof") if profile else ())) + self.irdbd_process = subprocess.Popen((prog_python, prog_irdbd, "-c", self.name + ".conf")) + + def kill_daemons(self): + """Kill daemons for this entity.""" + rpki.log.info("Killing daemons for %s" % self.name) + for proc in (self.rpkid_process, self.irdbd_process): + try: + os.kill(proc.pid, signal.SIGTERM) + except: + pass + proc.wait() + + def call_rpkid(self, *pdu): + """Send a left-right message to this entity's RPKI daemon and + return the response. + """ + rpki.log.info("Calling rpkid for %s" % self.name) + msg = rpki.left_right.msg(pdu) + msg.type = "query" + cms, xml = rpki.left_right.cms_msg.wrap(msg, self.irbe_key, self.irbe_cert, + pretty_print = True) + rpki.log.debug(xml) + url = "https://localhost:%d/left-right" % self.rpki_port + der = rpki.https.client( + client_key = self.irbe_key, + client_cert = self.irbe_cert, + server_ta = self.rpkid_ta, + url = url, + msg = cms) + msg, xml = rpki.left_right.cms_msg.unwrap(der, (self.rpkid_ta, self.rpkid_cert), + pretty_print = True) + rpki.log.debug(xml) + assert msg.type == "reply" + for pdu in msg: + assert not isinstance(pdu, rpki.left_right.report_error_elt) + return msg[0] if len(msg) == 1 else msg + + def cross_certify(self, certificant, reverse = False): + """Cross-certify and return the resulting certificate.""" + + if reverse: + certifier = certificant + certificant = self.name + "-SELF-1" + elif self.is_leaf(): + certifier = self.name + "-TA" + else: + certifier = self.name + "-SELF-1" + certfile = certifier + "-" + certificant + ".cer" + rpki.log.info("Cross certifying %s into %s's BPKI (%s)" % (certificant, certifier, certfile)) + signer = subprocess.Popen((prog_openssl, "x509", "-req", "-sha256", "-text", + "-extensions", "req_x509_ext", "-CAcreateserial", + "-in", certificant + ".req", + "-out", certfile, + "-extfile", certifier + ".conf", + "-CA", certifier + ".cer", + "-CAkey", certifier + ".key"), + stdout = subprocess.PIPE, + stderr = subprocess.PIPE) + errors = signer.communicate()[1] + if signer.returncode != 0: + msg = "Couldn't cross-certify %s into %s's BPKI: %s" % (certificant, certifier, errors) + rpki.log.error(msg) + raise RuntimeError, msg + return rpki.x509.X509(Auto_file = certfile) + + def create_rpki_objects(self): + """Create RPKI engine objects for this engine. + + Parent and child objects are tricky: + + - Parent object needs to know child_id by which parent refers to + this engine in order to set the contact URI correctly. + + - Child object needs to record the child_id by which this engine + refers to the child. + + This all just works so long as we walk the set of engines in the + right order (parents before their children). + + Root node of the engine tree is special, it too has a parent but + that one is the magic self-signed micro engine. + """ + + self_ca = rpki.x509.X509(Auto_file = self.name + "-SELF-1.cer") + rpki.log.info("Creating rpkid self object for %s" % self.name) + self.self_id = self.call_rpkid(rpki.left_right.self_elt.make_pdu( + action = "create", crl_interval = self.crl_interval, regen_margin = self.regen_margin, bpki_cert = self_ca)).self_id + + rpki.log.info("Creating rpkid BSC object for %s" % self.name) + pdu = self.call_rpkid(rpki.left_right.bsc_elt.make_pdu(action = "create", self_id = self.self_id, generate_keypair = True)) + self.bsc_id = pdu.bsc_id + + rpki.log.info("Issuing BSC EE cert for %s" % self.name) + cmd = (prog_openssl, "x509", "-req", "-sha256", "-extfile", self.name + "-RPKI.conf", "-extensions", "req_x509_ext", "-days", "30", + "-CA", self.name + "-SELF-1.cer", "-CAkey", self.name + "-SELF-1.key", "-CAcreateserial", "-text") + signer = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE) + signed = signer.communicate(input = pdu.pkcs10_request.get_PEM()) + if not signed[0]: + rpki.log.error(signed[1]) + raise RuntimeError, "Couldn't issue BSC EE certificate" + bsc_ee = rpki.x509.X509(PEM = signed[0]) + bsc_crl = rpki.x509.CRL(PEM_file = self.name + "-SELF-1.crl") + + rpki.log.info("Installing BSC EE cert for %s" % self.name) + self.call_rpkid(rpki.left_right.bsc_elt.make_pdu(action = "set", self_id = self.self_id, bsc_id = self.bsc_id, signing_cert = bsc_ee, signing_cert_crl = bsc_crl)) + + rpki.log.info("Creating pubd client object for %s" % self.name) + client_cert = self.cross_certify(pubd_name + "-TA", reverse = True) + client_id = call_pubd(rpki.publication.client_elt.make_pdu(action = "create", base_uri = self.sia_base, bpki_cert = client_cert)).client_id + + rpki.log.info("Creating rpkid repository object for %s" % self.name) + repository_cert = self.cross_certify(pubd_name + "-TA") + self.repository_id = self.call_rpkid(rpki.left_right.repository_elt.make_pdu( + action = "create", self_id = self.self_id, bsc_id = self.bsc_id, + bpki_cms_cert = repository_cert, bpki_https_cert = repository_cert, + peer_contact_uri = "https://localhost:%d/client/%d" % (pubd_port, client_id))).repository_id + + rpki.log.info("Creating rpkid parent object for %s" % self.name) + if self.is_root(): + rootd_cert = self.cross_certify(rootd_name + "-TA") + self.parent_id = self.call_rpkid(rpki.left_right.parent_elt.make_pdu( + action = "create", self_id = self.self_id, bsc_id = self.bsc_id, repository_id = self.repository_id, sia_base = self.sia_base, + bpki_cms_cert = rootd_cert, bpki_https_cert = rootd_cert, sender_name = self.name, recipient_name = "Walrus", + peer_contact_uri = "https://localhost:%s/" % rootd_port)).parent_id + else: + parent_cms_cert = self.cross_certify(self.parent.name + "-SELF-1") + parent_https_cert = self.cross_certify(self.parent.name + "-TA") + self.parent_id = self.call_rpkid(rpki.left_right.parent_elt.make_pdu( + action = "create", self_id = self.self_id, bsc_id = self.bsc_id, repository_id = self.repository_id, sia_base = self.sia_base, + bpki_cms_cert = parent_cms_cert, bpki_https_cert = parent_https_cert, sender_name = self.name, recipient_name = self.parent.name, + peer_contact_uri = "https://localhost:%s/up-down/%s" % (self.parent.rpki_port, self.child_id))).parent_id + + rpki.log.info("Creating rpkid child objects for %s" % self.name) + db = MySQLdb.connect(user = "irdb", db = self.irdb_db_name, passwd = irdb_db_pass) + cur = db.cursor() + for kid in self.kids: + if kid.is_leaf(): + bpki_cert = self.cross_certify(kid.name + "-TA") + else: + bpki_cert = self.cross_certify(kid.name + "-SELF-1") + rpki.log.info("Creating rpkid child object for %s as child of %s" % (kid.name, self.name)) + kid.child_id = self.call_rpkid(rpki.left_right.child_elt.make_pdu( + action = "create", self_id = self.self_id, bsc_id = self.bsc_id, bpki_cert = bpki_cert)).child_id + cur.execute("UPDATE registrant SET rpki_self_id = %s, rpki_child_id = %s WHERE IRBE_mapped_id = %s", (self.self_id, kid.child_id, kid.name)) + db.close() + + rpki.log.info("Creating rpkid route_origin objects for %s" % self.name) + for ro in self.route_origins: + ro.route_origin_id = self.call_rpkid(rpki.left_right.route_origin_elt.make_pdu( + action = "create", self_id = self.self_id, + as_number = ro.asn, ipv4 = ro.v4, ipv6 = ro.v6)).route_origin_id + + def setup_yaml_leaf(self): + """Generate certificates and write YAML scripts for leaf nodes. + We're cheating a bit here: properly speaking, we can't generate + issue or revoke requests without knowing the class, which is + generated on the fly, but at the moment the test case is + simplistic enough that the class will always be "1", so we just + wire in that value for now. + """ + + if not os.path.exists(self.name + ".key"): + rpki.log.info("Generating RPKI key for %s" % self.name) + subprocess.check_call((prog_openssl, "genrsa", "-out", self.name + ".key", "2048" ), + stdout = subprocess.PIPE, stderr = subprocess.STDOUT) + ski = rpki.x509.RSA(PEM_file = self.name + ".key").gSKI() + + self.cross_certify(self.parent.name + "-TA") + self.cross_certify(self.parent.name + "-SELF-1") + + rpki.log.info("Writing leaf YAML for %s" % self.name) + f = open(self.name + ".yaml", "w") + f.write(yaml_fmt_1 % { + "child_id" : self.child_id, + "parent_name" : self.parent.name, + "my_name" : self.name, + "https_port" : self.parent.rpki_port, + "sia" : self.sia_base, + "ski" : ski }) + f.close() + + def run_cron(self): + """Trigger cron run for this engine.""" + + rpki.log.info("Running cron for %s" % self.name) + rpki.https.client(client_key = self.irbe_key, + client_cert = self.irbe_cert, + server_ta = self.rpkid_ta, + url = "https://localhost:%d/cronjob" % self.rpki_port, + msg = "Run cron now, please") + + def run_yaml(self): + """Run YAML scripts for this leaf entity. Since we're not + bothering to check the class list returned by the list command, + the issue command may fail, so we treat failure of the list + command as an error, but only issue a warning when issue fails. + """ + + rpki.log.info("Running YAML for %s" % self.name) + subprocess.check_call((prog_python, prog_poke, "-y", self.name + ".yaml", "-r", "list")) + if subprocess.call((prog_python, prog_poke, "-y", self.name + ".yaml", "-r", "issue")) != 0: + rpki.log.warn("YAML issue command failed for %s, continuing" % self.name) + +def setup_bpki_cert_chain(name, ee = (), ca = ()): + """Build a set of BPKI certificates.""" + s = "exec >/dev/null 2>&1\n" + for kind in ("TA",) + ee + ca: + d = { "name" : name, + "kind" : kind, + "ca" : "false" if kind in ee else "true", + "openssl" : prog_openssl } + f = open("%(name)s-%(kind)s.conf" % d, "w") + f.write(bpki_cert_fmt_1 % d) + f.close() + if not os.path.exists("%(name)s-%(kind)s.key" % d): + s += bpki_cert_fmt_2 % d + s += bpki_cert_fmt_3 % d + d = { "name" : name, "openssl" : prog_openssl } + s += bpki_cert_fmt_4 % d + for kind in ee + ca: + d["kind"] = kind + s += bpki_cert_fmt_5 % d + for kind in ("TA",) + ca: + d["kind"] = kind + s += bpki_cert_fmt_6 % d + subprocess.check_call(s, shell = True) + +def setup_rootd(rpkid_name, rpkid_tag, rootd_yaml): + """Write the config files for rootd.""" + rpki.log.info("Writing config files for %s" % rootd_name) + d = { "rootd_name" : rootd_name, + "rootd_port" : rootd_port, + "rpkid_name" : rpkid_name, + "rpkid_tag" : rpkid_tag, + "rootd_sia" : rootd_sia, + "rsyncd_dir" : rsyncd_dir, + "openssl" : prog_openssl, + "lifetime" : rootd_yaml.get("lifetime", "30d") } + f = open(rootd_name + ".conf", "w") + f.write(rootd_fmt_1 % d) + f.close() + s = "exec >/dev/null 2>&1\n" + if not os.path.exists(rootd_name + ".key"): + s += rootd_fmt_2 % d + s += rootd_fmt_3 % d + subprocess.check_call(s, shell = True) + +def setup_rcynic(): + """Write the config file for rcynic.""" + rpki.log.info("Config file for rcynic") + d = { "rcynic_name" : rcynic_name, + "rootd_name" : rootd_name, + "rootd_sia" : rootd_sia } + f = open(rcynic_name + ".conf", "w") + f.write(rcynic_fmt_1 % d) + f.close() + +def setup_rsyncd(): + """Write the config file for rsyncd.""" + rpki.log.info("Config file for rsyncd") + d = { "rsyncd_name" : rsyncd_name, + "rsyncd_port" : rsyncd_port, + "rsyncd_module" : rsyncd_module, + "rsyncd_dir" : rsyncd_dir } + f = open(rsyncd_name + ".conf", "w") + f.write(rsyncd_fmt_1 % d) + f.close() + +def setup_publication(pubd_sql): + """Set up publication daemon.""" + rpki.log.info("Configure publication daemon") + pubd_dir = os.getcwd() + "/publication/" + assert rootd_sia.startswith("rsync://") + global rsyncd_dir + rsyncd_dir = pubd_dir + rootd_sia[len("rsync://"):] + os.makedirs(rsyncd_dir) + db = MySQLdb.connect(user = "pubd", db = "pubd", passwd = pubd_db_pass) + cur = db.cursor() + for sql in pubd_sql: + cur.execute(sql) + db.close() + d = { "pubd_name" : pubd_name, + "pubd_port" : pubd_port, + "pubd_pass" : pubd_db_pass, + "pubd_dir" : pubd_dir } + f = open(pubd_name + ".conf", "w") + f.write(pubd_fmt_1 % d) + f.close() + global pubd_ta + global pubd_irbe_key + global pubd_irbe_cert + global pubd_pubd_cert + pubd_ta = rpki.x509.X509(Auto_file = pubd_name + "-TA.cer") + pubd_irbe_key = rpki.x509.RSA( Auto_file = pubd_name + "-IRBE.key") + pubd_irbe_cert = rpki.x509.X509(Auto_file = pubd_name + "-IRBE.cer") + pubd_pubd_cert = rpki.x509.X509(Auto_file = pubd_name + "-PUBD.cer") + +def call_pubd(*pdu): + """Send a publication message to publication daemon and return the + response. + """ + rpki.log.info("Calling pubd") + msg = rpki.publication.msg(pdu) + msg.type = "query" + cms, xml = rpki.publication.cms_msg.wrap(msg, pubd_irbe_key, pubd_irbe_cert, + pretty_print = True) + rpki.log.debug(xml) + url = "https://localhost:%d/control" % pubd_port + der = rpki.https.client( + client_key = pubd_irbe_key, + client_cert = pubd_irbe_cert, + server_ta = pubd_ta, + url = url, + msg = cms) + msg, xml = rpki.publication.cms_msg.unwrap(der, (pubd_ta, pubd_pubd_cert), + pretty_print = True) + rpki.log.debug(xml) + assert msg.type == "reply" + for pdu in msg: + assert not isinstance(pdu, rpki.publication.report_error_elt) + return msg[0] if len(msg) == 1 else msg + +def set_pubd_crl(): + """Whack publication daemon's bpki_crl. This must be configured + before publication daemon starts talking to its clients, and must be + updated whenever we update the CRL. + """ + rpki.log.info("Setting pubd's BPKI CRL") + call_pubd(rpki.publication.config_elt.make_pdu( + action = "set", + bpki_crl = rpki.x509.CRL(Auto_file = pubd_name + "-TA.crl"))) + +def run_rcynic(): + """Run rcynic to see whether what was published makes sense.""" + rpki.log.info("Running rcynic") + env = os.environ.copy() + env["TZ"] = "" + subprocess.check_call((prog_rcynic, "-c", rcynic_name + ".conf"), env = env) + subprocess.call(rcynic_stats, shell = True, env = env) + +def mangle_sql(filename): + """Mangle an SQL file into a sequence of SQL statements.""" + # + # There is no pretty way to do this. Just shut your eyes, it'll be over soon. + # + f = open(filename) + statements = " ".join(" ".join(word for word in line.expandtabs().split(" ") if word) + for line in [line.strip(" \t\n") for line in f.readlines()] + if line and not line.startswith("--")).rstrip(";").split(";") + f.close() + return [stmt.strip() for stmt in statements] + +bpki_cert_fmt_1 = '''\ +[ req ] +distinguished_name = req_dn +x509_extensions = req_x509_ext +prompt = no +default_md = sha256 + +[ req_dn ] +CN = Test Certificate %(name)s %(kind)s + +[ req_x509_ext ] +basicConstraints = critical,CA:%(ca)s +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always + + +[ ca ] +default_ca = ca_default + +[ ca_default ] + +certificate = %(name)s-%(kind)s.cer +serial = %(name)s-%(kind)s.srl +private_key = %(name)s-%(kind)s.key +database = %(name)s-%(kind)s.idx +crlnumber = %(name)s-%(kind)s.cnm +default_crl_days = 30 +default_md = sha256 +''' + +bpki_cert_fmt_2 = '''\ +%(openssl)s genrsa -out %(name)s-%(kind)s.key 2048 && +''' + +bpki_cert_fmt_3 = '''\ +%(openssl)s req -new -sha256 -key %(name)s-%(kind)s.key -out %(name)s-%(kind)s.req -config %(name)s-%(kind)s.conf && +touch %(name)s-%(kind)s.idx && +echo >%(name)s-%(kind)s.cnm 01 && +''' + +bpki_cert_fmt_4 = '''\ +%(openssl)s x509 -req -sha256 -in %(name)s-TA.req -out %(name)s-TA.cer -extfile %(name)s-TA.conf -extensions req_x509_ext -signkey %(name)s-TA.key -days 60 -text \ +''' + +bpki_cert_fmt_5 = ''' && \ +%(openssl)s x509 -req -sha256 -in %(name)s-%(kind)s.req -out %(name)s-%(kind)s.cer -extfile %(name)s-%(kind)s.conf -extensions req_x509_ext -days 30 -text \ + -CA %(name)s-TA.cer -CAkey %(name)s-TA.key -CAcreateserial \ +''' + +bpki_cert_fmt_6 = ''' && \ +%(openssl)s ca -batch -gencrl -out %(name)s-%(kind)s.crl -config %(name)s-%(kind)s.conf \ +''' + +yaml_fmt_1 = '''--- +version: 1 +posturl: https://localhost:%(https_port)s/up-down/%(child_id)s +recipient-id: "%(parent_name)s" +sender-id: "%(my_name)s" + +cms-cert-file: %(my_name)s-RPKI.cer +cms-key-file: %(my_name)s-RPKI.key +cms-ca-cert-file: %(my_name)s-TA.cer +cms-crl-file: %(my_name)s-TA.crl +cms-ca-certs-file: + - %(my_name)s-TA-%(parent_name)s-TA.cer + - %(my_name)s-TA-%(parent_name)s-SELF-1.cer + +ssl-cert-file: %(my_name)s-RPKI.cer +ssl-key-file: %(my_name)s-RPKI.key +ssl-ca-cert-file: %(my_name)s-TA.cer +ssl-ca-certs-file: + - %(my_name)s-TA-%(parent_name)s-TA.cer + +# We're cheating here by hardwiring the class name + +requests: + list: + type: list + issue: + type: issue + class: 1 + sia: + - %(sia)s + cert-request-key-file: %(my_name)s.key + revoke: + type: revoke + class: 1 + ski: %(ski)s +''' + +conf_fmt_1 = '''\ + +[irdbd] + +startup-message = This is %(my_name)s irdbd + +sql-database = %(irdb_db_name)s +sql-username = irdb +sql-password = %(irdb_db_pass)s +bpki-ta = %(my_name)s-TA.cer +rpkid-cert = %(my_name)s-RPKI.cer +irdbd-cert = %(my_name)s-IRDB.cer +irdbd-key = %(my_name)s-IRDB.key +https-url = https://localhost:%(irdb_port)d/ + +[irbe_cli] + +rpkid-bpki-ta = %(my_name)s-TA.cer +rpkid-cert = %(my_name)s-RPKI.cer +rpkid-irbe-cert = %(my_name)s-IRBE.cer +rpkid-irbe-key = %(my_name)s-IRBE.key +rpkid-url = https://localhost:%(rpki_port)d/left-right + +[rpkid] + +startup-message = This is %(my_name)s rpkid + +sql-database = %(rpki_db_name)s +sql-username = rpki +sql-password = %(rpki_db_pass)s + +bpki-ta = %(my_name)s-TA.cer +rpkid-key = %(my_name)s-RPKI.key +rpkid-cert = %(my_name)s-RPKI.cer +irdb-cert = %(my_name)s-IRDB.cer +irbe-cert = %(my_name)s-IRBE.cer + +irdb-url = https://localhost:%(irdb_port)d/ + +server-host = localhost +server-port = %(rpki_port)d +''' + +rootd_fmt_1 = '''\ + +[rootd] + +bpki-ta = %(rootd_name)s-TA.cer +rootd-bpki-cert = %(rootd_name)s-RPKI.cer +rootd-bpki-key = %(rootd_name)s-RPKI.key +rootd-bpki-crl = %(rootd_name)s-TA.crl +child-bpki-cert = %(rootd_name)s-%(rpkid_name)s.cer + +server-port = %(rootd_port)s + +rpki-root-dir = %(rsyncd_dir)s +rpki-base-uri = %(rootd_sia)s +rpki-root-cert-uri = %(rootd_sia)s%(rootd_name)s.cer + +rpki-root-key = %(rootd_name)s.key +rpki-root-cert = %(rootd_name)s.cer + +rpki-subject-pkcs10 = %(rootd_name)s.subject.pkcs10 +rpki-subject-lifetime = %(lifetime)s + +rpki-root-crl = Bandicoot.crl +rpki-root-manifest = Bandicoot.mnf + +rpki-class-name = Wombat +rpki-subject-cert = Wombat.cer + +[req] +default_bits = 2048 +encrypt_key = no +distinguished_name = req_dn +#req_extensions = req_x509_ext +prompt = no +default_md = sha256 +default_days = 60 + +[req_dn] +CN = Completely Bogus Test Root (NOT FOR PRODUCTION USE) + +[req_x509_ext] +basicConstraints = critical,CA:true +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always + +[req_x509_rpki_ext] +basicConstraints = critical,CA:true +subjectKeyIdentifier = hash +keyUsage = critical,keyCertSign,cRLSign +subjectInfoAccess = 1.3.6.1.5.5.7.48.5;URI:%(rootd_sia)s,1.3.6.1.5.5.7.48.10;URI:%(rootd_sia)sBandicoot.mnf +sbgp-autonomousSysNum = critical,AS:0-4294967295 +sbgp-ipAddrBlock = critical,IPv4:0.0.0.0/0,IPv6:0::/0 +certificatePolicies = critical, @rpki_certificate_policy + +[rpki_certificate_policy] + +policyIdentifier = 1.3.6.1.5.5.7.14.2 +''' + +rootd_fmt_2 = '''\ +%(openssl)s genrsa -out %(rootd_name)s.key 2048 && +''' + +rootd_fmt_3 = '''\ +%(openssl)s rsa -pubout -outform DER -in %(rootd_name)s.key -out %(rootd_name)s.pkey && +%(openssl)s req -new -sha256 -key %(rootd_name)s.key -out %(rootd_name)s.req -config %(rootd_name)s.conf -text -extensions req_x509_rpki_ext && +%(openssl)s x509 -req -sha256 -in %(rootd_name)s.req -out %(rootd_name)s.cer -outform DER -extfile %(rootd_name)s.conf -extensions req_x509_rpki_ext \ + -signkey %(rootd_name)s.key && +ln -f %(rootd_name)s.cer %(rsyncd_dir)s && +%(openssl)s x509 -req -sha256 -in %(rpkid_name)s-%(rpkid_tag)s.req -out %(rootd_name)s-%(rpkid_name)s.cer -extfile %(rootd_name)s.conf -extensions req_x509_ext -text \ + -CA %(rootd_name)s-TA.cer -CAkey %(rootd_name)s-TA.key -CAcreateserial +''' + +rcynic_fmt_1 = '''\ +[rcynic] +xml-summary = %(rcynic_name)s.xml +jitter = 0 +use-links = yes +use-syslog = no +use-stderr = yes +log-level = log_debug +#trust-anchor = %(rootd_name)s.cer +trust-anchor-uri-with-key = %(rootd_sia)s%(rootd_name)s.cer %(rootd_name)s.pkey +''' + +rsyncd_fmt_1 = '''\ +port = %(rsyncd_port)d +address = localhost + +[%(rsyncd_module)s] +read only = yes +transfer logging = yes +use chroot = no +path = %(rsyncd_dir)s +comment = RPKI test +''' + +pubd_fmt_1 = '''\ +[pubd] + +sql-database = %(pubd_name)s +sql-username = pubd +sql-password = %(pubd_pass)s +bpki-ta = %(pubd_name)s-TA.cer +pubd-cert = %(pubd_name)s-PUBD.cer +pubd-key = %(pubd_name)s-PUBD.key +irbe-cert = %(pubd_name)s-IRBE.cer +server-host = localhost +server-port = %(pubd_port)d +publication-base = %(pubd_dir)s +''' + +main() |