aboutsummaryrefslogtreecommitdiff
path: root/ca/tests/yamltest.py
diff options
context:
space:
mode:
Diffstat (limited to 'ca/tests/yamltest.py')
-rw-r--r--ca/tests/yamltest.py298
1 files changed, 195 insertions, 103 deletions
diff --git a/ca/tests/yamltest.py b/ca/tests/yamltest.py
index 0932049b..e727789d 100644
--- a/ca/tests/yamltest.py
+++ b/ca/tests/yamltest.py
@@ -43,6 +43,7 @@ import re
import os
import logging
import argparse
+import webbrowser
import sys
import yaml
import signal
@@ -67,19 +68,21 @@ def cleanpath(*names):
"""
Construct normalized pathnames.
"""
+
return os.path.normpath(os.path.join(*names))
# Pathnames for various things we need
this_dir = os.getcwd()
test_dir = cleanpath(this_dir, "yamltest.dir")
-rpkid_dir = cleanpath(this_dir, "..")
+ca_dir = cleanpath(this_dir, "..")
-prog_rpkic = cleanpath(rpkid_dir, "rpkic")
-prog_rpkid = cleanpath(rpkid_dir, "rpkid")
-prog_irdbd = cleanpath(rpkid_dir, "irdbd")
-prog_pubd = cleanpath(rpkid_dir, "pubd")
-prog_rootd = cleanpath(rpkid_dir, "rootd")
+prog_rpkic = cleanpath(ca_dir, "rpkic")
+prog_rpkid = cleanpath(ca_dir, "rpkid")
+prog_irdbd = cleanpath(ca_dir, "irdbd")
+prog_pubd = cleanpath(ca_dir, "pubd")
+prog_rootd = cleanpath(ca_dir, "rootd")
+prog_rpki_manage = cleanpath(ca_dir, "rpki-manage")
class roa_request(object):
"""
@@ -110,6 +113,7 @@ class roa_request(object):
"""
Parse a ROA request from YAML format.
"""
+
return cls(y.get("asn"), y.get("ipv4"), y.get("ipv6"))
@@ -129,7 +133,7 @@ class router_cert(object):
def __init__(self, asn, router_id):
self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split()))
self.router_id = router_id
- self.keypair = rpki.x509.ECDSA.generate(self.ecparams())
+ self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True)
self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair)
self.gski = self.pkcs10.gSKI()
@@ -154,7 +158,7 @@ class allocation_db(list):
def __init__(self, yaml):
list.__init__(self)
self.root = allocation(yaml, self)
- assert self.root.is_root
+ assert self.root.is_root and not any(a.is_root for a in self if a is not self.root) and self[0] is self.root
if self.root.crl_interval is None:
self.root.crl_interval = 60 * 60
if self.root.regen_margin is None:
@@ -180,6 +184,7 @@ class allocation_db(list):
"""
Show contents of allocation database.
"""
+
for a in self:
a.dump()
@@ -210,6 +215,7 @@ class allocation(object):
"""
Allocate a TCP port.
"""
+
cls.base_port += 1
return cls.base_port
@@ -221,6 +227,7 @@ class allocation(object):
Allocate an engine number, mostly used to construct MySQL database
names.
"""
+
cls.base_engine += 1
return cls.base_engine
@@ -275,6 +282,7 @@ class allocation(object):
Compute resource closure of this node and its children, to avoid a
lot of tedious (and error-prone) duplication in the YAML file.
"""
+
resources = self.base
for kid in self.kids:
resources |= kid.closure()
@@ -285,6 +293,7 @@ class allocation(object):
"""
Show content of this allocation node.
"""
+
print str(self)
def __str__(self):
@@ -309,6 +318,7 @@ class allocation(object):
"""
Is this the root node?
"""
+
return self.parent is None
@property
@@ -316,6 +326,7 @@ class allocation(object):
"""
Is this entity hosted?
"""
+
return self.hosted_by is not None
@property
@@ -323,18 +334,21 @@ class allocation(object):
"""
Does this entity run a pubd?
"""
+
return self.is_root or not (self.is_hosted or only_one_pubd)
def path(self, *names):
"""
Construct pathnames in this entity's test directory.
"""
+
return cleanpath(test_dir, self.host.name, *names)
def csvout(self, fn):
"""
Open and log a CSV output file.
"""
+
path = self.path(fn)
print "Writing", path
return rpki.csv_utils.csv_writer(path)
@@ -343,6 +357,7 @@ class allocation(object):
"""
Construct service URL for this node's parent.
"""
+
return "http://localhost:%d/up-down/%s/%s" % (self.parent.host.rpkid_port,
self.parent.name,
self.name)
@@ -351,12 +366,12 @@ class allocation(object):
"""
Write Autonomous System Numbers CSV file.
"""
+
fn = "%s.asns.csv" % d.name
if not args.skip_config:
- f = self.csvout(fn)
- for k in self.kids:
- f.writerows((k.name, a) for a in k.resources.asn)
- f.close()
+ with self.csvout(fn) as f:
+ for k in self.kids:
+ f.writerows((k.name, a) for a in k.resources.asn)
if not args.stop_after_config:
self.run_rpkic("load_asns", fn)
@@ -364,12 +379,12 @@ class allocation(object):
"""
Write prefixes CSV file.
"""
+
fn = "%s.prefixes.csv" % d.name
if not args.skip_config:
- f = self.csvout(fn)
- for k in self.kids:
- f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6))
- f.close()
+ with self.csvout(fn) as f:
+ for k in self.kids:
+ f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6))
if not args.stop_after_config:
self.run_rpkic("load_prefixes", fn)
@@ -377,13 +392,13 @@ class allocation(object):
"""
Write ROA CSV file.
"""
+
fn = "%s.roas.csv" % d.name
if not args.skip_config:
- f = self.csvout(fn)
- for g1, r in enumerate(self.roa_requests):
- f.writerows((p, r.asn, "G%08d%08d" % (g1, g2))
- for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ())))
- f.close()
+ with self.csvout(fn) as f:
+ for g1, r in enumerate(self.roa_requests):
+ f.writerows((p, r.asn, "G%08d%08d" % (g1, g2))
+ for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ())))
if not args.stop_after_config:
self.run_rpkic("load_roa_requests", fn)
@@ -391,17 +406,14 @@ class allocation(object):
"""
Write Ghostbusters vCard file.
"""
+
if self.ghostbusters:
fn = "%s.ghostbusters.vcard" % d.name
if not args.skip_config:
path = self.path(fn)
print "Writing", path
- f = open(path, "w")
- for i, g in enumerate(self.ghostbusters):
- if i:
- f.write("\n")
- f.write(g)
- f.close()
+ with open(path, "w") as f:
+ f.write("\n".join(self.ghostbusters))
if not args.stop_after_config:
self.run_rpkic("load_ghostbuster_requests", fn)
@@ -409,6 +421,7 @@ class allocation(object):
"""
Write EE certificates (router certificates, etc).
"""
+
if self.router_certs:
fn = "%s.routercerts.xml" % d.name
if not args.skip_config:
@@ -440,6 +453,7 @@ class allocation(object):
"""
Walk up tree until we find somebody who runs pubd.
"""
+
s = self
while not s.runs_pubd:
s = s.parent
@@ -450,6 +464,7 @@ class allocation(object):
"""
Work out what pubd configure_publication_client will call us.
"""
+
path = []
s = self
if not args.flat_publication:
@@ -469,46 +484,45 @@ class allocation(object):
"""
r = dict(
- handle = self.name,
- run_rpkid = str(not self.is_hosted),
- run_pubd = str(self.runs_pubd),
- run_rootd = str(self.is_root),
- irdbd_sql_database = "irdb%d" % self.engine,
- irdbd_sql_username = "irdb",
- rpkid_sql_database = "rpki%d" % self.engine,
- rpkid_sql_username = "rpki",
- rpkid_server_host = "localhost",
- rpkid_server_port = str(self.rpkid_port),
- irdbd_server_host = "localhost",
- irdbd_server_port = str(self.irdbd_port),
- rootd_server_port = str(self.rootd_port),
- pubd_sql_database = "pubd%d" % self.engine,
- pubd_sql_username = "pubd",
- pubd_server_host = "localhost",
- pubd_server_port = str(self.pubd.pubd_port),
- publication_rsync_server = "localhost:%s" % self.pubd.rsync_port,
- bpki_servers_directory = self.path(),
- publication_base_directory = self.path("publication"),
- shared_sql_password = "fnord")
+ handle = self.name,
+ run_rpkid = str(not self.is_hosted),
+ run_pubd = str(self.runs_pubd),
+ run_rootd = str(self.is_root),
+ irdbd_sql_database = "irdb%d" % self.engine,
+ irdbd_sql_username = "irdb",
+ rpkid_sql_database = "rpki%d" % self.engine,
+ rpkid_sql_username = "rpki",
+ rpkid_server_host = "localhost",
+ rpkid_server_port = str(self.rpkid_port),
+ irdbd_server_host = "localhost",
+ irdbd_server_port = str(self.irdbd_port),
+ rootd_server_port = str(self.rootd_port),
+ pubd_sql_database = "pubd%d" % self.engine,
+ pubd_sql_username = "pubd",
+ pubd_server_host = "localhost",
+ pubd_server_port = str(self.pubd.pubd_port),
+ publication_rsync_server = "localhost:%s" % self.pubd.rsync_port,
+ bpki_servers_directory = self.path(),
+ publication_base_directory = self.path("publication"),
+ rrdp_publication_base_directory = self.path("rrdp-publication"),
+ shared_sql_password = "fnord")
r.update(config_overrides)
- f = open(self.path("rpki.conf"), "w")
- f.write("# Automatically generated, do not edit\n")
- print "Writing", f.name
-
- section = None
- for line in open(cleanpath(rpkid_dir, "examples/rpki.conf")):
- m = section_regexp.match(line)
- if m:
- section = m.group(1)
- m = variable_regexp.match(line)
- option = m.group(1) if m and section == "myrpki" else None
- if option and option in r:
- line = "%s = %s\n" % (option, r[option])
- f.write(line)
+ with open(self.path("rpki.conf"), "w") as f:
+ f.write("# Automatically generated, do not edit\n")
+ print "Writing", f.name
- f.close()
+ section = None
+ for line in open(cleanpath(ca_dir, "examples/rpki.conf")):
+ m = section_regexp.match(line)
+ if m:
+ section = m.group(1)
+ m = variable_regexp.match(line)
+ option = m.group(1) if m and section == "myrpki" else None
+ if option and option in r:
+ line = "%s = %s\n" % (option, r[option])
+ f.write(line)
def dump_rsyncd(self):
"""
@@ -516,25 +530,24 @@ class allocation(object):
"""
if self.runs_pubd:
- f = open(self.path("rsyncd.conf"), "w")
- print "Writing", f.name
- f.writelines(s + "\n" for s in
- ("# Automatically generated, do not edit",
- "port = %d" % self.rsync_port,
- "address = localhost",
- "[rpki]",
- "log file = rsyncd.log",
- "read only = yes",
- "use chroot = no",
- "path = %s" % self.path("publication"),
- "comment = RPKI test",
- "[root]",
- "log file = rsyncd_root.log",
- "read only = yes",
- "use chroot = no",
- "path = %s" % self.path("publication.root"),
- "comment = RPKI test root"))
- f.close()
+ with open(self.path("rsyncd.conf"), "w") as f:
+ print "Writing", f.name
+ f.writelines(s + "\n" for s in
+ ("# Automatically generated, do not edit",
+ "port = %d" % self.rsync_port,
+ "address = localhost",
+ "[rpki]",
+ "log file = rsyncd.log",
+ "read only = yes",
+ "use chroot = no",
+ "path = %s" % self.path("publication"),
+ "comment = RPKI test",
+ "[root]",
+ "log file = rsyncd_root.log",
+ "read only = yes",
+ "use chroot = no",
+ "path = %s" % self.path("publication.root"),
+ "comment = RPKI test root"))
@classmethod
def next_rpkic_counter(cls):
@@ -545,65 +558,116 @@ class allocation(object):
"""
Run rpkic for this entity.
"""
- cmd = [prog_rpkic, "-i", self.name, "-c", self.path("rpki.conf")]
+
+ cmd = [prog_rpkic, "-i", self.name]
if args.profile:
cmd.append("--profile")
cmd.append(self.path("rpkic.%s.prof" % rpki.sundial.now()))
cmd.extend(str(a) for a in argv if a is not None)
print 'Running "%s"' % " ".join(cmd)
- env = os.environ.copy()
- env["YAMLTEST_RPKIC_COUNTER"] = self.next_rpkic_counter()
+ env = dict(os.environ,
+ YAMLTEST_RPKIC_COUNTER = self.next_rpkic_counter(),
+ RPKI_CONF = self.path("rpki.conf"))
subprocess.check_call(cmd, cwd = self.host.path(), env = env)
+ def syncdb(self):
+ """
+ Run whatever Django ORM commands are necessary to set up the
+ database this week.
+ """
+
+ verbosity = 1
+
+ if verbosity > 0:
+ print "Running Django setup for", self.name
+
+ if not os.fork():
+ os.environ.update(RPKI_CONF = self.path("rpki.conf"),
+ RPKI_GUI_ENABLE = "yes")
+ logging.getLogger().setLevel(logging.WARNING)
+ import django.core.management
+ django.core.management.call_command("syncdb", migrate = True, verbosity = verbosity,
+ load_initial_data = False, interactive = False)
+ from django.contrib.auth.models import User
+ User.objects.create_superuser("root", "root@example.org", "fnord")
+ sys.exit(0)
+
+ if os.wait()[1]:
+ raise RuntimeError("Django setup failed for %s" % self.name)
+
def run_python_daemon(self, prog):
"""
Start a Python daemon and return a subprocess.Popen object
representing the running daemon.
"""
+
basename = os.path.splitext(os.path.basename(prog))[0]
cmd = [prog, "--foreground", "--log-level", "debug",
- "--log-file", self.path(basename + ".log"),
- "--config", self.path("rpki.conf")]
+ "--log-file", self.path(basename + ".log")]
if args.profile and basename != "rootd":
cmd.extend((
"--profile", self.path(basename + ".prof")))
- p = subprocess.Popen(cmd, cwd = self.path())
- print 'Running %s for %s: pid %d process %r' % (" ".join(cmd), self.name, p.pid, p)
+ env = dict(os.environ, RPKI_CONF = self.path("rpki.conf"))
+ p = subprocess.Popen(cmd, cwd = self.path(), env = env)
+ print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p)
return p
def run_rpkid(self):
"""
Run rpkid.
"""
+
return self.run_python_daemon(prog_rpkid)
def run_irdbd(self):
"""
Run irdbd.
"""
+
return self.run_python_daemon(prog_irdbd)
def run_pubd(self):
"""
Run pubd.
"""
+
return self.run_python_daemon(prog_pubd)
def run_rootd(self):
"""
Run rootd.
"""
+
return self.run_python_daemon(prog_rootd)
def run_rsyncd(self):
"""
Run rsyncd.
"""
+
p = subprocess.Popen(("rsync", "--daemon", "--no-detach", "--config", "rsyncd.conf"),
cwd = self.path())
print "Running rsyncd for %s: pid %d process %r" % (self.name, p.pid, p)
return p
+ def run_gui(self):
+ """
+ Start an instance of the RPKI GUI under the Django test server and
+ return a subprocess.Popen object representing the running daemon.
+ """
+
+ port = 8000 + self.engine
+ cmd = (prog_rpki_manage, "runserver", str(port))
+ env = dict(os.environ,
+ RPKI_CONF = self.path("rpki.conf"),
+ RPKI_DJANGO_DEBUG = "yes",
+ ALLOW_PLAIN_HTTP_FOR_TESTING = "I solemnly swear that I am not running this in production")
+ p = subprocess.Popen(cmd, cwd = self.path(), env = env,
+ stdout = open(self.path("gui.log"), "w"), stderr = subprocess.STDOUT)
+ print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p)
+ return p
+
+
def create_root_certificate(db_root):
print "Creating rootd RPKI root certificate"
@@ -615,9 +679,9 @@ def create_root_certificate(db_root):
root_key = rpki.x509.RSA.generate(quiet = True)
- root_uri = "rsync://localhost:%d/rpki/" % db_root.pubd.rsync_port
+ root_uri = "rsync://localhost:%d/rpki/%s-root/root" % (db_root.pubd.rsync_port, db_root.name)
- root_sia = (root_uri, root_uri + "root.mft", None)
+ root_sia = (root_uri + "/", root_uri + "/root.mft", None)
root_cert = rpki.x509.X509.self_certify(
keypair = root_key,
@@ -627,22 +691,21 @@ def create_root_certificate(db_root):
notAfter = rpki.sundial.now() + rpki.sundial.timedelta(days = 365),
resources = root_resources)
- f = open(db_root.path("publication.root/root.cer"), "wb")
- f.write(root_cert.get_DER())
- f.close()
+ with open(db_root.path("root.cer"), "wb") as f:
+ f.write(root_cert.get_DER())
- f = open(db_root.path("root.key"), "wb")
- f.write(root_key.get_DER())
- f.close()
+ with open(db_root.path("root.key"), "wb") as f:
+ f.write(root_key.get_DER())
- f = open(os.path.join(test_dir, "root.tal"), "w")
- f.write("rsync://localhost:%d/root/root.cer\n\n" % db_root.pubd.rsync_port)
- f.write(root_key.get_public().get_Base64())
- f.close()
+ with open(os.path.join(test_dir, "root.tal"), "w") as f:
+ f.write(root_uri + ".cer\n\n")
+ f.write(root_key.get_public().get_Base64())
+logger = logging.getLogger(__name__)
-os.environ["TZ"] = "UTC"
+os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings",
+ TZ = "UTC")
time.tzset()
parser = argparse.ArgumentParser(description = __doc__)
@@ -662,6 +725,12 @@ parser.add_argument("--synchronize", action = "store_true",
help = "synchronize IRDB with daemons")
parser.add_argument("--profile", action = "store_true",
help = "enable profiling")
+parser.add_argument("-g", "--run_gui", action = "store_true",
+ help = "enable GUI using django-admin runserver")
+parser.add_argument("--browser", action = "store_true",
+ help = "create web browser tabs for GUI")
+parser.add_argument("--notify-when-startup-complete", type = int,
+ help = "send SIGUSR1 to this process when startup is complete")
parser.add_argument("--store-router-private-keys", action = "store_true",
help = "write generate router private keys to disk")
parser.add_argument("yaml_file", type = argparse.FileType("r"),
@@ -680,7 +749,7 @@ try:
# passwords: this is mostly so that I can show a complete working
# example without publishing my own server's passwords.
- cfg = rpki.config.parser(args.config, "yamltest", allow_missing = True)
+ cfg = rpki.config.parser(set_filename = args.config, section = "yamltest", allow_missing = True)
only_one_pubd = cfg.getboolean("only_one_pubd", True)
allocation.base_port = cfg.getint("base_port", 4400)
@@ -721,6 +790,7 @@ try:
for d in db:
if not d.is_hosted:
+ print "Initializing", d.name
os.makedirs(d.path())
d.dump_conf()
if d.runs_pubd:
@@ -728,7 +798,9 @@ try:
d.dump_rsyncd()
if d.is_root:
os.makedirs(d.path("publication.root"))
+ d.syncdb()
d.run_rpkic("initialize_server_bpki")
+ print
# Initialize resource holding BPKI and generate self-descriptor
# for each entity.
@@ -766,6 +838,8 @@ try:
if d.runs_pubd:
progs.append(d.run_pubd())
progs.append(d.run_rsyncd())
+ if args.run_gui:
+ progs.append(d.run_gui())
if args.synchronize or not args.skip_config:
@@ -834,9 +908,27 @@ try:
d.dump_ghostbusters()
d.dump_router_certificates()
+ if args.run_gui:
+ print
+ print 'GUI user "root", password "fnord"'
+ for d in db:
+ if not d.is_hosted:
+ url = "http://127.0.0.1:%d/rpki/" % (8000 + d.engine)
+ print "GUI URL", url, "for", d.name
+ if args.browser:
+ if d is db.root:
+ webbrowser.open_new(url)
+ else:
+ webbrowser.open_new_tab(url)
+ time.sleep(2)
+
# Wait until something terminates.
if not args.stop_after_config or args.keep_going:
+ if args.notify_when_startup_complete:
+ print
+ print "Sending SIGUSR1 to process", args.notify_when_startup_complete
+ os.kill(args.notify_when_startup_complete, signal.SIGUSR1)
print
print "Waiting for daemons to exit"
signal.signal(signal.SIGCHLD, lambda *dont_care: None)
generated by cgit v1.2.3 (git 2.25.1) at 2025-05-10 13:57:25 +0000