path: root/myrpki/yamltest.py

"""
Convert testbed.py YAML configuration format to myrpki .conf and .csv
format.  Much of the YAML handling code lifted from testbed.py.

$Id$

Copyright (C) 2009  Internet Systems Consortium ("ISC")

Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.

THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.

Portions copyright (C) 2007--2008  American Registry for Internet Numbers ("ARIN")

Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.

THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS.  IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.

"""

import subprocess, csv, re, os, getopt, sys, base64, yaml, signal, errno, time
import rpki.resource_set, rpki.sundial, rpki.config, myrpki

section_regexp = re.compile("\s*\[\s*(.+?)\s*\]\s*$")
variable_regexp = re.compile("\s*([-a-zA-Z0-9_]+)\s*=\s*(.+?)\s*$")

def cleanpath(*names):
  return os.path.normpath(os.path.join(*names))

this_dir  = os.getcwd()
test_dir  = cleanpath(this_dir, "test")
rpkid_dir = cleanpath(this_dir, "../rpkid")

prog_myirbe = cleanpath(this_dir, "myirbe.py")
prog_myrpki = cleanpath(this_dir, "myrpki.py")
prog_rpkid  = cleanpath(rpkid_dir, "rpkid.py")
prog_irdbd  = cleanpath(rpkid_dir, "irdbd.py")
prog_pubd   = cleanpath(rpkid_dir, "pubd.py")
prog_rootd  = cleanpath(rpkid_dir, "rootd.py")

prog_openssl = cleanpath(this_dir, "../openssl/openssl/apps/openssl")

class roa_request(object):

  def __init__(self, asn, ipv4, ipv6):
    self.asn = asn
    self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None
    self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None

  def __eq__(self, other):
    return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6

  def __hash__(self):
    v4 = tuple(self.v4) if self.v4 is not None else None
    v6 = tuple(self.v6) if self.v6 is not None else None
    return self.asn.__hash__() + v4.__hash__() + v6.__hash__()

  def __str__(self):
    if self.v4 and self.v6:
      return "%s: %s,%s" % (self.asn, self.v4, self.v6)
    else:
      return "%s: %s" % (self.asn, self.v4 or self.v6)

  @classmethod
  def parse(cls, yaml):
    return cls(yaml.get("asn"), yaml.get("ipv4"), yaml.get("ipv6"))
    
class allocation_db(list):

  def __init__(self, yaml):
    list.__init__(self)
    self.root = allocation(yaml, self)
    assert self.root.is_root()
    if self.root.crl_interval is None:
      self.root.crl_interval = 24 * 60 * 60
    if self.root.regen_margin is None:
      self.root.regen_margin = 24 * 60 * 60
    for a in self:
      if a.sia_base is None:
        a.sia_base = ("rsync://rpki.example/" if a.is_root() else a.parent.sia_base) + a.name + "/"
      if a.base.valid_until is None:
        a.base.valid_until = a.parent.base.valid_until
      if a.crl_interval is None:
        a.crl_interval = a.parent.crl_interval
      if a.regen_margin is None:
        a.regen_margin = a.parent.regen_margin
    self.root.closure()
    self.map = dict((a.name, a) for a in self)
    for a in self:
      if a.is_hosted():
        a.hosted_by = self.map[a.hosted_by]
        a.hosted_by.hosts.append(a)
        assert not a.is_root() and not a.hosted_by.is_hosted()

  def dump(self):
    for a in self:
      a.dump()

  def make_rootd_openssl(self):
    env = { "PATH"           : os.environ["PATH"],
            "BPKI_DIRECTORY" : self.root.path("bpki.rootd"),
            "RANDFILE"       : ".OpenSSL.whines.unless.I.set.this" }
    cwd = self.root.path()
    return lambda *args: subprocess.check_call((prog_openssl,) + args, cwd = cwd, env = env)

class allocation(object):

  parent       = None
  crl_interval = None
  regen_margin = None

  base_port = 4400

  @classmethod
  def allocate_port(cls):
    cls.base_port += 1
    return cls.base_port

  base_engine = -1

  @classmethod
  def allocate_engine(cls):
    cls.base_engine += 1
    return cls.base_engine

  def __init__(self, yaml, db, parent = None):
    db.append(self)
    self.name = yaml["name"]
    self.parent = parent
    self.kids = [allocation(k, db, self) for k in yaml.get("kids", ())]
    valid_until = None
    if "valid_until" in yaml:
      valid_until = rpki.sundial.datetime.fromdatetime(yaml.get("valid_until"))
    if valid_until is None and "valid_for" in yaml:
      valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(yaml["valid_for"])
    self.base = rpki.resource_set.resource_bag(
      asn = rpki.resource_set.resource_set_as(yaml.get("asn")),
      v4 = rpki.resource_set.resource_set_ipv4(yaml.get("ipv4")),
      v6 = rpki.resource_set.resource_set_ipv6(yaml.get("ipv6")),
      valid_until = valid_until)
    self.sia_base = yaml.get("sia_base")
    if "crl_interval" in yaml:
      self.crl_interval = rpki.sundial.timedelta.parse(yaml["crl_interval"]).convert_to_seconds()
    if "regen_margin" in yaml:
      self.regen_margin = rpki.sundial.timedelta.parse(yaml["regen_margin"]).convert_to_seconds()
    self.roa_requests = [roa_request.parse(y) for y in yaml.get("roa_request", yaml.get("route_origin", ()))]
    for r in self.roa_requests:
      if r.v4:
        self.base.v4 = self.base.v4.union(r.v4.to_resource_set())
      if r.v6:
        self.base.v6 = self.base.v6.union(r.v6.to_resource_set())
    self.hosted_by = yaml.get("hosted_by")
    self.hosts = []
    if not self.is_hosted():
      self.engine = self.allocate_engine()
      self.rsync_port = self.allocate_port()
      self.rpkid_port = self.allocate_port()
      self.pubd_port  = self.allocate_port()
      self.irdbd_port = self.allocate_port()
    if self.is_root():
      self.rootd_port = self.allocate_port()

  def closure(self):
    resources = self.base
    for kid in self.kids:
      resources = resources.union(kid.closure())
    self.resources = resources
    return resources

  def dump(self):
    print str(self)

  def __str__(self):
    s = self.name + ":\n"
    if self.resources.asn:      s += "  ASNs: %s\n" % self.resources.asn
    if self.resources.v4:       s += "  IPv4: %s\n" % self.resources.v4
    if self.resources.v6:       s += "  IPv6: %s\n" % self.resources.v6
    if self.kids:               s += "  Kids: %s\n" % ", ".join(k.name for k in self.kids)
    if self.parent:             s += "    Up: %s\n" % self.parent.name
    #if self.sia_base:          s += "   SIA: %s\n" % self.sia_base
    if self.is_hosted():        s += "  Host: %s\n" % self.hosted_by.name
    if self.hosts:              s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts)
    for r in self.roa_requests: s += "   ROA: %s\n" % r
    if not self.is_hosted():    s += " IPort: %s\n" % self.irdbd_port
    if not self.is_hosted():    s += " PPort: %s\n" % self.pubd_port
    if not self.is_hosted():    s += " RPort: %s\n" % self.rpkid_port
    if not self.is_hosted():    s += " SPort: %s\n" % self.rsync_port
    if self.is_root():          s += " TPort: %s\n" % self.rootd_port
    return s + " Until: %s\n" % self.resources.valid_until

  def is_root(self):
    return self.parent is None

  def is_hosted(self):
    return self.hosted_by is not None

  def path(self, *names):
    return cleanpath(test_dir, self.name, *names)

  def outfile(self, filename):
    path = self.path(filename)
    print "Writing", path
    return open(path, "w")

  def up_down_url(self):
    parent_port = self.parent.hosted_by.rpkid_port if self.parent.is_hosted() else self.parent.rpkid_port
    return "https://localhost:%d/up-down/%s/%s" % (parent_port, self.parent.name, self.name)

  def dump_asns(self, fn):
    f = self.outfile(fn)
    for k in self.kids:
      for a in k.resources.asn:
        f.write("%s\t%s\n" % (k.name, a))
    f.close()

  def dump_children(self, fn):
    f = self.outfile(fn)
    for k in self.kids:
      f.write("%s\t%s\t%s\n" % (k.name, k.resources.valid_until, k.path("bpki.myrpki/ca.cer")))
    f.close()

  def dump_parents(self, fn):
    f = self.outfile(fn)
    if self.is_root():
      f.write("%s\t%s\t%s\t%s\n" % ("rootd", "https://localhost:%d/" % self.rootd_port, self.path("bpki.rootd/ca.cer"), self.path("bpki.rootd/ca.cer")))
    else:
      f.write("%s\t%s\t%s\t%s\n" % (self.parent.name, self.up_down_url(), self.parent.path("bpki.myrpki/ca.cer"), self.parent.path("bpki.rpkid/ca.cer")))
    f.close()

  def dump_prefixes(self, fn):
    f = self.outfile(fn)
    for k in self.kids:
      for p in k.resources.v4 + k.resources.v6:
        f.write("%s\t%s\n" % (k.name, p))
    f.close()

  def dump_roas(self, fn):
    f = self.outfile(fn)
    for r in self.roa_requests:
      for p in r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ():
        f.write("%s\t%s\n" % (p, r.asn))
    f.close()

  def dump_conf(self, fn):

    r = { ("myrpki", "handle") : self.name }

    if not self.is_hosted():
      r["irdbd",  "https-url"]     = "https://localhost:%d/" % self.irdbd_port
      r["irdbd",  "sql-database"]  = "irdb%d" % self.engine
      r["myirbe", "irdbd_conf"]    = "myrpki.conf"
      r["myirbe", "pubd_base"]     = "https://localhost:%d/" % self.pubd_port
      r["myirbe", "rpkid_base"]    = "https://localhost:%d/" % self.rpkid_port
      r["myirbe", "rsync_base"]    = "rsync://localhost:%d/" % self.rsync_port
      r["pubd",   "server-port"]   = "%d" % self.pubd_port
      r["pubd",   "sql-database"]  = "pubd%d" % self.engine
      r["rpkid",  "irdb-url"]      = "https://localhost:%d/" % self.irdbd_port
      r["rpkid",  "server-port"]   = "%d" % self.rpkid_port
      r["rpkid",  "sql-database"]  = "rpki%d" % self.engine
      r["rootd",  "rpki-root-dir"] = "publication/localhost:%d/" % self.rsync_port
      r["rootd",  "rpki-base-uri"] = "rsync://localhost:%d/" % self.rsync_port
      r["rootd",  "rpki-root-cert-uri"] = "rsync://localhost:%d/rootd.cer" % self.rsync_port
      r["rpki_x509_extensions",  "subjectInfoAccess"]  = (
        "1.3.6.1.5.5.7.48.5;URI:rsync://localhost:%d/,1.3.6.1.5.5.7.48.10;URI:rsync://localhost:%d/Bandicoot.mnf" %
        (self.rsync_port, self.rsync_port))

    if self.is_root():
      r["rootd", "server-port"] = "%d" % self.rootd_port

    if rpkid_password:
      r["rpkid", "sql-password"] = rpkid_password

    if irdbd_password:
      r["irdbd", "sql-password"] = irdbd_password

    if pubd_password:
      r["pubd", "sql-password"]  = pubd_password

    f = self.outfile(fn)
    f.write("# Automatically generated, do not edit\n")

    section = None
    for line in open("myrpki.conf"):
      m = section_regexp.match(line)
      if m:
        section = m.group(1)
      if section is None or (self.is_hosted() and section in ("myirbe", "rpkid", "irdbd", "pubd", "rootd")):
        continue
      m = variable_regexp.match(line) if m is None else None
      variable = m.group(1) if m else None
      if (section, variable) in r:
        line = variable + " = " +  r[section, variable] + "\n"
      f.write(line)

    f.close()

  def run_myirbe(self):
    if not self.is_hosted():
      print "Running myirbe.py for", self.name
      subprocess.check_call(("python", prog_myirbe), cwd = self.path())

  def run_myrpki(self):
    print "Running myrpki.py for", self.name
    subprocess.check_call(("python", prog_myrpki), cwd = self.path())

  def run_python_daemon(self, prog):
    basename = os.path.basename(prog)
    p = subprocess.Popen(("python", prog, "-c", self.path("myrpki.conf")),
                         cwd = self.path(),
                         stdout = open(self.path(os.path.splitext(basename)[0] + ".log"), "w"),
                         stderr = subprocess.STDOUT)
    print "Running %s for %s: pid %d process %r" % (basename, self.name, p.pid, p)
    return p
  
  def run_rpkid(self):
    return self.run_python_daemon(prog_rpkid)

  def run_irdbd(self):
    return self.run_python_daemon(prog_irdbd)

  def run_pubd(self):
    return self.run_python_daemon(prog_pubd)

  def run_rootd(self):
    return self.run_python_daemon(prog_rootd)

os.environ["TZ"] = "UTC"
time.tzset()

cfg_file = "yamltest.conf"

opts, argv = getopt.getopt(sys.argv[1:], "c:h?", ["config=", "help"])
for o, a in opts:
  if o in ("-h", "--help", "-?"):
    print __doc__
    sys.exit(0)
  if o in ("-c", "--config"):
    cfg_file = a

if len(argv) > 1:
  raise RuntimeError, "Unexpected arguments %r" % argv

yaml_file = argv[0] if argv else "../rpkid/testbed.1.yaml"

try:
  cfg = rpki.config.parser(cfg_file, "yamltest")
  rpkid_password = cfg.get("rpkid-password")
  irdbd_password = cfg.get("irdbd-password")
  pubd_password  = cfg.get("pubd-password")
except:
  rpkid_password = None
  irdbd_password = None
  pubd_password  = None

# Start clean

for root, dirs, files in os.walk(test_dir, topdown = False):
  for file in files:
    os.unlink(os.path.join(root, file))
  for dir in dirs:
    os.rmdir(os.path.join(root, dir))

# Read first YAML doc in file and process as compact description of
# test layout and resource allocations.  Ignore subsequent YAML docs,
# they're for testbed.py, not this script.

db = allocation_db(yaml.safe_load_all(open(yaml_file)).next())

# Show what we loaded

db.dump()

# Set up each entity in our test

for d in db:
  os.makedirs(d.path())
  d.dump_asns("asns.csv")
  d.dump_children("children.csv")
  d.dump_parents("parents.csv")
  d.dump_prefixes("prefixes.csv")
  d.dump_roas("roas.csv")
  d.dump_conf("myrpki.conf")

# Do initial myirbe.py run for each hosting entity to set up BPKI

for d in db:
  d.run_myirbe()

# Run myrpki.py several times for each entity.  First pass misses
# stuff that isn't generated until later in first pass.  Second pass
# should pick up everything and reach a stable state.  If anything
# changes during third pass, that's a bug.

for i in xrange(3):
  for d in db:
    d.run_myrpki()

# Set up a few things for rootd

rootd_openssl = db.make_rootd_openssl()

print "Creating rootd BPKI cross-certificate for its child"
rootd_openssl("ca", "-notext", "-batch",
              "-config",  "myrpki.conf",
              "-ss_cert", "bpki.myrpki/ca.cer",
              "-out",     "bpki.rootd/child.cer",
              "-extensions", "ca_x509_ext_xcert0")

print "Creating rootd RPKI root certificate"
rootd_openssl("x509", "-req", "-sha256", "-outform", "DER",
              "-signkey", "bpki.rootd/ca.key",
              "-in",      "bpki.rootd/ca.req",
              "-out",     "bpki.rootd/rpkiroot.cer",
              "-extfile", "myrpki.conf",
              "-extensions", "rpki_x509_extensions")

os.makedirs(db.root.path("publication/localhost:%d" % db.root.rsync_port))

# At this point we need to start a whole lotta daemons.

progs = []

try:
  print "Running daemons"
  progs.append(db.root.run_rootd())
  progs.extend(d.run_irdbd() for d in db if not d.is_hosted())
  progs.extend(d.run_pubd()  for d in db if not d.is_hosted())
  progs.extend(d.run_rpkid() for d in db if not d.is_hosted())

  print "Giving daemons time to start up"
  time.sleep(20)

  # Run myirbe again for each host, to set up IRDB and RPKI objects.
  # Need to run a second time to push BSC certs out to rpkid.
  # Nothing should happen on the third pass.

  for i in xrange(3):
    for d in db:
      d.run_myirbe()

  print "Done initializing daemons"

  # Wait until something terminates.

  signal.signal(signal.SIGCHLD, lambda *dont_care: None)
  if not [p for p in progs if p.poll() is not None]:
    signal.pause()

finally:

  # Shut everything down.

  signal.signal(signal.SIGCHLD, signal.SIG_DFL)
  for p in progs:
    if p.poll() is None:
      os.kill(p.pid, signal.SIGTERM)
    print "Program pid %d %r returned %d" % (p.pid, p, p.wait())