path: root/ca/tests/yamlconf.py

# $Id$
#
# Copyright (C) 2013--2014  Dragon Research Labs ("DRL")
# Portions copyright (C) 2009--2012  Internet Systems Consortium ("ISC")
# Portions copyright (C) 2007--2008  American Registry for Internet Numbers ("ARIN")
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notices and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND DRL, ISC, AND ARIN DISCLAIM ALL
# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS.  IN NO EVENT SHALL DRL,
# ISC, OR ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

"""
Test configuration tool, using the same YAML test description format
as smoketest.py and yamltest.py, but doing just the IRDB configuration
for a massive testbed, via direct use of the rpki.irdb library code.

For most purposes, you don't want this, but when building a
configuration for tens or hundreds of thousands of elements, being
able to do the initial configuration stage quickly can help a lot.
"""

# pylint: disable=W0702,W0621,W0602

import re
import os
import sys
import yaml
import time
import logging
import argparse
import subprocess
import lxml.etree
import rpki.resource_set
import rpki.sundial
import rpki.config
import rpki.log
import rpki.csv_utils
import rpki.x509
import rpki.sql_schemas

from rpki.mysql_import import MySQLdb

section_regexp = re.compile(r"\s*\[\s*(.+?)\s*\]\s*$")
variable_regexp = re.compile(r"\s*([-a-zA-Z0-9_]+)\s*=\s*(.+?)\s*$")

flat_publication = False
only_one_pubd = True
yaml_file = None
loopback = False
quiet = False
dns_suffix = None
mysql_rootuser = None
mysql_rootpass = None
publication_base = None
publication_root = None

# The SQL username mismatch between rpkid/examples/rpki.conf and
# rpkid/tests/smoketest.setup.sql is completely stupid and really
# should be cleaned up at some point...but not today, at least not as
# part of writing this program.  These default values are wired into
# yamltest to match smoketest.setup.sql, so wire them in here too but
# in a more obvious way.

config_overrides = {
  "irdbd_sql_username" : "irdb", "irdbd_sql_password" : "fnord",
  "rpkid_sql_username" : "rpki", "rpkid_sql_password" : "fnord",
  "pubd_sql_username"  : "pubd", "pubd_sql_password"  : "fnord" }

def cleanpath(*names):
  return os.path.normpath(os.path.join(*names))

this_dir  = os.getcwd()
test_dir  = None
rpki_conf = None

class roa_request(object):
  """
  Representation of a ROA request.
  """

  def __init__(self, asn, ipv4, ipv6):
    self.asn = asn
    self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None
    self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None

  def __eq__(self, other):
    return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6

  def __hash__(self):
    v4 = tuple(self.v4) if self.v4 is not None else None
    v6 = tuple(self.v6) if self.v6 is not None else None
    return self.asn.__hash__() + v4.__hash__() + v6.__hash__()

  def __str__(self):
    if self.v4 and self.v6:
      return "%s: %s,%s" % (self.asn, self.v4, self.v6)
    else:
      return "%s: %s" % (self.asn, self.v4 or self.v6)

  @classmethod
  def parse(cls, y):
    return cls(y.get("asn"), y.get("ipv4"), y.get("ipv6"))

class router_cert(object):
  """
  Representation for a router_cert object.
  """

  _ecparams = None

  @classmethod
  def ecparams(cls):
    if cls._ecparams is None:
      cls._ecparams = rpki.x509.KeyParams.generateEC()
    return cls._ecparams

  def __init__(self, asn, router_id):
    self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split()))
    self.router_id = router_id
    self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True)
    self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair)
    self.gski = self.pkcs10.gSKI()

  def __eq__(self, other):
    return self.asn == other.asn and self.router_id == other.router_id and self.gski == other.gski

  def __hash__(self):
    return tuple(self.asn).__hash__() + self.router_id.__hash__() + self.gski.__hash__()

  def __str__(self):
    return "%s: %s: %s" % (self.asn, self.router_id, self.gski)

  @classmethod
  def parse(cls, yaml):
    return cls(yaml.get("asn"), yaml.get("router_id"))


class allocation_db(list):
  """
  Allocation database.
  """

  def __init__(self, y):
    list.__init__(self)
    self.root = allocation(y, self)
    assert self.root.is_root
    if self.root.crl_interval is None:
      self.root.crl_interval = 60 * 60
    if self.root.regen_margin is None:
      self.root.regen_margin = 24 * 60 * 60
    if self.root.base.valid_until is None:
      self.root.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 2)
    for a in self:
      if a.base.valid_until is None:
        a.base.valid_until = a.parent.base.valid_until
      if a.crl_interval is None:
        a.crl_interval = a.parent.crl_interval
      if a.regen_margin is None:
        a.regen_margin = a.parent.regen_margin
    self.root.closure()
    self.map = dict((a.name, a) for a in self)
    for a in self:
      if a.is_hosted:
        a.hosted_by = self.map[a.hosted_by]
        a.hosted_by.hosts.append(a)
        assert not a.is_root and not a.hosted_by.is_hosted

  def dump(self):
    for a in self:
      a.dump()


class allocation(object):
  """
  One entity in our allocation database.  Every entity in the database
  is assumed to hold resources.  Entities that don't have the
  hosted_by property run their own copies of rpkid, irdbd, and pubd.
  """

  base_port     = 4400
  base_engine   = -1
  parent        = None
  crl_interval  = None
  regen_margin  = None
  engine        = -1
  rpkid_port    = 4404
  irdbd_port    = 4403
  pubd_port     = 4402
  rootd_port    = 4401
  rsync_port    = 873

  @classmethod
  def allocate_port(cls):
    cls.base_port += 1
    return cls.base_port

  @classmethod
  def allocate_engine(cls):
    cls.base_engine += 1
    return cls.base_engine

  def __init__(self, y, db, parent = None):
    db.append(self)
    self.name = y["name"]
    self.parent = parent
    self.kids = [allocation(k, db, self) for k in y.get("kids", ())]
    valid_until = None
    if "valid_until" in y:
      valid_until = rpki.sundial.datetime.from_datetime(y.get("valid_until"))
    if valid_until is None and "valid_for" in y:
      valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(y["valid_for"])
    self.base = rpki.resource_set.resource_bag(
      asn = rpki.resource_set.resource_set_as(y.get("asn")),
      v4 = rpki.resource_set.resource_set_ipv4(y.get("ipv4")),
      v6 = rpki.resource_set.resource_set_ipv6(y.get("ipv6")),
      valid_until = valid_until)
    if "crl_interval" in y:
      self.crl_interval = rpki.sundial.timedelta.parse(y["crl_interval"]).convert_to_seconds()
    if "regen_margin" in y:
      self.regen_margin = rpki.sundial.timedelta.parse(y["regen_margin"]).convert_to_seconds()
    if "ghostbusters" in y:
      self.ghostbusters = y.get("ghostbusters")
    elif "ghostbuster" in y:
      self.ghostbusters = [y.get("ghostbuster")]
    else:
      self.ghostbusters = []
    self.roa_requests = [roa_request.parse(r) for r in y.get("roa_request", ())]
    self.router_certs = [router_cert.parse(r) for r in y.get("router_cert", ())]
    for r in self.roa_requests:
      if r.v4:
        self.base.v4 |= r.v4.to_resource_set()
      if r.v6:
        self.base.v6 |= r.v6.to_resource_set()
    for r in self.router_certs:
      self.base.asn |= r.asn
    self.hosted_by = y.get("hosted_by")
    self.hosts = []
    if not self.is_hosted:
      self.engine = self.allocate_engine()
    if loopback and not self.is_hosted:
      self.rpkid_port = self.allocate_port()
      self.irdbd_port = self.allocate_port()
    if loopback and self.runs_pubd:
      self.pubd_port  = self.allocate_port()
      self.rsync_port = self.allocate_port()
    if loopback and self.is_root:
      self.rootd_port = self.allocate_port()

  def closure(self):
    resources = self.base
    for kid in self.kids:
      resources |= kid.closure()
    self.resources = resources
    return resources

  @property
  def hostname(self):
    if loopback:
      return "localhost"
    elif dns_suffix:
      return self.name + "." + dns_suffix.lstrip(".")
    else:
      return self.name

  @property
  def rsync_server(self):
    if loopback:
      return "%s:%s" % (self.pubd.hostname, self.pubd.rsync_port)
    else:
      return self.pubd.hostname

  def dump(self):
    if not quiet:
      print str(self)

  def __str__(self):
    s = self.name + ":\n"
    if self.resources.asn:      s += "  ASNs: %s\n" % self.resources.asn
    if self.resources.v4:       s += "  IPv4: %s\n" % self.resources.v4
    if self.resources.v6:       s += "  IPv6: %s\n" % self.resources.v6
    if self.kids:               s += "  Kids: %s\n" % ", ".join(k.name for k in self.kids)
    if self.parent:             s += "    Up: %s\n" % self.parent.name
    if self.is_hosted:          s += "  Host: %s\n" % self.hosted_by.name
    if self.hosts:              s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts)
    for r in self.roa_requests: s += "   ROA: %s\n" % r
    if not self.is_hosted:      s += " IPort: %s\n" % self.irdbd_port
    if self.runs_pubd:          s += " PPort: %s\n" % self.pubd_port
    if not self.is_hosted:      s += " RPort: %s\n" % self.rpkid_port
    if self.runs_pubd:          s += " SPort: %s\n" % self.rsync_port
    if self.is_root:            s += " TPort: %s\n" % self.rootd_port
    return s + " Until: %s\n" % self.resources.valid_until

  @property
  def is_root(self):
    return self.parent is None

  @property
  def is_hosted(self):
    return self.hosted_by is not None

  @property
  def runs_pubd(self):
    return self.is_root or not (self.is_hosted or only_one_pubd)

  def path(self, *names):
    return cleanpath(test_dir, self.host.name, *names)

  def csvout(self, fn):
    path = self.path(fn)
    if not quiet:
      print "Writing", path
    return rpki.csv_utils.csv_writer(path)

  def up_down_url(self):
    return "http://%s:%d/up-down/%s/%s" % (self.parent.host.hostname,
                                           self.parent.host.rpkid_port,
                                           self.parent.name,
                                           self.name)

  def dump_asns(self, fn):
    with self.csvout(fn) as f:
      for k in self.kids:
        f.writerows((k.name, a) for a in k.resources.asn)

  def dump_prefixes(self, fn):
    with self.csvout(fn) as f:
      for k in self.kids:
        f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6))

  def dump_roas(self, fn):
    with self.csvout(fn) as f:
      for g1, r in enumerate(self.roa_requests):
        f.writerows((p, r.asn, "G%08d%08d" % (g1, g2))
                    for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ())))

  def dump_ghostbusters(self, fn):
    if self.ghostbusters:
      path = self.path(fn)
      if not quiet:
        print "Writing", path
      with open(path, "w") as f:
        for i, g in enumerate(self.ghostbusters):
          if i > 0:
            f.write("\n")
          f.write(g)

  def dump_router_certificates(self, fn):
    if self.router_certs:
      path = self.path(fn)
      if not quiet:
        print "Writing", path
      xmlns = rpki.relaxng.router_certificate.xmlns
      xml = lxml.etree.Element(xmlns + "router_certificate_requests",
                               version = rpki.relaxng.router_certificate.version,
                               nsmap = rpki.relaxng.router_certificate.nsmap)
      for r in self.router_certs:
        x = lxml.etree.SubElement(xml, xmlns + "router_certificate_request",
                                  router_id   = str(r.router_id),
                                  asn         = str(r.asn),
                                  valid_until = str(self.resources.valid_until))
        x.text = r.pkcs10.get_Base64()
      rpki.relaxng.router_certificate.assertValid(xml)
      lxml.etree.ElementTree(xml).write(path, pretty_print = True)

  @property
  def pubd(self):
    s = self
    while not s.runs_pubd:
      s = s.parent
    return s

  @property
  def client_handle(self):
    path = []
    s = self
    if not flat_publication:
      while not s.runs_pubd:
        path.append(s)
        s = s.parent
    path.append(s)
    return ".".join(i.name for i in reversed(path))

  @property
  def host(self):
    return self.hosted_by or self

  @property
  def publication_base_directory(self):
    if not loopback and publication_base is not None:
      return publication_base
    else:
      return self.path("publication")

  @property
  def publication_root_directory(self):
    if not loopback and publication_root is not None:
      return publication_root
    else:
      return self.path("publication.root")

  def dump_conf(self):

    r = dict(
      handle                    = self.name,
      run_rpkid                 = str(not self.is_hosted),
      run_pubd                  = str(self.runs_pubd),
      run_rootd                 = str(self.is_root),
      irdbd_sql_username        = "irdb",
      rpkid_sql_username        = "rpki",
      rpkid_server_host         = self.hostname,
      rpkid_server_port         = str(self.rpkid_port),
      irdbd_server_host         = "localhost",
      irdbd_server_port         = str(self.irdbd_port),
      rootd_server_port         = str(self.rootd_port),
      pubd_sql_username         = "pubd",
      pubd_server_host          = self.pubd.hostname,
      pubd_server_port          = str(self.pubd.pubd_port),
      publication_rsync_server  = self.rsync_server)

    if loopback:
      r.update(
        irdbd_sql_database      = self.irdb_name,
        rpkid_sql_database      = "rpki%d" % self.engine,
        pubd_sql_database       = "pubd%d" % self.engine,
        bpki_servers_directory  = self.path(),
        publication_base_directory = self.publication_base_directory)

    r.update(config_overrides)

    with open(self.path("rpki.conf"), "w") as f:
      f.write("# Automatically generated, do not edit\n")
      if not quiet:
        print "Writing", f.name

      section = None
      for line in open(rpki_conf):
        m = section_regexp.match(line)
        if m:
          section = m.group(1)
        m = variable_regexp.match(line)
        option = m.group(1) if m and section == "myrpki" else None
        if option and option in r:
          line = "%s = %s\n" % (option, r[option])
        f.write(line)

  def dump_rsyncd(self):
    lines = []
    if self.runs_pubd:
      lines.extend((
        "# Automatically generated, do not edit",
        "port         = %d"           % self.rsync_port,
        "address      = %s"           % self.hostname,
        "log file     = rsyncd.log",
        "read only    = yes",
        "use chroot   = no",
        "[rpki]",
        "path         = %s"           % self.publication_base_directory,
        "comment      = RPKI test"))
    if self.is_root:
      assert self.runs_pubd
      lines.extend((
        "[root]",
        "path         = %s"           % self.publication_root_directory,
        "comment      = RPKI test root"))
    if lines:
      with open(self.path("rsyncd.conf"), "w") as f:
        if not quiet:
          print "Writing", f.name
        f.writelines(line + "\n" for line in lines)

  @property
  def irdb_name(self):
    return "irdb%d" % self.host.engine

  @property
  def irdb(self):
    prior_name = self.zoo.handle
    return rpki.irdb.database(
      self.irdb_name,
      on_entry = lambda: self.zoo.reset_identity(self.name),
      on_exit  = lambda: self.zoo.reset_identity(prior_name))

  def syncdb(self):
    import django.core.management
    assert not self.is_hosted
    django.core.management.call_command(
      "syncdb",
      verbosity = 0,
      database = self.irdb_name,
      migrate = True,
      load_initial_data = False,
      interactive = False)

  def hire_zookeeper(self):
    assert not self.is_hosted
    self._zoo = rpki.irdb.Zookeeper(
      cfg = rpki.config.parser(filename = self.path("rpki.conf")),
      logstream = None if quiet else sys.stdout)

  @property
  def zoo(self):
    return self.host._zoo

  def dump_root(self):

    assert self.is_root and not self.is_hosted

    root_resources = rpki.resource_set.resource_bag(
      asn = rpki.resource_set.resource_set_as("0-4294967295"),
      v4  = rpki.resource_set.resource_set_ipv4("0.0.0.0/0"),
      v6  = rpki.resource_set.resource_set_ipv6("::/0"))

    root_key = rpki.x509.RSA.generate(quiet = True)

    root_uri = "rsync://%s/rpki/" % self.rsync_server

    root_sia = (root_uri, root_uri + "root.mft", None)

    root_cert = rpki.x509.X509.self_certify(
      keypair     = root_key,
      subject_key = root_key.get_public(),
      serial      = 1,
      sia         = root_sia,
      notAfter    = rpki.sundial.now() + rpki.sundial.timedelta(days = 365),
      resources   = root_resources)

    with open(self.path("publication.root", "root.cer"), "wb") as f:
      f.write(root_cert.get_DER())

    with open(self.path("root.key"), "wb") as f:
      f.write(root_key.get_DER())

    with open(cleanpath(test_dir, "root.tal"), "w") as f:
      f.write("rsync://%s/root/root.cer\n\n%s" % (
        self.rsync_server, root_key.get_public().get_Base64()))

  def mkdir(self, *path):
    path = self.path(*path)
    if not quiet:
      print "Creating directory", path
    os.makedirs(path)

  def dump_sql(self):
    if not self.is_hosted:
      with open(self.path("rpkid.sql"), "w") as f:
        if not quiet:
          print "Writing", f.name
        f.write(rpki.sql_schemas.rpkid)
    if self.runs_pubd:
      with open(self.path("pubd.sql"), "w") as f:
        if not quiet:
          print "Writing", f.name
        f.write(rpki.sql_schemas.pubd)
    if not self.is_hosted:
      username = config_overrides["irdbd_sql_username"]
      password = config_overrides["irdbd_sql_password"]
      cmd = ("mysqldump", "-u", username, "-p" + password, self.irdb_name)
      with open(self.path("irdbd.sql"), "w") as f:
        if not quiet:
          print "Writing", f.name
        subprocess.check_call(cmd, stdout = f)


def pre_django_sql_setup(needed):

  username = config_overrides["irdbd_sql_username"]
  password = config_overrides["irdbd_sql_password"]

  # If we have the MySQL root password, just blow away and recreate
  # the required databases.  Otherwise, check for missing databases,
  # then blow away all tables in the required databases.  In either
  # case, we assume that the Django syncdb code will populate
  # databases as necessary, all we need to do here is provide empty
  # databases for the Django code to fill in.

  if mysql_rootpass is not None:
    if mysql_rootpass:
      db = MySQLdb.connect(user = mysql_rootuser, passwd = mysql_rootpass)
    else:
      db = MySQLdb.connect(user = mysql_rootuser)
    cur = db.cursor()
    for database in needed:
      try:
        cur.execute("DROP DATABASE IF EXISTS %s" % database)
      except:
        pass
      cur.execute("CREATE DATABASE %s" % database)
      cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % (
        database, username), (password,))

  else:
    db = MySQLdb.connect(user = username, passwd = password)
    cur = db.cursor()
    cur.execute("SHOW DATABASES")
    existing = set(r[0] for r in cur.fetchall())
    if needed - existing:
      sys.stderr.write("The following databases are missing:\n")
      for database in sorted(needed - existing):
        sys.stderr.write("  %s\n" % database)
      sys.stderr.write("Please create them manually or put MySQL root password in my config file\n")
      sys.exit("Missing databases and MySQL root password not known, can't continue")
    for database in needed:
      db.select_db(database)
      cur.execute("SHOW TABLES")
      tables = [r[0] for r in cur.fetchall()]
      cur.execute("SET foreign_key_checks = 0")
      for table in tables:
        cur.execute("DROP TABLE %s" % table)
      cur.execute("SET foreign_key_checks = 1")

  cur.close()
  db.commit()
  db.close()

class timestamp(object):

  def __init__(self, *args):
    self.count = 0
    self.start = self.tick = rpki.sundial.now()

  def __call__(self, *args):
    now = rpki.sundial.now()
    if not quiet:
      print "[Count %s last %s total %s now %s]" % (
        self.count, now - self.tick, now - self.start, now)
    self.tick = now
    self.count += 1


def main():

  global flat_publication
  global config_overrides
  global only_one_pubd
  global loopback
  global dns_suffix
  global mysql_rootuser
  global mysql_rootpass
  global yaml_file
  global test_dir
  global rpki_conf
  global publication_base
  global publication_root
  global quiet

  os.environ["TZ"] = "UTC"
  time.tzset()

  parser = argparse.ArgumentParser(description = "yamlconf")
  parser.add_argument("-c", "--config", help = "configuration file")
  parser.add_argument("--dns_suffix",
                      help = "DNS suffix to add to hostnames")
  parser.add_argument("-l", "--loopback", action = "store_true",
                      help = "Configure for use with yamltest on localhost")
  parser.add_argument("-f", "--flat_publication", action = "store_true",
                      help = "Use flat publication model")
  parser.add_argument("-q", "--quiet", action = "store_true",
                      help = "Work more quietly")
  parser.add_argument("--profile",
                      help = "Filename for profile output")
  parser.add_argument("yaml_file", type = argparse.FileType("r"),
                      help = "YAML file describing network to build")
  args = parser.parse_args()

  dns_suffix = args.dns_suffix
  loopback = args.loopback
  flat_publication = args.flat_publication
  quiet = args.quiet
  yaml_file = args.yaml_file

  rpki.log.init("yamlconf", argparse.Namespace(log_level   = logging.DEBUG,
                                               log_handler = lambda: logging.StreamHandler(sys.stdout)))

  # Allow optional config file for this tool to override default
  # passwords: this is mostly so that I can show a complete working
  # example without publishing my own server's passwords.

  cfg = rpki.config.parser(set_filename = args.config, section = "yamlconf", allow_missing = True)
  try:
    cfg.set_global_flags()
  except:
    pass

  # Use of "yamltest.dir" is deliberate: intent is for what we write to
  # be usable with "yamltest --skip_config".

  only_one_pubd = cfg.getboolean("only_one_pubd", True)
  test_dir = cfg.get("test_directory", cleanpath(this_dir, "yamltest.dir"))
  rpki_conf = cfg.get("rpki_conf", cleanpath(this_dir, "..", "examples/rpki.conf"))
  mysql_rootuser = cfg.get("mysql_rootuser", "root")

  try:
    mysql_rootpass = cfg.get("mysql_rootpass")
  except:
    pass

  try:
    publication_base = cfg.get("publication_base")
  except:
    pass

  try:
    publication_root = cfg.get("publication_root")
  except:
    pass

  for k in ("rpkid_sql_password", "irdbd_sql_password", "pubd_sql_password",
            "rpkid_sql_username", "irdbd_sql_username", "pubd_sql_username"):
    if cfg.has_option(k):
      config_overrides[k] = cfg.get(k)

  if args.profile:
    import cProfile
    prof = cProfile.Profile()
    try:
      prof.runcall(body)
    finally:
      prof.dump_stats(args.profile)
      if not quiet:
        print
        print "Dumped profile data to %s" % args.profile
  else:
    body()

def body():

  global rpki

  ts = timestamp()

  for root, dirs, files in os.walk(test_dir, topdown = False):
    for fn in files:
      os.unlink(os.path.join(root, fn))
    for d in dirs:
      os.rmdir(os.path.join(root, d))

  if not quiet:
    print
    print "Reading YAML", yaml_file.name

  db = allocation_db(yaml.safe_load_all(yaml_file).next())

  # Show what we loaded

  #db.dump()

  # Do pre-Django SQL setup

  pre_django_sql_setup(set(d.irdb_name for d in db if not d.is_hosted))

  # Now ready for fun with multiple databases in Django!
  #
  # https://docs.djangoproject.com/en/1.4/topics/db/multi-db/
  # https://docs.djangoproject.com/en/1.4/topics/db/sql/
  #
  # This program's use of the ORM is sufficiently different that it's
  # not worth straining to use rpki.django_settings, so we just use
  # Django's settings API directly.

  database_template = {
    "ENGINE"   : "django.db.backends.mysql",
    "USER"     : config_overrides["irdbd_sql_username"],
    "PASSWORD" : config_overrides["irdbd_sql_password"],
    "HOST"     : "",
    "PORT"     : "",
    "OPTIONS"  : { "init_command": "SET storage_engine=INNODB" }}

  databases = dict((d.irdb_name, dict(database_template, NAME = d.irdb_name))
                   for d in db if not d.is_hosted)

  databases["default"] = databases[db.root.irdb_name]

  from django.conf import settings

  settings.configure(
    DATABASES = databases,
    DATABASE_ROUTERS = ["rpki.irdb.router.DBContextRouter"],
    INSTALLED_APPS = ("rpki.irdb", "south"))

  import rpki.irdb

  rpki.irdb.models.ca_certificate_lifetime = rpki.sundial.timedelta(days = 3652 * 2)
  rpki.irdb.models.ee_certificate_lifetime = rpki.sundial.timedelta(days = 3652)

  ts()

  for d in db:
    if not quiet:
      print
      print "Configuring", d.name

    if not d.is_hosted:
      d.mkdir()
    if d.runs_pubd:
      d.mkdir("publication")
    if d.is_root:
      d.mkdir("publication.root")

    if not d.is_hosted:
      d.dump_conf()
      d.dump_rsyncd()

    d.dump_asns("%s.asns.csv" % d.name)
    d.dump_prefixes("%s.prefixes.csv" % d.name)
    d.dump_roas("%s.roas.csv" % d.name)
    d.dump_ghostbusters("%s.ghostbusters.vcard" % d.name)
    d.dump_router_certificates("%s.routercerts.xml" % d.name)

    if not d.is_hosted:
      if not quiet:
        print "Initializing SQL"
      d.syncdb()
      if not quiet:
        print "Hiring zookeeper"
      d.hire_zookeeper()

    with d.irdb:
      if not quiet:
        print "Creating identity"
      x = d.zoo.initialize()

      if d.is_root:
        if not quiet:
          print "Creating RPKI root certificate and TAL"
        d.dump_root()
        x = d.zoo.configure_rootd()

      else:
        with d.parent.irdb:
          x = d.parent.zoo.configure_child(x.file)[0]
        x = d.zoo.configure_parent(x.file)[0]

      with d.pubd.irdb:
        x = d.pubd.zoo.configure_publication_client(x.file, flat = flat_publication)[0]
      d.zoo.configure_repository(x.file)

    if loopback and not d.is_hosted:
      with d.irdb:
        d.zoo.write_bpki_files()

    ts()

  if not loopback:
    if not quiet:
      print
    for d in db:
      d.dump_sql()

if __name__ == "__main__":
  main()