path: root/rpkid/rpki/myrpki.py

"""
This (oversized) module used to be an (oversized) program.
Refactoring in progress, some doc still needs updating.


This program is now the merger of three different tools: the old
myrpki.py script, the old myirbe.py script, and the newer setup.py CLI
tool.  As such, it is still in need of some cleanup, but the need to
provide a saner user interface is more urgent than internal code
prettiness at the moment.  In the long run, 90% of the code in this
file probably ought to move to well-designed library modules.

Overall goal here is to build up the configuration necessary to run
rpkid and friends, by reading a config file, a collection of .CSV
files, and the results of a few out-of-band XML setup messages
exchanged with one's parents, children, and so forth.

The config file is in an OpenSSL-compatible format, the CSV files are
simple tab-delimited text.  The XML files are all generated by this
program, either the local instance or an instance being run by another
player in the system; the mechanism used to exchange these setup
messages is outside the scope of this program, feel free to use
PGP-signed mail, a web interface (not provided), USB stick, carrier
pigeons, whatever works.

With one exception, the commands in this program avoid using any
third-party Python code other than the rpki libraries themselves; with
the same one exception, all OpenSSL work is done with the OpenSSL
command line tool (the one built as a side effect of building rcynic
will do, if your platform has no system copy or the system copy is too
old).  This is all done in an attempt to make the code more portable,
so one can run most of the RPKI back end software on a laptop or
whatever.  The one exception is the configure_daemons command, which
must, of necessity, use the same communication libraries as the
daemons with which it is conversing.  So that one command will not
work if the correct Python modules are not available.


$Id$

Copyright (C) 2009--2010  Internet Systems Consortium ("ISC")

Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.

THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
"""

from __future__ import with_statement

import subprocess, csv, re, os, getopt, sys, base64, time, glob, copy, warnings
import rpki.config, rpki.cli, rpki.sundial, rpki.log, rpki.oids

try:
  from lxml.etree            import (Element, SubElement, ElementTree,
                                     fromstring as ElementFromString,
                                     tostring   as ElementToString)
except ImportError:
  from xml.etree.ElementTree import (Element, SubElement, ElementTree,
                                     fromstring as ElementFromString,
                                     tostring   as ElementToString)



# Our XML namespace and protocol version.

namespace      = "http://www.hactrn.net/uris/rpki/myrpki/"
version        = "2"
namespaceQName = "{" + namespace + "}"

# Whether to include incomplete entries when rendering to XML.

allow_incomplete = False

# Whether to whine about incomplete entries while rendering to XML.

whine = False

class comma_set(set):
  """
  Minor customization of set(), to provide a print syntax.
  """

  def __str__(self):
    return ",".join(self)

class EntityDB(object):
  """
  Wrapper for entitydb path lookups.  Hmm, maybe some or all of the
  entitydb glob stuff should end up here too?  Later.
  """

  def __init__(self, cfg):
    self.dir = cfg.get("entitydb_dir", "entitydb")

  def __call__(self, *args):
    return os.path.join(self.dir, *args)

  def iterate(self, *args):
    return glob.iglob(os.path.join(self.dir, *args))

class roa_request(object):
  """
  Representation of a ROA request.
  """

  v4re = re.compile("^([0-9]{1,3}\.){3}[0-9]{1,3}/[0-9]+(-[0-9]+)?$", re.I)
  v6re = re.compile("^([0-9a-f]{0,4}:){0,15}[0-9a-f]{0,4}/[0-9]+(-[0-9]+)?$", re.I)

  def __init__(self, asn, group):
    self.asn = asn
    self.group = group
    self.v4 = comma_set()
    self.v6 = comma_set()

  def __repr__(self):
    s = "<%s asn %s group %s" % (self.__class__.__name__, self.asn, self.group)
    if self.v4:
      s += " v4 %s" % self.v4
    if self.v6:
      s += " v6 %s" % self.v6
    return s + ">"

  def add(self, prefix):
    """
    Add one prefix to this ROA request.
    """
    if self.v4re.match(prefix):
      self.v4.add(prefix)
    elif self.v6re.match(prefix):
      self.v6.add(prefix)
    else:
      raise RuntimeError, "Bad prefix syntax: %r" % (prefix,)

  def xml(self, e):
    """
    Generate XML element represeting representing this ROA request.
    """
    e = SubElement(e, "roa_request",
                   asn = self.asn,
                   v4 = str(self.v4),
                   v6 = str(self.v6))
    e.tail = "\n"

class roa_requests(dict):
  """
  Database of ROA requests.
  """

  def add(self, asn, group, prefix):
    """
    Add one <ASN, group, prefix> set to ROA request database.
    """
    key = (asn, group)
    if key not in self:
      self[key] = roa_request(asn, group)
    self[key].add(prefix)

  def xml(self, e):
    """
    Render ROA requests as XML elements.
    """
    for r in self.itervalues():
      r.xml(e)

  @classmethod
  def from_csv(cls, roa_csv_file):
    """
    Parse ROA requests from CSV file.
    """
    self = cls()
    # format:  p/n-m asn group
    for pnm, asn, group in csv_reader(roa_csv_file, columns = 3):
      self.add(asn = asn, group = group, prefix = pnm)
    return self

class child(object):
  """
  Representation of one child entity.
  """

  v4re = re.compile("^(([0-9]{1,3}\.){3}[0-9]{1,3}/[0-9]+)|(([0-9]{1,3}\.){3}[0-9]{1,3}-([0-9]{1,3}\.){3}[0-9]{1,3})$", re.I)
  v6re = re.compile("^(([0-9a-f]{0,4}:){0,15}[0-9a-f]{0,4}/[0-9]+)|(([0-9a-f]{0,4}:){0,15}[0-9a-f]{0,4}-([0-9a-f]{0,4}:){0,15}[0-9a-f]{0,4})$", re.I)

  def __init__(self, handle):
    self.handle = handle
    self.asns = comma_set()
    self.v4 = comma_set()
    self.v6 = comma_set()
    self.validity = None
    self.bpki_certificate = None

  def __repr__(self):
    s = "<%s %s" % (self.__class__.__name__, self.handle)
    if self.asns:
      s += " asn %s" % self.asns
    if self.v4:
      s += " v4 %s" % self.v4
    if self.v6:
      s += " v6 %s" % self.v6
    if self.validity:
      s += " valid %s" % self.validity
    if self.bpki_certificate:
      s += " cert %s" % self.bpki_certificate
    return s + ">"

  def add(self, prefix = None, asn = None, validity = None, bpki_certificate = None):
    """
    Add prefix, autonomous system number, validity date, or BPKI
    certificate for this child.
    """
    if prefix is not None:
      if self.v4re.match(prefix):
        self.v4.add(prefix)
      elif self.v6re.match(prefix):
        self.v6.add(prefix)
      else:
        raise RuntimeError, "Bad prefix syntax: %r" % (prefix,)
    if asn is not None:
      self.asns.add(asn)
    if validity is not None:
      self.validity = validity
    if bpki_certificate is not None:
      self.bpki_certificate = bpki_certificate

  def xml(self, e):
    """
    Render this child as an XML element.
    """
    complete = self.bpki_certificate and self.validity
    if whine and not complete:
      print "Incomplete child entry %s" % self
    if complete or allow_incomplete:
      e = SubElement(e, "child",
                     handle = self.handle,
                     valid_until = self.validity,
                     asns = str(self.asns),
                     v4 = str(self.v4),
                     v6 = str(self.v6))
      e.tail = "\n"
      if self.bpki_certificate:
        PEMElement(e, "bpki_certificate", self.bpki_certificate)

class children(dict):
  """
  Database of children.
  """

  def add(self, handle, prefix = None, asn = None, validity = None, bpki_certificate = None):
    """
    Add resources to a child, creating the child object if necessary.
    """
    if handle not in self:
      self[handle] = child(handle)
    self[handle].add(prefix = prefix, asn = asn, validity = validity, bpki_certificate = bpki_certificate)

  def xml(self, e):
    """
    Render children database to XML.
    """
    for c in self.itervalues():
      c.xml(e)

  @classmethod
  def from_csv(cls, prefix_csv_file, asn_csv_file, fxcert, entitydb):
    """
    Parse child resources, certificates, and validity dates from CSV files.
    """
    self = cls()
    for f in entitydb.iterate("children", "*.xml"):
      c = etree_read(f)
      self.add(handle = os.path.splitext(os.path.split(f)[-1])[0],
               validity = c.get("valid_until"),
               bpki_certificate = fxcert(c.findtext("bpki_child_ta")))
    # childname p/n
    for handle, pn in csv_reader(prefix_csv_file, columns = 2):
      self.add(handle = handle, prefix = pn)
    # childname asn
    for handle, asn in csv_reader(asn_csv_file, columns = 2):
      self.add(handle = handle, asn = asn)
    return self

class parent(object):
  """
  Representation of one parent entity.
  """

  def __init__(self, handle):
    self.handle = handle
    self.service_uri = None
    self.bpki_cms_certificate = None
    self.myhandle = None
    self.sia_base = None

  def __repr__(self):
    s = "<%s %s" % (self.__class__.__name__, self.handle)
    if self.myhandle:
      s += " myhandle %s" % self.myhandle
    if self.service_uri:
      s += " uri %s" % self.service_uri
    if self.sia_base:
      s += " sia %s" % self.sia_base
    if self.bpki_cms_certificate:
      s += " cms %s" % self.bpki_cms_certificate
    return s + ">"

  def add(self, service_uri = None,
          bpki_cms_certificate = None,
          myhandle = None,
          sia_base = None):
    """
    Add service URI or BPKI certificates to this parent object.
    """
    if service_uri is not None:
      self.service_uri = service_uri
    if bpki_cms_certificate is not None:
      self.bpki_cms_certificate = bpki_cms_certificate
    if myhandle is not None:
      self.myhandle = myhandle
    if sia_base is not None:
      self.sia_base = sia_base

  def xml(self, e):
    """
    Render this parent object to XML.
    """
    complete = self.bpki_cms_certificate and self.myhandle and self.service_uri and self.sia_base
    if whine and not complete:
      print "Incomplete parent entry %s" % self
    if complete or allow_incomplete:
      e = SubElement(e, "parent",
                     handle = self.handle,
                     myhandle = self.myhandle,
                     service_uri = self.service_uri,
                     sia_base = self.sia_base)
      e.tail = "\n"
      if self.bpki_cms_certificate:
        PEMElement(e, "bpki_cms_certificate", self.bpki_cms_certificate)

class parents(dict):
  """
  Database of parent objects.
  """

  def add(self, handle,
          service_uri = None,
          bpki_cms_certificate = None,
          myhandle = None,
          sia_base = None):
    """
    Add service URI or certificates to parent object, creating it if necessary.
    """
    if handle not in self:
      self[handle] = parent(handle)
    self[handle].add(service_uri = service_uri,
                     bpki_cms_certificate = bpki_cms_certificate,
                     myhandle = myhandle,
                     sia_base = sia_base)

  def xml(self, e):
    for c in self.itervalues():
      c.xml(e)

  @classmethod
  def from_csv(cls, fxcert, entitydb):
    """
    Parse parent data from entitydb.
    """
    self = cls()
    for f in entitydb.iterate("parents", "*.xml"):
      h = os.path.splitext(os.path.split(f)[-1])[0]
      p = etree_read(f)
      r = etree_read(f.replace(os.path.sep + "parents"      + os.path.sep,
                               os.path.sep + "repositories" + os.path.sep))
      assert r.get("type") == "confirmed"
      self.add(handle = h,
               service_uri = p.get("service_uri"),
               bpki_cms_certificate = fxcert(p.findtext("bpki_resource_ta")),
               myhandle = p.get("child_handle"),
               sia_base = r.get("sia_base"))
    return self

class repository(object):
  """
  Representation of one repository entity.
  """

  def __init__(self, handle):
    self.handle = handle
    self.service_uri = None
    self.bpki_certificate = None

  def __repr__(self):
    s = "<%s %s" % (self.__class__.__name__, self.handle)
    if self.service_uri:
      s += " uri %s" % self.service_uri
    if self.bpki_certificate:
      s += " cert %s" % self.bpki_certificate
    return s + ">"

  def add(self, service_uri = None, bpki_certificate = None):
    """
    Add service URI or BPKI certificates to this repository object.
    """
    if service_uri is not None:
      self.service_uri = service_uri
    if bpki_certificate is not None:
      self.bpki_certificate = bpki_certificate

  def xml(self, e):
    """
    Render this repository object to XML.
    """
    complete = self.bpki_certificate and self.service_uri
    if whine and not complete:
      print "Incomplete repository entry %s" % self
    if complete or allow_incomplete:
      e = SubElement(e, "repository",
                     handle = self.handle,
                     service_uri = self.service_uri)
      e.tail = "\n"
      if self.bpki_certificate:
        PEMElement(e, "bpki_certificate", self.bpki_certificate)

class repositories(dict):
  """
  Database of repository objects.
  """

  def add(self, handle,
          service_uri = None,
          bpki_certificate = None):
    """
    Add service URI or certificate to repository object, creating it if necessary.
    """
    if handle not in self:
      self[handle] = repository(handle)
    self[handle].add(service_uri = service_uri,
                     bpki_certificate = bpki_certificate)

  def xml(self, e):
    for c in self.itervalues():
      c.xml(e)

  @classmethod
  def from_csv(cls, fxcert, entitydb):
    """
    Parse repository data from entitydb.
    """
    self = cls()
    for f in entitydb.iterate("repositories", "*.xml"):
      h = os.path.splitext(os.path.split(f)[-1])[0]
      r = etree_read(f)
      assert r.get("type") == "confirmed"
      self.add(handle = h,
               service_uri = r.get("service_uri"),
               bpki_certificate = fxcert(r.findtext("bpki_server_ta")))
    return self

class csv_reader(object):
  """
  Reader for tab-delimited text that's (slightly) friendlier than the
  stock Python csv module (which isn't intended for direct use by
  humans anyway, and neither was this package originally, but that
  seems to be the way that it has evolved...).

  Columns parameter specifies how many columns users of the reader
  expect to see; lines with fewer columns will be padded with None
  values.

  Original API design for this class courtesy of Warren Kumari, but
  don't blame him if you don't like what I did with his ideas.
  """

  def __init__(self, filename, columns = None, min_columns = None, comment_characters = "#;"):
    assert columns is None or isinstance(columns, int)
    assert min_columns is None or isinstance(min_columns, int)
    if columns is not None and min_columns is None:
      min_columns = columns
    self.filename = filename
    self.columns = columns
    self.min_columns = min_columns
    self.comment_characters = comment_characters 
    self.file = open(filename, "r")

  def __iter__(self):
    line_number = 0
    for line in self.file:
      line_number += 1
      line = line.strip()
      if not line or line[0] in self.comment_characters:
        continue
      fields = line.split()
      if self.min_columns is not None and len(fields) < self.min_columns:
        raise RuntimeError, "%s:%d: Not enough columns in line %r" % (self.filename, line_number, line)
      if self.columns is not None and len(fields) > self.columns:
        raise RuntimeError, "%s:%d: Too many  columns in line %r" % (self.filename, line_number, line)
      if self.columns is not None and len(fields) < self.columns:
        fields += tuple(None for i in xrange(self.columns - len(fields)))
      yield fields

class csv_writer(object):
  """
  Writer object for tab delimited text.  We just use the stock CSV
  module in excel-tab mode for this.

  If "renmwo" is set (default), the file will be written to
  a temporary name and renamed to the real filename after closing.
  """

  def __init__(self, filename, renmwo = True):
    self.filename = filename
    self.renmwo = "%s.~renmwo%d~" % (filename, os.getpid()) if renmwo else filename
    self.file = open(self.renmwo, "w")
    self.writer = csv.writer(self.file, dialect = csv.get_dialect("excel-tab"))

  def close(self):
    """
    Close this writer.
    """
    if self.file is not None:
      self.file.close()
      self.file = None
      if self.filename != self.renmwo:
        os.rename(self.renmwo, self.filename)

  def __getattr__(self, attr):
    """
    Fake inheritance from whatever object csv.writer deigns to give us.
    """
    return getattr(self.writer, attr)

def PEMElement(e, tag, filename, **kwargs):
  """
  Create an XML element containing Base64 encoded data taken from a
  PEM file.
  """
  lines = open(filename).readlines()
  while lines:
    if lines.pop(0).startswith("-----BEGIN "):
      break
  while lines:
    if lines.pop(-1).startswith("-----END "):
      break
  if e.text is None:
    e.text = "\n"
  se = SubElement(e, tag, **kwargs)
  se.text = "\n" + "".join(lines)
  se.tail = "\n"
  return se

class CA(object):
  """
  Representation of one certification authority.
  """

  # Mapping of path restriction values we use to OpenSSL config file
  # section names.

  path_restriction = { 0 : "ca_x509_ext_xcert0",
                       1 : "ca_x509_ext_xcert1" }

  def __init__(self, cfg_file, dir):
    self.cfg    = cfg_file
    self.dir    = dir
    self.cer    = dir + "/ca.cer"
    self.key    = dir + "/ca.key"
    self.req    = dir + "/ca.req"
    self.crl    = dir + "/ca.crl"
    self.index  = dir + "/index"
    self.serial = dir + "/serial"
    self.crlnum = dir + "/crl_number"

    cfg = rpki.config.parser(cfg_file, "myrpki")
    self.openssl = cfg.get("openssl", "openssl")

    self.env = { "PATH" : os.environ["PATH"],
                 "BPKI_DIRECTORY" : dir,
                 "RANDFILE" : ".OpenSSL.whines.unless.I.set.this",
                 "OPENSSL_CONF" : cfg_file }

  def run_openssl(self, *cmd, **kwargs):
    """
    Run an OpenSSL command, suppresses stderr unless OpenSSL returns
    failure, and returns stdout.
    """
    stdin = kwargs.pop("stdin", None)
    env = self.env.copy()
    env.update(kwargs)
    cmd = (self.openssl,) + cmd
    p = subprocess.Popen(cmd, env = env, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
    stdout, stderr = p.communicate(stdin)
    if p.wait() != 0:
      sys.stderr.write("OpenSSL command failed: " + stderr + "\n")
      raise subprocess.CalledProcessError(returncode = p.returncode, cmd = cmd)
    return stdout

  def run_ca(self, *args):
    """
    Run OpenSSL "ca" command with common initial arguments.
    """
    self.run_openssl("ca", "-batch", "-config",  self.cfg, *args)

  def run_req(self, key_file, req_file, log_key = sys.stdout):
    """
    Run OpenSSL "genrsa" and  "req" commands.
    """
    if not os.path.exists(key_file):
      if log_key:
        log_key.write("Generating 2048-bit RSA key %s\n" % os.path.realpath(key_file))
      self.run_openssl("genrsa", "-out", key_file, "2048")
    if not os.path.exists(req_file):
      self.run_openssl("req", "-new", "-sha256", "-config", self.cfg, "-key", key_file, "-out", req_file)

  def run_dgst(self, input, algorithm = "md5"):
    """
    Run OpenSSL "dgst" command, return cleaned-up result.
    """
    hash = self.run_openssl("dgst", "-" + algorithm, stdin = input)
    #
    # Twits just couldn't leave well enough alone, grr.
    hash = "".join(hash.split())
    if hash.startswith("(stdin)="):
      hash =  hash[len("(stdin)="):]
    return hash

  @staticmethod
  def touch_file(filename, content = None):
    """
    Create dumb little text files expected by OpenSSL "ca" utility.
    """
    if not os.path.exists(filename):
      f = open(filename, "w")
      if content is not None:
        f.write(content)
      f.close()

  def setup(self, ca_name):
    """
    Set up this CA.  ca_name is an X.509 distinguished name in
    /tag=val/tag=val format.
    """

    modified = False

    if not os.path.exists(self.dir):
      os.makedirs(self.dir)
      self.touch_file(self.index)
      self.touch_file(self.serial, "01\n")
      self.touch_file(self.crlnum, "01\n")

    self.run_req(key_file = self.key, req_file = self.req)

    if not os.path.exists(self.cer):
      modified = True
      self.run_ca("-selfsign", "-extensions", "ca_x509_ext_ca", "-subj", ca_name, "-in", self.req, "-out", self.cer)

    if not os.path.exists(self.crl):
      modified = True
      self.run_ca("-gencrl", "-out", self.crl)

    return modified

  def ee(self, ee_name, base_name):
    """
    Issue an end-enity certificate.
    """
    key_file = "%s/%s.key" % (self.dir, base_name)
    req_file = "%s/%s.req" % (self.dir, base_name)
    cer_file = "%s/%s.cer" % (self.dir, base_name)
    self.run_req(key_file = key_file, req_file = req_file)
    if not os.path.exists(cer_file):
      self.run_ca("-extensions", "ca_x509_ext_ee", "-subj", ee_name, "-in", req_file, "-out", cer_file)
      return True
    else:
      return False

  def cms_xml_sign(self, ee_name, base_name, elt):
    """
    Sign an XML object with CMS, return Base64 text.
    """
    self.ee(ee_name, base_name)
    return base64.b64encode(self.run_openssl(
      "cms", "-sign", "-binary", "-outform", "DER",
      "-keyid", "-md", "sha256", "-nodetach", "-nosmimecap",
      "-econtent_type", ".".join(str(i) for i in rpki.oids.name2oid["id-ct-xml"]),
      "-inkey",  "%s/%s.key" % (self.dir, base_name),
      "-signer", "%s/%s.cer" % (self.dir, base_name),
      stdin = ElementToString(etree_pre_write(elt))))

  def cms_xml_verify(self, b64, ca):
    """
    Attempt to verify and extract XML from a Base64-encoded signed CMS
    object.  CA is the filename of a certificate that we expect to be
    the issuer of the EE certificate bundled with the CMS, and must
    previously have been cross-certified under our trust anchor.
    """
    # In theory, we should be able to use the -certfile parameter to
    # pass in the CA certificate, but in practice, I have never gotten
    # this to work, either with the command line tool or in the
    # OpenSSL C API.  Dunno why.  Passing both TA and CA via -CAfile
    # does work, so we do that, using a temporary file, sigh.
    CAfile = os.path.join(self.dir, "temp.%s.pem" % os.getpid())
    try:
      f = open(CAfile, "w")
      f.write(open(self.cer).read())
      f.write(open(ca).read())
      f.close()
      return etree_post_read(ElementFromString(self.run_openssl(
        "cms", "-verify", "-inform", "DER", "-CAfile", CAfile,
        stdin = base64.b64decode(b64))))
    finally:
      if os.path.exists(CAfile):
        os.unlink(CAfile)

  def bsc(self, pkcs10):
    """
    Issue BSC certificiate, if we have a PKCS #10 request for it.
    """

    if pkcs10 is None:
      return None, None

    pkcs10 = base64.b64decode(pkcs10)

    hash = self.run_dgst(pkcs10)

    req_file = "%s/bsc.%s.req" % (self.dir, hash)
    cer_file = "%s/bsc.%s.cer" % (self.dir, hash)

    if not os.path.exists(cer_file):
      self.run_openssl("req", "-inform", "DER", "-out", req_file, stdin = pkcs10)
      self.run_ca("-extensions", "ca_x509_ext_ee", "-in", req_file, "-out", cer_file)

    return req_file, cer_file

  def fxcert(self, b64, filename = None, path_restriction = 0):
    """
    Write PEM certificate to file, then cross-certify.
    """
    fn = os.path.join(self.dir, filename or "temp.%s.cer" % os.getpid())
    try:
      self.run_openssl("x509", "-inform", "DER", "-out", fn,
                       stdin = base64.b64decode(b64))
      return self.xcert(fn, path_restriction)
    finally:
      if not filename and os.path.exists(fn):
        os.unlink(fn)

  def xcert_filename(self, cert):
    """
    Generate filename for a cross-certification.

    Extracts public key and subject name from PEM file and hash it so
    we can use the result as a tag for cross-certifying this cert.
    """

    if cert and os.path.exists(cert):
      return "%s/xcert.%s.cer" % (self.dir, self.run_dgst(self.run_openssl(
        "x509", "-noout", "-pubkey", "-subject", "-in", cert)).strip())
    else:
      return None

  def xcert(self, cert, path_restriction = 0):
    """
    Cross-certify a certificate represented as a PEM file, if we
    haven't already.  This only works for self-signed certs, due to
    limitations of the OpenSSL command line tool, but that suffices
    for our purposes.
    """

    xcert = self.xcert_filename(cert)
    if not os.path.exists(xcert):
      self.run_ca("-ss_cert", cert, "-out", xcert, "-extensions", self.path_restriction[path_restriction])
    return xcert

  def xcert_revoke(self, cert):
    """
    Revoke a cross-certification and regenerate CRL.
    """

    xcert = self.xcert_filename(cert)
    if xcert:
      self.run_ca("-revoke", xcert)
      self.run_ca("-gencrl", "-out", self.crl)

def etree_validate(e):
  # This is a kludge, schema should be loaded as module or configured
  # in .conf, but it will do as a temporary debugging hack.
  schema = os.getenv("MYRPKI_RNG")
  if schema:
    try:
      import lxml.etree
    except ImportError:
      return
    try:
      lxml.etree.RelaxNG(file = schema).assertValid(e)
    except lxml.etree.RelaxNGParseError:
      return
    except lxml.etree.DocumentInvalid:
      print lxml.etree.tostring(e, pretty_print = True)
      raise

def etree_write(e, filename, verbose = False, validate = True, msg = None):
  """
  Write out an etree to a file, safely.

  I still miss SYSCAL(RENMWO).
  """
  filename = os.path.realpath(filename)
  tempname = filename
  if not filename.startswith("/dev/"):
    tempname += ".tmp"
  if verbose or msg:
    print "Writing", filename
  if msg:
    print msg
  e = etree_pre_write(e, validate)
  ElementTree(e).write(tempname)
  if tempname != filename:
    os.rename(tempname, filename)

def etree_pre_write(e, validate = True):
  """
  Do the namespace frobbing needed on write; broken out of
  etree_write() because also needed with ElementToString().
  """
  e = copy.deepcopy(e)
  e.set("version", version)
  for i in e.getiterator():
    if i.tag[0] != "{":
      i.tag = namespaceQName + i.tag
    assert i.tag.startswith(namespaceQName)
  if validate:
    etree_validate(e)
  return e

def etree_read(filename, verbose = False, validate = True):
  """
  Read an etree from a file, verifying then stripping XML namespace
  cruft.
  """
  if verbose:
    print "Reading", filename
  e = ElementTree(file = filename).getroot()
  return etree_post_read(e, validate)

def etree_post_read(e, validate = True):
  """
  Do the namespace frobbing needed on read; broken out of etree_read()
  beause also needed by ElementFromString().
  """
  if validate:
    etree_validate(e)
  for i in e.getiterator():
    if i.tag.startswith(namespaceQName):
      i.tag = i.tag[len(namespaceQName):]
    else:
      raise RuntimeError, "XML tag %r is not in namespace %r" % (i.tag, namespace)
  return e

def b64_equal(thing1, thing2):
  """
  Compare two Base64-encoded values for equality.
  """
  return "".join(thing1.split()) == "".join(thing2.split())



class main(rpki.cli.Cmd):

  prompt = "myrpki> "

  completedefault = rpki.cli.Cmd.filename_complete

  show_xml = False

  def __init__(self):
    os.environ["TZ"] = "UTC"
    time.tzset()

    rpki.log.use_syslog = False

    self.cfg_file = os.getenv("MYRPKI_CONF", "myrpki.conf")

    opts, argv = getopt.getopt(sys.argv[1:], "c:h?", ["config=", "help"])
    for o, a in opts:
      if o in ("-c", "--config"):
        self.cfg_file = a
      elif o in ("-h", "--help", "-?"):
        argv = ["help"]

    if not argv or argv[0] != "help":
      rpki.log.init("myrpki")
      self.read_config()

    rpki.cli.Cmd.__init__(self, argv)


  def help_overview(self):
    """
    Show program __doc__ string.  Perhaps there's some clever way to
    do this using the textwrap module, but for now something simple
    and crude will suffice.
    """
    for line in __doc__.splitlines(True):
      self.stdout.write(" " * 4 + line)
    self.stdout.write("\n")

  def entitydb_complete(self, prefix, text, line, begidx, endidx):
    """
    Completion helper for entitydb filenames.
    """
    names = []
    for name in self.entitydb.iterate(prefix, "*.xml"):
      name = os.path.splitext(os.path.basename(name))[0]
      if name.startswith(text):
        names.append(name)
    return names

  def read_config(self):

    self.cfg = rpki.config.parser(self.cfg_file, "myrpki")

    self.histfile  = self.cfg.get("history_file", ".myrpki_history")
    self.handle    = self.cfg.get("handle")
    self.run_rpkid = self.cfg.getboolean("run_rpkid")
    self.run_pubd  = self.cfg.getboolean("run_pubd")
    self.run_rootd = self.cfg.getboolean("run_rootd")
    self.entitydb  = EntityDB(self.cfg)

    if self.run_rootd and (not self.run_pubd or not self.run_rpkid):
      raise RuntimeError, "Can't run rootd unless also running rpkid and pubd"

    self.bpki_resources = CA(self.cfg_file, self.cfg.get("bpki_resources_directory"))
    if self.run_rpkid or self.run_pubd or self.run_rootd:
      self.bpki_servers = CA(self.cfg_file, self.cfg.get("bpki_servers_directory"))

    self.pubd_contact_info = self.cfg.get("pubd_contact_info", "")

    self.rsync_module = self.cfg.get("publication_rsync_module")
    self.rsync_server = self.cfg.get("publication_rsync_server")


  def do_initialize(self, arg):
    """
    Initialize an RPKI installation.  This command reads the
    configuration file, creates the BPKI and EntityDB directories,
    generates the initial BPKI certificates, and creates an XML file
    describing the resource-holding aspect of this RPKI installation.
    """

    if arg:
      raise RuntimeError, "This command takes no arguments"

    self.bpki_resources.setup(self.cfg.get("bpki_resources_ta_dn",
                                           "/CN=%s BPKI Resource Trust Anchor" % self.handle))
    if self.run_rpkid or self.run_pubd or self.run_rootd:
      self.bpki_servers.setup(self.cfg.get("bpki_servers_ta_dn",
                                           "/CN=%s BPKI Server Trust Anchor" % self.handle))

    # Create entitydb directories.

    for i in ("parents", "children", "repositories", "pubclients"):
      d = self.entitydb(i)
      if not os.path.exists(d):
        os.makedirs(d)

    if self.run_rpkid or self.run_pubd or self.run_rootd:

      if self.run_rpkid:
        self.bpki_servers.ee(self.cfg.get("bpki_rpkid_ee_dn",
                                          "/CN=%s rpkid server certificate" % self.handle), "rpkid")
        self.bpki_servers.ee(self.cfg.get("bpki_irdbd_ee_dn",
                                          "/CN=%s irdbd server certificate" % self.handle), "irdbd")
      if self.run_pubd:
        self.bpki_servers.ee(self.cfg.get("bpki_pubd_ee_dn",
                                          "/CN=%s pubd server certificate" % self.handle), "pubd")
      if self.run_rpkid or self.run_pubd:
        self.bpki_servers.ee(self.cfg.get("bpki_irbe_ee_dn",
                                          "/CN=%s irbe client certificate" % self.handle), "irbe")
      if self.run_rootd:
        self.bpki_servers.ee(self.cfg.get("bpki_rootd_ee_dn",
                                          "/CN=%s rootd server certificate" % self.handle), "rootd")

    # Build the identity.xml file.  Need to check for existing file so we don't
    # overwrite?  Worry about that later.

    e = Element("identity", handle = self.handle)
    PEMElement(e, "bpki_ta", self.bpki_resources.cer)
    etree_write(e, self.entitydb("identity.xml"),
                msg = None if self.run_rootd else 'This is the "identity" file you will need to send to your parent')

    # If we're running rootd, construct a fake parent to go with it,
    # and cross-certify in both directions so we can talk to rootd.

    if self.run_rootd:

      e = Element("parent", parent_handle = self.handle, child_handle = self.handle,
                  service_uri = "http://localhost:%s/" % self.cfg.get("rootd_server_port"),
                  valid_until = str(rpki.sundial.now() + rpki.sundial.timedelta(days = 365)))
      PEMElement(e, "bpki_resource_ta", self.bpki_servers.cer)
      PEMElement(e, "bpki_server_ta", self.bpki_servers.cer)
      PEMElement(e, "bpki_child_ta", self.bpki_resources.cer)
      SubElement(e, "repository", type = "offer")
      etree_write(e, self.entitydb("parents", "%s.xml" % self.handle))

      self.bpki_resources.xcert(self.bpki_servers.cer)

      rootd_child_fn = self.cfg.get("child-bpki-cert", None, "rootd")
      if not os.path.exists(rootd_child_fn):
        os.link(self.bpki_servers.xcert(self.bpki_resources.cer), rootd_child_fn)

      repo_file_name = self.entitydb("repositories", "%s.xml" % self.handle)

      try:
        want_offer = etree_read(repo_file_name).get("type") != "confirmed"
      except IOError:
        want_offer = True

      if want_offer:
        e = Element("repository", type = "offer", handle = self.handle, parent_handle = self.handle)
        PEMElement(e, "bpki_client_ta", self.bpki_resources.cer)
        etree_write(e, repo_file_name,
                    msg = 'This is the "repository offer" file for you to use if you want to publish in your own repository')


  def do_update_bpki(self, arg):
    """
    Update BPKI certificates.  Assumes an existing RPKI installation.

    Basic plan here is to reissue all BPKI certificates we can, right
    now.  In the long run we might want to be more clever about only
    touching ones that need maintenance, but this will do for a start.

    Most likely this should be run under cron.
    """

    if self.bpki_servers:
      bpkis = (self.bpki_resources, self.bpki_servers)
    else:
      bpkis = (self.bpki_resources,)

    for bpki in bpkis:
      for cer in glob.iglob("%s/*.cer" % bpki.dir):
        key = cer[0:-4] + ".key"
        req = cer[0:-4] + ".req"
        if os.path.exists(key):
          print "Regenerating BPKI PKCS #10", req
          bpki.run_openssl("x509", "-x509toreq", "-in", cer, "-out", req, "-signkey", key)
        print "Clearing BPKI certificate", cer
        os.unlink(cer)
        if cer == bpki.cer:
          assert req == bpki.req
          print "Regenerating certificate", cer
          bpki.run_ca("-selfsign", "-extensions", "ca_x509_ext_ca", "-in", req, "-out", cer)

    print "Regenerating CRLs"
    for bpki in bpkis:
      bpki.run_ca("-gencrl", "-out", bpki.crl)

    self.do_initialize(None)
    if self.run_rpkid or self.run_pubd or self.run_rootd:
      self.do_configure_daemons(arg)
    else:
      self.do_configure_resources(None)


  def do_configure_child(self, arg):
    """
    Configure a new child of this RPKI entity, given the child's XML
    identity file as an input.  This command extracts the child's data
    from the XML, cross-certifies the child's resource-holding BPKI
    certificate, and generates an XML file describing the relationship
    between the child and this parent, including this parent's BPKI
    data and up-down protocol service URI.
    """

    child_handle = None

    opts, argv = getopt.getopt(arg.split(), "", ["child_handle="])
    for o, a in opts:
      if o == "--child_handle":
        child_handle = a
    
    if len(argv) != 1:
      raise RuntimeError, "Need to specify filename for child.xml"

    c = etree_read(argv[0])

    if child_handle is None:
      child_handle = c.get("handle")

    try:
      e = etree_read(self.cfg.get("xml_filename"))
      service_uri_base = e.get("service_uri")
      server_ta = e.findtext("bpki_server_ta")
    except IOError:
      service_uri_base = None      
      server_ta = None

    if not service_uri_base and self.run_rpkid:
      service_uri_base = "http://%s:%s/up-down/%s" % (self.cfg.get("rpkid_server_host"),
                                                      self.cfg.get("rpkid_server_port"),
                                                      self.handle)
    if not service_uri_base or not server_ta:
      print "Sorry, you can't set up children of a hosted config that itself has not yet been set up"
      return

    print "Child calls itself %r, we call it %r" % (c.get("handle"), child_handle)

    if self.run_rpkid or self.run_pubd or self.run_rootd:
      self.bpki_servers.fxcert(c.findtext("bpki_ta"))

    e = Element("parent", parent_handle = self.handle, child_handle = child_handle,
                service_uri = "%s/%s" % (service_uri_base, child_handle),
                valid_until = str(rpki.sundial.now() + rpki.sundial.timedelta(days = 365)))

    PEMElement(e, "bpki_resource_ta", self.bpki_resources.cer)
    if self.run_rpkid or self.run_pubd or self.run_rootd:
      PEMElement(e, "bpki_server_ta",   self.bpki_servers.cer)
    else:
      assert server_ta is not None
      SubElement(e, "bpki_server_ta").text = server_ta
    SubElement(e, "bpki_child_ta").text = c.findtext("bpki_ta")

    try:
      repo = None
      for f in self.entitydb.iterate("repositories", "*.xml"):
        r = etree_read(f)
        if r.get("type") == "confirmed":
          if repo is not None:
            raise RuntimeError, "Too many repositories, I don't know what to do, not giving referral"
          repo_handle = os.path.splitext(os.path.split(f)[-1])[0]
          repo = r
      if repo is None:
        raise RuntimeError, "Couldn't find any usable repositories, not giving referral"

      if repo_handle == self.handle:
        SubElement(e, "repository", type = "offer")
      else:
        proposed_sia_base = repo.get("sia_base") + child_handle + "/"
        r = Element("referral", authorized_sia_base = proposed_sia_base)
        r.text = c.findtext("bpki_ta")
        auth = self.bpki_resources.cms_xml_sign(
          "/CN=%s Publication Referral" % self.handle, "referral", r)
        r = SubElement(e, "repository", type = "referral")
        SubElement(r, "authorization", referrer = repo.get("client_handle")).text = auth
        SubElement(r, "contact_info").text = repo.findtext("contact_info")

    except RuntimeError, err:
      print err

    etree_write(e, self.entitydb("children", "%s.xml" % child_handle),
                msg = "Send this file back to the child you just configured")


  def do_delete_child(self, arg):
    """
    Delete a child of this RPKI entity.

    This should check that the XML file it's deleting really is a
    child, but doesn't, yet.
    """

    try:
      os.unlink(self.entitydb("children", "%s.xml" % arg))
    except OSError:
      print "No such child \"%s\"" % arg

  def complete_delete_child(self, *args):
    return self.entitydb_complete("children", *args)


  def do_configure_parent(self, arg):
    """
    Configure a new parent of this RPKI entity, given the output of
    the parent's configure_child command as input.  This command reads
    the parent's response XML, extracts the parent's BPKI and service
    URI information, cross-certifies the parent's BPKI data into this
    entity's BPKI, and checks for offers or referrals of publication
    service.  If a publication offer or referral is present, we
    generate a request-for-service message to that repository, in case
    the user wants to avail herself of the referral or offer.
    """

    parent_handle = None

    opts, argv = getopt.getopt(arg.split(), "", ["parent_handle="])
    for o, a in opts:
      if o == "--parent_handle":
        parent_handle = a

    if len(argv) != 1:
      raise RuntimeError, "Need to specify filename for parent.xml on command line"

    p = etree_read(argv[0])

    if parent_handle is None:
      parent_handle = p.get("parent_handle")

    print "Parent calls itself %r, we call it %r" % (p.get("parent_handle"), parent_handle)
    print "Parent calls us %r" % p.get("child_handle")

    self.bpki_resources.fxcert(p.findtext("bpki_resource_ta"))
    self.bpki_resources.fxcert(p.findtext("bpki_server_ta"))

    etree_write(p, self.entitydb("parents", "%s.xml" % parent_handle))

    r = p.find("repository")

    if r is not None and r.get("type") in ("offer", "referral"):
      r.set("handle", self.handle)
      r.set("parent_handle", parent_handle)
      PEMElement(r, "bpki_client_ta", self.bpki_resources.cer)
      etree_write(r, self.entitydb("repositories", "%s.xml" % parent_handle),
                  msg = 'This is the "repository %s" file to send to the repository operator' % r.get("type"))
    else:
      print "Couldn't find repository offer or referral"


  def do_delete_parent(self, arg):
    """
    Delete a parent of this RPKI entity.

    This should check that the XML file it's deleting really is a
    parent, but doesn't, yet.
    """

    try:
      os.unlink(self.entitydb("parents", "%s.xml" % arg))
    except OSError:
      print "No such parent \"%s\"" % arg

  def complete_delete_parent(self, *args):
    return self.entitydb_complete("parents", *args)


  def do_configure_publication_client(self, arg):
    """
    Configure publication server to know about a new client, given the
    client's request-for-service message as input.  This command reads
    the client's request for service, cross-certifies the client's
    BPKI data, and generates a response message containing the
    repository's BPKI data and service URI.
    """

    sia_base = None

    opts, argv = getopt.getopt(arg.split(), "", ["sia_base="])
    for o, a in opts:
      if o == "--sia_base":
        sia_base = a
    
    if len(argv) != 1:
      raise RuntimeError, "Need to specify filename for client.xml"

    client = etree_read(argv[0])

    if sia_base is None:

      auth = client.find("authorization")
      if auth is not None:
        print "Found <authorization/> element, this looks like a referral"
        referrer = etree_read(self.entitydb("pubclients", "%s.xml" % auth.get("referrer").replace("/",".")))
        referrer = self.bpki_servers.fxcert(referrer.findtext("bpki_client_ta"))
        referral = self.bpki_servers.cms_xml_verify(auth.text, referrer)
        if not b64_equal(referral.text, client.findtext("bpki_client_ta")):
          raise RuntimeError, "Referral trust anchor does not match"
        sia_base = referral.get("authorized_sia_base")

      elif client.get("parent_handle") == self.handle:
        print "Client claims to be our child, checking"
        client_ta = client.findtext("bpki_client_ta")
        assert client_ta
        for child in self.entitydb.iterate("children", "*.xml"):
          c = etree_read(child)
          if b64_equal(c.findtext("bpki_child_ta"), client_ta):
            sia_base = "rsync://%s/%s/%s/%s/" % (self.rsync_server, self.rsync_module,
                                                 self.handle, client.get("handle"))
            break

    # If we still haven't figured out what to do with this client, it
    # gets a top-level tree of its own, no attempt at nesting.

    if sia_base is None:
      print "Don't know where to nest this client, defaulting to top-level"
      sia_base = "rsync://%s/%s/%s/" % (self.rsync_server, self.rsync_module, client.get("handle"))
      
    assert sia_base.startswith("rsync://")

    client_handle = "/".join(sia_base.rstrip("/").split("/")[4:])

    parent_handle = client.get("parent_handle")

    print "Client calls itself %r, we call it %r" % (client.get("handle"), client_handle)
    print "Client says its parent handle is %r" % parent_handle

    self.bpki_servers.fxcert(client.findtext("bpki_client_ta"))

    e = Element("repository", type = "confirmed",
                client_handle = client_handle,
                parent_handle = parent_handle,
                sia_base = sia_base,
                service_uri = "http://%s:%s/client/%s" % (self.cfg.get("pubd_server_host"),
                                                          self.cfg.get("pubd_server_port"),
                                                          client_handle))

    PEMElement(e, "bpki_server_ta", self.bpki_servers.cer)
    SubElement(e, "bpki_client_ta").text = client.findtext("bpki_client_ta")
    SubElement(e, "contact_info").text = self.pubd_contact_info
    etree_write(e, self.entitydb("pubclients", "%s.xml" % client_handle.replace("/", ".")),
                msg = "Send this file back to the publication client you just configured")


  def do_delete_publication_client(self, arg):
    """
    Delete a publication client of this RPKI entity.

    This should check that the XML file it's deleting really is a
    client, but doesn't, yet.
    """

    try:
      os.unlink(self.entitydb("pubclients", "%s.xml" % arg))
    except OSError:
      print "No such client \"%s\"" % arg

  def complete_delete_publication_client(self, *args):
    return self.entitydb_complete("pubclients", *args)


  def do_configure_repository(self, arg):
    """
    Configure a publication repository for this RPKI entity, given the
    repository's response to our request-for-service message as input.
    This command reads the repository's response, extracts and
    cross-certifies the BPKI data and service URI, and links the
    repository data with the corresponding parent data in our local
    database.
    """

    parent_handle = None

    opts, argv = getopt.getopt(arg.split(), "", ["parent_handle="])
    for o, a in opts:
      if o == "--parent_handle":
        parent_handle = a

    if len(argv) != 1:
      raise RuntimeError, "Need to specify filename for repository.xml on command line"

    r = etree_read(argv[0])

    if parent_handle is None:
      parent_handle = r.get("parent_handle")

    print "Repository calls us %r" % (r.get("client_handle"))
    print "Repository response associated with parent_handle %r" % parent_handle

    etree_write(r, self.entitydb("repositories", "%s.xml" % parent_handle))


  def do_delete_repository(self, arg):
    """
    Delete a repository of this RPKI entity.

    This should check that the XML file it's deleting really is a
    repository, but doesn't, yet.
    """

    try:
      os.unlink(self.entitydb("repositories", "%s.xml" % arg))
    except OSError:
      print "No such repository \"%s\"" % arg

  def complete_delete_repository(self, *args):
    return self.entitydb_complete("repositories", *args)




  def configure_resources_main(self, msg = None):
    """
    Main program of old myrpki.py script.  This remains separate
    because it's called from more than one place.
    """

    roa_csv_file                  = self.cfg.get("roa_csv")
    prefix_csv_file               = self.cfg.get("prefix_csv")
    asn_csv_file                  = self.cfg.get("asn_csv")

    # This probably should become an argument instead of (or in
    # addition to a default from?) a config file option.
    xml_filename                  = self.cfg.get("xml_filename")

    try:
      e = etree_read(xml_filename)
      bsc_req, bsc_cer = self.bpki_resources.bsc(e.findtext("bpki_bsc_pkcs10"))
      service_uri = e.get("service_uri")
      server_ta = e.findtext("bpki_server_ta")
    except IOError:
      bsc_req, bsc_cer = None, None
      service_uri = None
      server_ta = None

    e = Element("myrpki", handle = self.handle)

    if service_uri:
      e.set("service_uri", service_uri)

    roa_requests.from_csv(roa_csv_file).xml(e)

    children.from_csv(
      prefix_csv_file = prefix_csv_file,
      asn_csv_file = asn_csv_file,
      fxcert = self.bpki_resources.fxcert,
      entitydb = self.entitydb).xml(e)

    parents.from_csv(     fxcert = self.bpki_resources.fxcert, entitydb = self.entitydb).xml(e)
    repositories.from_csv(fxcert = self.bpki_resources.fxcert, entitydb = self.entitydb).xml(e)

    PEMElement(e, "bpki_ca_certificate", self.bpki_resources.cer)
    PEMElement(e, "bpki_crl",            self.bpki_resources.crl)

    if bsc_cer:
      PEMElement(e, "bpki_bsc_certificate", bsc_cer)

    if bsc_req:
      PEMElement(e, "bpki_bsc_pkcs10", bsc_req)

    if server_ta:
      SubElement(e, "bpki_server_ta").text = server_ta

    etree_write(e, xml_filename, msg = msg)


  def do_configure_resources(self, arg):
    """
    Read CSV files and all the descriptions of parents and children
    that we've built up, package the result up as a single XML file to
    be shipped to a hosting rpkid.
    """

    if arg:
      raise RuntimeError, "Unexpected argument %r" % arg
    self.configure_resources_main(msg = "Send this file to the rpkid operator who is hosting you")



  def do_configure_daemons(self, arg):
    """
    Configure RPKI daemons with the data built up by the other
    commands in this program.

    The basic model here is that each entity with resources to certify
    runs the myrpki tool, but not all of them necessarily run their
    own RPKI engines.  The entities that do run RPKI engines get data
    from the entities they host via the XML files output by the
    configure_resources command.  Those XML files are the input to
    this command, which uses them to do all the work of configuring
    daemons, populating SQL databases, and so forth.  A few operations
    (eg, BSC construction) generate data which has to be shipped back
    to the resource holder, which we do by updating the same XML file.

    In essence, the XML files are a sneakernet (or email, or carrier
    pigeon) communication channel between the resource holders and the
    RPKI engine operators.

    As a convenience, for the normal case where the RPKI engine
    operator is itself a resource holder, this command in effect runs
    the configure_resources command automatically to process the RPKI
    engine operator's own resources.

    Note that, due to the back and forth nature of some of these
    operations, it may take several cycles for data structures to stablize
    and everything to reach a steady state.  This is normal.
    """

    argv = arg.split()

    try:
      import rpki.http, rpki.resource_set, rpki.relaxng, rpki.exceptions
      import rpki.left_right, rpki.x509, rpki.async
      if hasattr(warnings, "catch_warnings"):
        with warnings.catch_warnings():
          warnings.simplefilter("ignore", DeprecationWarning)
          import MySQLdb
      else:
        import MySQLdb

    except ImportError, e:
      print "Sorry, you appear to be missing some of the Python modules needed to run this command"
      print "[Error: %r]" % e

    def findbase64(tree, name, b64type = rpki.x509.X509):
      x = tree.findtext(name)
      return b64type(Base64 = x) if x else None

    # We can use a single BSC for everything -- except BSC key
    # rollovers.  Drive off that bridge when we get to it.

    bsc_handle = "bsc"

    self.cfg.set_global_flags()

    # Default values for CRL parameters are low, for testing.  Not
    # quite as low as they once were, too much expired CRL whining.

    self_crl_interval = self.cfg.getint("self_crl_interval", 2 * 60 * 60)
    self_regen_margin = self.cfg.getint("self_regen_margin", self_crl_interval / 4)
    pubd_base         = "http://%s:%s/" % (self.cfg.get("pubd_server_host"), self.cfg.get("pubd_server_port"))
    rpkid_base        = "http://%s:%s/" % (self.cfg.get("rpkid_server_host"), self.cfg.get("rpkid_server_port"))

    # Wrappers to simplify calling rpkid and pubd.

    call_rpkid = rpki.async.sync_wrapper(rpki.http.caller(
      proto       = rpki.left_right,
      client_key  = rpki.x509.RSA( PEM_file = self.bpki_servers.dir + "/irbe.key"),
      client_cert = rpki.x509.X509(PEM_file = self.bpki_servers.dir + "/irbe.cer"),
      server_ta   = rpki.x509.X509(PEM_file = self.bpki_servers.cer),
      server_cert = rpki.x509.X509(PEM_file = self.bpki_servers.dir + "/rpkid.cer"),
      url         = rpkid_base + "left-right",
      debug       = self.show_xml))

    if self.run_pubd:

      call_pubd = rpki.async.sync_wrapper(rpki.http.caller(
        proto       = rpki.publication,
        client_key  = rpki.x509.RSA( PEM_file = self.bpki_servers.dir + "/irbe.key"),
        client_cert = rpki.x509.X509(PEM_file = self.bpki_servers.dir + "/irbe.cer"),
        server_ta   = rpki.x509.X509(PEM_file = self.bpki_servers.cer),
        server_cert = rpki.x509.X509(PEM_file = self.bpki_servers.dir + "/pubd.cer"),
        url         = pubd_base + "control",
        debug       = self.show_xml))

      # Make sure that pubd's BPKI CRL is up to date.

      call_pubd(rpki.publication.config_elt.make_pdu(
        action = "set",
        bpki_crl = rpki.x509.CRL(PEM_file = self.bpki_servers.crl)))

    irdbd_cfg = rpki.config.parser(self.cfg.get("irdbd_conf", self.cfg_file), "irdbd")

    db = MySQLdb.connect(user   = irdbd_cfg.get("sql-username"),
                         db     = irdbd_cfg.get("sql-database"),
                         passwd = irdbd_cfg.get("sql-password"))

    cur = db.cursor()

    xmlfiles = []

    # If [myrpki] section includes an "xml_filename" setting, run
    # myrpki.py internally, as a convenience, and include its output at
    # the head of our list of XML files to process.

    my_xmlfile = self.cfg.get("xml_filename", "")
    if my_xmlfile:
      self.configure_resources_main()
      xmlfiles.append(my_xmlfile)
    else:
      my_xmlfile = None

    # Add any other XML files specified on the command line

    xmlfiles.extend(argv)

    for xmlfile in xmlfiles:

      # Parse XML file and validate it against our scheme

      tree = etree_read(xmlfile, validate = True)

      handle = tree.get("handle")

      # Update IRDB with parsed resource and roa-request data.

      cur.execute(
        """
        DELETE
        FROM  roa_request_prefix
        USING roa_request, roa_request_prefix
        WHERE roa_request.roa_request_id = roa_request_prefix.roa_request_id AND roa_request.roa_request_handle = %s
        """, (handle,))

      cur.execute("DELETE FROM roa_request WHERE roa_request.roa_request_handle = %s", (handle,))

      for x in tree.getiterator("roa_request"):
        cur.execute("INSERT roa_request (roa_request_handle, asn) VALUES (%s, %s)", (handle, x.get("asn")))
        roa_request_id = cur.lastrowid
        for version, prefix_set in ((4, rpki.resource_set.roa_prefix_set_ipv4(x.get("v4"))), (6, rpki.resource_set.roa_prefix_set_ipv6(x.get("v6")))):
          if prefix_set:
            cur.executemany("INSERT roa_request_prefix (roa_request_id, prefix, prefixlen, max_prefixlen, version) VALUES (%s, %s, %s, %s, %s)",
                            ((roa_request_id, p.prefix, p.prefixlen, p.max_prefixlen, version) for p in prefix_set))

      cur.execute(
        """
        DELETE
        FROM   registrant_asn
        USING registrant, registrant_asn
        WHERE registrant.registrant_id = registrant_asn.registrant_id AND registrant.registry_handle = %s
        """ , (handle,))

      cur.execute(
        """
        DELETE FROM registrant_net USING registrant, registrant_net
        WHERE registrant.registrant_id = registrant_net.registrant_id AND registrant.registry_handle = %s
        """ , (handle,))

      cur.execute("DELETE FROM registrant WHERE registrant.registry_handle = %s" , (handle,))

      for x in tree.getiterator("child"):
        child_handle = x.get("handle")
        asns = rpki.resource_set.resource_set_as(x.get("asns"))
        ipv4 = rpki.resource_set.resource_set_ipv4(x.get("v4"))
        ipv6 = rpki.resource_set.resource_set_ipv6(x.get("v6"))

        cur.execute("INSERT registrant (registrant_handle, registry_handle, registrant_name, valid_until) VALUES (%s, %s, %s, %s)",
                    (child_handle, handle, child_handle, rpki.sundial.datetime.fromXMLtime(x.get("valid_until")).to_sql()))
        child_id = cur.lastrowid
        if asns:
          cur.executemany("INSERT registrant_asn (start_as, end_as, registrant_id) VALUES (%s, %s, %s)",
                          ((a.min, a.max, child_id) for a in asns))
        if ipv4:
          cur.executemany("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 4, %s)",
                          ((a.min, a.max, child_id) for a in ipv4))
        if ipv6:
          cur.executemany("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 6, %s)",
                          ((a.min, a.max, child_id) for a in ipv6))

      db.commit()

      # Check for certificates before attempting anything else

      hosted_cacert = findbase64(tree, "bpki_ca_certificate")
      if not hosted_cacert:
        print "Nothing else I can do without a trust anchor for the entity I'm hosting."
        continue

      rpkid_xcert = rpki.x509.X509(PEM_file = self.bpki_servers.fxcert(b64 = hosted_cacert.get_Base64(),
                                                                       #filename = handle + ".cacert.cer",
                                                                       path_restriction = 1))

      # See what rpkid and pubd already have on file for this entity.

      if self.run_pubd:
        client_pdus = dict((x.client_handle, x)
                           for x in call_pubd(rpki.publication.client_elt.make_pdu(action = "list"))
                           if isinstance(x, rpki.publication.client_elt))

      rpkid_reply = call_rpkid(
        rpki.left_right.self_elt.make_pdu(      action = "get",  tag = "self",       self_handle = handle),
        rpki.left_right.bsc_elt.make_pdu(       action = "list", tag = "bsc",        self_handle = handle),
        rpki.left_right.repository_elt.make_pdu(action = "list", tag = "repository", self_handle = handle),
        rpki.left_right.parent_elt.make_pdu(    action = "list", tag = "parent",     self_handle = handle),
        rpki.left_right.child_elt.make_pdu(     action = "list", tag = "child",      self_handle = handle))

      self_pdu        = rpkid_reply[0]
      bsc_pdus        = dict((x.bsc_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.bsc_elt))
      repository_pdus = dict((x.repository_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.repository_elt))
      parent_pdus     = dict((x.parent_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.parent_elt))
      child_pdus      = dict((x.child_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.child_elt))

      pubd_query = []
      rpkid_query = []

      # There should be exactly one <self/> object per hosted entity, by definition

      if (isinstance(self_pdu, rpki.left_right.report_error_elt) or
          self_pdu.crl_interval != self_crl_interval or
          self_pdu.regen_margin != self_regen_margin or
          self_pdu.bpki_cert != rpkid_xcert):
        rpkid_query.append(rpki.left_right.self_elt.make_pdu(
          action = "create" if isinstance(self_pdu, rpki.left_right.report_error_elt) else "set",
          tag = "self",
          self_handle = handle,
          bpki_cert = rpkid_xcert,
          crl_interval = self_crl_interval,
          regen_margin = self_regen_margin))

      # In general we only need one <bsc/> per <self/>.  BSC objects are a
      # little unusual in that the PKCS #10 subelement is generated by rpkid
      # in response to generate_keypair, so there's more of a separation
      # between create and set than with other objects.

      bsc_cert = findbase64(tree, "bpki_bsc_certificate")
      bsc_crl  = findbase64(tree, "bpki_crl", rpki.x509.CRL)

      bsc_pdu = bsc_pdus.pop(bsc_handle, None)

      if bsc_pdu is None:
        rpkid_query.append(rpki.left_right.bsc_elt.make_pdu(
          action = "create",
          tag = "bsc",
          self_handle = handle,
          bsc_handle = bsc_handle,
          generate_keypair = "yes"))
      elif bsc_pdu.signing_cert != bsc_cert or bsc_pdu.signing_cert_crl != bsc_crl:
        rpkid_query.append(rpki.left_right.bsc_elt.make_pdu(
          action = "set",
          tag = "bsc",
          self_handle = handle,
          bsc_handle = bsc_handle,
          signing_cert = bsc_cert,
          signing_cert_crl = bsc_crl))

      rpkid_query.extend(rpki.left_right.bsc_elt.make_pdu(
        action = "destroy", self_handle = handle, bsc_handle = b) for b in bsc_pdus)

      bsc_req = None

      if bsc_pdu and bsc_pdu.pkcs10_request:
        bsc_req = bsc_pdu.pkcs10_request

      # At present we need one <repository/> per <parent/>, not because
      # rpkid requires that, but because pubd does.  pubd probably should
      # be fixed to support a single client allowed to update multiple
      # trees, but for the moment the easiest way forward is just to
      # enforce a 1:1 mapping between <parent/> and <repository/> objects

      for repository in tree.getiterator("repository"):

        repository_handle = repository.get("handle")
        repository_pdu = repository_pdus.pop(repository_handle, None)
        repository_uri = repository.get("service_uri")
        repository_cert = findbase64(repository, "bpki_certificate")

        if (repository_pdu is None or
            repository_pdu.bsc_handle != bsc_handle or
            repository_pdu.peer_contact_uri != repository_uri or
            repository_pdu.bpki_cert != repository_cert):
          rpkid_query.append(rpki.left_right.repository_elt.make_pdu(
            action = "create" if repository_pdu is None else "set",
            tag = repository_handle,
            self_handle = handle,
            repository_handle = repository_handle,
            bsc_handle = bsc_handle,
            peer_contact_uri = repository_uri,
            bpki_cert = repository_cert))

      rpkid_query.extend(rpki.left_right.repository_elt.make_pdu(
        action = "destroy", self_handle = handle, repository_handle = r) for r in repository_pdus)

      # <parent/> setup code currently assumes 1:1 mapping between
      # <repository/> and <parent/>, and further assumes that the handles
      # for an associated pair are the identical (that is:
      # parent.repository_handle == parent.parent_handle).

      for parent in tree.getiterator("parent"):

        parent_handle = parent.get("handle")
        parent_pdu = parent_pdus.pop(parent_handle, None)
        parent_uri = parent.get("service_uri")
        parent_myhandle = parent.get("myhandle")
        parent_sia_base = parent.get("sia_base")
        parent_cms_cert = findbase64(parent, "bpki_cms_certificate")

        if (parent_pdu is None or
            parent_pdu.bsc_handle != bsc_handle or
            parent_pdu.repository_handle != parent_handle or
            parent_pdu.peer_contact_uri != parent_uri or
            parent_pdu.sia_base != parent_sia_base or
            parent_pdu.sender_name != parent_myhandle or
            parent_pdu.recipient_name != parent_handle or
            parent_pdu.bpki_cms_cert != parent_cms_cert):
          rpkid_query.append(rpki.left_right.parent_elt.make_pdu(
            action = "create" if parent_pdu is None else "set",
            tag = parent_handle,
            self_handle = handle,
            parent_handle = parent_handle,
            bsc_handle = bsc_handle,
            repository_handle = parent_handle,
            peer_contact_uri = parent_uri,
            sia_base = parent_sia_base,
            sender_name = parent_myhandle,
            recipient_name = parent_handle,
            bpki_cms_cert = parent_cms_cert))

      rpkid_query.extend(rpki.left_right.parent_elt.make_pdu(
        action = "destroy", self_handle = handle, parent_handle = p) for p in parent_pdus)

      # Children are simpler than parents, because they call us, so no URL
      # to construct and figuring out what certificate to use is their
      # problem, not ours.

      for child in tree.getiterator("child"):

        child_handle = child.get("handle")
        child_pdu = child_pdus.pop(child_handle, None)
        child_cert = findbase64(child, "bpki_certificate")

        if (child_pdu is None or
            child_pdu.bsc_handle != bsc_handle or
            child_pdu.bpki_cert != child_cert):
          rpkid_query.append(rpki.left_right.child_elt.make_pdu(
            action = "create" if child_pdu is None else "set",
            tag = child_handle,
            self_handle = handle,
            child_handle = child_handle,
            bsc_handle = bsc_handle,
            bpki_cert = child_cert))

      rpkid_query.extend(rpki.left_right.child_elt.make_pdu(
        action = "destroy", self_handle = handle, child_handle = c) for c in child_pdus)

      # Publication setup.

      if self.run_pubd:

        for f in self.entitydb.iterate("pubclients", "*.xml"):
          c = etree_read(f)

          client_handle = c.get("client_handle")
          client_base_uri = c.get("sia_base")
          client_bpki_cert = rpki.x509.X509(PEM_file = self.bpki_servers.fxcert(c.findtext("bpki_client_ta")))
          client_pdu = client_pdus.pop(client_handle, None)

          if (client_pdu is None or
              client_pdu.base_uri != client_base_uri or
              client_pdu.bpki_cert != client_bpki_cert):
            pubd_query.append(rpki.publication.client_elt.make_pdu(
              action = "create" if client_pdu is None else "set",
              client_handle = client_handle,
              bpki_cert = client_bpki_cert,
              base_uri = client_base_uri))

        pubd_query.extend(rpki.publication.client_elt.make_pdu(
            action = "destroy", client_handle = p) for p in client_pdus)

      # If we changed anything, ship updates off to daemons

      failed = False

      if rpkid_query:
        rpkid_reply = call_rpkid(*rpkid_query)
        bsc_pdus = dict((x.bsc_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.bsc_elt))
        if bsc_handle in bsc_pdus and bsc_pdus[bsc_handle].pkcs10_request:
          bsc_req = bsc_pdus[bsc_handle].pkcs10_request
        for r in rpkid_reply:
          if isinstance(r, rpki.left_right.report_error_elt):
            failed = True
            print "rpkid reported failure:", r.error_code
            if r.error_text:
              print r.error_text

      if failed:
        raise RuntimeError

      if pubd_query:
        assert self.run_pubd
        pubd_reply = call_pubd(*pubd_query)
        for r in pubd_reply:
          if isinstance(r, rpki.publication.report_error_elt):
            failed = True
            print "pubd reported failure:", r.error_code
            if r.error_text:
              print r.error_text
            
      if failed:
        raise RuntimeError

      # Rewrite XML.

      e = tree.find("bpki_bsc_pkcs10")
      if e is not None:
        tree.remove(e)
      if bsc_req is not None:
        SubElement(tree, "bpki_bsc_pkcs10").text = bsc_req.get_Base64()

      tree.set("service_uri", rpkid_base + "up-down/" + handle)

      e = tree.find("bpki_server_ta")
      if e is not None:
        tree.remove(e)
      PEMElement(tree, "bpki_server_ta", self.bpki_resources.cer)

      etree_write(tree, xmlfile, validate = True,
                  msg = None if xmlfile is my_xmlfile else 'Send this file back to the hosted entity ("%s")' % handle)

    db.close()

    # We used to run event loop again to give TLS connections a chance to shut down cleanly.
    # Seems not to be needed (and sometimes hangs forever, which is odd) with TLS out of the picture.
    #rpki.async.event_loop()