path: root/rpkid/rpki/rpkic.py

"""
This (oversized) module used to be an (oversized) program.
Refactoring in progress, some doc still needs updating.


This program is now the merger of three different tools: the old
myrpki.py script, the old myirbe.py script, and the newer setup.py CLI
tool.  As such, it is still in need of some cleanup, but the need to
provide a saner user interface is more urgent than internal code
prettiness at the moment.  In the long run, 90% of the code in this
file probably ought to move to well-designed library modules.

Overall goal here is to build up the configuration necessary to run
rpkid and friends, by reading a config file, a collection of .CSV
files, and the results of a few out-of-band XML setup messages
exchanged with one's parents, children, and so forth.

The config file is in an OpenSSL-compatible format, the CSV files are
simple tab-delimited text.  The XML files are all generated by this
program, either the local instance or an instance being run by another
player in the system; the mechanism used to exchange these setup
messages is outside the scope of this program, feel free to use
PGP-signed mail, a web interface (not provided), USB stick, carrier
pigeons, whatever works.

With one exception, the commands in this program avoid using any
third-party Python code other than the rpki libraries themselves; with
the same one exception, all OpenSSL work is done with the OpenSSL
command line tool (the one built as a side effect of building rcynic
will do, if your platform has no system copy or the system copy is too
old).  This is all done in an attempt to make the code more portable,
so one can run most of the RPKI back end software on a laptop or
whatever.  The one exception is the configure_daemons command, which
must, of necessity, use the same communication libraries as the
daemons with which it is conversing.  So that one command will not
work if the correct Python modules are not available.


$Id$

Copyright (C) 2009--2011  Internet Systems Consortium ("ISC")

Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.

THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
"""

import subprocess, csv, re, os, getopt, sys, base64, time, glob, copy, warnings
import rpki.config, rpki.cli, rpki.sundial, rpki.log, rpki.oids
import rpki.http, rpki.resource_set, rpki.relaxng, rpki.exceptions
import rpki.left_right, rpki.x509, rpki.async

from lxml.etree import (Element, SubElement, ElementTree,
                        fromstring as ElementFromString,
                        tostring   as ElementToString)

from rpki.csv_utils import (csv_reader, csv_writer, BadCSVSyntax)



# Our XML namespace and protocol version.

namespace      = "http://www.hactrn.net/uris/rpki/myrpki/"
version        = "2"
namespaceQName = "{" + namespace + "}"

## @var allow_incomplete
# Whether to include incomplete entries when rendering to XML.

allow_incomplete = False

## @var whine
# Whether to whine about incomplete entries while rendering to XML.

whine = True

# A whole lot of exceptions

class BadCommandSyntax(Exception):
  """
  Bad command line syntax.
  """

class BadPrefixSyntax(Exception):
  """
  Bad prefix syntax.
  """

class CouldntTalkToDaemon(Exception):
  """
  Couldn't talk to daemon.
  """

class BadXMLMessage(Exception):
  """
  Bad XML message.
  """

class PastExpiration(Exception):
  """
  Expiration date has already passed.
  """

class CantRunRootd(Exception):
  """
  Can't run rootd.
  """


def B64Element(e, tag, obj, **kwargs):
  """
  Create an XML element containing Base64 encoded data taken from a
  DER object.
  """

  if e.text is None:
    e.text = "\n"
  se = SubElement(e, tag, **kwargs)
  se.text = "\n" + obj.get_Base64()
  se.tail = "\n"
  return se



class CA(object):
  """
  Representation of one certification authority.

  This used to be a big complicated wrapper around the awfulness of
  running the OpenSSL command line tool in a subprocess.

  With the new Django-model-based IRDB, this is mostly just a wrapper
  around the IRDB objects and their associated crypto methods.

  For the moment we keep the BPKI directories, because we need some
  place to write cert and key files for the daemons.  Not yet sure
  quite where this is going in the long run.
  """

  def __init__(self, dir, identity, purpose):
    self.dir     = dir
    self.cer     = dir + "/ca.cer"
    self.crl     = dir + "/ca.crl"
    self.identiy = identity
    self.purpose = purpose
    self.ca      = None

  def setup(self, ca_name):
    """
    Set up this CA.  I no longer remember why this is not part of
    __init__().  Maybe it will come back to me
    """

    self.ca = rpki.irdb.CA.objects.get_or_certify(
      identity = self.identity,
      purpose = self.purpose)[0]

    f = open(self.cer, "w")
    f.write(self.ca.certificate.get_PEM())
    f.close()

    f = open(self.crl, "w")
    f.write(self.ca.latest_crl.get_PEM())
    f.close()

  def ee(self, purpose):
    """
    Issue an end-enity certificate.
    """

    return rpki.irdb.EECertificate.objects.get_or_certify(
      issuer = self.ca,
      purpose = purpose)[0]

  def cms_xml_sign(self, elt):
    """
    Sign an XML object with CMS, return Base64 text.
    """

    ee = self.ee("referral")
    return rpki.x509.SignedReferral().wrap(
      msg = elt,
      keypair = ee.private_key,
      certs = ee.certificate,
      crls = self.ca.latest_crl)

  def cms_xml_verify(self, b64, ca):
    """
    Attempt to verify and extract XML from a Base64-encoded signed CMS
    object.

    ca is a rpki.x509.X509 CA certificate which we expect to be the
    issuer of the EE certificate bundled with the CMS; this CA
    certificate must previously have been cross-certified under our
    trust anchor.
    """

    return rpki.x509.SignedReferral(Base64 = b64).unwrap(
      ta = (ca, self.ca.certificate))

  def bsc(self, handle, pkcs10):
    """
    Issue BSC certificate, if we have a PKCS #10 request for it.

    Returns IRDB BSC object, or None if we don't have one and can't
    make one because we don't have the PKCS #10 yet.
    """

    if pkcs10 is None:
      return None

    return rpki.irdb.BSC.objects.get_or_certify(
      issuer = self.ca,
      handle = handle,
      pkcs10 = rpki.x509.PKCS10(Base64 = pkcs10))[0]



def etree_write(e, filename, verbose = False, msg = None):
  """
  Write out an etree to a file, safely.

  I still miss SYSCAL(RENMWO).
  """

  filename = os.path.realpath(filename)
  tempname = filename
  if not filename.startswith("/dev/"):
    tempname += ".%s.tmp" % os.getpid()
  if verbose or msg:
    print "Writing", filename
  if msg:
    print msg
  e = copy.deepcopy(e)
  e.set("version", version)
  for i in e.getiterator():
    if i.tag[0] != "{":
      i.tag = namespaceQName + i.tag
    assert i.tag.startswith(namespaceQName)
  rpki.relaxng.myrpki.assertValid(e)
  ElementTree(e).write(tempname)
  if tempname != filename:
    os.rename(tempname, filename)

def etree_read(filename, verbose = False):
  """
  Read an etree from a file, verifying then stripping XML namespace
  cruft.
  """

  if verbose:
    print "Reading", filename
  e = ElementTree(file = filename).getroot()
  rpki.relaxng.myrpki.assertValid(e)
  for i in e.getiterator():
    if i.tag.startswith(namespaceQName):
      i.tag = i.tag[len(namespaceQName):]
    else:
      raise BadXMLMessage, "XML tag %r is not in namespace %r" % (i.tag, namespace)
  return e



class main(rpki.cli.Cmd):

  prompt = "rpkic> "

  completedefault = rpki.cli.Cmd.filename_complete

  show_xml = False

  def __init__(self):
    os.environ["TZ"] = "UTC"
    time.tzset()

    rpki.log.use_syslog = False

    self.cfg_file = None

    opts, argv = getopt.getopt(sys.argv[1:], "c:h?", ["config=", "help"])
    for o, a in opts:
      if o in ("-c", "--config"):
        self.cfg_file = a
      elif o in ("-h", "--help", "-?"):
        argv = ["help"]

    if not argv or argv[0] != "help":
      rpki.log.init("rpkic")
      self.read_config()

    rpki.cli.Cmd.__init__(self, argv)


  def help_overview(self):
    """
    Show program __doc__ string.  Perhaps there's some clever way to
    do this using the textwrap module, but for now something simple
    and crude will suffice.
    """

    for line in __doc__.splitlines(True):
      self.stdout.write(" " * 4 + line)
    self.stdout.write("\n")

  def irdb_handle_complete(self, klass, text, line, begidx, endidx):
    return [obj.handle for obj in klass.objects.all() if obj.handle.startswith(text)]

  def read_config(self):

    # For reasons I don't understand, importing this at the global
    # level isn't working properly today.  Importing it here works
    # fine.  WTF?

    import rpki.config
    
    self.cfg = rpki.config.parser(self.cfg_file, "myrpki")

    self.histfile  = self.cfg.get("history_file", ".rpkic_history")
    self.handle    = self.cfg.get("handle")
    self.run_rpkid = self.cfg.getboolean("run_rpkid")
    self.run_pubd  = self.cfg.getboolean("run_pubd")
    self.run_rootd = self.cfg.getboolean("run_rootd")

    from django.conf import settings

    irdbd_section = "irdbd"

    settings.configure(
      DATABASES = { "default" : {
        "ENGINE"   : "django.db.backends.mysql",
        "NAME"     : self.cfg.get("sql-database", section = irdbd_section),
        "USER"     : self.cfg.get("sql-username", section = irdbd_section),
        "PASSWORD" : self.cfg.get("sql-password", section = irdbd_section),
        "HOST"     : "",
        "PORT"     : "" }},
      INSTALLED_APPS = ("rpki.irdb",),
    )

    import rpki.irdb

    if self.run_rootd and (not self.run_pubd or not self.run_rpkid):
      raise CantRunRootd, "Can't run rootd unless also running rpkid and pubd"

    self.default_repository = self.cfg.get("default_repository", "")
    self.pubd_contact_info = self.cfg.get("pubd_contact_info", "")

    self.rsync_module = self.cfg.get("publication_rsync_module")
    self.rsync_server = self.cfg.get("publication_rsync_server")

    try:
      self.identity = rpki.irdb.Identity.objects.get(handle = self.handle)
    except rpki.irdb.Identity.DoesNotExist:
      self.identity = None
      self.resource_ca = None
      self.server_ca = None
    else:
      try:
        self.resource_ca = self.identity.ca_set.get(purpose = "resources")
      except rpki.irdb.CA.DoesNotExist:
        self.resource_ca = None
      try:
        self.server_ca = self.identity.ca_set.get(purpose = "servers")
      except rpki.irdb.CA.DoesNotExist:
        self.server_ca = None

  def do_initialize(self, arg):
    """
    Initialize an RPKI installation.  This command reads the
    configuration file, creates the BPKI and EntityDB directories,
    generates the initial BPKI certificates, and creates an XML file
    describing the resource-holding aspect of this RPKI installation.
    """

    if arg:
      raise BadCommandSyntax, "This command takes no arguments"

    self.identity, created = rpki.irdb.Identity.objects.get_or_create(handle = self.handle)
    if created:
      print 'Created new identity for "%s"' % self.handle

    self.resource_ca, created = rpki.irdb.CA.objects.get_or_certify(
      identity = self.identity, purpose = "resources")
    if created:
      print "Created new BPKI resource CA"

    if not self.run_rpkid and not self.run_pubd and not self.run_rootd:
      self.server_ca = None
    else:
      self.server_ca, created = rpki.irdb.CA.objects.get_or_certify(
        identity = self.identity, purpose = "servers")
      if created:
        print "Created new BPKI server CA"

      if self.run_rpkid:
        rpki.irdb.EECertificate.objects.get_or_certify(issuer = self.server_ca, purpose = "rpkid")
        rpki.irdb.EECertificate.objects.get_or_certify(issuer = self.server_ca, purpose = "irdbd")
      if self.run_pubd:
        rpki.irdb.EECertificate.objects.get_or_certify(issuer = self.server_ca, purpose = "pubd")
      if self.run_rpkid or self.run_pubd:
        rpki.irdb.EECertificate.objects.get_or_certify(issuer = self.server_ca, purpose = "irbe")

      ## @todo
      # Why do we issue root's EE certificate under our server CA?
      # We've "always" done this, but does it make sense now?  rootd
      # only speaks up-down, so it's really just another resource
      # holder.  If we just issued it under our resource CA, we
      # wouldn't have to cross certify anything to talk to it.  Which
      # might in itself break something, as it'd be the only parent we
      # -didn't- have to cross-certify.  Leave alone for now, but
      # think about this later.

      if self.run_rootd:
        rpki.irdb.EECertificate.objects.get_or_certify(issuer = self.server_ca, purpose = "rootd")

    # Build the identity.xml file.  Need to check for existing file so we don't
    # overwrite?  Worry about that later.

    e = Element("identity", handle = self.handle)
    B64Element(e, "bpki_ta", self.resource_ca.certificate)
    etree_write(e, "identity.xml",
                msg = None if self.run_rootd else 'This is the "identity" file you will need to send to your parent')

    # If we're running rootd, construct a fake parent to go with it,
    # and cross-certify in both directions so we can talk to rootd.

    if self.run_rootd:

      rpki.irdb.Parent.objects.get_or_certify(
        issuer = self.resource_ca, 
        handle = self.handle,
        parent_handle = self.handle,
        child_handle = self.handle,
        ta = self.server_ca.certificate,
        service_uri = "http://localhost:%s/" % self.cfg.get("rootd_server_port"),
        repository_type = "offer")
      
      rpki.irdb.Child.objects.get_or_certify(
        issuer = self.server_ca,
        handle = self.handle,
        ta = self.resource_ca.certificate,
        valid_until = self.resource_ca.certificate.getNotAfter())

      try:
        self.resource_ca.repositories.get(handle = self.handle)

      except rpki.irdb.Repository.DoesNotExist:
        e = Element("repository", type = "offer", handle = self.handle, parent_handle = self.handle)
        B64Element(e, "bpki_client_ta", self.resource_ca.certificate)
        etree_write(e, "rootd_repository_offer.xml",
                    msg = 'This is the "repository offer" file for you to use if you want to publish in your own repository')


  def do_update_bpki(self, arg):
    """
    Update BPKI certificates.  Assumes an existing RPKI installation.

    Basic plan here is to reissue all BPKI certificates we can, right
    now.  In the long run we might want to be more clever about only
    touching ones that need maintenance, but this will do for a start.

    We also reissue CRLs for all CAs.

    Most likely this should be run under cron.
    """

    for model in (rpki.irdb.CA,
                  rpki.irdb.EECertificate,
                  rpki.irdb.BSC,
                  rpki.irdb.Child,
                  rpki.irdb.Parent,
                  rpki.irdb.Client,
                  rpki.irdb.Repository):
      for obj in model.all():
        print "Regenerating certificate", obj.certificate.getSubject()
        obj.avow()

    for ca in rpki.irdb.CA.all():
      print "Regenerating CRL for", ca.identity.handle, ca.purpose
      ca.generate_crl()

  def do_configure_child(self, arg):
    """
    Configure a new child of this RPKI entity, given the child's XML
    identity file as an input.  This command extracts the child's data
    from the XML, cross-certifies the child's resource-holding BPKI
    certificate, and generates an XML file describing the relationship
    between the child and this parent, including this parent's BPKI
    data and up-down protocol service URI.
    """

    child_handle = None

    opts, argv = getopt.getopt(arg.split(), "", ["child_handle="])
    for o, a in opts:
      if o == "--child_handle":
        child_handle = a
    
    if len(argv) != 1:
      raise BadCommandSyntax, "Need to specify filename for child.xml"

    c = etree_read(argv[0])

    if child_handle is None:
      child_handle = c.get("handle")

    service_uri = "http://%s:%s/up-down/%s/%s" % (self.cfg.get("rpkid_server_host"),
                                                  self.cfg.get("rpkid_server_port"),
                                                  self.handle, child_handle)

    valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 365)

    print "Child calls itself %r, we call it %r" % (c.get("handle"), child_handle)

    rpki.irdb.Child.objects.get_or_certify(
      issuer = self.resource_ca,
      handle = child_handle,
      ta = rpki.x509.X509(Base64 = c.findtext("bpki_ta")),
      valid_until = valid_until.toXMLtime())

    e = Element("parent", parent_handle = self.handle, child_handle = child_handle,
                service_uri = service_uri, valid_until = valid_until)
    B64Element(e, "bpki_resource_ta", self.resource_ca.certificate)
    SubElement(e, "bpki_child_ta").text = c.findtext("bpki_ta")

    try:
      repo = self.resource_ca.repositories.get(handle = self.default_repository)
    except rpki.irdb.Repository.DoesNotExist:
      try:
        repo = self.resource_ca.repositories[0]
      except rpki.irdb.Repository.DoesNotExist:
        repo = None

    if repo is None:
      print "Couldn't find any usable repositories, not giving referral"

    elif repo_handle == self.handle:
      SubElement(e, "repository", type = "offer")

    else:
      proposed_sia_base = repo.get("sia_base") + child_handle + "/"
      r = Element("referral", authorized_sia_base = proposed_sia_base)
      r.text = c.findtext("bpki_ta")
      auth = self.bpki_resources.cms_xml_sign(r)

      r = SubElement(e, "repository", type = "referral")
      SubElement(r, "authorization", referrer = repo.get("client_handle")).text = auth
      SubElement(r, "contact_info").text = repo.findtext("contact_info")

    etree_write(e, "parent-response-to-%s.xml" % child_handle,
                msg = "Send this file back to the child you just configured")


  def do_delete_child(self, arg):
    """
    Delete a child of this RPKI entity.
    """

    try:
      self.resource_ca.children.get(handle = arg).delete()
    except rpki.irdb.Child.DoesNotExist:
      print "No such child \"%s\"" % arg

  def complete_delete_child(self, *args):
    return self.irdb_handle_complete(rpki.irdb.Child, *args)


  def do_configure_parent(self, arg):
    """
    Configure a new parent of this RPKI entity, given the output of
    the parent's configure_child command as input.  This command reads
    the parent's response XML, extracts the parent's BPKI and service
    URI information, cross-certifies the parent's BPKI data into this
    entity's BPKI, and checks for offers or referrals of publication
    service.  If a publication offer or referral is present, we
    generate a request-for-service message to that repository, in case
    the user wants to avail herself of the referral or offer.
    """

    parent_handle = None

    opts, argv = getopt.getopt(arg.split(), "", ["parent_handle="])
    for o, a in opts:
      if o == "--parent_handle":
        parent_handle = a

    if len(argv) != 1:
      raise BadCommandSyntax, "Need to specify filename for parent.xml on command line"

    p = etree_read(argv[0])

    if parent_handle is None:
      parent_handle = p.get("parent_handle")

    r = p.find("repository")

    repository_type = "none"
    referrer = None
    referral_authorization = None

    if r is not None:
      repository_type = r.get("type")

    if repository_type == "referral":
      a = r.find("authorization")
      referrer = a.get("referrer")
      referral_authorization = rpki.x509.SignedReferral(Base64 = a.text)

    print "Parent calls itself %r, we call it %r" % (p.get("parent_handle"), parent_handle)
    print "Parent calls us %r" % p.get("child_handle")

    rpki.irdb.Parent.get_or_certify(
      issuer = self.resource_ca,
      handle = parent_handle,
      child_handle = p.get("child_handle"),
      parent_handle = p.get("parent_handle"),
      service_uri = p.get("service_uri"),
      ta = rpki.x509.X509(Base64 = p.findtext("bpki_resource_ta")),
      repository_type = repository_type,
      referrer = referrer,
      referral_authorization = referral_authorization)[0]

    if repository_type == "none":
      r = Element("repository", type = "none")
    r.set("handle", self.handle)
    r.set("parent_handle", parent_handle)
    B64Element(r, "bpki_client_ta", self.resource_ca.certificate)
    etree_write(r, "repository-request-for-%s.xml" % parent_handle,
                msg = "This is the file to send to the repository operator")


  def do_delete_parent(self, arg):
    """
    Delete a parent of this RPKI entity.
    """

    try:
      self.resource_ca.parents.get(handle = arg).delete()
    except rpki.irdb.Parent.DoesNotExist:
      print "No such parent \"%s\"" % arg

  def complete_delete_parent(self, *args):
    return self.irdb_handle_complete(rpki.irdb.Parent, *args)


  def do_configure_publication_client(self, arg):
    """
    Configure publication server to know about a new client, given the
    client's request-for-service message as input.  This command reads
    the client's request for service, cross-certifies the client's
    BPKI data, and generates a response message containing the
    repository's BPKI data and service URI.
    """

    sia_base = None

    opts, argv = getopt.getopt(arg.split(), "", ["sia_base="])
    for o, a in opts:
      if o == "--sia_base":
        sia_base = a
    
    if len(argv) != 1:
      raise BadCommandSyntax, "Need to specify filename for client.xml"

    client = etree_read(argv[0])

    client_ta = rpki.x509.X509(Base64 = client.findtext("bpki_client_ta"))

    if sia_base is None and client.get("handle") == self.handle and self.bpki_resources.ca.certificate == client_ta:
      print "This looks like self-hosted publication"
      sia_base = "rsync://%s/%s/%s/" % (self.rsync_server, self.rsync_module, self.handle)

    if sia_base is None and client.get("type") == "referral":
      print "This looks like a referral, checking"
      try:
        auth = client.find("authorization")
        referrer = self.resource_ca.clients.get(handle = auth.get("referrer"))
        referral_cms = rpki.x509.SignedReferral(Base64 = auth.text)
        referral_xml = referral_cms.unwrap(ta = (referrer.certificate, self.server_ca.certificate))
        if rpki.x509.X509(Base64 = referral_xml.text) != client_ta:
          raise BadXMLMessage, "Referral trust anchor does not match"
        sia_base = referral.get("authorized_sia_base")
      except rpki.irdb.Client.DoesNotExist:
        print "We have no record of the client (%s) alleged to have made this referral" % auth.get("referrer")

    if sia_base is None and client.get("type") == "offer" and client.get("parent_handle") == self.handle:
      print "This looks like an offer, client claims to be our child, checking"
      try:
        child = self.resource_ca.children.get(ta = client_ta)
      except rpki.irdb.Child.DoesNotExist:
        print "Can't find a child matching this client"
      else:
        sia_base = "rsync://%s/%s/%s/%s/" % (self.rsync_server, self.rsync_module,
                                             self.handle, client.get("handle"))

    # If we still haven't figured out what to do with this client, it
    # gets a top-level tree of its own, no attempt at nesting.

    if sia_base is None:
      print "Don't know where to nest this client, defaulting to top-level"
      sia_base = "rsync://%s/%s/%s/" % (self.rsync_server, self.rsync_module, client.get("handle"))
      
    if not sia_base.startswith("rsync://"):
      raise BadXMLMessage, "Malformed sia_base parameter %r, should start with 'rsync://'" % sia_base

    client_handle = "/".join(sia_base.rstrip("/").split("/")[4:])

    parent_handle = client.get("parent_handle")

    print "Client calls itself %r, we call it %r" % (client.get("handle"), client_handle)
    print "Client says its parent handle is %r" % parent_handle

    rpki.irdb.Client.get_or_certify(
      issuer = self.server_ca,
      handle = client_handle,
      ta = client_ta)

    e = Element("repository", type = "confirmed",
                client_handle = client_handle,
                parent_handle = parent_handle,
                sia_base = sia_base,
                service_uri = "http://%s:%s/client/%s" % (self.cfg.get("pubd_server_host"),
                                                          self.cfg.get("pubd_server_port"),
                                                          client_handle))

    B64Element(e, "bpki_server_ta", self.server_ca.certificate)
    B64Element(e, "bpki_client_ta", client_ta)
    SubElement(e, "contact_info").text = self.pubd_contact_info
    etree_write(e, "repository-response-to-%s.xml" % client_handle.replace("/", "."),
                msg = "Send this file back to the publication client you just configured")


  def do_delete_publication_client(self, arg):
    """
    Delete a publication client of this RPKI entity.
    """

    try:
      self.resource_ca.clients.get(handle = arg).delete()
    except rpki.irdb.Client.DoesNotExist:
      print "No such client \"%s\"" % arg

  def complete_delete_publication_client(self, *args):
    return self.irdb_handle_complete(rpki.irdb.Client, *args)


  def do_configure_repository(self, arg):
    """
    Configure a publication repository for this RPKI entity, given the
    repository's response to our request-for-service message as input.
    This command reads the repository's response, extracts and
    cross-certifies the BPKI data and service URI, and links the
    repository data with the corresponding parent data in our local
    database.
    """

    parent_handle = None

    opts, argv = getopt.getopt(arg.split(), "", ["parent_handle="])
    for o, a in opts:
      if o == "--parent_handle":
        parent_handle = a

    if len(argv) != 1:
      raise BadCommandSyntax, "Need to specify filename for repository.xml on command line"

    r = etree_read(argv[0])

    if parent_handle is None:
      parent_handle = r.get("parent_handle")

    print "Repository calls us %r" % (r.get("client_handle"))
    print "Repository response associated with parent_handle %r" % parent_handle

    try:
      parent = self.resource_ca.parents.get(handle = parent_handle)

    except rpki.irdb.Parent.DoesNotExist:
      print "Could not find parent %r in our database" % parent_handle

    else:
      rpki.irdb.Repository.get_or_certify(
        issuer = self.resource_ca,
        handle = parent_handle,
        client_handle = r.get("client_handle"),
        service_uri = r.get("service_uri"),
        sia_base = r.get("sia_base"),
        ta = rpki.x509.X509(Base64 = r.findtext("bpki_server_ta")),
        parent = parent)

  def do_delete_repository(self, arg):
    """
    Delete a repository of this RPKI entity.

    This should check that the XML file it's deleting really is a
    repository, but doesn't, yet.
    """

    try:
      self.resource_ca.repositories.get(handle = arg).delete()
    except rpki.irdb.Repository.DoesNotExist:
      print "No such repository \"%s\"" % arg

  def complete_delete_repository(self, *args):
    return self.irdb_handle_complete(rpki.irdb.Repository, *args)


  def renew_children_common(self, arg, plural):
    """
    Common code for renew_child and renew_all_children commands.
    """

    valid_until = None

    opts, argv = getopt.getopt(arg.split(), "", ["valid_until"])
    for o, a in opts:
      if o == "--valid_until":
        valid_until = a

    if plural:
      if len(argv) != 0:
        raise BadCommandSyntax, "Unexpected arguments"
      children = self.resource_ca.children
    else:
      if len(argv) != 1:
        raise BadCommandSyntax, "Need to specify child handle"
      children = self.resource_ca.children.filter(handle = argv[0])

    if valid_until is None:
      valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 365)
    else:
      valid_until = rpki.sundial.fromXMLtime(valid_until)
      if valid_until < rpki.sundial.now():
        raise PastExpiration, "Specified new expiration time %s has passed" % valid_until

    print "New validity date", valid_until

    for child in children:
      child.valid_until = valid_until
      child.save()

  def do_renew_child(self, arg):
    """
    Update validity period for one child entity.
    """

    return self.renew_children_common(arg, False)

  def complete_renew_child(self, *args):
    return self.irdb_handle_complete(rpki.irdb.Child, *args)

  def do_renew_all_children(self, arg):
    """
    Update validity period for all child entities.
    """

    return self.renew_children_common(arg, True)




  def configure_resources_main(self, msg = None):
    """
    Main program of old myrpki.py script.  This remains separate
    because it's called from more than one place.
    """

    roa_csv_file                  = self.cfg.get("roa_csv")
    prefix_csv_file               = self.cfg.get("prefix_csv")
    asn_csv_file                  = self.cfg.get("asn_csv")

    # This probably should become an argument instead of (or in
    # addition to a default from?) a config file option.
    xml_filename                  = self.cfg.get("xml_filename")

    try:
      e = etree_read(xml_filename)
      bsc = self.bpki_resources.bsc(e.findtext("bpki_bsc_pkcs10"))
      service_uri = e.get("service_uri")
    except IOError:
      bsc = None
      service_uri = None

    e = Element("myrpki", handle = self.handle)

    if service_uri:
      e.set("service_uri", service_uri)

    roa_requests.from_csv(roa_csv_file).xml(e)

    children.from_entitydb(
      prefix_csv_file = prefix_csv_file,
      asn_csv_file = asn_csv_file,
      fxcert = self.bpki_resources.fxcert,
      entitydb = self.entitydb).xml(e)

    parents.from_entitydb(
      fxcert = self.bpki_resources.fxcert,
      entitydb = self.entitydb).xml(e)

    repositories.from_entitydb(
      fxcert = self.bpki_resources.fxcert,
      entitydb = self.entitydb).xml(e)

    B64Element(e, "bpki_ca_certificate", self.bpki_resources.cer)
    B64Element(e, "bpki_crl",            self.bpki_resources.crl)

    if bsc is not None:
      B64Element(e, "bpki_bsc_certificate", bsc.certificate)
      B64Element(e, "bpki_bsc_pkcs10",      bsc.pkcs10)

    etree_write(e, xml_filename, msg = msg)


  def do_configure_resources(self, arg):
    """
    Read CSV files and all the descriptions of parents and children
    that we've built up, package the result up as a single XML file to
    be shipped to a hosting rpkid.
    """

    if arg:
      raise BadCommandSyntax, "Unexpected argument %r" % arg
    self.configure_resources_main(msg = "Send this file to the rpkid operator who is hosting you")



  def do_configure_daemons(self, arg):
    """
    Configure RPKI daemons with the data built up by the other
    commands in this program.

    The basic model here is that each entity with resources to certify
    runs the rpkic tool, but not all of them necessarily run their
    own RPKI engines.  The entities that do run RPKI engines get data
    from the entities they host via the XML files output by the
    configure_resources command.  Those XML files are the input to
    this command, which uses them to do all the work of configuring
    daemons, populating SQL databases, and so forth.  A few operations
    (eg, BSC construction) generate data which has to be shipped back
    to the resource holder, which we do by updating the same XML file.

    In essence, the XML files are a sneakernet (or email, or carrier
    pigeon) communication channel between the resource holders and the
    RPKI engine operators.

    As a convenience, for the normal case where the RPKI engine
    operator is itself a resource holder, this command in effect runs
    the configure_resources command automatically to process the RPKI
    engine operator's own resources.

    Note that, due to the back and forth nature of some of these
    operations, it may take several cycles for data structures to stablize
    and everything to reach a steady state.  This is normal.
    """

    argv = arg.split()

    def findbase64(tree, name, b64type = rpki.x509.X509):
      x = tree.findtext(name)
      return b64type(Base64 = x) if x else None

    # We can use a single BSC for everything -- except BSC key
    # rollovers.  Drive off that bridge when we get to it.

    bsc_handle = "bsc"

    self.cfg.set_global_flags()

    # Default values for CRL parameters are low, for testing.  Not
    # quite as low as they once were, too much expired CRL whining.

    self_crl_interval = self.cfg.getint("self_crl_interval", 2 * 60 * 60)
    self_regen_margin = self.cfg.getint("self_regen_margin", self_crl_interval / 4)
    pubd_base         = "http://%s:%s/" % (self.cfg.get("pubd_server_host"), self.cfg.get("pubd_server_port"))
    rpkid_base        = "http://%s:%s/" % (self.cfg.get("rpkid_server_host"), self.cfg.get("rpkid_server_port"))

    # Wrappers to simplify calling rpkid and pubd.

    call_rpkid = rpki.async.sync_wrapper(rpki.http.caller(
      proto       = rpki.left_right,
      client_key  = rpki.x509.RSA( PEM_file = self.bpki_servers.dir + "/irbe.key"),
      client_cert = rpki.x509.X509(PEM_file = self.bpki_servers.dir + "/irbe.cer"),
      server_ta   = rpki.x509.X509(PEM_file = self.bpki_servers.cer),
      server_cert = rpki.x509.X509(PEM_file = self.bpki_servers.dir + "/rpkid.cer"),
      url         = rpkid_base + "left-right",
      debug       = self.show_xml))

    if self.run_pubd:

      call_pubd = rpki.async.sync_wrapper(rpki.http.caller(
        proto       = rpki.publication,
        client_key  = rpki.x509.RSA( PEM_file = self.bpki_servers.dir + "/irbe.key"),
        client_cert = rpki.x509.X509(PEM_file = self.bpki_servers.dir + "/irbe.cer"),
        server_ta   = rpki.x509.X509(PEM_file = self.bpki_servers.cer),
        server_cert = rpki.x509.X509(PEM_file = self.bpki_servers.dir + "/pubd.cer"),
        url         = pubd_base + "control",
        debug       = self.show_xml))

      # Make sure that pubd's BPKI CRL is up to date.

      call_pubd(rpki.publication.config_elt.make_pdu(
        action = "set",
        bpki_crl = rpki.x509.CRL(PEM_file = self.bpki_servers.crl)))

    irdb = IRDB(self.cfg)

    xmlfiles = []

    # If [myrpki] section includes an "xml_filename" setting, run
    # myrpki.py internally, as a convenience, and include its output at
    # the head of our list of XML files to process.

    my_xmlfile = self.cfg.get("xml_filename", "")
    if my_xmlfile:
      self.configure_resources_main()
      xmlfiles.append(my_xmlfile)
    else:
      my_xmlfile = None

    # Add any other XML files specified on the command line

    xmlfiles.extend(argv)

    for xmlfile in xmlfiles:

      # Parse XML file and validate it against our scheme

      tree = etree_read(xmlfile)

      handle = tree.get("handle")

      # Update IRDB with parsed resource and roa-request data.

      roa_requests = [
        (x.get('asn'),
         rpki.resource_set.roa_prefix_set_ipv4(x.get("v4")),
         rpki.resource_set.roa_prefix_set_ipv6(x.get("v6")))
        for x in tree.getiterator("roa_request") ]

      children = [
        (x.get("handle"),
         rpki.resource_set.resource_set_as(x.get("asns")),
         rpki.resource_set.resource_set_ipv4(x.get("v4")),
         rpki.resource_set.resource_set_ipv6(x.get("v6")),
         rpki.sundial.datetime.fromXMLtime(x.get("valid_until")))
        for x in tree.getiterator("child") ]

      # ghostbusters are ignored for now
      irdb.update(handle, roa_requests, children)

      # Check for certificates before attempting anything else

      hosted_cacert = findbase64(tree, "bpki_ca_certificate")
      if not hosted_cacert:
        print "Nothing else I can do without a trust anchor for the entity I'm hosting."
        continue

      rpkid_xcert = rpki.x509.X509(PEM_file = self.bpki_servers.fxcert(
        b64 = hosted_cacert.get_Base64(),
        path_restriction = 1))

      # See what rpkid and pubd already have on file for this entity.

      if self.run_pubd:
        client_pdus = dict((x.client_handle, x)
                           for x in call_pubd(rpki.publication.client_elt.make_pdu(action = "list"))
                           if isinstance(x, rpki.publication.client_elt))

      rpkid_reply = call_rpkid(
        rpki.left_right.self_elt.make_pdu(      action = "get",  tag = "self",       self_handle = handle),
        rpki.left_right.bsc_elt.make_pdu(       action = "list", tag = "bsc",        self_handle = handle),
        rpki.left_right.repository_elt.make_pdu(action = "list", tag = "repository", self_handle = handle),
        rpki.left_right.parent_elt.make_pdu(    action = "list", tag = "parent",     self_handle = handle),
        rpki.left_right.child_elt.make_pdu(     action = "list", tag = "child",      self_handle = handle))

      self_pdu        = rpkid_reply[0]
      bsc_pdus        = dict((x.bsc_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.bsc_elt))
      repository_pdus = dict((x.repository_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.repository_elt))
      parent_pdus     = dict((x.parent_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.parent_elt))
      child_pdus      = dict((x.child_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.child_elt))

      pubd_query = []
      rpkid_query = []

      # There should be exactly one <self/> object per hosted entity, by definition

      if (isinstance(self_pdu, rpki.left_right.report_error_elt) or
          self_pdu.crl_interval != self_crl_interval or
          self_pdu.regen_margin != self_regen_margin or
          self_pdu.bpki_cert != rpkid_xcert):
        rpkid_query.append(rpki.left_right.self_elt.make_pdu(
          action = "create" if isinstance(self_pdu, rpki.left_right.report_error_elt) else "set",
          tag = "self",
          self_handle = handle,
          bpki_cert = rpkid_xcert,
          crl_interval = self_crl_interval,
          regen_margin = self_regen_margin))

      # In general we only need one <bsc/> per <self/>.  BSC objects are a
      # little unusual in that the PKCS #10 subelement is generated by rpkid
      # in response to generate_keypair, so there's more of a separation
      # between create and set than with other objects.

      bsc_cert = findbase64(tree, "bpki_bsc_certificate")
      bsc_crl  = findbase64(tree, "bpki_crl", rpki.x509.CRL)

      bsc_pdu = bsc_pdus.pop(bsc_handle, None)

      if bsc_pdu is None:
        rpkid_query.append(rpki.left_right.bsc_elt.make_pdu(
          action = "create",
          tag = "bsc",
          self_handle = handle,
          bsc_handle = bsc_handle,
          generate_keypair = "yes"))
      elif bsc_pdu.signing_cert != bsc_cert or bsc_pdu.signing_cert_crl != bsc_crl:
        rpkid_query.append(rpki.left_right.bsc_elt.make_pdu(
          action = "set",
          tag = "bsc",
          self_handle = handle,
          bsc_handle = bsc_handle,
          signing_cert = bsc_cert,
          signing_cert_crl = bsc_crl))

      rpkid_query.extend(rpki.left_right.bsc_elt.make_pdu(
        action = "destroy", self_handle = handle, bsc_handle = b) for b in bsc_pdus)

      bsc_req = None

      if bsc_pdu and bsc_pdu.pkcs10_request:
        bsc_req = bsc_pdu.pkcs10_request

      # At present we need one <repository/> per <parent/>, not because
      # rpkid requires that, but because pubd does.  pubd probably should
      # be fixed to support a single client allowed to update multiple
      # trees, but for the moment the easiest way forward is just to
      # enforce a 1:1 mapping between <parent/> and <repository/> objects

      for repository in tree.getiterator("repository"):

        repository_handle = repository.get("handle")
        repository_pdu = repository_pdus.pop(repository_handle, None)
        repository_uri = repository.get("service_uri")
        repository_cert = findbase64(repository, "bpki_certificate")

        if (repository_pdu is None or
            repository_pdu.bsc_handle != bsc_handle or
            repository_pdu.peer_contact_uri != repository_uri or
            repository_pdu.bpki_cert != repository_cert):
          rpkid_query.append(rpki.left_right.repository_elt.make_pdu(
            action = "create" if repository_pdu is None else "set",
            tag = repository_handle,
            self_handle = handle,
            repository_handle = repository_handle,
            bsc_handle = bsc_handle,
            peer_contact_uri = repository_uri,
            bpki_cert = repository_cert))

      rpkid_query.extend(rpki.left_right.repository_elt.make_pdu(
        action = "destroy", self_handle = handle, repository_handle = r) for r in repository_pdus)

      # <parent/> setup code currently assumes 1:1 mapping between
      # <repository/> and <parent/>, and further assumes that the handles
      # for an associated pair are the identical (that is:
      # parent.repository_handle == parent.parent_handle).

      for parent in tree.getiterator("parent"):

        parent_handle = parent.get("handle")
        parent_pdu = parent_pdus.pop(parent_handle, None)
        parent_uri = parent.get("service_uri")
        parent_myhandle = parent.get("myhandle")
        parent_sia_base = parent.get("sia_base")
        parent_cms_cert = findbase64(parent, "bpki_cms_certificate")

        if (parent_pdu is None or
            parent_pdu.bsc_handle != bsc_handle or
            parent_pdu.repository_handle != parent_handle or
            parent_pdu.peer_contact_uri != parent_uri or
            parent_pdu.sia_base != parent_sia_base or
            parent_pdu.sender_name != parent_myhandle or
            parent_pdu.recipient_name != parent_handle or
            parent_pdu.bpki_cms_cert != parent_cms_cert):
          rpkid_query.append(rpki.left_right.parent_elt.make_pdu(
            action = "create" if parent_pdu is None else "set",
            tag = parent_handle,
            self_handle = handle,
            parent_handle = parent_handle,
            bsc_handle = bsc_handle,
            repository_handle = parent_handle,
            peer_contact_uri = parent_uri,
            sia_base = parent_sia_base,
            sender_name = parent_myhandle,
            recipient_name = parent_handle,
            bpki_cms_cert = parent_cms_cert))

      rpkid_query.extend(rpki.left_right.parent_elt.make_pdu(
        action = "destroy", self_handle = handle, parent_handle = p) for p in parent_pdus)

      # Children are simpler than parents, because they call us, so no URL
      # to construct and figuring out what certificate to use is their
      # problem, not ours.

      for child in tree.getiterator("child"):

        child_handle = child.get("handle")
        child_pdu = child_pdus.pop(child_handle, None)
        child_cert = findbase64(child, "bpki_certificate")

        if (child_pdu is None or
            child_pdu.bsc_handle != bsc_handle or
            child_pdu.bpki_cert != child_cert):
          rpkid_query.append(rpki.left_right.child_elt.make_pdu(
            action = "create" if child_pdu is None else "set",
            tag = child_handle,
            self_handle = handle,
            child_handle = child_handle,
            bsc_handle = bsc_handle,
            bpki_cert = child_cert))

      rpkid_query.extend(rpki.left_right.child_elt.make_pdu(
        action = "destroy", self_handle = handle, child_handle = c) for c in child_pdus)

      # Publication setup.

      if self.run_pubd:

        for f in self.entitydb.iterate("pubclients"):
          c = etree_read(f)

          client_handle = c.get("client_handle")
          client_base_uri = c.get("sia_base")
          client_bpki_cert = rpki.x509.X509(PEM_file = self.bpki_servers.fxcert(
            b64 = c.findtext("bpki_client_ta"),
            handle = client_handle,
            bpki_type = rpki.irdb.Client))
          client_pdu = client_pdus.pop(client_handle, None)

          if (client_pdu is None or
              client_pdu.base_uri != client_base_uri or
              client_pdu.bpki_cert != client_bpki_cert):
            pubd_query.append(rpki.publication.client_elt.make_pdu(
              action = "create" if client_pdu is None else "set",
              client_handle = client_handle,
              bpki_cert = client_bpki_cert,
              base_uri = client_base_uri))

        pubd_query.extend(rpki.publication.client_elt.make_pdu(
            action = "destroy", client_handle = p) for p in client_pdus)

      # If we changed anything, ship updates off to daemons

      failed = False

      if rpkid_query:
        rpkid_reply = call_rpkid(*rpkid_query)
        bsc_pdus = dict((x.bsc_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.bsc_elt))
        if bsc_handle in bsc_pdus and bsc_pdus[bsc_handle].pkcs10_request:
          bsc_req = bsc_pdus[bsc_handle].pkcs10_request
        for r in rpkid_reply:
          if isinstance(r, rpki.left_right.report_error_elt):
            failed = True
            print "rpkid reported failure:", r.error_code
            if r.error_text:
              print r.error_text

      if failed:
        raise CouldntTalkToDaemon

      if pubd_query:
        assert self.run_pubd
        pubd_reply = call_pubd(*pubd_query)
        for r in pubd_reply:
          if isinstance(r, rpki.publication.report_error_elt):
            failed = True
            print "pubd reported failure:", r.error_code
            if r.error_text:
              print r.error_text
            
      if failed:
        raise CouldntTalkToDaemon

      # Rewrite XML.

      e = tree.find("bpki_bsc_pkcs10")
      if e is not None:
        tree.remove(e)
      if bsc_req is not None:
        SubElement(tree, "bpki_bsc_pkcs10").text = bsc_req.get_Base64()

      tree.set("service_uri", rpkid_base + "up-down/" + handle)

      etree_write(tree, xmlfile,
                  msg = None if xmlfile is my_xmlfile else 'Send this file back to the hosted entity ("%s")' % handle)

    irdb.close()

    # We used to run event loop again to give TLS connections a chance to shut down cleanly.
    # Seems not to be needed (and sometimes hangs forever, which is odd) with TLS out of the picture.
    #rpki.async.event_loop()