Revoke certificates and clean up published objects when processing a

left-right destroy action.  Clean up empty publication directories on
withdrawal. Fix rootd's handling of CRLs and manifests.  See #197.

svn path=/trunk/; revision=4354

5 files changed, 110 insertions, 58 deletions

diff --git a/rpkid/rpki/http.py b/rpkid/rpki/http.py
index d8afd44c..0df7e6f2 100644
--- a/rpkid/rpki/http.py
+++ b/rpkid/rpki/http.py
@@ -766,7 +766,14 @@ class http_client(http_stream):
       self.update_timeout()
 
     if self.msg.code != 200:
-      raise rpki.exceptions.HTTPRequestFailed, "HTTP request failed with status %s, reason %s, response %s" % (self.msg.code, self.msg.reason, self.msg.body)
+      errmsg = "HTTP request failed"
+      if self.msg.code is not None:
+        errmsg += " with status %s" % self.msg.code
+      if self.msg.reason:
+        errmsg += ", reason %s" % self.msg.reason
+      if self.msg.body:
+        errmsg += ", response %s" % self.msg.body
+      raise rpki.exceptions.HTTPRequestFailed(errmsg)
     self.queue.return_result(self, self.msg, detach = self.expect_close)
 
   def handle_close(self):
diff --git a/rpkid/rpki/left_right.py b/rpkid/rpki/left_right.py
index 7d056b7b..7cb18f8c 100644
--- a/rpkid/rpki/left_right.py
+++ b/rpkid/rpki/left_right.py
@@ -253,18 +253,9 @@ class self_elt(data_elt):
     """
     Extra cleanup actions when destroying a self_elt.
     """
-
+    rpki.log.trace()
     def loop(iterator, parent):
-
-      def revoked_forgotten():
-        parent.delete(iterator)
-
-      def revoke_forgotten_failed(e):
-        rpki.log.warn("Couldn't revoke forgotten certificates, blundering onwards: %s" % e)
-        revoked_forgotten()
-
-      parent.serve_revoke_forgotten(revoked_forgotten, revoke_forgotten_failed)
-
+      parent.delete(iterator)
     rpki.async.iterator(self.parents, loop, cb)
 
 
@@ -952,6 +943,39 @@ class parent_elt(data_elt):
       ca.reissue(cb = iterator, eb = eb)
     rpki.async.iterator(self.cas, loop, cb)
 
+
+  def get_skis(self, cb, eb):
+    """
+    Fetch SKIs that this parent thinks we have.  In theory this should
+    agree with our own database, but in practice stuff can happen, so
+    sometimes we need to know what our parent thinks.
+
+    Result is a dictionary with the resource class name as key and a
+    set of SKIs as value.
+    """
+
+    def done(r_msg):
+      cb(dict((rc.class_name, set(c.cert.gSKI() for c in rc.certs))
+              for rc in r_msg.payload.classes))
+
+    rpki.up_down.list_pdu.query(self, done, eb)
+
+
+  def revoke_skis(self, rc_name, skis_to_revoke, cb, eb):
+    """
+    Revoke a set of SKIs within a particular resource class.
+    """
+
+    def loop(iterator, ski):
+      rpki.log.debug("Asking parent %r to revoke class %r, SKI %s" % (self, rc_name, ski))
+      q_pdu = rpki.up_down.revoke_pdu()
+      q_pdu.class_name = rc_name
+      q_pdu.ski = ski
+      self.query_up_down(q_pdu, lambda r_pdu: iterator(), eb)
+
+    rpki.async.iterator(skis_to_revoke, loop, cb)
+
+
   def serve_revoke_forgotten(self, cb, eb):
     """
     Handle a left-right revoke_forgotten action for this parent.
@@ -966,30 +990,19 @@ class parent_elt(data_elt):
     require an explicit trigger.
     """
 
-    def got_list(r_msg):
-
-      ca_map = dict((ca.parent_resource_class, ca) for ca in self.cas)
-
-      def rc_loop(rc_iterator, rc):
-
-        if rc.class_name in ca_map:
-
-          def ski_loop(ski_iterator, ski):
-            rpki.log.warn("Revoking certificates missing from our database, class %r, SKI %s" % (rc.class_name, ski))
-            rpki.up_down.revoke_pdu.query(ca, ski, lambda x: ski_iterator(), eb)
+    def got_skis(skis_from_parent):
 
-          ca = ca_map[rc.class_name]
-          skis_parent_knows_about = set(c.cert.gSKI() for c in rc.certs)
-          skis_ca_knows_about = set(ca_detail.latest_ca_cert.gSKI() for ca_detail in ca.issue_response_candidate_ca_details)
-          skis_only_parent_knows_about = skis_parent_knows_about - skis_ca_knows_about
-          rpki.async.iterator(skis_only_parent_knows_about, ski_loop, rc_iterator)
+      def loop(iterator, item):
+        rc_name, skis_to_revoke = item
+        if rc_name in ca_map:
+          for ca_detail in ca_map[rc_name].issue_response_candidate_ca_details:
+            skis_to_revoke.discard(ca_detail.latest_ca_cert.gSKI())
+        self.revoke_skis(rc_name, skis_to_revoke, iterator, eb)
 
-        else:
-          rc_iterator()
-
-      rpki.async.iterator(r_msg.payload.classes, rc_loop, cb)
+      ca_map = dict((ca.parent_resource_class, ca) for ca in self.cas)
+      rpki.async.iterator(skis_from_parent.items(), loop, cb)
 
-    rpki.up_down.list_pdu.query(self, got_list, eb)
+    self.get_skis(got_skis, eb)
 
 
   def delete(self, cb, delete_parent = True):
@@ -999,22 +1012,25 @@ class parent_elt(data_elt):
     """
 
     def loop(iterator, ca):
+      self.gctx.checkpoint()
+      ca.delete(self, iterator)
 
-      def revoked():
-        ca.delete(self, iterator)
-
-      def revoke_failed(e):
-        rpki.log.warn("Couldn't revoke CA certificate, blundering onwards: %s" %  e)
-        revoked()
+    def revoke():
+      self.gctx.checkpoint()
+      self.serve_revoke_forgotten(done, fail)
 
-      ca.revoke(revoked, revoke_failed, revoke_all = True)
+    def fail(e):
+      rpki.log.warn("Trouble getting parent to revoke certificates, blundering onwards: %s" % e)
+      done()
 
     def done():
+      self.gctx.checkpoint()
+      self.gctx.sql.sweep()
       if delete_parent:
         self.sql_delete()
       cb()
 
-    rpki.async.iterator(self.cas, loop, done)
+    rpki.async.iterator(self.cas, loop, revoke)
 
 
   def serve_destroy_hook(self, cb, eb):
diff --git a/rpkid/rpki/publication.py b/rpkid/rpki/publication.py
index 14e3d36a..f60e3af5 100644
--- a/rpkid/rpki/publication.py
+++ b/rpkid/rpki/publication.py
@@ -219,7 +219,7 @@ class publication_object_elt(rpki.xml_utils.base_elt, publication_namespace):
 
   def serve_withdraw(self):
     """
-    Withdraw an object.
+    Withdraw an object, then recursively delete empty directories.
     """
     rpki.log.info("Withdrawing %s" % self.uri)
     filename = self.uri_to_filename()
@@ -230,6 +230,15 @@ class publication_object_elt(rpki.xml_utils.base_elt, publication_namespace):
         raise rpki.exceptions.NoObjectAtURI, "No object published at %s" % self.uri
       else:
         raise
+    min_path_len = len(self.gctx.publication_base.rstrip("/"))
+    dirname = os.path.dirname(filename)
+    while len(dirname) > min_path_len:
+      try:
+        os.rmdir(dirname)
+      except OSError:
+        break
+      else:
+        dirname = os.path.dirname(dirname)
 
   def uri_to_filename(self):
     """
diff --git a/rpkid/rpki/rootd.py b/rpkid/rpki/rootd.py
index 5e26d41c..6b774eef 100644
--- a/rpkid/rpki/rootd.py
+++ b/rpkid/rpki/rootd.py
@@ -58,11 +58,15 @@ class issue_pdu(rpki.up_down.issue_pdu):
 
 class revoke_pdu(rpki.up_down.revoke_pdu):
   def serve_pdu(self, q_msg, r_msg, ignored, callback, errback):
-    rootd.subject_cert = get_subject_cert()
+    rpki.log.debug("Revocation requested for SKI %s" % self.ski)
+    subject_cert = rootd.get_subject_cert()
     if subject_cert is None or subject_cert.gSKI() != self.ski:
       raise rpki.exceptions.NotInDatabase
+    now = rpki.sundial.now()
+    rootd.revoke_subject_cert(now)
     rootd.del_subject_cert()
     rootd.del_subject_pkcs10()
+    rootd.generate_crl_and_manifest(now)
     r_msg.payload = rpki.up_down.revoke_response_pdu()
     r_msg.payload.class_name = self.class_name
     r_msg.payload.ski = self.ski
@@ -95,8 +99,6 @@ class cms_msg(rpki.up_down.cms_msg):
 
 class main(object):
 
-  rpki_root_cert = None
-
   def get_root_cert(self):
     rpki.log.debug("Read root cert %s" % self.rpki_root_cert_file)
     self.rpki_root_cert = rpki.x509.X509(Auto_file = self.rpki_root_cert_file)
@@ -172,54 +174,65 @@ class main(object):
     rpki.log.info("Generating subject cert with resources " + str(resources))
     req_key = pkcs10.getPublicKey()
     req_sia = pkcs10.get_SIA()
-    crldp = self.rpki_base_uri + self.rpki_root_crl
-    serial = now.totimestamp()
+    self.serial_number += 1
     subject_cert = self.rpki_root_cert.issue(
       keypair     = self.rpki_root_key,
       subject_key = req_key,
-      serial      = serial,
+      serial      = self.serial_number,
       sia         = req_sia,
       aia         = self.rpki_root_cert_uri,
-      crldp       = crldp,
+      crldp       = self.rpki_base_uri + self.rpki_root_crl,
       resources   = resources,
       notAfter    = now + self.rpki_subject_lifetime)
+    self.set_subject_cert(subject_cert)
+    self.generate_crl_and_manifest(now)
+    return subject_cert
+
+  def generate_crl_and_manifest(self, now):
+    subject_cert = self.get_subject_cert()
+    self.serial_number += 1
+    self.crl_number += 1
     crl = rpki.x509.CRL.generate(
       keypair             = self.rpki_root_key,
       issuer              = self.rpki_root_cert,
-      serial              = serial,
+      serial              = self.crl_number,
       thisUpdate          = now,
       nextUpdate          = now + self.rpki_subject_lifetime,
-      revokedCertificates = ())
+      revokedCertificates = self.revoked)
     rpki.log.debug("Writing CRL %s" % (self.rpki_root_dir + self.rpki_root_crl))
     f = open(self.rpki_root_dir + self.rpki_root_crl, "wb")
     f.write(crl.get_DER())
     f.close()
+    manifest_content = [(self.rpki_root_crl, crl)]
+    if subject_cert is not None:
+      manifest_content.append((self.rpki_subject_cert, subject_cert))
     manifest_resources = rpki.resource_set.resource_bag.from_inheritance()
     manifest_keypair = rpki.x509.RSA.generate()
     manifest_cert = self.rpki_root_cert.issue(
       keypair     = self.rpki_root_key,
       subject_key = manifest_keypair.get_RSApublic(),
-      serial      = serial + 1,
+      serial      = self.serial_number,
       sia         = ((rpki.oids.name2oid["id-ad-signedObject"],
                       ("uri", self.rpki_base_uri + self.rpki_root_manifest)),),
       aia         = self.rpki_root_cert_uri,
-      crldp       = crldp,
+      crldp       = self.rpki_base_uri + self.rpki_root_crl,
       resources   = manifest_resources,
       notAfter    = now + self.rpki_subject_lifetime,
       is_ca       = False)
     manifest = rpki.x509.SignedManifest.build(
-      serial         = serial,
+      serial         = self.crl_number,
       thisUpdate     = now,
       nextUpdate     = now + self.rpki_subject_lifetime,
-      names_and_objs = [(self.rpki_subject_cert, subject_cert), (self.rpki_root_crl, crl)],
+      names_and_objs = manifest_content,
       keypair        = manifest_keypair,
       certs          = manifest_cert)
     rpki.log.debug("Writing manifest %s" % (self.rpki_root_dir + self.rpki_root_manifest))
     f = open(self.rpki_root_dir + self.rpki_root_manifest, "wb")
     f.write(manifest.get_DER())
     f.close()
-    self.set_subject_cert(subject_cert)
-    return subject_cert
+
+  def revoke_subject_cert(self, now):
+    self.revoked.append((self.get_subject_cert().getSerial(), now.toASN1tuple(), ()))
 
   def compose_response(self, r_msg, pkcs10 = None):
     subject_cert = self.issue_subject_cert_maybe(pkcs10)
@@ -265,6 +278,11 @@ class main(object):
     global rootd
     rootd = self                        # Gross, but simpler than what we'd have to do otherwise
 
+    self.rpki_root_cert = None
+    self.serial_number = 0
+    self.crl_number = 0
+    self.revoked = []
+
     os.environ["TZ"] = "UTC"
     time.tzset()
 
@@ -286,6 +304,8 @@ class main(object):
 
     self.cfg = rpki.config.parser(self.cfg_file, "rootd")
 
+    rpki.log.enable_tracebacks = True
+
     self.cfg.set_global_flags()
 
     self.bpki_ta                 = rpki.x509.X509(Auto_update = self.cfg.get("bpki-ta"))
diff --git a/rpkid/rpki/rpkid.py b/rpkid/rpki/rpkid.py
index e2530813..01b37988 100644
--- a/rpkid/rpki/rpkid.py
+++ b/rpkid/rpki/rpkid.py
@@ -1158,7 +1158,7 @@ class child_cert_obj(rpki.sql.sql_persistent):
     """
     return self.ca_detail.ca.sia_uri + self.uri_tail
 
-  def revoke(self, publisher, generate_crl_and_manifest = False):
+  def revoke(self, publisher, generate_crl_and_manifest = True):
     """
     Revoke a child cert.
     """