rootd with publciation protocol mostly working with modern IRDB.

svn path=/branches/tk705/; revision=5913

6 files changed, 109 insertions, 110 deletions

diff --git a/ca/rpki-confgen.xml b/ca/rpki-confgen.xml
index 7eb62111..864da8b2 100644
--- a/ca/rpki-confgen.xml
+++ b/ca/rpki-confgen.xml
@@ -186,20 +186,6 @@
       </doc>
     </option>
 
-    <option name  = "publication_root_cert_directory"
-            value = "${myrpki::publication_base_directory}.root">
-      <doc>
-        Root of local directory tree where rootd (sigh) should write out
-        published data.  This is just like publication_base_directory, but
-        rootd is too dumb to use pubd and needs its own directory in
-        which to write one certificate, one CRL, and one manifest.
-        Neither rootd nor rsyncd much cares //where// you tell them to put
-        this stuff, the important thing is that the rsync URIs in
-        generated certificates match up with the published objects so that
-        relying parties can find and verify rootd's published outputs.
-      </doc>
-    </option>
-
     <option name  = "publication_rsync_module"
             value = "rpki">
       <doc>
@@ -209,15 +195,6 @@
       </doc>
     </option>
 
-    <option name  = "publication_root_module"
-            value = "root">
-      <doc>
-        rsyncd module name corresponding to publication_root_cert_directory.
-        This has to match the module you configured into `rsyncd.conf`.
-        Leave this alone unless you have some need to change it.
-      </doc>
-    </option>
-
     <option name  = "publication_rsync_server"
             value = "${myrpki::pubd_server_host}">
       <doc>
@@ -720,6 +697,14 @@
       </doc>
     </option>
 
+    <option name  = "pubd-bpki-cert"
+            value = "${myrpki::bpki_servers_directory}/pubd.cer">
+      <doc>
+        BPKI certificate for pubd.  Don't change this unless you
+	really know what you are doing.
+      </doc>
+    </option>
+
     <option name  = "server-host"
             value = "${myrpki::rootd_server_host}">
       <doc>
@@ -737,19 +722,23 @@
     <option name  = "rpki_data_dir"
             value = "${myrpki::bpki_servers_directory}">
       <doc>
-        Directory where rootd should store its RPKI data files.
+        Directory where rootd should store its RPKI data files.  This
+        is only used to construct other variables, rootd itself
+        doesn't read it.
       </doc>
     </option>
 
     <option name  = "rpki_base_uri"
-            value = "rsync://${myrpki::publication_rsync_server}/${myrpki::publication_rsync_module}/">
+            value = "rsync://${myrpki::publication_rsync_server}/${myrpki::publication_rsync_module}/${myrpki::handle}-root/root">
       <doc>
-        rsync URI corresponding to directory containing rootd's outputs.
+        rsync URI corresponding to directory containing rootd's
+        outputs.  This is only used to construct other variables,
+        rootd itself doesn't read it.
       </doc>
     </option>
 
     <option name  = "rpki-root-cert-uri"
-            value = "rsync://${myrpki::publication_rsync_server}/${myrpki::publication_root_module}/root.cer">
+            value = "${rootd::rpki_base_uri}.cer">
       <doc>
         rsync URI for rootd's root (self-signed) RPKI certificate.
       </doc>
@@ -758,8 +747,7 @@
     <option name  = "rpki-root-cert-file"
             value = "${rootd::rpki_data_dir}/root.cer">
       <doc>
-        Filename (as opposed to rsync URI) of rootd's root RPKI
-        certificate.
+        Filename of rootd's root RPKI certificate.
       </doc>
     </option>
 
@@ -794,8 +782,7 @@
     <option name  = "rpki-root-manifest-file"
             value = "${rootd::rpki_data_dir}/root.mft">
       <doc>
-        Filename (relative to rootd-base-uri and rpki-root-dir) of the
-        manifest for rootd's root RPKI certificate.
+        Filename of the manifest for rootd's root RPKI certificate.
       </doc>
     </option>
 
@@ -836,6 +823,13 @@
       </doc>
     </option>
 
+    <option name  = "pubd-contact-uri"
+	    value = "http://${myrpki::pubd_server_host}:${myrpki::pubd_server_port}/client/${myrpki::handle}-root">
+      <doc>
+	URI at which rootd should contact pubd for service.
+      </doc>
+    </option>
+
   </section>
 
   <section name = "web_portal">
diff --git a/ca/tests/yamlconf.py b/ca/tests/yamlconf.py
index 1b339a89..8f956c6b 100644
--- a/ca/tests/yamlconf.py
+++ b/ca/tests/yamlconf.py
@@ -530,15 +530,15 @@ class allocation(object):
       notAfter    = rpki.sundial.now() + rpki.sundial.timedelta(days = 365),
       resources   = root_resources)
 
-    with open(self.path("publication.root", "root.cer"), "wb") as f:
+    with open(self.path("root.cer"), "wb") as f:
       f.write(root_cert.get_DER())
 
     with open(self.path("root.key"), "wb") as f:
       f.write(root_key.get_DER())
 
     with open(cleanpath(test_dir, "root.tal"), "w") as f:
-      f.write("rsync://%s/root/root.cer\n\n%s" % (
-        self.rsync_server, root_key.get_public().get_Base64()))
+      f.write("rsync://%s/root/root.cer\n\n" % self.rsync_server)
+      f.write(root_key.get_public().get_Base64())
 
   def mkdir(self, *path):
     path = self.path(*path)
diff --git a/ca/tests/yamltest.py b/ca/tests/yamltest.py
index 0d427cf3..0afa172b 100644
--- a/ca/tests/yamltest.py
+++ b/ca/tests/yamltest.py
@@ -367,10 +367,9 @@ class allocation(object):
 
     fn = "%s.asns.csv" % d.name
     if not args.skip_config:
-      f = self.csvout(fn)
-      for k in self.kids:
-        f.writerows((k.name, a) for a in k.resources.asn)
-      f.close()
+      with self.csvout(fn) as f:
+        for k in self.kids:
+          f.writerows((k.name, a) for a in k.resources.asn)
     if not args.stop_after_config:
       self.run_rpkic("load_asns", fn)
 
@@ -381,10 +380,9 @@ class allocation(object):
 
     fn = "%s.prefixes.csv" % d.name
     if not args.skip_config:
-      f = self.csvout(fn)
-      for k in self.kids:
-        f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6))
-      f.close()
+      with self.csvout(fn) as f:
+        for k in self.kids:
+          f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6))
     if not args.stop_after_config:
       self.run_rpkic("load_prefixes", fn)
 
@@ -395,11 +393,10 @@ class allocation(object):
 
     fn = "%s.roas.csv" % d.name
     if not args.skip_config:
-      f = self.csvout(fn)
-      for g1, r in enumerate(self.roa_requests):
-        f.writerows((p, r.asn, "G%08d%08d" % (g1, g2))
-                    for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ())))
-      f.close()
+      with self.csvout(fn) as f:
+        for g1, r in enumerate(self.roa_requests):
+          f.writerows((p, r.asn, "G%08d%08d" % (g1, g2))
+                      for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ())))
     if not args.stop_after_config:
       self.run_rpkic("load_roa_requests", fn)
 
@@ -413,12 +410,8 @@ class allocation(object):
       if not args.skip_config:
         path = self.path(fn)
         print "Writing", path
-        f = open(path, "w")
-        for i, g in enumerate(self.ghostbusters):
-          if i:
-            f.write("\n")
-          f.write(g)
-        f.close()
+        with open(path, "w") as f:
+          f.write("\n".join(self.ghostbusters))
       if not args.stop_after_config:
         self.run_rpkic("load_ghostbuster_requests", fn)
 
@@ -507,22 +500,20 @@ class allocation(object):
 
     r.update(config_overrides)
 
-    f = open(self.path("rpki.conf"), "w")
-    f.write("# Automatically generated, do not edit\n")
-    print "Writing", f.name
-
-    section = None
-    for line in open(cleanpath(rpkid_dir, "examples/rpki.conf")):
-      m = section_regexp.match(line)
-      if m:
-        section = m.group(1)
-      m = variable_regexp.match(line)
-      option = m.group(1) if m and section == "myrpki" else None
-      if option and option in r:
-        line = "%s = %s\n" % (option, r[option])
-      f.write(line)
+    with open(self.path("rpki.conf"), "w") as f:
+      f.write("# Automatically generated, do not edit\n")
+      print "Writing", f.name
 
-    f.close()
+      section = None
+      for line in open(cleanpath(rpkid_dir, "examples/rpki.conf")):
+        m = section_regexp.match(line)
+        if m:
+          section = m.group(1)
+        m = variable_regexp.match(line)
+        option = m.group(1) if m and section == "myrpki" else None
+        if option and option in r:
+          line = "%s = %s\n" % (option, r[option])
+        f.write(line)
 
   def dump_rsyncd(self):
     """
@@ -530,25 +521,24 @@ class allocation(object):
     """
 
     if self.runs_pubd:
-      f = open(self.path("rsyncd.conf"), "w")
-      print "Writing", f.name
-      f.writelines(s + "\n" for s in
-                   ("# Automatically generated, do not edit",
-                    "port         = %d"           % self.rsync_port,
-                    "address      = localhost",
-                    "[rpki]",
-                    "log file     = rsyncd.log",
-                    "read only    = yes",
-                    "use chroot   = no",
-                    "path         = %s"           % self.path("publication"),
-                    "comment      = RPKI test",
-                    "[root]",
-                    "log file     = rsyncd_root.log",
-                    "read only    = yes",
-                    "use chroot   = no",
-                    "path         = %s"           % self.path("publication.root"),
-                    "comment      = RPKI test root"))
-      f.close()
+      with open(self.path("rsyncd.conf"), "w") as f:
+        print "Writing", f.name
+        f.writelines(s + "\n" for s in
+                     ("# Automatically generated, do not edit",
+                      "port         = %d"           % self.rsync_port,
+                      "address      = localhost",
+                      "[rpki]",
+                      "log file     = rsyncd.log",
+                      "read only    = yes",
+                      "use chroot   = no",
+                      "path         = %s"           % self.path("publication"),
+                      "comment      = RPKI test",
+                      "[root]",
+                      "log file     = rsyncd_root.log",
+                      "read only    = yes",
+                      "use chroot   = no",
+                      "path         = %s"           % self.path("publication.root"),
+                      "comment      = RPKI test root"))
 
   @classmethod
   def next_rpkic_counter(cls):
@@ -636,9 +626,9 @@ def create_root_certificate(db_root):
 
   root_key = rpki.x509.RSA.generate(quiet = True)
 
-  root_uri = "rsync://localhost:%d/rpki/" % db_root.pubd.rsync_port
+  root_uri = "rsync://localhost:%d/rpki/%s-root/root" % (db_root.pubd.rsync_port, db_root.name)
 
-  root_sia = (root_uri, root_uri + "root.mft", None)
+  root_sia = (root_uri + "/", root_uri + "/root.mft", None)
 
   root_cert = rpki.x509.X509.self_certify(
     keypair     = root_key,
@@ -648,18 +638,15 @@ def create_root_certificate(db_root):
     notAfter    = rpki.sundial.now() + rpki.sundial.timedelta(days = 365),
     resources   = root_resources)
 
-  f = open(db_root.path("publication.root/root.cer"), "wb")
-  f.write(root_cert.get_DER())
-  f.close()
+  with open(db_root.path("root.cer"), "wb") as f:
+    f.write(root_cert.get_DER())
 
-  f = open(db_root.path("root.key"), "wb")
-  f.write(root_key.get_DER())
-  f.close()
+  with open(db_root.path("root.key"), "wb") as f:
+    f.write(root_key.get_DER())
 
-  f = open(os.path.join(test_dir, "root.tal"), "w")
-  f.write("rsync://localhost:%d/root/root.cer\n\n" % db_root.pubd.rsync_port)
-  f.write(root_key.get_public().get_Base64())
-  f.close()
+  with open(os.path.join(test_dir, "root.tal"), "w") as f:
+    f.write(root_uri + ".cer\n\n")
+    f.write(root_key.get_public().get_Base64())
 
 
 
diff --git a/rpki/irdb/zookeeper.py b/rpki/irdb/zookeeper.py
index 0effe213..c9f7d78e 100644
--- a/rpki/irdb/zookeeper.py
+++ b/rpki/irdb/zookeeper.py
@@ -1543,6 +1543,23 @@ class Zookeeper(object):
           bpki_cert = client.certificate,
           base_uri = client.sia_base))
 
+    # rootd instances are also a weird sort of client
+
+    for rootd in rpki.irdb.Rootd.objects.all():
+
+      client_handle = rootd.issuer.handle + "-root"
+      client_pdu = client_pdus.pop(client_handle, None)
+      sia_base = "rsync://%s/%s/%s/" % (self.rsync_server, self.rsync_module, client_handle)
+
+      if (client_pdu is None or
+          client_pdu.base_uri != sia_base or
+          client_pdu.bpki_cert != rootd.issuer.certificate):
+        pubd_query.append(rpki.publication_control.client_elt.make_pdu(
+          action = "create" if client_pdu is None else "set",
+          client_handle = client_handle,
+          bpki_cert = rootd.issuer.certificate,
+          base_uri = sia_base))
+
     # Delete any unknown clients
 
     pubd_query.extend(rpki.publication_control.client_elt.make_pdu(
diff --git a/rpki/pubd.py b/rpki/pubd.py
index 22335ce8..8b58747e 100644
--- a/rpki/pubd.py
+++ b/rpki/pubd.py
@@ -181,9 +181,8 @@ class main(object):
       if not q_msg.is_query():
         raise rpki.exceptions.BadQuery("Message type is not query")
       r_msg = q_msg.__class__.reply()
-      delta = self.session.new_delta()
+      delta = None
       failed = False
-      did_something = False
       for q_pdu in q_msg:
         try:
           if isinstance(q_pdu, rpki.publication.list_elt):
@@ -194,6 +193,8 @@ class main(object):
               r_pdu.hash = obj.hash
               r_msg.append(r_pdu)
           else:
+            if delta is None and not failed:
+              delta = self.session.new_delta()
             q_pdu.gctx = self
             q_pdu.client = client
             q_pdu.client.check_allowed_uri(q_pdu.uri)
@@ -202,7 +203,6 @@ class main(object):
             r_pdu.tag = q_pdu.tag
             r_pdu.uri = q_pdu.uri
             r_msg.append(r_pdu)
-            did_something = True
         except (rpki.async.ExitNow, SystemExit):
           raise
         except Exception, e:
@@ -210,15 +210,16 @@ class main(object):
             logger.exception("Exception processing PDU %r", q_pdu)
           r_msg.append(rpki.publication.report_error_elt.from_exception(e, q_pdu.tag))
           failed = True
+          if delta is not None:
+            delta.sql_delete()
+            self.session.serial -= 1
+            self.session.sql_mark_dirty()
       #
-      # This isn't really right as long as we're using SQL autocommit
+      # This isn't really right as long as we're using SQL autocommit;
+      # there should be an SQL ROLLBACK somewhere if anything above fails.
       #
-      if failed or not did_something:
-        # This should SQL rollback
-        #
-        # Under current scheme I don't think delta is in SQL yet so this may be wrong
-        delta.sql_delete()
-      else:
+      if delta is not None:
+        assert not failed
         delta.activate()
         self.sql.sweep()
         self.session.generate_snapshot()
diff --git a/rpki/rootd.py b/rpki/rootd.py
index b1b29487..bd940534 100644
--- a/rpki/rootd.py
+++ b/rpki/rootd.py
@@ -337,7 +337,7 @@ class main(object):
         try:
           logger.debug("Received response from pubd")
           r_cms = rpki.publication.cms_msg(DER = r_der)
-          r_msg = r_cms.unwrap((self.bpki_ta, self.pubd_bpki_cert))
+          r_msg = r_cms.unwrap(self.bpki_ta)
           self.pubd_cms_timestamp = r_cms.check_replay(self.pubd_cms_timestamp, self.pubd_contact_uri)
           for r_pdu in r_msg:
             r_pdu.raise_if_error()