Sort out BPKI mess with rootd talking to pubd. Underlying problem is

that smoketest.py uses an obsolete BPKI model which requires different
configuration than modern code.  Real fix is several different major
refactoring jobs (merging rootd into rpkid, merging all three
YAML-based test tools...).  As usual, not today.

svn path=/branches/tk705/; revision=5914

6 files changed, 120 insertions, 11 deletions

diff --git a/ca/rpki-confgen.xml b/ca/rpki-confgen.xml
index 864da8b2..13c2c798 100644
--- a/ca/rpki-confgen.xml
+++ b/ca/rpki-confgen.xml
@@ -697,11 +697,10 @@
       </doc>
     </option>
 
-    <option name  = "pubd-bpki-cert"
-            value = "${myrpki::bpki_servers_directory}/pubd.cer">
+    <option name  = "pubd-bpki-cert">
       <doc>
-        BPKI certificate for pubd.  Don't change this unless you
-	really know what you are doing.
+        BPKI certificate for pubd.  Don't set this unless you really
+	know what you are doing.
       </doc>
     </option>
 
diff --git a/ca/tests/test-rrdp.py b/ca/tests/test-rrdp.py
new file mode 100755
index 00000000..cff17a87
--- /dev/null
+++ b/ca/tests/test-rrdp.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+# $Id$
+#
+# Copyright (C) 2013  Dragon Research Labs ("DRL")
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH
+# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS.  IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+
+"""
+Proof-of-concept test driver for RRDP code.  Still fairly kludgy in places.
+"""
+
+import os
+import sys
+import glob
+import time
+import textwrap
+import argparse
+import subprocess 
+
+parser = argparse.ArgumentParser(description = __doc__)
+parser.add_argument("--use-smoketest", action = "store_true")
+parser.add_argument("--yaml-file", default = "smoketest.2.yaml")
+parser.add_argument("--delay", type = int, default = 300)
+parser.add_argument("--exhaustive", action = "store_true")
+args = parser.parse_args()
+
+def log(msg):
+  sys.stdout.write(msg + "\n")
+  sys.stdout.flush()
+
+def run(*argv):
+  log("Running: " + " ".join(argv))
+  subprocess.check_call(argv)
+
+def dataglob(pattern):
+  return glob.iglob(os.path.join(("smoketest.dir" if args.use_smoketest else "yamltest.dir/RIR"), pattern))
+
+def snapshot_to_serial(fn):
+  return os.path.splitext(os.path.basename(fn))[0]
+
+def delta_to_serial(fn):
+  return os.path.splitext(os.path.basename(fn))[0].split("-")[1]
+
+top = os.path.expanduser("~/rpki/subvert-rpki.hactrn.net/branches/tk705")
+
+rrdp_test_tool = os.path.join(top, "potpourri/rrdp-test-tool")
+rcynic         = os.path.join(top, "rp/rcynic/rcynic")
+rcynic_text    = os.path.join(top, "rp/rcynic/rcynic-text")
+
+with open("rcynic-rrdp.conf", "w") as f:
+  f.write(textwrap.dedent('''# Automatically generated for RRDP tests, do not edit.
+    [rcynic]
+    xml-summary  = rcynic.xml
+    jitter       = 0
+    use-links    = yes
+    use-syslog   = no
+    use-stderr   = yes
+    log-level    = log_debug
+    run-rsync    = no
+    '''))
+  if args.use_smoketest:
+    f.write("trust-anchor = smoketest.dir/root.cer\n")
+  else:
+    f.write("trust-anchor = yamltest.dir/RIR/publication/RIR-root/root.cer\n")
+
+if args.use_smoketest:
+  run("python", "smoketest.py", args.yaml_file)
+else:
+  run("python", "sql-cleaner.py")
+  argv = ("python", "yamltest.py", args.yaml_file)
+  log("Running: " + " ".join(argv))
+  yamltest = subprocess.Popen(argv)
+  log("Sleeping %s" % args.delay)
+  time.sleep(args.delay)
+  yamltest.terminate()
+
+snapshots = dict((int(snapshot_to_serial(fn)), fn) for fn in dataglob("rrdp-publication/*/snapshot/*.xml"))
+deltas    = dict((int(delta_to_serial(fn)),    fn) for fn in dataglob("rrdp-publication/*/deltas/*.xml"))
+
+for snapshot in sorted(snapshots):
+
+  time.sleep(1)
+  run("rm", "-rf", "rcynic-data")
+  run(rrdp_test_tool, snapshots[snapshot])
+  run(rcynic, "-c", "rcynic-rrdp.conf")
+  run(rcynic_text, "rcynic.xml")
+
+  for delta in sorted(deltas):
+    if delta > snapshot:
+      time.sleep(1)
+      run(rrdp_test_tool, deltas[delta])
+      run(rcynic, "-c", "rcynic-rrdp.conf")
+      run(rcynic_text, "rcynic.xml")
+
+  if not args.exhaustive:
+    break
diff --git a/potpourri/rrdp-test-tool b/potpourri/rrdp-test-tool
index d83e188d..9bdb53b5 100755
--- a/potpourri/rrdp-test-tool
+++ b/potpourri/rrdp-test-tool
@@ -120,7 +120,7 @@ class main(object):
       raise RuntimeError("Can't apply deltas: current %s old %s new %s" % (cur, old, new))
     for i, delta in enumerate(xml.iterchildren(tags.delta)):
       serial = int(delta.get("serial"))
-      print " Delta %3d serial %ds" % (i, serial)
+      print " Delta %3d serial %d" % (i, serial)
       if cur != serial - 1:
         raise RuntimeError("Can't apply delta: current %s delta serial %s" % (cur, serial))
       for j, elt in enumerate(delta.iterchildren(tags.withdraw)):
diff --git a/rpki/rootd.py b/rpki/rootd.py
index bd940534..0508dc49 100644
--- a/rpki/rootd.py
+++ b/rpki/rootd.py
@@ -337,7 +337,7 @@ class main(object):
         try:
           logger.debug("Received response from pubd")
           r_cms = rpki.publication.cms_msg(DER = r_der)
-          r_msg = r_cms.unwrap(self.bpki_ta)
+          r_msg = r_cms.unwrap((self.bpki_ta, self.pubd_bpki_cert))
           self.pubd_cms_timestamp = r_cms.check_replay(self.pubd_cms_timestamp, self.pubd_contact_uri)
           for r_pdu in r_msg:
             r_pdu.raise_if_error()
@@ -450,7 +450,11 @@ class main(object):
     self.rootd_bpki_cert         = rpki.x509.X509(Auto_update = self.cfg.get("rootd-bpki-cert"))
     self.rootd_bpki_crl          = rpki.x509.CRL( Auto_update = self.cfg.get("rootd-bpki-crl"))
     self.child_bpki_cert         = rpki.x509.X509(Auto_update = self.cfg.get("child-bpki-cert"))
-    self.pubd_bpki_cert          = rpki.x509.X509(Auto_update = self.cfg.get("pubd-bpki-cert"))
+
+    if self.cfg.has_option("pubd-bpki-cert"):
+      self.pubd_bpki_cert        = rpki.x509.X509(Auto_update = self.cfg.get("pubd-bpki-cert"))
+    else:
+      self.pubd_bpki_cert        = None
 
     self.http_server_host        = self.cfg.get("server-host", "")
     self.http_server_port        = self.cfg.getint("server-port")
diff --git a/rpki/sql_schemas.py b/rpki/sql_schemas.py
index ad469204..38ce8fc8 100644
--- a/rpki/sql_schemas.py
+++ b/rpki/sql_schemas.py
@@ -294,7 +294,7 @@ CREATE TABLE session (
         session_id              SERIAL NOT NULL,
         uuid                    VARCHAR(36) NOT NULL,
         serial                  BIGINT UNSIGNED NOT NULL,
-        snapshot                TEXT,
+        snapshot                LONGTEXT,
         hash                    CHAR(64),
         PRIMARY KEY             (session_id),
         UNIQUE                  (uuid)
@@ -303,7 +303,7 @@ CREATE TABLE session (
 CREATE TABLE delta (
         delta_id                SERIAL NOT NULL,
         serial                  BIGINT UNSIGNED NOT NULL,
-        xml                     TEXT NOT NULL,
+        xml                     LONGTEXT NOT NULL,
         hash                    CHAR(64) NOT NULL,
         expires                 DATETIME NOT NULL,
         session_id              BIGINT UNSIGNED NOT NULL,
diff --git a/schemas/sql/pubd.sql b/schemas/sql/pubd.sql
index de857bf4..2a0e2851 100644
--- a/schemas/sql/pubd.sql
+++ b/schemas/sql/pubd.sql
@@ -47,7 +47,7 @@ CREATE TABLE session (
         session_id              SERIAL NOT NULL,
         uuid                    VARCHAR(36) NOT NULL,
         serial                  BIGINT UNSIGNED NOT NULL,
-        snapshot                TEXT,
+        snapshot                LONGTEXT,
         hash                    CHAR(64),
         PRIMARY KEY             (session_id),
         UNIQUE                  (uuid)
@@ -56,7 +56,7 @@ CREATE TABLE session (
 CREATE TABLE delta (
         delta_id                SERIAL NOT NULL,
         serial                  BIGINT UNSIGNED NOT NULL,
-        xml                     TEXT NOT NULL,
+        xml                     LONGTEXT NOT NULL,
         hash                    CHAR(64) NOT NULL,
         expires                 DATETIME NOT NULL,
         session_id              BIGINT UNSIGNED NOT NULL,