Add multiple-URI TAL support to rcynicng.

Whack CA tools with a club until rcynicng works with our CA RRDP
implementation.

Add makemigrations target to top-level Makefile; this is a bit of a
kludge, but sure beats doing this by hand.

svn path=/branches/tk705/; revision=6230

10 files changed, 86 insertions, 126 deletions

diff --git a/Makefile.in b/Makefile.in
index 9d53cf7a..8f53c2f4 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -214,9 +214,11 @@ lint: .FORCE
 tags: Makefile .FORCE
 	{ find rpki rp ca schemas -type f \( -name '*.[ch]' -o -name '*.py' -o -name '*.sql' -o -name '*.rnc' \) ! -name relaxng.py -print; find rp ca -type f -perm -1 ! -name '*~' -print | xargs grep -El '^#!.+python'; } | etags -
 
-# This isn't all that useful until SQL has been set up.  Might want to
-# hack up something using ca/rpki-confgen and ca/rpki-sql-setup.
 makemigrations:
-	for i in rpkid pubd irdb; do RPKI_CONF=ca/examples/rpki.conf ca/rpki-manage makemigrations --settings rpki.django_settings.$$i; done
+	RPKI_CONF=makemigrations.conf.$$$$ TEMP_DB=makemigrations.db.$$$$; export RPKI_CONF TEMP_DB; trap "rm -f $$RPKI_CONF $$TEMP_DB" 0; \
+	${PYTHON} ca/rpki-confgen --read-xml ca/rpki-confgen.xml --autoconf --set myrpki::shared_sql_engine=sqlite3 \
+		--set myrpki::rpkid_sql_database=$$TEMP_DB --set myrpki::irdbd_sql_database=$$TEMP_DB --set myrpki::pubd_sql_database=$$TEMP_DB \
+		--pwgen myrpki::shared_sql_password --pwgen web_portal::secret-key --write-conf $$RPKI_CONF; \
+	for i in rpkid pubd irdb; do ca/rpki-manage makemigrations --settings rpki.django_settings.$$i; done
 
 .FORCE:
diff --git a/ca/rpki-confgen.xml b/ca/rpki-confgen.xml
index c100b7b9..ba33c7c9 100644
--- a/ca/rpki-confgen.xml
+++ b/ca/rpki-confgen.xml
@@ -217,15 +217,22 @@
       </doc>
     </option>
 
-    <option name  = "publication_rrdp_notification_uri"
-	    value = "https://${myrpki::pubd_server_host}/rrdp/notify.xml">
-
+    <option name  = "publication_rrdp_base_uri"
+	    value = "https://${myrpki::pubd_server_host}/rrdp/">
       <doc>
-	URI for RRDP notification file.  In most cases this should be
-	a HTTPS URL for the notify.xml file on the publication server.
+	Base URI for RRDP notification, snapshot, and delta files.
+	In most cases this should be a HTTPS URL for the directory
+	on the publication server where the notify.xml lives.
       </doc>
     </option>
 
+    <option name  = "publication_rrdp_notification_uri"
+	    value = "${myrpki::publication_rrdp_base_uri}notify.xml">
+       <doc>
+	 URI for RRDP notification file.  You shouldn't need to change this.
+       </doc>
+    </option>
+
     <option name  = "start_rpkid"
             value = "${myrpki::run_rpkid}">
       <doc>
@@ -709,6 +716,13 @@
       </doc>
     </option>
 
+    <option name  = "rrdp-base-uri"
+	    value = "${myrpki::publication_rrdp_base_uri}">
+      <doc>
+	RRDP base URI for naming snapshots and deltas.
+      </doc>
+    </option>
+
   </section>
 
   <section name = "rootd">
diff --git a/ca/tests/yamltest.py b/ca/tests/yamltest.py
index 3f7e97a4..80ba59d5 100755
--- a/ca/tests/yamltest.py
+++ b/ca/tests/yamltest.py
@@ -50,7 +50,6 @@ import textwrap
 import lxml.etree
 import rpki.resource_set
 import rpki.sundial
-import rpki.config
 import rpki.log
 import rpki.csv_utils
 import rpki.x509
@@ -339,7 +338,7 @@ class allocation(object):
         Does this entity run a pubd?
         """
 
-        return self.is_root or not (self.is_hosted or only_one_pubd)
+        return self.is_root or (args.one_pubd_per_rpkid and not self.is_hosted)
 
     def path(self, *names):
         """
@@ -500,7 +499,7 @@ class allocation(object):
             pubd_server_host                  = "localhost",
             pubd_server_port                  = str(self.pubd.pubd_port),
             publication_rsync_server          = "localhost:%s" % self.pubd.rsync_port,
-            publication_rrdp_notification_uri = "http://localhost:%s/rrdp/notify.xml" % self.pubd.rrdp_port,
+            publication_rrdp_base_uri         = "https://localhost:%s/" % self.pubd.rrdp_port,
             bpki_servers_directory            = self.path(),
             publication_base_directory        = self.path("publication.rsync"),
             rrdp_publication_base_directory   = self.path("publication.rrdp"),
@@ -521,8 +520,6 @@ class allocation(object):
                 rpkid_sql_database            = "rpki%d" % self.engine,
                 pubd_sql_database             = "pubd%d" % self.engine)
 
-        r.update(config_overrides)
-
         with open(self.path("rpki.conf"), "w") as f:
             f.write("# Automatically generated, do not edit\n")
             print "Writing", f.name
@@ -786,11 +783,11 @@ def create_root_certificate(db_root):
 
     root_key = rpki.x509.RSA.generate(quiet = True)
 
-    root_uri = "rsync://localhost:%d/rpki/%s-root/root" % (db_root.pubd.rsync_port, db_root.name)
+    rsync_uri = "rsync://localhost:%d/rpki/%s-root/root" % (db_root.pubd.rsync_port, db_root.name)
 
-    rrdp_uri = "http://localhost:%s/rrdp/notify.xml" % db.root.pubd.rrdp_port
+    https_uri = "https://localhost:%s/" % db.root.pubd.rrdp_port
 
-    root_sia = (root_uri + "/", root_uri + "/root.mft", None, rrdp_uri)
+    root_sia = (rsync_uri + "/", rsync_uri + "/root.mft", None, https_uri + "notify.xml")
 
     root_cert = rpki.x509.X509.self_certify(
         keypair     = root_key,
@@ -806,8 +803,12 @@ def create_root_certificate(db_root):
     with open(db_root.path("root.key"), "wb") as f:
         f.write(root_key.get_DER())
 
+    os.link(db_root.path("root.cer"),
+            db_root.path("publication.rrdp", "root.cer"))
+
     with open(os.path.join(test_dir, "root.tal"), "w") as f:
-        f.write(root_uri + ".cer\n\n")
+        f.write(rsync_uri + ".cer\n")
+        f.write(https_uri + "root.cer\n")
         f.write(root_key.get_public().get_Base64())
 
 
@@ -818,23 +819,21 @@ os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings.irdb",
 time.tzset()
 
 parser = argparse.ArgumentParser(description = __doc__)
-parser.add_argument("-c", "--config",
-                    help = "configuration file")
-parser.add_argument("-f", "--flat_publication", action = "store_true",
+parser.add_argument("-f", "--flat-publication", action = "store_true",
                     help = "disable hierarchical publication")
-parser.add_argument("-k", "--keep_going", action = "store_true",
+parser.add_argument("-k", "--keep-going", action = "store_true",
                     help = "keep going until all subprocesses exit")
 parser.add_argument("-p", "--pidfile",
                     help = "save pid to this file")
-parser.add_argument("--skip_config", action = "store_true",
+parser.add_argument("--skip-config", action = "store_true",
                     help = "skip over configuration phase")
-parser.add_argument("--stop_after_config", action = "store_true",
+parser.add_argument("--stop-after-config", action = "store_true",
                     help = "stop after configuration phase")
 parser.add_argument("--synchronize", action = "store_true",
                     help = "synchronize IRDB with daemons")
 parser.add_argument("--profile", action = "store_true",
                     help = "enable profiling")
-parser.add_argument("-g", "--run_gui", "--gui", action = "store_true",
+parser.add_argument("-g", "--run-gui", "--gui", action = "store_true",
                     help = "enable GUI using django-admin runserver")
 parser.add_argument("--no-browser", action = "store_true",
                     help = "don't create web browser tabs for GUI")
@@ -844,6 +843,10 @@ parser.add_argument("--store-router-private-keys", action = "store_true",
                     help = "write generate router private keys to disk")
 parser.add_argument("--sql-engine", choices = ("mysql", "sqlite3", "postgresql"), default = "sqlite3",
                     help = "select SQL engine to use")
+parser.add_argument("--one-pubd-per-rpkid", action = "store_true",
+                    help = "enable separate a pubd process for each rpkid process")
+parser.add_argument("--base-port", type = int, default = 4400,
+                    help = "base port number for allocated TCP ports")
 parser.add_argument("yaml_file", type = argparse.FileType("r"),
                     help = "YAML description of test network")
 args = parser.parse_args()
@@ -858,20 +861,7 @@ try:
     rpki.log.init("yamltest", argparse.Namespace(log_level   = logging.DEBUG,
                                                  log_handler = lambda: logging.StreamHandler(sys.stdout)))
 
-    # Allow optional config file for this tool to override default
-    # passwords: this is mostly so that I can show a complete working
-    # example without publishing my own server's passwords.
-
-    cfg = rpki.config.parser(set_filename = args.config, section = "yamltest", allow_missing = True)
-
-    only_one_pubd = cfg.getboolean("only_one_pubd", True)
-    allocation.base_port = cfg.getint("base_port", 4400)
-
-    config_overrides = dict(
-        (k, cfg.get(k))
-        for k in ("rpkid_sql_password", "irdbd_sql_password", "pubd_sql_password",
-                  "rpkid_sql_username", "irdbd_sql_username", "pubd_sql_username")
-        if cfg.has_option(k))
+    allocation.base_port = args.base_port
 
     # Start clean, maybe
 
@@ -911,8 +901,6 @@ try:
                     os.makedirs(d.path("publication.rrdp"))
                     d.dump_rsyncd()
                     d.dump_httpsd()
-                if d.is_root:
-                    os.makedirs(d.path("publication.root"))
                 d.syncdb()
                 d.run_rpkic("initialize_server_bpki")
                 print
diff --git a/rp/rcynic/rcynicng b/rp/rcynic/rcynicng
index 1b1cd067..bf041145 100755
--- a/rp/rcynic/rcynicng
+++ b/rp/rcynic/rcynicng
@@ -701,13 +701,13 @@ def read_tals():
                 furi = "file://" + os.path.abspath(os.path.join(head, fn))
                 try:
                     with open(os.path.join(head, fn), "r") as f:
-                        lines = f.readlines()
-                    uri = lines.pop(0).strip()
-                    b64 = "".join(lines[lines.index("\n"):])
-                    key = rpki.POW.Asymmetric.derReadPublic(b64.decode("base64"))
-                    if not uri.endswith(".cer"):
+                        lines = [line.strip() for line in f]
+                    blank = lines.index("")
+                    uris = lines[:blank]
+                    key  = rpki.POW.Asymmetric.derReadPublic("".join(lines[blank:]).decode("base64"))
+                    if not uris or not all(uri.endswith(".cer") for uri in uris):
                         Status.add(furi, None, codes.MALFORMED_TAL_URI)
-                    yield uri, key
+                    yield uris, key
                 except:
                     Status.add(furi, None, codes.UNREADABLE_TRUST_ANCHOR_LOCATOR)
 
@@ -1072,7 +1072,7 @@ class Fetcher(object):
                     logger.debug("RRDP %s committing snapshot %s serial %s", self.uri, url, serial)
 
             else:
-                logger.debug("RRDP %s %s deltas (%s -- %s)", self.uri, 
+                logger.debug("RRDP %s %s deltas (%s--%s)", self.uri, 
                              (serial - snapshot.serial), snapshot.serial, serial)
 
                 for serial in xrange(snapshot.serial + 1, serial + 1):
@@ -1133,8 +1133,15 @@ class Fetcher(object):
 
 class CheckTALTask(object):
 
-    def __init__(self, uri, key):
-        self.uri = uri
+    def __init__(self, uris, key):
+        rsync_uri = first_rsync_uri(uris)
+        https_uri = first_https_uri(uris)
+
+        if args.prefer_rsync:
+            self.uri = rsync_uri or https_uri
+        else:
+            self.uri = https_uri or rsync_uri 
+
         self.key = key
 
     def __repr__(self):
@@ -1301,7 +1308,7 @@ def launcher():
     for i in xrange(args.workers):
         tornado.ioloop.IOLoop.current().spawn_callback(worker, i)
 
-    yield [task_queue.put(CheckTALTask(uri, key)) for uri, key in read_tals()]
+    yield [task_queue.put(CheckTALTask(uris, key)) for uris, key in read_tals()]
     yield task_queue.join()
 
 
diff --git a/rpki/irdb/zookeeper.py b/rpki/irdb/zookeeper.py
index cf4dbda6..e7d9965c 100644
--- a/rpki/irdb/zookeeper.py
+++ b/rpki/irdb/zookeeper.py
@@ -414,6 +414,7 @@ class Zookeeper(object):
         if self.run_pubd:
             pubd = self.server_ca.ee_certificates.get(purpose = "pubd")
             writer(self.cfg.get("bpki-ta",   section = pubd_section), self.server_ca.certificate)
+            writer(self.cfg.get("pubd-crl",  section = pubd_section), self.server_ca.latest_crl)
             writer(self.cfg.get("pubd-key",  section = pubd_section), pubd.private_key)
             writer(self.cfg.get("pubd-cert", section = pubd_section), pubd.certificate)
             writer(self.cfg.get("irbe-cert", section = pubd_section),
@@ -423,10 +424,10 @@ class Zookeeper(object):
             try:
                 rootd = rpki.irdb.models.ResourceHolderCA.objects.get(handle = self.handle).rootd
                 writer(self.cfg.get("bpki-ta",         section = rootd_section), self.server_ca.certificate)
-                writer(self.cfg.get("rootd-bpki-crl",  section = rootd_section), self.server_ca.latest_crl)
                 writer(self.cfg.get("rootd-bpki-key",  section = rootd_section), rootd.private_key)
                 writer(self.cfg.get("rootd-bpki-cert", section = rootd_section), rootd.certificate)
                 writer(self.cfg.get("child-bpki-cert", section = rootd_section), rootd.issuer.certificate)
+                # rootd-bpki-crl is the same as pubd-crl, already written
             except rpki.irdb.models.ResourceHolderCA.DoesNotExist:
                 self.log("rootd enabled but resource holding entity not yet configured, skipping rootd setup")
             except rpki.irdb.models.Rootd.DoesNotExist:
@@ -844,15 +845,14 @@ class Zookeeper(object):
             port   = self.cfg.get("pubd_server_port", section = myrpki_section),
             handle = client.handle)
 
-        rrdp_uri = self.cfg.get("publication_rrdp_notification_uri", section = myrpki_section,
-                                default = "") or None
+        rrdp_uri = self.cfg.get("publication_rrdp_notification_uri", section = myrpki_section, default = "")
 
         e = Element(tag_oob_repository_response, nsmap = oob_nsmap, version = oob_version,
                     service_uri = service_uri,
                     publisher_handle = client.handle,
                     sia_base = client.sia_base)
 
-        if rrdp_uri is not None:
+        if rrdp_uri:
             e.set("rrdp_notification_uri", rrdp_uri)
 
         B64Element(e, tag_oob_repository_bpki_ta, self.server_ca.certificate)
diff --git a/rpki/pubd.py b/rpki/pubd.py
index 3ae8645a..c8e812a8 100644
--- a/rpki/pubd.py
+++ b/rpki/pubd.py
@@ -112,11 +112,9 @@ class main(object):
 
         self.publication_base = self.cfg.get("publication-base", "publication/")
 
-        self.rrdp_uri_base = self.cfg.get("rrdp-uri-base",
-                                          "http://%s/rrdp/" % socket.getfqdn())
+        self.rrdp_base_uri = self.cfg.get("rrdp-base-uri", "https://%s/rrdp/" % socket.getfqdn())
         self.rrdp_expiration_interval = rpki.sundial.timedelta.parse(self.cfg.get("rrdp-expiration-interval", "6h"))
-        self.rrdp_publication_base = self.cfg.get("rrdp-publication-base",
-                                                  "rrdp-publication/")
+        self.rrdp_publication_base = self.cfg.get("rrdp-publication-base", "rrdp-publication/")
 
         try:
             self.session = rpki.pubdb.models.Session.objects.get()
@@ -286,7 +284,7 @@ class main(object):
 
             else:
                 if delta is not None:
-                    self.session.synchronize_rrdp_files(self.rrdp_publication_base, self.rrdp_uri_base)
+                    self.session.synchronize_rrdp_files(self.rrdp_publication_base, self.rrdp_base_uri)
                     delta.update_rsync_files(self.publication_base)
 
             request.send_cms_response(rpki.publication.cms_msg().wrap(r_msg, self.pubd_key, self.pubd_cert, self.pubd_crl))
diff --git a/rpki/pubdb/models.py b/rpki/pubdb/models.py
index 43600a5e..9d614ceb 100644
--- a/rpki/pubdb/models.py
+++ b/rpki/pubdb/models.py
@@ -88,8 +88,16 @@ class Session(models.Model):
     # Debugging flag to prevent expiration of old RRDP files.
     # This simplifies debugging delta code.  Need for this
     # may go away once RRDP is fully integrated into rcynic.
+
     keep_all_rrdp_files = False
 
+    ## @var keep_these_files
+    # Filenames which should not be deleted during cleanup.
+    # Expected use is to allow us to store a root certificate
+    # in in the RRDP base directory.
+
+    keep_these_files = set(["root.cer"])
+
     def new_delta(self, expires):
         """
         Construct a new delta associated with this session.
@@ -158,28 +166,28 @@ class Session(models.Model):
 
 
     @staticmethod
-    def _rrdp_filename_to_uri(fn, rrdp_uri_base):
-        return "%s/%s" % (rrdp_uri_base.rstrip("/"), fn)
+    def _rrdp_filename_to_uri(fn, rrdp_base_uri):
+        return "%s/%s" % (rrdp_base_uri.rstrip("/"), fn)
 
 
-    def _generate_update_xml(self, rrdp_uri_base):
+    def _generate_update_xml(self, rrdp_base_uri):
         xml = Element(rrdp_tag_notification, nsmap = rrdp_nsmap,
                       version = rrdp_version,
                       session_id = self.uuid,
                       serial = str(self.serial))
         SubElement(xml, rrdp_tag_snapshot,
-                   uri = self._rrdp_filename_to_uri(self.snapshot_fn, rrdp_uri_base),
+                   uri = self._rrdp_filename_to_uri(self.snapshot_fn, rrdp_base_uri),
                    hash = self.hash)
         for delta in self.delta_set.all():
             SubElement(xml, rrdp_tag_delta,
-                       uri = self._rrdp_filename_to_uri(delta.fn, rrdp_uri_base),
+                       uri = self._rrdp_filename_to_uri(delta.fn, rrdp_base_uri),
                        hash =  delta.hash,
                        serial = str(delta.serial))
         rpki.relaxng.rrdp.assertValid(xml)
         return ElementToString(xml, pretty_print = True)
 
 
-    def synchronize_rrdp_files(self, rrdp_publication_base, rrdp_uri_base):
+    def synchronize_rrdp_files(self, rrdp_publication_base, rrdp_base_uri):
         """
         Write current RRDP files to disk, clean up old files and directories.
         """
@@ -193,11 +201,12 @@ class Session(models.Model):
         self._write_rrdp_file(self.snapshot_fn, self.snapshot, rrdp_publication_base)
         current_filenames.add(self.snapshot_fn)
 
-        self._write_rrdp_file(self.notification_fn, self._generate_update_xml(rrdp_uri_base),
+        self._write_rrdp_file(self.notification_fn, self._generate_update_xml(rrdp_base_uri),
                               rrdp_publication_base, overwrite = True)
         current_filenames.add(self.notification_fn)
 
         if not self.keep_all_rrdp_files:
+            current_filenames |= self.keep_these_files
             for root, dirs, files in os.walk(rrdp_publication_base, topdown = False):
                 for fn in files:
                     fn = os.path.join(root, fn)
diff --git a/rpki/rpkidb/migrations/0001_initial.py b/rpki/rpkidb/migrations/0001_initial.py
index a313ba63..274775e3 100644
--- a/rpki/rpkidb/migrations/0001_initial.py
+++ b/rpki/rpkidb/migrations/0001_initial.py
@@ -27,10 +27,7 @@ class Migration(migrations.Migration):
             name='CA',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('last_crl_sn', models.BigIntegerField(default=1)),
-                ('last_manifest_sn', models.BigIntegerField(default=1)),
-                ('next_manifest_update', rpki.fields.SundialField(null=True)),
-                ('next_crl_update', rpki.fields.SundialField(null=True)),
+                ('last_crl_manifest_number', models.BigIntegerField(default=1)),
                 ('last_issued_sn', models.BigIntegerField(default=1)),
                 ('sia_uri', models.TextField(null=True)),
                 ('parent_resource_class', models.TextField(null=True)),
@@ -47,9 +44,9 @@ class Migration(migrations.Migration):
                 ('latest_ca_cert', rpki.fields.CertificateField(null=True)),
                 ('manifest_private_key_id', rpki.fields.RSAPrivateKeyField(null=True)),
                 ('manifest_public_key', rpki.fields.PublicKeyField(null=True)),
-                ('latest_manifest_cert', rpki.fields.CertificateField(null=True)),
                 ('latest_manifest', rpki.fields.ManifestField(null=True)),
                 ('manifest_published', rpki.fields.SundialField(null=True)),
+                ('next_crl_manifest_update', rpki.fields.SundialField(null=True)),
                 ('state', rpki.fields.EnumField(choices=[(1, 'pending'), (2, 'active'), (3, 'deprecated'), (4, 'revoked')])),
                 ('ca_cert_uri', models.TextField(null=True)),
                 ('ca', models.ForeignKey(related_name='ca_details', to='rpkidb.CA')),
diff --git a/rpki/rpkidb/migrations/0002_remove_cadetail_latest_manifest_cert.py b/rpki/rpkidb/migrations/0002_remove_cadetail_latest_manifest_cert.py
deleted file mode 100644
index a96f1805..00000000
--- a/rpki/rpkidb/migrations/0002_remove_cadetail_latest_manifest_cert.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('rpkidb', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='cadetail',
-            name='latest_manifest_cert',
-        ),
-    ]
diff --git a/rpki/rpkidb/migrations/0003_auto_20151111_1230.py b/rpki/rpkidb/migrations/0003_auto_20151111_1230.py
deleted file mode 100644
index 8ed021e8..00000000
--- a/rpki/rpkidb/migrations/0003_auto_20151111_1230.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-import rpki.fields
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('rpkidb', '0002_remove_cadetail_latest_manifest_cert'),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name='ca',
-            old_name='last_crl_sn',
-            new_name='last_crl_manifest_number',
-        ),
-        migrations.RemoveField(
-            model_name='ca',
-            name='last_manifest_sn',
-        ),
-        migrations.RemoveField(
-            model_name='ca',
-            name='next_crl_update',
-        ),
-        migrations.RemoveField(
-            model_name='ca',
-            name='next_manifest_update',
-        ),
-        migrations.AddField(
-            model_name='cadetail',
-            name='next_crl_manifest_update',
-            field=rpki.fields.SundialField(null=True),
-        ),
-    ]