diff options
| -rw-r--r-- | Makefile.in | 8 | ||||
| -rw-r--r-- | ca/rpki-confgen.xml | 24 | ||||
| -rwxr-xr-x | ca/tests/yamltest.py | 52 | ||||
| -rwxr-xr-x | rp/rcynic/rcynicng | 27 | ||||
| -rw-r--r-- | rpki/irdb/zookeeper.py | 8 | ||||
| -rw-r--r-- | rpki/pubd.py | 8 | ||||
| -rw-r--r-- | rpki/pubdb/models.py | 23 | ||||
| -rw-r--r-- | rpki/rpkidb/migrations/0001_initial.py | 7 | ||||
| -rw-r--r-- | rpki/rpkidb/migrations/0002_remove_cadetail_latest_manifest_cert.py | 18 | ||||
| -rw-r--r-- | rpki/rpkidb/migrations/0003_auto_20151111_1230.py | 37 |
10 files changed, 86 insertions, 126 deletions
diff --git a/Makefile.in b/Makefile.in index 9d53cf7a..8f53c2f4 100644 --- a/Makefile.in +++ b/Makefile.in @@ -214,9 +214,11 @@ lint: .FORCE tags: Makefile .FORCE { find rpki rp ca schemas -type f \( -name '*.[ch]' -o -name '*.py' -o -name '*.sql' -o -name '*.rnc' \) ! -name relaxng.py -print; find rp ca -type f -perm -1 ! -name '*~' -print | xargs grep -El '^#!.+python'; } | etags - -# This isn't all that useful until SQL has been set up. Might want to -# hack up something using ca/rpki-confgen and ca/rpki-sql-setup. makemigrations: - for i in rpkid pubd irdb; do RPKI_CONF=ca/examples/rpki.conf ca/rpki-manage makemigrations --settings rpki.django_settings.$$i; done + RPKI_CONF=makemigrations.conf.$$$$ TEMP_DB=makemigrations.db.$$$$; export RPKI_CONF TEMP_DB; trap "rm -f $$RPKI_CONF $$TEMP_DB" 0; \ + ${PYTHON} ca/rpki-confgen --read-xml ca/rpki-confgen.xml --autoconf --set myrpki::shared_sql_engine=sqlite3 \ + --set myrpki::rpkid_sql_database=$$TEMP_DB --set myrpki::irdbd_sql_database=$$TEMP_DB --set myrpki::pubd_sql_database=$$TEMP_DB \ + --pwgen myrpki::shared_sql_password --pwgen web_portal::secret-key --write-conf $$RPKI_CONF; \ + for i in rpkid pubd irdb; do ca/rpki-manage makemigrations --settings rpki.django_settings.$$i; done .FORCE: diff --git a/ca/rpki-confgen.xml b/ca/rpki-confgen.xml index c100b7b9..ba33c7c9 100644 --- a/ca/rpki-confgen.xml +++ b/ca/rpki-confgen.xml @@ -217,15 +217,22 @@ </doc> </option> - <option name = "publication_rrdp_notification_uri" - value = "https://${myrpki::pubd_server_host}/rrdp/notify.xml"> - + <option name = "publication_rrdp_base_uri" + value = "https://${myrpki::pubd_server_host}/rrdp/"> <doc> - URI for RRDP notification file. In most cases this should be - a HTTPS URL for the notify.xml file on the publication server. + Base URI for RRDP notification, snapshot, and delta files. + In most cases this should be a HTTPS URL for the directory + on the publication server where the notify.xml lives. </doc> </option> + <option name = "publication_rrdp_notification_uri" + value = "${myrpki::publication_rrdp_base_uri}notify.xml"> + <doc> + URI for RRDP notification file. You shouldn't need to change this. + </doc> + </option> + <option name = "start_rpkid" value = "${myrpki::run_rpkid}"> <doc> @@ -709,6 +716,13 @@ </doc> </option> + <option name = "rrdp-base-uri" + value = "${myrpki::publication_rrdp_base_uri}"> + <doc> + RRDP base URI for naming snapshots and deltas. + </doc> + </option> + </section> <section name = "rootd"> diff --git a/ca/tests/yamltest.py b/ca/tests/yamltest.py index 3f7e97a4..80ba59d5 100755 --- a/ca/tests/yamltest.py +++ b/ca/tests/yamltest.py @@ -50,7 +50,6 @@ import textwrap import lxml.etree import rpki.resource_set import rpki.sundial -import rpki.config import rpki.log import rpki.csv_utils import rpki.x509 @@ -339,7 +338,7 @@ class allocation(object): Does this entity run a pubd? """ - return self.is_root or not (self.is_hosted or only_one_pubd) + return self.is_root or (args.one_pubd_per_rpkid and not self.is_hosted) def path(self, *names): """ @@ -500,7 +499,7 @@ class allocation(object): pubd_server_host = "localhost", pubd_server_port = str(self.pubd.pubd_port), publication_rsync_server = "localhost:%s" % self.pubd.rsync_port, - publication_rrdp_notification_uri = "http://localhost:%s/rrdp/notify.xml" % self.pubd.rrdp_port, + publication_rrdp_base_uri = "https://localhost:%s/" % self.pubd.rrdp_port, bpki_servers_directory = self.path(), publication_base_directory = self.path("publication.rsync"), rrdp_publication_base_directory = self.path("publication.rrdp"), @@ -521,8 +520,6 @@ class allocation(object): rpkid_sql_database = "rpki%d" % self.engine, pubd_sql_database = "pubd%d" % self.engine) - r.update(config_overrides) - with open(self.path("rpki.conf"), "w") as f: f.write("# Automatically generated, do not edit\n") print "Writing", f.name @@ -786,11 +783,11 @@ def create_root_certificate(db_root): root_key = rpki.x509.RSA.generate(quiet = True) - root_uri = "rsync://localhost:%d/rpki/%s-root/root" % (db_root.pubd.rsync_port, db_root.name) + rsync_uri = "rsync://localhost:%d/rpki/%s-root/root" % (db_root.pubd.rsync_port, db_root.name) - rrdp_uri = "http://localhost:%s/rrdp/notify.xml" % db.root.pubd.rrdp_port + https_uri = "https://localhost:%s/" % db.root.pubd.rrdp_port - root_sia = (root_uri + "/", root_uri + "/root.mft", None, rrdp_uri) + root_sia = (rsync_uri + "/", rsync_uri + "/root.mft", None, https_uri + "notify.xml") root_cert = rpki.x509.X509.self_certify( keypair = root_key, @@ -806,8 +803,12 @@ def create_root_certificate(db_root): with open(db_root.path("root.key"), "wb") as f: f.write(root_key.get_DER()) + os.link(db_root.path("root.cer"), + db_root.path("publication.rrdp", "root.cer")) + with open(os.path.join(test_dir, "root.tal"), "w") as f: - f.write(root_uri + ".cer\n\n") + f.write(rsync_uri + ".cer\n") + f.write(https_uri + "root.cer\n") f.write(root_key.get_public().get_Base64()) @@ -818,23 +819,21 @@ os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings.irdb", time.tzset() parser = argparse.ArgumentParser(description = __doc__) -parser.add_argument("-c", "--config", - help = "configuration file") -parser.add_argument("-f", "--flat_publication", action = "store_true", +parser.add_argument("-f", "--flat-publication", action = "store_true", help = "disable hierarchical publication") -parser.add_argument("-k", "--keep_going", action = "store_true", +parser.add_argument("-k", "--keep-going", action = "store_true", help = "keep going until all subprocesses exit") parser.add_argument("-p", "--pidfile", help = "save pid to this file") -parser.add_argument("--skip_config", action = "store_true", +parser.add_argument("--skip-config", action = "store_true", help = "skip over configuration phase") -parser.add_argument("--stop_after_config", action = "store_true", +parser.add_argument("--stop-after-config", action = "store_true", help = "stop after configuration phase") parser.add_argument("--synchronize", action = "store_true", help = "synchronize IRDB with daemons") parser.add_argument("--profile", action = "store_true", help = "enable profiling") -parser.add_argument("-g", "--run_gui", "--gui", action = "store_true", +parser.add_argument("-g", "--run-gui", "--gui", action = "store_true", help = "enable GUI using django-admin runserver") parser.add_argument("--no-browser", action = "store_true", help = "don't create web browser tabs for GUI") @@ -844,6 +843,10 @@ parser.add_argument("--store-router-private-keys", action = "store_true", help = "write generate router private keys to disk") parser.add_argument("--sql-engine", choices = ("mysql", "sqlite3", "postgresql"), default = "sqlite3", help = "select SQL engine to use") +parser.add_argument("--one-pubd-per-rpkid", action = "store_true", + help = "enable separate a pubd process for each rpkid process") +parser.add_argument("--base-port", type = int, default = 4400, + help = "base port number for allocated TCP ports") parser.add_argument("yaml_file", type = argparse.FileType("r"), help = "YAML description of test network") args = parser.parse_args() @@ -858,20 +861,7 @@ try: rpki.log.init("yamltest", argparse.Namespace(log_level = logging.DEBUG, log_handler = lambda: logging.StreamHandler(sys.stdout))) - # Allow optional config file for this tool to override default - # passwords: this is mostly so that I can show a complete working - # example without publishing my own server's passwords. - - cfg = rpki.config.parser(set_filename = args.config, section = "yamltest", allow_missing = True) - - only_one_pubd = cfg.getboolean("only_one_pubd", True) - allocation.base_port = cfg.getint("base_port", 4400) - - config_overrides = dict( - (k, cfg.get(k)) - for k in ("rpkid_sql_password", "irdbd_sql_password", "pubd_sql_password", - "rpkid_sql_username", "irdbd_sql_username", "pubd_sql_username") - if cfg.has_option(k)) + allocation.base_port = args.base_port # Start clean, maybe @@ -911,8 +901,6 @@ try: os.makedirs(d.path("publication.rrdp")) d.dump_rsyncd() d.dump_httpsd() - if d.is_root: - os.makedirs(d.path("publication.root")) d.syncdb() d.run_rpkic("initialize_server_bpki") print diff --git a/rp/rcynic/rcynicng b/rp/rcynic/rcynicng index 1b1cd067..bf041145 100755 --- a/rp/rcynic/rcynicng +++ b/rp/rcynic/rcynicng @@ -701,13 +701,13 @@ def read_tals(): furi = "file://" + os.path.abspath(os.path.join(head, fn)) try: with open(os.path.join(head, fn), "r") as f: - lines = f.readlines() - uri = lines.pop(0).strip() - b64 = "".join(lines[lines.index("\n"):]) - key = rpki.POW.Asymmetric.derReadPublic(b64.decode("base64")) - if not uri.endswith(".cer"): + lines = [line.strip() for line in f] + blank = lines.index("") + uris = lines[:blank] + key = rpki.POW.Asymmetric.derReadPublic("".join(lines[blank:]).decode("base64")) + if not uris or not all(uri.endswith(".cer") for uri in uris): Status.add(furi, None, codes.MALFORMED_TAL_URI) - yield uri, key + yield uris, key except: Status.add(furi, None, codes.UNREADABLE_TRUST_ANCHOR_LOCATOR) @@ -1072,7 +1072,7 @@ class Fetcher(object): logger.debug("RRDP %s committing snapshot %s serial %s", self.uri, url, serial) else: - logger.debug("RRDP %s %s deltas (%s -- %s)", self.uri, + logger.debug("RRDP %s %s deltas (%s--%s)", self.uri, (serial - snapshot.serial), snapshot.serial, serial) for serial in xrange(snapshot.serial + 1, serial + 1): @@ -1133,8 +1133,15 @@ class Fetcher(object): class CheckTALTask(object): - def __init__(self, uri, key): - self.uri = uri + def __init__(self, uris, key): + rsync_uri = first_rsync_uri(uris) + https_uri = first_https_uri(uris) + + if args.prefer_rsync: + self.uri = rsync_uri or https_uri + else: + self.uri = https_uri or rsync_uri + self.key = key def __repr__(self): @@ -1301,7 +1308,7 @@ def launcher(): for i in xrange(args.workers): tornado.ioloop.IOLoop.current().spawn_callback(worker, i) - yield [task_queue.put(CheckTALTask(uri, key)) for uri, key in read_tals()] + yield [task_queue.put(CheckTALTask(uris, key)) for uris, key in read_tals()] yield task_queue.join() diff --git a/rpki/irdb/zookeeper.py b/rpki/irdb/zookeeper.py index cf4dbda6..e7d9965c 100644 --- a/rpki/irdb/zookeeper.py +++ b/rpki/irdb/zookeeper.py @@ -414,6 +414,7 @@ class Zookeeper(object): if self.run_pubd: pubd = self.server_ca.ee_certificates.get(purpose = "pubd") writer(self.cfg.get("bpki-ta", section = pubd_section), self.server_ca.certificate) + writer(self.cfg.get("pubd-crl", section = pubd_section), self.server_ca.latest_crl) writer(self.cfg.get("pubd-key", section = pubd_section), pubd.private_key) writer(self.cfg.get("pubd-cert", section = pubd_section), pubd.certificate) writer(self.cfg.get("irbe-cert", section = pubd_section), @@ -423,10 +424,10 @@ class Zookeeper(object): try: rootd = rpki.irdb.models.ResourceHolderCA.objects.get(handle = self.handle).rootd writer(self.cfg.get("bpki-ta", section = rootd_section), self.server_ca.certificate) - writer(self.cfg.get("rootd-bpki-crl", section = rootd_section), self.server_ca.latest_crl) writer(self.cfg.get("rootd-bpki-key", section = rootd_section), rootd.private_key) writer(self.cfg.get("rootd-bpki-cert", section = rootd_section), rootd.certificate) writer(self.cfg.get("child-bpki-cert", section = rootd_section), rootd.issuer.certificate) + # rootd-bpki-crl is the same as pubd-crl, already written except rpki.irdb.models.ResourceHolderCA.DoesNotExist: self.log("rootd enabled but resource holding entity not yet configured, skipping rootd setup") except rpki.irdb.models.Rootd.DoesNotExist: @@ -844,15 +845,14 @@ class Zookeeper(object): port = self.cfg.get("pubd_server_port", section = myrpki_section), handle = client.handle) - rrdp_uri = self.cfg.get("publication_rrdp_notification_uri", section = myrpki_section, - default = "") or None + rrdp_uri = self.cfg.get("publication_rrdp_notification_uri", section = myrpki_section, default = "") e = Element(tag_oob_repository_response, nsmap = oob_nsmap, version = oob_version, service_uri = service_uri, publisher_handle = client.handle, sia_base = client.sia_base) - if rrdp_uri is not None: + if rrdp_uri: e.set("rrdp_notification_uri", rrdp_uri) B64Element(e, tag_oob_repository_bpki_ta, self.server_ca.certificate) diff --git a/rpki/pubd.py b/rpki/pubd.py index 3ae8645a..c8e812a8 100644 --- a/rpki/pubd.py +++ b/rpki/pubd.py @@ -112,11 +112,9 @@ class main(object): self.publication_base = self.cfg.get("publication-base", "publication/") - self.rrdp_uri_base = self.cfg.get("rrdp-uri-base", - "http://%s/rrdp/" % socket.getfqdn()) + self.rrdp_base_uri = self.cfg.get("rrdp-base-uri", "https://%s/rrdp/" % socket.getfqdn()) self.rrdp_expiration_interval = rpki.sundial.timedelta.parse(self.cfg.get("rrdp-expiration-interval", "6h")) - self.rrdp_publication_base = self.cfg.get("rrdp-publication-base", - "rrdp-publication/") + self.rrdp_publication_base = self.cfg.get("rrdp-publication-base", "rrdp-publication/") try: self.session = rpki.pubdb.models.Session.objects.get() @@ -286,7 +284,7 @@ class main(object): else: if delta is not None: - self.session.synchronize_rrdp_files(self.rrdp_publication_base, self.rrdp_uri_base) + self.session.synchronize_rrdp_files(self.rrdp_publication_base, self.rrdp_base_uri) delta.update_rsync_files(self.publication_base) request.send_cms_response(rpki.publication.cms_msg().wrap(r_msg, self.pubd_key, self.pubd_cert, self.pubd_crl)) diff --git a/rpki/pubdb/models.py b/rpki/pubdb/models.py index 43600a5e..9d614ceb 100644 --- a/rpki/pubdb/models.py +++ b/rpki/pubdb/models.py @@ -88,8 +88,16 @@ class Session(models.Model): # Debugging flag to prevent expiration of old RRDP files. # This simplifies debugging delta code. Need for this # may go away once RRDP is fully integrated into rcynic. + keep_all_rrdp_files = False + ## @var keep_these_files + # Filenames which should not be deleted during cleanup. + # Expected use is to allow us to store a root certificate + # in in the RRDP base directory. + + keep_these_files = set(["root.cer"]) + def new_delta(self, expires): """ Construct a new delta associated with this session. @@ -158,28 +166,28 @@ class Session(models.Model): @staticmethod - def _rrdp_filename_to_uri(fn, rrdp_uri_base): - return "%s/%s" % (rrdp_uri_base.rstrip("/"), fn) + def _rrdp_filename_to_uri(fn, rrdp_base_uri): + return "%s/%s" % (rrdp_base_uri.rstrip("/"), fn) - def _generate_update_xml(self, rrdp_uri_base): + def _generate_update_xml(self, rrdp_base_uri): xml = Element(rrdp_tag_notification, nsmap = rrdp_nsmap, version = rrdp_version, session_id = self.uuid, serial = str(self.serial)) SubElement(xml, rrdp_tag_snapshot, - uri = self._rrdp_filename_to_uri(self.snapshot_fn, rrdp_uri_base), + uri = self._rrdp_filename_to_uri(self.snapshot_fn, rrdp_base_uri), hash = self.hash) for delta in self.delta_set.all(): SubElement(xml, rrdp_tag_delta, - uri = self._rrdp_filename_to_uri(delta.fn, rrdp_uri_base), + uri = self._rrdp_filename_to_uri(delta.fn, rrdp_base_uri), hash = delta.hash, serial = str(delta.serial)) rpki.relaxng.rrdp.assertValid(xml) return ElementToString(xml, pretty_print = True) - def synchronize_rrdp_files(self, rrdp_publication_base, rrdp_uri_base): + def synchronize_rrdp_files(self, rrdp_publication_base, rrdp_base_uri): """ Write current RRDP files to disk, clean up old files and directories. """ @@ -193,11 +201,12 @@ class Session(models.Model): self._write_rrdp_file(self.snapshot_fn, self.snapshot, rrdp_publication_base) current_filenames.add(self.snapshot_fn) - self._write_rrdp_file(self.notification_fn, self._generate_update_xml(rrdp_uri_base), + self._write_rrdp_file(self.notification_fn, self._generate_update_xml(rrdp_base_uri), rrdp_publication_base, overwrite = True) current_filenames.add(self.notification_fn) if not self.keep_all_rrdp_files: + current_filenames |= self.keep_these_files for root, dirs, files in os.walk(rrdp_publication_base, topdown = False): for fn in files: fn = os.path.join(root, fn) diff --git a/rpki/rpkidb/migrations/0001_initial.py b/rpki/rpkidb/migrations/0001_initial.py index a313ba63..274775e3 100644 --- a/rpki/rpkidb/migrations/0001_initial.py +++ b/rpki/rpkidb/migrations/0001_initial.py @@ -27,10 +27,7 @@ class Migration(migrations.Migration): name='CA', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('last_crl_sn', models.BigIntegerField(default=1)), - ('last_manifest_sn', models.BigIntegerField(default=1)), - ('next_manifest_update', rpki.fields.SundialField(null=True)), - ('next_crl_update', rpki.fields.SundialField(null=True)), + ('last_crl_manifest_number', models.BigIntegerField(default=1)), ('last_issued_sn', models.BigIntegerField(default=1)), ('sia_uri', models.TextField(null=True)), ('parent_resource_class', models.TextField(null=True)), @@ -47,9 +44,9 @@ class Migration(migrations.Migration): ('latest_ca_cert', rpki.fields.CertificateField(null=True)), ('manifest_private_key_id', rpki.fields.RSAPrivateKeyField(null=True)), ('manifest_public_key', rpki.fields.PublicKeyField(null=True)), - ('latest_manifest_cert', rpki.fields.CertificateField(null=True)), ('latest_manifest', rpki.fields.ManifestField(null=True)), ('manifest_published', rpki.fields.SundialField(null=True)), + ('next_crl_manifest_update', rpki.fields.SundialField(null=True)), ('state', rpki.fields.EnumField(choices=[(1, 'pending'), (2, 'active'), (3, 'deprecated'), (4, 'revoked')])), ('ca_cert_uri', models.TextField(null=True)), ('ca', models.ForeignKey(related_name='ca_details', to='rpkidb.CA')), diff --git a/rpki/rpkidb/migrations/0002_remove_cadetail_latest_manifest_cert.py b/rpki/rpkidb/migrations/0002_remove_cadetail_latest_manifest_cert.py deleted file mode 100644 index a96f1805..00000000 --- a/rpki/rpkidb/migrations/0002_remove_cadetail_latest_manifest_cert.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('rpkidb', '0001_initial'), - ] - - operations = [ - migrations.RemoveField( - model_name='cadetail', - name='latest_manifest_cert', - ), - ] diff --git a/rpki/rpkidb/migrations/0003_auto_20151111_1230.py b/rpki/rpkidb/migrations/0003_auto_20151111_1230.py deleted file mode 100644 index 8ed021e8..00000000 --- a/rpki/rpkidb/migrations/0003_auto_20151111_1230.py +++ /dev/null @@ -1,37 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import migrations, models -import rpki.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('rpkidb', '0002_remove_cadetail_latest_manifest_cert'), - ] - - operations = [ - migrations.RenameField( - model_name='ca', - old_name='last_crl_sn', - new_name='last_crl_manifest_number', - ), - migrations.RemoveField( - model_name='ca', - name='last_manifest_sn', - ), - migrations.RemoveField( - model_name='ca', - name='next_crl_update', - ), - migrations.RemoveField( - model_name='ca', - name='next_manifest_update', - ), - migrations.AddField( - model_name='cadetail', - name='next_crl_manifest_update', - field=rpki.fields.SundialField(null=True), - ), - ] |