diff options
139 files changed, 10565 insertions, 6045 deletions
diff --git a/Makefile.in b/Makefile.in index 8908ae32..faba9054 100644 --- a/Makefile.in +++ b/Makefile.in @@ -45,11 +45,14 @@ SETUP_PY_ROOT = `${PYTHON} -c 'import sys; print "--root " + sys.argv[1] if sys. POW_SO = rpki/POW/_POW.so -RNGS = schemas/relaxng/left-right-schema.rng \ - schemas/relaxng/up-down-schema.rng \ - schemas/relaxng/publication-schema.rng \ +RNGS = schemas/relaxng/left-right.rng \ + schemas/relaxng/up-down.rng \ + schemas/relaxng/publication.rng \ + schemas/relaxng/publication-control.rng \ schemas/relaxng/myrpki.rng \ - schemas/relaxng/router-certificate-schema.rng + schemas/relaxng/router-certificate.rng \ + schemas/relaxng/rrdp.rng \ + schemas/relaxng/oob-setup.rng SQLS = schemas/sql/rpkid.sql \ schemas/sql/pubd.sql @@ -187,29 +190,45 @@ ${abs_top_srcdir}/rpki/sql_schemas.py: buildtools/make-sql-schemas.py ${SQLS} cd schemas/sql; ${PYTHON} ${abs_top_srcdir}/buildtools/make-sql-schemas.py >$@.tmp mv $@.tmp $@ -schemas/relaxng/left-right-schema.rng: schemas/relaxng/left-right-schema.rnc - ${TRANG} schemas/relaxng/left-right-schema.rnc schemas/relaxng/left-right-schema.rng +schemas/relaxng/left-right.rng: schemas/relaxng/left-right.rnc + ${TRANG} schemas/relaxng/left-right.rnc schemas/relaxng/left-right.rng -schemas/relaxng/up-down-schema.rng: schemas/relaxng/up-down-schema.rnc - ${TRANG} schemas/relaxng/up-down-schema.rnc schemas/relaxng/up-down-schema.rng +schemas/relaxng/up-down.rng: schemas/relaxng/up-down.rnc + ${TRANG} schemas/relaxng/up-down.rnc schemas/relaxng/up-down.rng -schemas/relaxng/publication-schema.rng: schemas/relaxng/publication-schema.rnc - ${TRANG} schemas/relaxng/publication-schema.rnc schemas/relaxng/publication-schema.rng +schemas/relaxng/publication.rng: schemas/relaxng/publication.rnc + ${TRANG} schemas/relaxng/publication.rnc schemas/relaxng/publication.rng + +schemas/relaxng/publication-control.rng: schemas/relaxng/publication-control.rnc + ${TRANG} schemas/relaxng/publication-control.rnc schemas/relaxng/publication-control.rng schemas/relaxng/myrpki.rng: schemas/relaxng/myrpki.rnc ${TRANG} schemas/relaxng/myrpki.rnc schemas/relaxng/myrpki.rng -schemas/relaxng/router-certificate-schema.rng: schemas/relaxng/router-certificate-schema.rnc - ${TRANG} schemas/relaxng/router-certificate-schema.rnc schemas/relaxng/router-certificate-schema.rng +schemas/relaxng/router-certificate.rng: schemas/relaxng/router-certificate.rnc + ${TRANG} schemas/relaxng/router-certificate.rnc schemas/relaxng/router-certificate.rng + +schemas/relaxng/rrdp.rng: schemas/relaxng/rrdp.rnc + ${TRANG} schemas/relaxng/rrdp.rnc schemas/relaxng/rrdp.rng + +schemas/relaxng/oob-setup.rng: schemas/relaxng/oob-setup.rnc + ${TRANG} schemas/relaxng/oob-setup.rnc schemas/relaxng/oob-setup.rng # Eg: PYLINT_FLAGS='--disable=W0311' -lint: - { find rpki rp ca -name '*.py' -print; find rp ca -type f -perm -1 -print | xargs grep -El '^#!.+python'; } | \ - sort -u | xargs pylint --rcfile ${abs_top_srcdir}/buildtools/pylint.rc ${PYLINT_FLAGS} +lint: .FORCE + pylint --rcfile ${abs_top_srcdir}/buildtools/pylint.rc ${PYLINT_FLAGS} \ + rpki `find rp ca -type f -perm -1 -print | xargs grep -El '^#!.+python'` -tags: Makefile +tags: Makefile .FORCE find rpki rp ca schemas -type f \ \( -name '*.[ch]' -o -name '*.py' -o -name '*.sql' -o -name '*.rnc' \) \ ! -name relaxng.py ! -name sql_schemas.py -print | \ etags - + +# This isn't all that useful until SQL has been set up. Might want to +# hack up something using ca/rpki-confgen and ca/rpki-sql-setup. +makemigrations: + for i in rpkid pubd irdb; do RPKI_CONF=ca/examples/rpki.conf ca/rpki-manage makemigrations --settings rpki.django_settings.$$i; done + +.FORCE: diff --git a/buildtools/build-freebsd-ports.py b/buildtools/build-freebsd-ports.py index fc35c94b..b4031302 100644 --- a/buildtools/build-freebsd-ports.py +++ b/buildtools/build-freebsd-ports.py @@ -23,36 +23,40 @@ This is a script because we need to generate package lists and update version numbers in the Makefiles. """ -import sys import os import re -import subprocess -import errno +import sys import glob +import errno import shutil import argparse +import subprocess def check_dir(s): if not os.path.isdir(s): raise argparse.ArgumentTypeError("%r is not a directory" % s) return s -parser = argparse.ArgumentParser(description = __doc__) -parser.add_argument("--allow-dirty", action = "store_true", - help = "don't insist on pristine subversion checkout") +parser = argparse.ArgumentParser(description = __doc__, + formatter_class = argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument("--local-dist", action = "store_true", + help = "generate local distribution from subversion working tree (implies --make-package)") parser.add_argument("--make-package", action = "store_true", help = "build binary package") parser.add_argument("--no-clean", action = "store_true", help = "don't clean port after staging etc (implies --no-tarball)") parser.add_argument("--no-tarball", action = "store_true", help = "don't create tarball of generated port") +parser.add_argument("--portsdir", type = os.path.abspath, + default = os.path.abspath("freebsd-ports"), + help = "where to build FreeBSD port trees") parser.add_argument("svndir", metavar = "subversion-working-directory", type = check_dir, help = "directory containing subversion working tree") args = parser.parse_args() svnversion = subprocess.check_output(("svnversion", "-c", args.svndir)).strip().split(":")[-1] -if args.allow_dirty: +if args.local_dist: svnversion = svnversion.translate(None, "M") if not svnversion.isdigit(): @@ -66,53 +70,59 @@ if branch != "trunk" and (branch[:2] != "tk" or not branch[2:].isdigit()): version = "0." + svnversion tarname = "rpki-%s-r%s" % (branch, svnversion) tarball = tarname + ".tar.xz" -url = "http://download.rpki.net/" + tarball - -portsdir = os.path.abspath("freebsd-ports") -portsdir_old = portsdir + ".old" -# Could perhaps use distutils.sysconfig.get_python_lib() instead of -# this regexp hack, but would be just as complicated in its own way, -# so just go with this for the moment. - -py_lib = re.compile(r"^lib/python\d+\.\d+") -py_sitelib = re.compile(r"^lib/python\d+\.\d+/site-packages") +portsdir_old = args.portsdir + ".old" if os.path.isdir(portsdir_old): shutil.rmtree(portsdir_old) -if os.path.isdir(portsdir): - os.rename(portsdir, portsdir_old) +if os.path.isdir(args.portsdir): + os.rename(args.portsdir, portsdir_old) -shutil.copytree(os.path.join(args.svndir, "buildtools", "freebsd-skeleton"), portsdir) +shutil.copytree(os.path.join(args.svndir, "buildtools", "freebsd-skeleton"), args.portsdir) -if os.path.exists(os.path.join(portsdir_old, tarball)): - os.link(os.path.join(portsdir_old, tarball), os.path.join(portsdir, tarball)) +if args.local_dist: + subprocess.check_call(("svn", "export", args.svndir, os.path.join(args.portsdir, tarname))) + for fn, fmt in (("VERSION", "%s\n"), ("rpki/version.py", "VERSION = \"%s\"\n")): + with open(os.path.join(args.portsdir, tarname, fn), "w") as f: + f.write(fmt % version) + subprocess.check_call(("tar", "cJvvf", tarball, tarname), cwd = args.portsdir) + shutil.rmtree(os.path.join(args.portsdir, tarname)) +elif os.path.exists(os.path.join(portsdir_old, tarball)): + os.link(os.path.join(portsdir_old, tarball), os.path.join(args.portsdir, tarball)) elif os.path.exists(os.path.join("/usr/ports/distfiles", tarball)): - shutil.copy(os.path.join("/usr/ports/distfiles", tarball), os.path.join(portsdir, tarball)) + shutil.copy(os.path.join("/usr/ports/distfiles", tarball), os.path.join(args.portsdir, tarball)) if os.path.isdir(portsdir_old): shutil.rmtree(portsdir_old) -if args.make_package: - pkgdir = os.path.join(portsdir, "packages") +if args.make_package or args.local_dist: + pkgdir = os.path.join(args.portsdir, "packages") os.mkdir(pkgdir) -formatdict = dict(SVNVERSION = svnversion, SVNBRANCH = branch) +py_lib = re.compile(r"^lib/python\d+\.\d+") +py_sitelib = re.compile(r"^lib/python\d+\.\d+/site-packages") + +if args.local_dist: + master_site = "file://" + args.portsdir + "/" +else: + master_site = "http://download.rpki.net/" + +formatdict = dict(SVNVERSION = svnversion, SVNBRANCH = branch, MASTER_SITE = master_site) keepdirs = ("usr", "etc", "bin", "var", "lib", "libexec", "sbin", "share", "etc/rc.d", "%%PYTHON_SITELIBDIR%%") for port in ("rpki-rp", "rpki-ca"): - base = os.path.join(portsdir, port) + base = os.path.join(args.portsdir, port) stage = os.path.join(base, "work", "stage") - fn = os.path.join(portsdir, port, "Makefile") + fn = os.path.join(args.portsdir, port, "Makefile") with open(fn, "r") as f: template = f.read() with open(fn, "w") as f: f.write(template % formatdict) - subprocess.check_call(("make", "makesum", "stage", "DISTDIR=" + portsdir, "NO_DEPENDS=yes"), + subprocess.check_call(("make", "makesum", "stage", "DISTDIR=" + args.portsdir, "NO_DEPENDS=yes"), cwd = base) with open(os.path.join(base, "pkg-plist"), "w") as f: @@ -136,11 +146,11 @@ for port in ("rpki-rp", "rpki-ca"): if dn and dn not in keepdirs and not py_lib.match(dn): f.write("@dirrm %s\n" % dn) - if args.make_package: - subprocess.check_call(("make", "clean", "package", "PKGREPOSITORY=" + pkgdir), cwd = base) + if args.make_package or args.local_dist: + subprocess.check_call(("make", "clean", "package", "DISTDIR=" + args.portsdir, "PKGREPOSITORY=" + pkgdir), cwd = base) if not args.no_clean: subprocess.check_call(("make", "clean"), cwd = base) if not args.no_tarball and not args.no_clean: - subprocess.check_call(("tar", "czf", "%s-port.tgz" % port, port), cwd = portsdir) + subprocess.check_call(("tar", "czf", "%s-port.tgz" % port, port), cwd = args.portsdir) diff --git a/buildtools/freebsd-skeleton/rpki-ca/Makefile b/buildtools/freebsd-skeleton/rpki-ca/Makefile index 1bbc1355..c8ceaea1 100644 --- a/buildtools/freebsd-skeleton/rpki-ca/Makefile +++ b/buildtools/freebsd-skeleton/rpki-ca/Makefile @@ -1,7 +1,7 @@ PORTNAME= rpki-ca PORTVERSION= 0.%(SVNVERSION)s CATEGORIES= net -MASTER_SITES= http://download.rpki.net/ +MASTER_SITES= %(MASTER_SITE)s DISTFILES= rpki-%(SVNBRANCH)s-r%(SVNVERSION)s.tar.xz WRKSRC= ${WRKDIR}/rpki-%(SVNBRANCH)s-r%(SVNVERSION)s MAINTAINER= sra@hactrn.net diff --git a/buildtools/freebsd-skeleton/rpki-ca/files/rpki-ca.in b/buildtools/freebsd-skeleton/rpki-ca/files/rpki-ca.in index d6234a12..0c021e6d 100644 --- a/buildtools/freebsd-skeleton/rpki-ca/files/rpki-ca.in +++ b/buildtools/freebsd-skeleton/rpki-ca/files/rpki-ca.in @@ -22,13 +22,12 @@ stop_cmd="rpkica_stop" load_rc_config $name : ${rpkica_enable="NO"} - : ${rpkica_pid_dir="/var/run/rpki"} rpkica_start() { - /usr/bin/install -m 755 -d $rpkica_pid_dir - /usr/local/sbin/rpki-start-servers + /usr/bin/install -m 755 -d ${rpkica_pid_dir} + /usr/local/sbin/rpki-start-servers ${rpkica_flags} return 0 } @@ -36,9 +35,9 @@ rpkica_stop() { for i in rpkid pubd irdbd rootd do - if /bin/test -f $rpkica_pid_dir/$i.pid + if /bin/test -f ${rpkica_pid_dir}/${i}.pid then - /bin/kill `/bin/cat $rpkica_pid_dir/$i.pid` + /bin/kill `/bin/cat ${rpkica_pid_dir}/${i}.pid` fi done return 0 diff --git a/buildtools/freebsd-skeleton/rpki-rp/Makefile b/buildtools/freebsd-skeleton/rpki-rp/Makefile index 16537fdc..e020eaf3 100644 --- a/buildtools/freebsd-skeleton/rpki-rp/Makefile +++ b/buildtools/freebsd-skeleton/rpki-rp/Makefile @@ -1,7 +1,7 @@ PORTNAME= rpki-rp PORTVERSION= 0.%(SVNVERSION)s CATEGORIES= net -MASTER_SITES= http://download.rpki.net/ +MASTER_SITES= %(MASTER_SITE)s DISTFILES= rpki-%(SVNBRANCH)s-r%(SVNVERSION)s.tar.xz WRKSRC= ${WRKDIR}/rpki-%(SVNBRANCH)s-r%(SVNVERSION)s MAINTAINER= sra@hactrn.net diff --git a/buildtools/pylint.rc b/buildtools/pylint.rc index ed296108..34302f74 100644 --- a/buildtools/pylint.rc +++ b/buildtools/pylint.rc @@ -19,11 +19,21 @@ [MASTER] profile=no -ignore=.svn + +# Including "gui" here is a temporary measure: it's risky, but so is +# making ten zillion cosmetic changes in a co-worker's code on a +# long-running development branch. + +ignore=.svn,.git,migrations,south_migrations,irbe_cli,gui + persistent=yes cache-size=500 load-plugins= +# Extension (C, etc) modules that pylint should trust enough to import. + +extension-pkg-whitelist=lxml,rpki.POW + [MESSAGES CONTROL] # Enable only checker(s) with the given id(s). This option conflicts with the @@ -44,14 +54,13 @@ disable-msg-cat= #enable-msg= # Disable the message(s) with the given id(s). -disable=R0801,R0903,R0913,C0321,R0904,W0201,E1101,W0614,C0301,R0901,C0302,R0902,R0201,W0613,R0912,R0915,W0703,W0212,R0914,W0603,W0142,I0011,C0111,C0103,R0401,C0326,R0911,C0325 +disable=R0801,R0903,R0913,C0321,R0904,W0201,E1101,W0614,C0301,R0901,C0302,R0902,R0201,W0613,R0912,R0915,W0703,W0212,R0914,W0603,W0142,I0011,C0111,C0103,R0401,C0326,R0911,C0325,C0330,W0311,E1124 [REPORTS] #output-format=parseable msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg} -include-ids=yes files-output=no reports=no evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) diff --git a/ca/irbe_cli b/ca/irbe_cli index 91c12aa9..2edde024 100755 --- a/ca/irbe_cli +++ b/ca/irbe_cli @@ -41,12 +41,11 @@ import sys import getopt import textwrap import rpki.left_right -import rpki.http +import rpki.http_simple import rpki.x509 import rpki.config import rpki.log import rpki.publication -import rpki.async pem_out = None @@ -305,7 +304,7 @@ for o, a in opts: if not argv: usage(1) -cfg = rpki.config.parser(cfg_file, "irbe_cli") +cfg = rpki.config.parser(set_filename = cfg_file, section = "irbe_cli") q_msg_left_right = [] q_msg_publication = [] @@ -322,26 +321,7 @@ while argv: argv = q_pdu.client_getopt(argv[1:]) q_msg.append(q_pdu) -import django - -from django.conf import settings - -settings.configure( - DATABASES = { "default" : { - "ENGINE" : "django.db.backends.mysql", - "NAME" : cfg.get("sql-database", section = "irdbd"), - "USER" : cfg.get("sql-username", section = "irdbd"), - "PASSWORD" : cfg.get("sql-password", section = "irdbd"), - "HOST" : "", - "PORT" : "", - "OPTIONS" : { "init_command": "SET storage_engine=INNODB" }}}, - INSTALLED_APPS = ("rpki.irdb",), - MIDDLEWARE_CLASSES = (), -) - -if django.VERSION >= (1, 7): - from django.apps import apps - apps.populate(settings.INSTALLED_APPS) +os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings.irdb") import rpki.irdb @@ -350,46 +330,36 @@ irbe = server_ca.ee_certificates.get(purpose = "irbe") if q_msg_left_right: - class left_right_proto(object): - cms_msg = left_right_cms_msg - msg = left_right_msg - rpkid = server_ca.ee_certificates.get(purpose = "rpkid") rpkid_url = "http://%s:%s/left-right/" % ( cfg.get("server-host", section = "rpkid"), cfg.get("server-port", section = "rpkid")) - call_rpkid = rpki.async.sync_wrapper(rpki.http.caller( - proto = left_right_proto, - client_key = irbe.private_key, - client_cert = irbe.certificate, - server_ta = server_ca.certificate, - server_cert = rpkid.certificate, - url = rpkid_url, - debug = verbose)) - - call_rpkid(*q_msg_left_right) + rpki.http_simple.client( + proto_cms_msg = left_right_cms_msg, + client_key = irbe.private_key, + client_cert = irbe.certificate, + server_ta = server_ca.certificate, + server_cert = rpkid.certificate, + url = rpkid_url, + debug = verbose, + q_msg = left_right_msg(*q_msg_left_right)) if q_msg_publication: - class publication_proto(object): - msg = publication_msg - cms_msg = publication_cms_msg - pubd = server_ca.ee_certificates.get(purpose = "pubd") pubd_url = "http://%s:%s/control/" % ( cfg.get("server-host", section = "pubd"), cfg.get("server-port", section = "pubd")) - call_pubd = rpki.async.sync_wrapper(rpki.http.caller( - proto = publication_proto, - client_key = irbe.private_key, - client_cert = irbe.certificate, - server_ta = server_ca.certificate, - server_cert = pubd.certificate, - url = pubd_url, - debug = verbose)) - - call_pubd(*q_msg_publication) + rpki.http_simple.client( + proto_cms_msg = publication_cms_msg, + client_key = irbe.private_key, + client_cert = irbe.certificate, + server_ta = server_ca.certificate, + server_cert = pubd.certificate, + url = pubd_url, + debug = verbose, + q_msg = publication_msg(*q_msg_publication)) diff --git a/ca/rpki-confgen.xml b/ca/rpki-confgen.xml index ac36d3db..63e9d793 100644 --- a/ca/rpki-confgen.xml +++ b/ca/rpki-confgen.xml @@ -186,17 +186,17 @@ </doc> </option> - <option name = "publication_root_cert_directory" - value = "${myrpki::publication_base_directory}.root"> + <option name = "rrdp_publication_base_directory" + value = "${autoconf::datarootdir}/rpki/rrdp-publication"> <doc> - Root of local directory tree where rootd (sigh) should write out - published data. This is just like publication_base_directory, but - rootd is too dumb to use pubd and needs its own directory in - which to write one certificate, one CRL, and one manifest. - Neither rootd nor rsyncd much cares //where// you tell them to put - this stuff, the important thing is that the rsync URIs in - generated certificates match up with the published objects so that - relying parties can find and verify rootd's published outputs. + Root of local directory tree where pubd should write out RRDP + files. You need to configure this, and the configuration + should match up with the directory where you point the web + server (usually Apache) that serves the RRDP files. Neither + pubd nor Apache much cares //where// you tell it to put this + stuff, the important thing is that all the URIs match up so + that relying parties can find and verify rpkid's published + outputs. </doc> </option> @@ -209,15 +209,6 @@ </doc> </option> - <option name = "publication_root_module" - value = "root"> - <doc> - rsyncd module name corresponding to publication_root_cert_directory. - This has to match the module you configured into `rsyncd.conf`. - Leave this alone unless you have some need to change it. - </doc> - </option> - <option name = "publication_rsync_server" value = "${myrpki::pubd_server_host}"> <doc> @@ -226,6 +217,15 @@ </doc> </option> + <option name = "publication_rrdp_notification_uri" + value = "https://${myrpki::pubd_server_host}/rrdp/notify.xml"> + + <doc> + URI for RRDP notification file. In most cases this should be + a HTTPS URL for the notify.xml file on the publication server. + </doc> + </option> + <option name = "start_rpkid" value = "${myrpki::run_rpkid}"> <doc> @@ -577,6 +577,20 @@ </doc> </option> + <option name = "rrdp-publication-base" + value = "${myrpki::rrdp_publication_base_directory}"> + <doc> + Root of local directory tree where pubd should write out RRDP + files. You need to configure this, and the configuration + should match up with the directory where you point the web + server (usually Apache) that serves the RRDP files. Neither + pubd nor Apache much cares //where// you tell it to put this + stuff, the important thing is that all the URIs match up so + that relying parties can find and verify rpkid's published + outputs. + </doc> + </option> + <option name = "server-host" value = "${myrpki::pubd_server_host}"> <doc> @@ -618,6 +632,15 @@ </doc> </option> + <option name = "pubd-crl" + value = "${myrpki::bpki_servers_directory}/ca.crl"> + <doc> + Where pubd should look for the CRL covering its own BPKI EE + certificate. Don't change this unless you really know what + you are doing. + </doc> + </option> + <option name = "irbe-cert" value = "${myrpki::bpki_servers_directory}/irbe.cer"> <doc> @@ -638,10 +661,9 @@ </doc> <doc> - Ok, if that wasn't enough to scare you off: rootd is a mess, and - needs to be rewritten, or, better, merged into rpkid. It - doesn't use the publication protocol, and it requires far too - many configuration parameters. + Ok, if that wasn't enough to scare you off: rootd is a mess, + needs to be rewritten, or, better, merged into rpkid, and + requires far too many configuration parameters. </doc> <doc> @@ -712,6 +734,13 @@ </doc> </option> + <option name = "pubd-bpki-cert"> + <doc> + BPKI certificate for pubd. Don't set this unless you really + know what you are doing. + </doc> + </option> + <option name = "server-host" value = "${myrpki::rootd_server_host}"> <doc> @@ -726,72 +755,85 @@ </doc> </option> - <option name = "rpki-root-dir" - value = "${myrpki::publication_base_directory}"> + <option name = "rpki_data_dir" + value = "${myrpki::bpki_servers_directory}"> <doc> - Where rootd should write its output. Yes, rootd should be using - pubd instead of publishing directly, but it doesn't. This - needs to match pubd's configuration. + Directory where rootd should store its RPKI data files. This + is only used to construct other variables, rootd itself + doesn't read it. </doc> </option> - <option name = "rpki-base-uri" - value = "rsync://${myrpki::publication_rsync_server}/${myrpki::publication_rsync_module}/"> + <option name = "rpki_base_uri" + value = "rsync://${myrpki::publication_rsync_server}/${myrpki::publication_rsync_module}/${myrpki::handle}-root/root"> <doc> - rsync URI corresponding to directory containing rootd's outputs. + rsync URI corresponding to directory containing rootd's + outputs. This is only used to construct other variables, + rootd itself doesn't read it. </doc> </option> <option name = "rpki-root-cert-uri" - value = "rsync://${myrpki::publication_rsync_server}/${myrpki::publication_root_module}/root.cer"> + value = "${rootd::rpki_base_uri}.cer"> <doc> rsync URI for rootd's root (self-signed) RPKI certificate. </doc> </option> - <option name = "rpki-root-key" - value = "${myrpki::bpki_servers_directory}/root.key"> + <option name = "rpki-root-cert-file" + value = "${rootd::rpki_data_dir}/root.cer"> + <doc> + Filename of rootd's root RPKI certificate. + </doc> + </option> + + <option name = "rpki-root-key-file" + value = "${rootd::rpki_data_dir}/root.key"> <doc> Private key corresponding to rootd's root RPKI certificate. </doc> </option> - <option name = "rpki-root-cert" - value = "${myrpki::publication_root_cert_directory}/root.cer"> + <option name = "rpki-root-crl-uri" + value = "${rootd::rpki_base_uri}/root.crl"> <doc> - Filename (as opposed to rsync URI) of rootd's root RPKI - certificate. + URI of the CRL for rootd's root RPKI certificate. </doc> </option> - <option name = "rpki-subject-pkcs10" - value = "${myrpki::bpki_servers_directory}/rootd.subject.pkcs10"> + <option name = "rpki-root-crl-file" + value = "${rootd::rpki_data_dir}/root.crl"> <doc> - Where rootd should stash a copy of the PKCS #10 request it gets - from its one (and only) child + Filename of the CRL for rootd's root RPKI certificate. </doc> </option> - <option name = "rpki-subject-lifetime" - value = "30d"> + <option name = "rpki-root-manifest-uri" + value = "${rootd::rpki_base_uri}/root.mft"> <doc> - Lifetime of the one and only RPKI certificate rootd issues. + URI of the manifest for rootd's root RPKI certificate. + </doc> + </option> + + <option name = "rpki-root-manifest-file" + value = "${rootd::rpki_data_dir}/root.mft"> + <doc> + Filename of the manifest for rootd's root RPKI certificate. </doc> </option> - <option name = "rpki-root-crl" - value = "root.crl"> + <option name = "rpki-subject-pkcs10-file" + value = "${rootd::rpki_data_dir}/subject.pkcs10"> <doc> - Filename (relative to rootd-base-uri and rpki-root-dir) of the CRL - for rootd's root RPKI certificate. + Where rootd should stash a copy of the PKCS #10 request it gets + from its one (and only) child </doc> </option> - <option name = "rpki-root-manifest" - value = "root.mft"> + <option name = "rpki-subject-lifetime" + value = "30d"> <doc> - Filename (relative to rootd-base-uri and rpki-root-dir) of the - manifest for rootd's root RPKI certificate. + Lifetime of the one and only RPKI certificate rootd issues. </doc> </option> @@ -803,44 +845,52 @@ </doc> </option> - <option name = "rpki-subject-cert" - value = "${myrpki::handle}.cer"> + <option name = "rpki-subject-cert-uri" + value = "${rootd::rpki_base_uri}/${myrpki::handle}.cer"> <doc> - Filename (relative to rootd-base-uri and rpki-root-dir) of the one - (and only) RPKI certificate rootd issues. + URI of the one (and only) RPKI certificate rootd issues. </doc> </option> - </section> - - <section name = "web_portal"> - - <doc> - Glue to allow the Django application to pull user configuration - from this file rather than directly editing settings.py. - </doc> - - <option name = "sql-database" - value = "${myrpki::irdbd_sql_database}"> + <option name = "rpki-subject-cert-file" + value = "${rootd::rpki_data_dir}/${myrpki::handle}.cer"> <doc> - SQL database name the web portal should use. + Filename of the one (and only) RPKI certificate rootd issues. </doc> </option> - <option name = "sql-username" - value = "${myrpki::irdbd_sql_username}"> + <option name = "pubd-contact-uri" + value = "http://${myrpki::pubd_server_host}:${myrpki::pubd_server_port}/client/${myrpki::handle}-root"> <doc> - SQL user name the web portal should use. + URI at which rootd should contact pubd for service. </doc> </option> - <option name = "sql-password" - value = "${myrpki::irdbd_sql_password}"> + <option name = "rrdp-notification-uri" + value = "${myrpki::publication_rrdp_notification_uri"> <doc> - SQL password the web portal should use. + RRDP URI for inclusion in generated objects. </doc> </option> + </section> + + <section name = "web_portal"> + + <doc> + Glue to allow Django to pull user configuration from this file + rather than requiring the user to edit settings.py. + </doc> + + <!-- + We used to have SQL settings for the GUI here, but since + they're pretty much required to be identical to the ones for + irdbd at this point, the duplicate entries were just another + chance to misconfigure something, so I removed them. Not yet + sure whether this was the right approach. Too much historical + baggage in this file. + --> + <option name = "secret-key"> <doc> Site-specific secret key for Django. diff --git a/ca/rpki-manage b/ca/rpki-manage index 0d581ce9..16d0990d 100755 --- a/ca/rpki-manage +++ b/ca/rpki-manage @@ -1,13 +1,15 @@ #!/usr/bin/env python import os -from django.core.management import execute_from_command_line # django-admin seems to have problems creating the superuser account when # $LANG is unset or is set to something totally incompatible with UTF-8. -if os.environ.get('LANG') in (None, "", "C"): - os.environ['LANG'] = 'en_US.UTF-8' -os.environ['DJANGO_SETTINGS_MODULE'] = 'rpki.gui.default_settings' +if os.environ.get("LANG") in (None, "", "C"): + os.environ["LANG"] = "en_US.UTF-8" + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "rpki.django_settings.gui") + +from django.core.management import execute_from_command_line execute_from_command_line() diff --git a/ca/rpki-sql-backup b/ca/rpki-sql-backup index e60f9ae3..02835956 100755 --- a/ca/rpki-sql-backup +++ b/ca/rpki-sql-backup @@ -41,7 +41,7 @@ parser.add_argument("-o", "--output", help = "destination for SQL dump (default: stdout)") args = parser.parse_args() -cfg = rpki.config.parser(args.config, "myrpki") +cfg = rpki.config.parser(set_filename = args.config, section = "myrpki") for name in ("rpkid", "irdbd", "pubd"): if cfg.getboolean("start_" + name, False): diff --git a/ca/rpki-sql-setup b/ca/rpki-sql-setup index edc2c242..848e3d0f 100755 --- a/ca/rpki-sql-setup +++ b/ca/rpki-sql-setup @@ -54,7 +54,7 @@ class RootDB(object): user = "root", passwd = getpass.getpass("Please enter your MySQL root password: ")) else: - mysql_cfg = rpki.config.parser(self.mysql_defaults, "client") + mysql_cfg = rpki.config.parser(set_filename = self.mysql_defaults, section = "client") self.db = MySQLdb.connect(db = "mysql", user = mysql_cfg.get("user"), passwd = mysql_cfg.get("password")) @@ -299,7 +299,7 @@ parser.set_defaults(dispatch = do_create_if_missing) args = parser.parse_args() try: - cfg = rpki.config.parser(args.config, "myrpki") + cfg = rpki.config.parser(set_filename = args.config, section = "myrpki") root = RootDB(args.mysql_defaults) current_version = Version(rpki.version.VERSION) for program_name in ("irdbd", "rpkid", "pubd"): diff --git a/ca/rpki-start-servers b/ca/rpki-start-servers index 8a745896..f1f70aa8 100755 --- a/ca/rpki-start-servers +++ b/ca/rpki-start-servers @@ -64,13 +64,13 @@ group.add_argument("--log-syslog", default = "daemon", nargs = "?", help = "log syslog") args = parser.parse_args() -cfg = rpki.config.parser(args.config, "myrpki") +cfg = rpki.config.parser(set_filename = args.config, section = "myrpki") def run(name, old_flag = None): if cfg.getboolean("start_" + name, cfg.getboolean("run_" + name if old_flag is None else old_flag, False)): # pylint: disable=E1103 log_file = os.path.join(args.log_directory, name + ".log") - cmd = (os.path.join(rpki.autoconf.libexecdir, name), "--config", cfg.filename, "--log-level", args.log_level) + cmd = (os.path.join(rpki.autoconf.libexecdir, name), "--log-level", args.log_level) if args.log_file: cmd += ("--log-file", log_file) elif args.log_rotating_file_kbytes: diff --git a/ca/rpki.wsgi b/ca/rpki.wsgi index 72ba75ac..487650f7 100644 --- a/ca/rpki.wsgi +++ b/ca/rpki.wsgi @@ -21,7 +21,7 @@ import sys import os import rpki.autoconf -os.environ['DJANGO_SETTINGS_MODULE'] = 'rpki.gui.default_settings' +os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings.gui") # Needed for local_settings.py sys.path.insert(1, rpki.autoconf.sysconfdir + '/rpki') @@ -39,7 +39,7 @@ os.environ['DISABLE_SETPROCTITLE'] = 'yes' if not os.environ.get('PYTHON_EGG_CACHE') and rpki.autoconf.WSGI_PYTHON_EGG_CACHE_DIR: os.environ['PYTHON_EGG_CACHE'] = rpki.autoconf.WSGI_PYTHON_EGG_CACHE_DIR -import django.core.handlers.wsgi -application = django.core.handlers.wsgi.WSGIHandler() +from django.core.wsgi import get_wsgi_application +application = get_wsgi_application() # vim:ft=python diff --git a/ca/rpkigui-apache-conf-gen b/ca/rpkigui-apache-conf-gen index 6f71c7b1..8ac9c94a 100755 --- a/ca/rpkigui-apache-conf-gen +++ b/ca/rpkigui-apache-conf-gen @@ -29,6 +29,50 @@ import rpki.autoconf fqdn = socket.getfqdn() vhost_template = """\ + +# +# Stuff that should be visible with both HTTP and HTTPS is (now) +# outside the vhost block (see if this works properly...). +# + +# +# Allow access to the directory where rcynic-html writes +# its output files. +# +<Directory %(RCYNIC_HTML_DIR)s> +%(allow)s +</Directory> + +# +# Add alias pointing to rcynic-html's output files. +# +# If for some reason you need to change this, be careful to leave +# the trailing slash off the URL, otherwise /rcynic will be +# swallowed by the WSGIScriptAlias +# +Alias /rcynic %(RCYNIC_HTML_DIR)s/ + +# +# Allow access to the directory where pubd writes RRDP files. +# +<Directory %(datarootdir)s/rpki/rrdp-publication/> +%(allow)s +</Directory> + +# +# Add alias pointing to pubd's RRD output files. +# +Alias /rrdp %(datarootdir)s/rpki/rrdp-publication/ + +# +# RRDP "notification" file needs a short expiration: this is +# a critical part of how RRDP interacts with HTTP caching. +# +<LocationMatch ^/rrdp/updates[.]xml$> + ExpiresActive on + ExpiresDefault "access plus 5 minutes" +</LocationMatch> + # # By default, this configuration assumes that you use name-based # virtual hosting. If that's not what you want, you may need @@ -78,23 +122,6 @@ vhost_template = """\ Alias /site_media/ %(datarootdir)s/rpki/media/ # - # Allow access to the directory where rcynic-html writes - # its output files. - # - <Directory %(RCYNIC_HTML_DIR)s> -%(allow)s - </Directory> - - # - # Add alias pointing to rcynic-html's output files. - # - # If for some reason you need to change this, be careful to leave - # the trailing slash off the URL, otherwise /rcynic will be - # swallowed by the WSGIScriptAlias - # - Alias /rcynic %(RCYNIC_HTML_DIR)s/ - - # # Redirect to the GUI dashboard when someone hits the bare vhost. # RedirectMatch ^/$ /rpki/ @@ -102,7 +129,7 @@ vhost_template = """\ # # Enable HTTPS # - SSLEngine on + SSLEngine on # # Specify HTTPS server certificate and key files for this virtual host. @@ -434,6 +461,7 @@ class Debian(Platform): def enable(self): self.run("a2enmod", "ssl") + self.run("a2enmod", "expires") self.run("a2ensite", "rpki") # # In light of BREACH and CRIME attacks, mod_deflate is looking @@ -447,7 +475,7 @@ class Debian(Platform): self.run("service", "apache2", "restart") class Ubuntu(Debian): - + # On Ubuntu, the filename must end in .conf on Trusty and must not # end in .conf on Precise. @property diff --git a/ca/rpkigui-query-routes b/ca/rpkigui-query-routes index 1f698f23..179f8c2c 100755 --- a/ca/rpkigui-query-routes +++ b/ca/rpkigui-query-routes @@ -49,7 +49,6 @@ qs = rv.RouteOrigin.objects.filter( prefix_max__gte=r.max ) - def validity_marker(route, roa, roa_prefix): "Return + if the roa would cause the route to be accepted, or - if not" # we already know the ROA covers this route because they are returned diff --git a/ca/tests/Makefile.in b/ca/tests/Makefile.in index 9796dd2b..618a741e 100644 --- a/ca/tests/Makefile.in +++ b/ca/tests/Makefile.in @@ -3,12 +3,11 @@ PYTHON = @PYTHON@ abs_top_builddir = @abs_top_builddir@ -all: protocol-samples +all: + @true clean: - rm -rf smoketest.dir left-right-protocol-samples publication-protocol-samples yamltest.dir rcynic.xml rcynic-data - -protocol-samples: left-right-protocol-samples/.stamp publication-protocol-samples/.stamp + rm -rf smoketest.dir left-right-protocol-samples publication-protocol-samples publication-control-protocol-samples rrdp-samples yamltest.dir rcynic.xml rcynic-data left-right-protocol-samples/.stamp: left-right-protocol-samples.xml split-protocol-samples.xsl rm -rf left-right-protocol-samples @@ -16,20 +15,44 @@ left-right-protocol-samples/.stamp: left-right-protocol-samples.xml split-protoc xsltproc --param verbose 0 --stringparam dir left-right-protocol-samples split-protocol-samples.xsl left-right-protocol-samples.xml touch $@ +left-right-relaxng: left-right-protocol-samples/.stamp + xmllint --noout --relaxng ../../schemas/relaxng/left-right.rng left-right-protocol-samples/*.xml + publication-protocol-samples/.stamp: publication-protocol-samples.xml split-protocol-samples.xsl rm -rf publication-protocol-samples mkdir publication-protocol-samples xsltproc --param verbose 0 --stringparam dir publication-protocol-samples split-protocol-samples.xsl publication-protocol-samples.xml touch $@ -relaxng: protocol-samples - xmllint --noout --relaxng ../../schemas/relaxng/left-right-schema.rng left-right-protocol-samples/*.xml - xmllint --noout --relaxng ../../schemas/relaxng/up-down-schema.rng up-down-protocol-samples/*.xml - xmllint --noout --relaxng ../../schemas/relaxng/publication-schema.rng publication-protocol-samples/*.xml +publication-relaxng: publication-protocol-samples/.stamp + xmllint --noout --relaxng ../../schemas/relaxng/publication.rng publication-protocol-samples/*.xml + +publication-control-protocol-samples/.stamp: publication-control-protocol-samples.xml split-protocol-samples.xsl + rm -rf publication-control-protocol-samples + mkdir publication-control-protocol-samples + xsltproc --param verbose 0 --stringparam dir publication-control-protocol-samples split-protocol-samples.xsl publication-control-protocol-samples.xml + touch $@ + +publication-control-relaxng: publication-control-protocol-samples/.stamp + xmllint --noout --relaxng ../../schemas/relaxng/publication-control.rng publication-control-protocol-samples/*.xml + +rrdp-samples/.stamp: rrdp-samples.xml split-protocol-samples.xsl + rm -rf rrdp-samples + mkdir rrdp-samples + xsltproc --param verbose 0 --stringparam dir rrdp-samples split-protocol-samples.xsl rrdp-samples.xml + touch $@ + +rrdp-relaxng: rrdp-samples/.stamp + xmllint --noout --relaxng ../../schemas/relaxng/rrdp.rng rrdp-samples/*.xml + +up-down-relaxng: + xmllint --noout --relaxng ../../schemas/relaxng/up-down.rng up-down-protocol-samples/*.xml + +relaxng: up-down-relaxng left-right-relaxng publication-relaxng publication-control-relaxng rrdp-relaxng all-tests:: relaxng -parse-test: protocol-samples +parse-test: left-right-protocol-samples publication-protocol-samples publication-control-protocol-samples ${PYTHON} xml-parse-test.py all-tests:: parse-test diff --git a/ca/tests/bgpsec-yaml.py b/ca/tests/bgpsec-yaml.py index 1562f86e..d33184bf 100755 --- a/ca/tests/bgpsec-yaml.py +++ b/ca/tests/bgpsec-yaml.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # # $Id$ -# +# # Copyright (C) 2014 Dragon Research Labs ("DRL") # # Permission to use, copy, modify, and distribute this software for any @@ -30,11 +30,11 @@ root = "Root" class Kid(object): - def __init__(self, n): - self.name = "ISP-%03d" % n - self.ipv4 = "10.%d.0.0/16" % n - self.asn = n - self.router_id = n * 10000 + def __init__(self, i): + self.name = "ISP-%03d" % i + self.ipv4 = "10.%d.0.0/16" % i + self.asn = i + self.router_id = i * 10000 @property def declare(self): @@ -72,12 +72,12 @@ docs.append([shell_first, gym = kids[50:70] for kid in gym: - docs.append([shell_next, + docs.append([shell_next, kid.del_routercert, sleeper]) for kid in gym: - docs.append([shell_next, + docs.append([shell_next, kid.add_routercert, sleeper]) diff --git a/ca/tests/publication-control-protocol-samples.xml b/ca/tests/publication-control-protocol-samples.xml new file mode 100644 index 00000000..e094f3f6 --- /dev/null +++ b/ca/tests/publication-control-protocol-samples.xml @@ -0,0 +1,155 @@ +<!-- -*- SGML -*- + - $Id$ + - + - Copyright (C) 2008 American Registry for Internet Numbers ("ARIN") + - + - Permission to use, copy, modify, and distribute this software for any + - purpose with or without fee is hereby granted, provided that the above + - copyright notice and this permission notice appear in all copies. + - + - THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH + - REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + - AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, + - INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + - LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + - OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + - PERFORMANCE OF THIS SOFTWARE. + - + - + - This is a collection of sample publication protocol PDU samples + - to use as test cases for the publication protocol RelaxNG schema. + --> + +<completely_gratuitous_wrapper_element_to_let_me_run_this_through_xmllint> + + <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-control/"> + <client action="create" client_handle="3" base_uri="rsync://wombat.invalid/"> + <bpki_cert> + MIIDGzCCAgOgAwIBAgIJAKi+/+wUhQlxMA0GCSqGSIb3DQEBBQUAMCQxIjAgBgNV + BAMTGVRlc3QgQ2VydGlmaWNhdGUgQm9iIFJvb3QwHhcNMDcwODAxMTk1MzEwWhcN + MDcwODMxMTk1MzEwWjAkMSIwIAYDVQQDExlUZXN0IENlcnRpZmljYXRlIEJvYiBS + b290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArKYUtJaM5PH5917S + G2ACc7iBYdQO2HYyu8Gb6i9Q2Gxc3cWEX7RTBvgOL79pWf3GIdnoupzMnoZVtY3G + Ux2G/0WkmLui2TCeDhcfXdQ4rcp8J3V/6ESj+yuEPPOG8UN17mUKKgujrch6ZvgC + DO9AyOK/uXu+ABQXTPsn2pVe2EVh3V004ShLi8GKgVdqb/rW/6GTg0Xb/zLT6WWM + uT++6sXTlztJdQYkRamJvKfQDU1naC8mAkGf79Tba0xyBGAUII0GfREY6t4/+NAP + 2Yyb3xNlBqcJoTov0JfNKHZcCZePr79j7LK/hkZxxip+Na9xDpE+oQRV+DRukCRJ + diqg+wIDAQABo1AwTjAMBgNVHRMEBTADAQH/MB0GA1UdDgQWBBTDEsXJe6pjAQD4 + ULlB7+GMDBlimTAfBgNVHSMEGDAWgBTDEsXJe6pjAQD4ULlB7+GMDBlimTANBgkq + hkiG9w0BAQUFAAOCAQEAWWkNcW6S1tKKqtzJsdfhjJiAAPQmOXJskv0ta/8f6Acg + cum1YieNdtT0n96P7CUHOWP8QBb91JzeewR7b6WJLwb1Offs3wNq3kk75pJe89r4 + XY39EZHhMW+Dv0PhIKu2CgD4LeyH1FVTQkF/QObGEmkn+s+HTsuzd1l2VLwcP1Sm + sqep6LAlFj62qqaIJzNeQ9NVkBqtkygnYlBOkaBTHfQTux3jYNpEo8JJB5e/WFdH + YyMNrG2xMOtIC7T4+IOHgT8PgrNhaeDg9ctewj0X8Qi9nI9nXeinicLX8vj6hdEq + 3ORv7RZMJNYqv1HQ3wUE2B7fCPFv7EUwzaCds1kgRQ== + </bpki_cert> + </client> + </msg> + + <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-control/"> + <client action="create" client_handle="3"/> + </msg> + + <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-control/"> + <client action="set" client_handle="3"> + <bpki_glue> + MIIDGzCCAgOgAwIBAgIJAKi+/+wUhQlxMA0GCSqGSIb3DQEBBQUAMCQxIjAgBgNV + BAMTGVRlc3QgQ2VydGlmaWNhdGUgQm9iIFJvb3QwHhcNMDcwODAxMTk1MzEwWhcN + MDcwODMxMTk1MzEwWjAkMSIwIAYDVQQDExlUZXN0IENlcnRpZmljYXRlIEJvYiBS + b290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArKYUtJaM5PH5917S + G2ACc7iBYdQO2HYyu8Gb6i9Q2Gxc3cWEX7RTBvgOL79pWf3GIdnoupzMnoZVtY3G + Ux2G/0WkmLui2TCeDhcfXdQ4rcp8J3V/6ESj+yuEPPOG8UN17mUKKgujrch6ZvgC + DO9AyOK/uXu+ABQXTPsn2pVe2EVh3V004ShLi8GKgVdqb/rW/6GTg0Xb/zLT6WWM + uT++6sXTlztJdQYkRamJvKfQDU1naC8mAkGf79Tba0xyBGAUII0GfREY6t4/+NAP + 2Yyb3xNlBqcJoTov0JfNKHZcCZePr79j7LK/hkZxxip+Na9xDpE+oQRV+DRukCRJ + diqg+wIDAQABo1AwTjAMBgNVHRMEBTADAQH/MB0GA1UdDgQWBBTDEsXJe6pjAQD4 + ULlB7+GMDBlimTAfBgNVHSMEGDAWgBTDEsXJe6pjAQD4ULlB7+GMDBlimTANBgkq + hkiG9w0BAQUFAAOCAQEAWWkNcW6S1tKKqtzJsdfhjJiAAPQmOXJskv0ta/8f6Acg + cum1YieNdtT0n96P7CUHOWP8QBb91JzeewR7b6WJLwb1Offs3wNq3kk75pJe89r4 + XY39EZHhMW+Dv0PhIKu2CgD4LeyH1FVTQkF/QObGEmkn+s+HTsuzd1l2VLwcP1Sm + sqep6LAlFj62qqaIJzNeQ9NVkBqtkygnYlBOkaBTHfQTux3jYNpEo8JJB5e/WFdH + YyMNrG2xMOtIC7T4+IOHgT8PgrNhaeDg9ctewj0X8Qi9nI9nXeinicLX8vj6hdEq + 3ORv7RZMJNYqv1HQ3wUE2B7fCPFv7EUwzaCds1kgRQ== + </bpki_glue> + </client> + </msg> + + <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-control/"> + <client action="set" client_handle="3"/> + </msg> + + <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-control/"> + <client action="get" client_handle="3"/> + </msg> + + <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-control/"> + <client action="get" client_handle="3" base_uri="rsync://wombat.invalid/"> + <bpki_cert> + MIIDGzCCAgOgAwIBAgIJAKi+/+wUhQlxMA0GCSqGSIb3DQEBBQUAMCQxIjAgBgNV + BAMTGVRlc3QgQ2VydGlmaWNhdGUgQm9iIFJvb3QwHhcNMDcwODAxMTk1MzEwWhcN + MDcwODMxMTk1MzEwWjAkMSIwIAYDVQQDExlUZXN0IENlcnRpZmljYXRlIEJvYiBS + b290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArKYUtJaM5PH5917S + G2ACc7iBYdQO2HYyu8Gb6i9Q2Gxc3cWEX7RTBvgOL79pWf3GIdnoupzMnoZVtY3G + Ux2G/0WkmLui2TCeDhcfXdQ4rcp8J3V/6ESj+yuEPPOG8UN17mUKKgujrch6ZvgC + DO9AyOK/uXu+ABQXTPsn2pVe2EVh3V004ShLi8GKgVdqb/rW/6GTg0Xb/zLT6WWM + uT++6sXTlztJdQYkRamJvKfQDU1naC8mAkGf79Tba0xyBGAUII0GfREY6t4/+NAP + 2Yyb3xNlBqcJoTov0JfNKHZcCZePr79j7LK/hkZxxip+Na9xDpE+oQRV+DRukCRJ + diqg+wIDAQABo1AwTjAMBgNVHRMEBTADAQH/MB0GA1UdDgQWBBTDEsXJe6pjAQD4 + ULlB7+GMDBlimTAfBgNVHSMEGDAWgBTDEsXJe6pjAQD4ULlB7+GMDBlimTANBgkq + hkiG9w0BAQUFAAOCAQEAWWkNcW6S1tKKqtzJsdfhjJiAAPQmOXJskv0ta/8f6Acg + cum1YieNdtT0n96P7CUHOWP8QBb91JzeewR7b6WJLwb1Offs3wNq3kk75pJe89r4 + XY39EZHhMW+Dv0PhIKu2CgD4LeyH1FVTQkF/QObGEmkn+s+HTsuzd1l2VLwcP1Sm + sqep6LAlFj62qqaIJzNeQ9NVkBqtkygnYlBOkaBTHfQTux3jYNpEo8JJB5e/WFdH + YyMNrG2xMOtIC7T4+IOHgT8PgrNhaeDg9ctewj0X8Qi9nI9nXeinicLX8vj6hdEq + 3ORv7RZMJNYqv1HQ3wUE2B7fCPFv7EUwzaCds1kgRQ== + </bpki_cert> + </client> + </msg> + + <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-control/"> + <client action="list"/> + </msg> + + <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-control/"> + <client action="list" client_handle="3"> + <bpki_cert> + MIIDGzCCAgOgAwIBAgIJAKi+/+wUhQlxMA0GCSqGSIb3DQEBBQUAMCQxIjAgBgNV + BAMTGVRlc3QgQ2VydGlmaWNhdGUgQm9iIFJvb3QwHhcNMDcwODAxMTk1MzEwWhcN + MDcwODMxMTk1MzEwWjAkMSIwIAYDVQQDExlUZXN0IENlcnRpZmljYXRlIEJvYiBS + b290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArKYUtJaM5PH5917S + G2ACc7iBYdQO2HYyu8Gb6i9Q2Gxc3cWEX7RTBvgOL79pWf3GIdnoupzMnoZVtY3G + Ux2G/0WkmLui2TCeDhcfXdQ4rcp8J3V/6ESj+yuEPPOG8UN17mUKKgujrch6ZvgC + DO9AyOK/uXu+ABQXTPsn2pVe2EVh3V004ShLi8GKgVdqb/rW/6GTg0Xb/zLT6WWM + uT++6sXTlztJdQYkRamJvKfQDU1naC8mAkGf79Tba0xyBGAUII0GfREY6t4/+NAP + 2Yyb3xNlBqcJoTov0JfNKHZcCZePr79j7LK/hkZxxip+Na9xDpE+oQRV+DRukCRJ + diqg+wIDAQABo1AwTjAMBgNVHRMEBTADAQH/MB0GA1UdDgQWBBTDEsXJe6pjAQD4 + ULlB7+GMDBlimTAfBgNVHSMEGDAWgBTDEsXJe6pjAQD4ULlB7+GMDBlimTANBgkq + hkiG9w0BAQUFAAOCAQEAWWkNcW6S1tKKqtzJsdfhjJiAAPQmOXJskv0ta/8f6Acg + cum1YieNdtT0n96P7CUHOWP8QBb91JzeewR7b6WJLwb1Offs3wNq3kk75pJe89r4 + XY39EZHhMW+Dv0PhIKu2CgD4LeyH1FVTQkF/QObGEmkn+s+HTsuzd1l2VLwcP1Sm + sqep6LAlFj62qqaIJzNeQ9NVkBqtkygnYlBOkaBTHfQTux3jYNpEo8JJB5e/WFdH + YyMNrG2xMOtIC7T4+IOHgT8PgrNhaeDg9ctewj0X8Qi9nI9nXeinicLX8vj6hdEq + 3ORv7RZMJNYqv1HQ3wUE2B7fCPFv7EUwzaCds1kgRQ== + </bpki_cert> + </client> + </msg> + + <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-control/"> + <client action="destroy" client_handle="3"/> + </msg> + + <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-control/"> + <client action="destroy" client_handle="3"/> + </msg> + + <!-- === --> + + <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-control/"> + <report_error error_code="your_hair_is_on_fire">text string</report_error> + </msg> + + <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-control/"> + <report_error error_code="your_hair_is_on_fire"/> + </msg> + +</completely_gratuitous_wrapper_element_to_let_me_run_this_through_xmllint> diff --git a/ca/tests/publication-protocol-samples.xml b/ca/tests/publication-protocol-samples.xml index 96b095a7..6d0a99a9 100644 --- a/ca/tests/publication-protocol-samples.xml +++ b/ca/tests/publication-protocol-samples.xml @@ -1,370 +1,107 @@ <!-- -*- SGML -*- - - $Id$ + - $Id$ - - - Copyright (C) 2008 American Registry for Internet Numbers ("ARIN") + - Sample PDUs for RPKI publication protocol, from current I-D. - - - Permission to use, copy, modify, and distribute this software for any - - purpose with or without fee is hereby granted, provided that the above - - copyright notice and this permission notice appear in all copies. + - Copyright (c) 2014 IETF Trust and the persons identified as authors + - of the code. All rights reserved. - - - THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH - - REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - - AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, - - INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM - - LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE - - OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR - - PERFORMANCE OF THIS SOFTWARE. + - Redistribution and use in source and binary forms, with or without + - modification, are permitted provided that the following conditions + - are met: - + - * Redistributions of source code must retain the above copyright + - notice, this list of conditions and the following disclaimer. - - - This is a collection of sample publication protocol PDU samples - - to use as test cases for the publication protocol RelaxNG schema. + - * Redistributions in binary form must reproduce the above copyright + - notice, this list of conditions and the following disclaimer in + - the documentation and/or other materials provided with the + - distribution. + - + - * Neither the name of Internet Society, IETF or IETF Trust, nor the + - names of specific contributors, may be used to endorse or promote + - products derived from this software without specific prior written + - permission. + - + - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + - FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + - COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + - BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + - LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + - ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + - POSSIBILITY OF SUCH DAMAGE. --> <completely_gratuitous_wrapper_element_to_let_me_run_this_through_xmllint> - <msg xmlns="http://www.hactrn.net/uris/rpki/publication-spec/" type="query" version="1"> - <config action="set"> - <bpki_crl> - MIIBezBlAgEBMA0GCSqGSIb3DQEBCwUAMCMxITAfBgNVBAMTGFRlc3QgQ2VydGlm - aWNhdGUgcHViZCBUQRcNMDgwNjAyMjE0OTQ1WhcNMDgwNzAyMjE0OTQ1WqAOMAww - CgYDVR0UBAMCAQEwDQYJKoZIhvcNAQELBQADggEBAFWCWgBl4ljVqX/CHo+RpqYt - vmKMnjPVflMXUB7i28RGP4DAq4l7deDU7Q82xEJyE4TXMWDWAV6UG6uUGum0VHWO - cj9ohqyiZUGfOsKg2hbwkETm8sAENOsi1yNdyKGk6jZ16aF5fubxQqZa1pdGCSac - 1/ZYC5sLLhEz3kmz+B9z9mXFVc5TgAh4dN3Gy5ftF8zZAFpDGnS4biCnRVqhGv6R - 0Lh/5xmii+ZU6kNDhbeMsjJg+ZOmtN+wMeHSIbjiy0WuuaZ3k2xSh0C94anrHBZA - vvCRhbazjR0Ef5OMZ5lcllw3uO8IHuoisHKkehy4Y0GySdj98fV+OuiRTH9vt/M= - </bpki_crl> - </config> - </msg> - - <msg xmlns="http://www.hactrn.net/uris/rpki/publication-spec/" type="reply" version="1"> - <config action="set"/> - </msg> - - <msg xmlns="http://www.hactrn.net/uris/rpki/publication-spec/" type="query" version="1"> - <config action="get"/> - </msg> - - <msg xmlns="http://www.hactrn.net/uris/rpki/publication-spec/" type="reply" version="1"> - <config action="get"> - <bpki_crl> - MIIBezBlAgEBMA0GCSqGSIb3DQEBCwUAMCMxITAfBgNVBAMTGFRlc3QgQ2VydGlm - aWNhdGUgcHViZCBUQRcNMDgwNjAyMjE0OTQ1WhcNMDgwNzAyMjE0OTQ1WqAOMAww - CgYDVR0UBAMCAQEwDQYJKoZIhvcNAQELBQADggEBAFWCWgBl4ljVqX/CHo+RpqYt - vmKMnjPVflMXUB7i28RGP4DAq4l7deDU7Q82xEJyE4TXMWDWAV6UG6uUGum0VHWO - cj9ohqyiZUGfOsKg2hbwkETm8sAENOsi1yNdyKGk6jZ16aF5fubxQqZa1pdGCSac - 1/ZYC5sLLhEz3kmz+B9z9mXFVc5TgAh4dN3Gy5ftF8zZAFpDGnS4biCnRVqhGv6R - 0Lh/5xmii+ZU6kNDhbeMsjJg+ZOmtN+wMeHSIbjiy0WuuaZ3k2xSh0C94anrHBZA - vvCRhbazjR0Ef5OMZ5lcllw3uO8IHuoisHKkehy4Y0GySdj98fV+OuiRTH9vt/M= - </bpki_crl> - </config> - </msg> - - <!-- === --> - - <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <client action="create" client_handle="3" base_uri="rsync://wombat.invalid/"> - <bpki_cert> - MIIDGzCCAgOgAwIBAgIJAKi+/+wUhQlxMA0GCSqGSIb3DQEBBQUAMCQxIjAgBgNV - BAMTGVRlc3QgQ2VydGlmaWNhdGUgQm9iIFJvb3QwHhcNMDcwODAxMTk1MzEwWhcN - MDcwODMxMTk1MzEwWjAkMSIwIAYDVQQDExlUZXN0IENlcnRpZmljYXRlIEJvYiBS - b290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArKYUtJaM5PH5917S - G2ACc7iBYdQO2HYyu8Gb6i9Q2Gxc3cWEX7RTBvgOL79pWf3GIdnoupzMnoZVtY3G - Ux2G/0WkmLui2TCeDhcfXdQ4rcp8J3V/6ESj+yuEPPOG8UN17mUKKgujrch6ZvgC - DO9AyOK/uXu+ABQXTPsn2pVe2EVh3V004ShLi8GKgVdqb/rW/6GTg0Xb/zLT6WWM - uT++6sXTlztJdQYkRamJvKfQDU1naC8mAkGf79Tba0xyBGAUII0GfREY6t4/+NAP - 2Yyb3xNlBqcJoTov0JfNKHZcCZePr79j7LK/hkZxxip+Na9xDpE+oQRV+DRukCRJ - diqg+wIDAQABo1AwTjAMBgNVHRMEBTADAQH/MB0GA1UdDgQWBBTDEsXJe6pjAQD4 - ULlB7+GMDBlimTAfBgNVHSMEGDAWgBTDEsXJe6pjAQD4ULlB7+GMDBlimTANBgkq - hkiG9w0BAQUFAAOCAQEAWWkNcW6S1tKKqtzJsdfhjJiAAPQmOXJskv0ta/8f6Acg - cum1YieNdtT0n96P7CUHOWP8QBb91JzeewR7b6WJLwb1Offs3wNq3kk75pJe89r4 - XY39EZHhMW+Dv0PhIKu2CgD4LeyH1FVTQkF/QObGEmkn+s+HTsuzd1l2VLwcP1Sm - sqep6LAlFj62qqaIJzNeQ9NVkBqtkygnYlBOkaBTHfQTux3jYNpEo8JJB5e/WFdH - YyMNrG2xMOtIC7T4+IOHgT8PgrNhaeDg9ctewj0X8Qi9nI9nXeinicLX8vj6hdEq - 3ORv7RZMJNYqv1HQ3wUE2B7fCPFv7EUwzaCds1kgRQ== - </bpki_cert> - </client> - </msg> - - <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <client action="create" client_handle="3"/> - </msg> - - <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <client action="set" client_handle="3"> - <bpki_glue> - MIIDGzCCAgOgAwIBAgIJAKi+/+wUhQlxMA0GCSqGSIb3DQEBBQUAMCQxIjAgBgNV - BAMTGVRlc3QgQ2VydGlmaWNhdGUgQm9iIFJvb3QwHhcNMDcwODAxMTk1MzEwWhcN - MDcwODMxMTk1MzEwWjAkMSIwIAYDVQQDExlUZXN0IENlcnRpZmljYXRlIEJvYiBS - b290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArKYUtJaM5PH5917S - G2ACc7iBYdQO2HYyu8Gb6i9Q2Gxc3cWEX7RTBvgOL79pWf3GIdnoupzMnoZVtY3G - Ux2G/0WkmLui2TCeDhcfXdQ4rcp8J3V/6ESj+yuEPPOG8UN17mUKKgujrch6ZvgC - DO9AyOK/uXu+ABQXTPsn2pVe2EVh3V004ShLi8GKgVdqb/rW/6GTg0Xb/zLT6WWM - uT++6sXTlztJdQYkRamJvKfQDU1naC8mAkGf79Tba0xyBGAUII0GfREY6t4/+NAP - 2Yyb3xNlBqcJoTov0JfNKHZcCZePr79j7LK/hkZxxip+Na9xDpE+oQRV+DRukCRJ - diqg+wIDAQABo1AwTjAMBgNVHRMEBTADAQH/MB0GA1UdDgQWBBTDEsXJe6pjAQD4 - ULlB7+GMDBlimTAfBgNVHSMEGDAWgBTDEsXJe6pjAQD4ULlB7+GMDBlimTANBgkq - hkiG9w0BAQUFAAOCAQEAWWkNcW6S1tKKqtzJsdfhjJiAAPQmOXJskv0ta/8f6Acg - cum1YieNdtT0n96P7CUHOWP8QBb91JzeewR7b6WJLwb1Offs3wNq3kk75pJe89r4 - XY39EZHhMW+Dv0PhIKu2CgD4LeyH1FVTQkF/QObGEmkn+s+HTsuzd1l2VLwcP1Sm - sqep6LAlFj62qqaIJzNeQ9NVkBqtkygnYlBOkaBTHfQTux3jYNpEo8JJB5e/WFdH - YyMNrG2xMOtIC7T4+IOHgT8PgrNhaeDg9ctewj0X8Qi9nI9nXeinicLX8vj6hdEq - 3ORv7RZMJNYqv1HQ3wUE2B7fCPFv7EUwzaCds1kgRQ== - </bpki_glue> - </client> - </msg> - - <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <client action="set" client_handle="3"/> - </msg> - - <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <client action="get" client_handle="3"/> - </msg> - - <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <client action="get" client_handle="3" base_uri="rsync://wombat.invalid/"> - <bpki_cert> - MIIDGzCCAgOgAwIBAgIJAKi+/+wUhQlxMA0GCSqGSIb3DQEBBQUAMCQxIjAgBgNV - BAMTGVRlc3QgQ2VydGlmaWNhdGUgQm9iIFJvb3QwHhcNMDcwODAxMTk1MzEwWhcN - MDcwODMxMTk1MzEwWjAkMSIwIAYDVQQDExlUZXN0IENlcnRpZmljYXRlIEJvYiBS - b290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArKYUtJaM5PH5917S - G2ACc7iBYdQO2HYyu8Gb6i9Q2Gxc3cWEX7RTBvgOL79pWf3GIdnoupzMnoZVtY3G - Ux2G/0WkmLui2TCeDhcfXdQ4rcp8J3V/6ESj+yuEPPOG8UN17mUKKgujrch6ZvgC - DO9AyOK/uXu+ABQXTPsn2pVe2EVh3V004ShLi8GKgVdqb/rW/6GTg0Xb/zLT6WWM - uT++6sXTlztJdQYkRamJvKfQDU1naC8mAkGf79Tba0xyBGAUII0GfREY6t4/+NAP - 2Yyb3xNlBqcJoTov0JfNKHZcCZePr79j7LK/hkZxxip+Na9xDpE+oQRV+DRukCRJ - diqg+wIDAQABo1AwTjAMBgNVHRMEBTADAQH/MB0GA1UdDgQWBBTDEsXJe6pjAQD4 - ULlB7+GMDBlimTAfBgNVHSMEGDAWgBTDEsXJe6pjAQD4ULlB7+GMDBlimTANBgkq - hkiG9w0BAQUFAAOCAQEAWWkNcW6S1tKKqtzJsdfhjJiAAPQmOXJskv0ta/8f6Acg - cum1YieNdtT0n96P7CUHOWP8QBb91JzeewR7b6WJLwb1Offs3wNq3kk75pJe89r4 - XY39EZHhMW+Dv0PhIKu2CgD4LeyH1FVTQkF/QObGEmkn+s+HTsuzd1l2VLwcP1Sm - sqep6LAlFj62qqaIJzNeQ9NVkBqtkygnYlBOkaBTHfQTux3jYNpEo8JJB5e/WFdH - YyMNrG2xMOtIC7T4+IOHgT8PgrNhaeDg9ctewj0X8Qi9nI9nXeinicLX8vj6hdEq - 3ORv7RZMJNYqv1HQ3wUE2B7fCPFv7EUwzaCds1kgRQ== - </bpki_cert> - </client> - </msg> - - <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <client action="list"/> - </msg> - - <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <client action="list" client_handle="3"> - <bpki_cert> - MIIDGzCCAgOgAwIBAgIJAKi+/+wUhQlxMA0GCSqGSIb3DQEBBQUAMCQxIjAgBgNV - BAMTGVRlc3QgQ2VydGlmaWNhdGUgQm9iIFJvb3QwHhcNMDcwODAxMTk1MzEwWhcN - MDcwODMxMTk1MzEwWjAkMSIwIAYDVQQDExlUZXN0IENlcnRpZmljYXRlIEJvYiBS - b290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArKYUtJaM5PH5917S - G2ACc7iBYdQO2HYyu8Gb6i9Q2Gxc3cWEX7RTBvgOL79pWf3GIdnoupzMnoZVtY3G - Ux2G/0WkmLui2TCeDhcfXdQ4rcp8J3V/6ESj+yuEPPOG8UN17mUKKgujrch6ZvgC - DO9AyOK/uXu+ABQXTPsn2pVe2EVh3V004ShLi8GKgVdqb/rW/6GTg0Xb/zLT6WWM - uT++6sXTlztJdQYkRamJvKfQDU1naC8mAkGf79Tba0xyBGAUII0GfREY6t4/+NAP - 2Yyb3xNlBqcJoTov0JfNKHZcCZePr79j7LK/hkZxxip+Na9xDpE+oQRV+DRukCRJ - diqg+wIDAQABo1AwTjAMBgNVHRMEBTADAQH/MB0GA1UdDgQWBBTDEsXJe6pjAQD4 - ULlB7+GMDBlimTAfBgNVHSMEGDAWgBTDEsXJe6pjAQD4ULlB7+GMDBlimTANBgkq - hkiG9w0BAQUFAAOCAQEAWWkNcW6S1tKKqtzJsdfhjJiAAPQmOXJskv0ta/8f6Acg - cum1YieNdtT0n96P7CUHOWP8QBb91JzeewR7b6WJLwb1Offs3wNq3kk75pJe89r4 - XY39EZHhMW+Dv0PhIKu2CgD4LeyH1FVTQkF/QObGEmkn+s+HTsuzd1l2VLwcP1Sm - sqep6LAlFj62qqaIJzNeQ9NVkBqtkygnYlBOkaBTHfQTux3jYNpEo8JJB5e/WFdH - YyMNrG2xMOtIC7T4+IOHgT8PgrNhaeDg9ctewj0X8Qi9nI9nXeinicLX8vj6hdEq - 3ORv7RZMJNYqv1HQ3wUE2B7fCPFv7EUwzaCds1kgRQ== - </bpki_cert> - </client> - </msg> - - <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <client action="destroy" client_handle="3"/> - </msg> - - <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <client action="destroy" client_handle="3"/> - </msg> - - <!-- === --> - - <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <certificate action="publish" uri="rsync://wombat.invalid/testbed/RIR/1/j7ghjwblCrcCp9ltyPDNzYKPfxc.cer"> - MIIE+jCCA+KgAwIBAgIBDTANBgkqhkiG9w0BAQsFADAzMTEwLwYDVQQDEyhERjRBODAxN0U2 - NkE5RTkxNzJFNDYxMkQ4Q0Y0QzgzRjIzOERFMkEzMB4XDTA4MDUyMjE4MDUxMloXDTA4MDUy - NDE3NTQ1M1owMzExMC8GA1UEAxMoOEZCODIxOEYwNkU1MEFCNzAyQTdEOTZEQzhGMENEQ0Q4 - MjhGN0YxNzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMeziKp0k5nP7v6SZoNs - XIMQYRgNtC6Fr/9Xm/1yQHomiPqHUk47rHhGojYiK5AhkrwoYhkH4UjJl2iwklDYczXuaBU3 - F5qrKlZ4aZnjIxdlP7+hktVpeApL6yuJTUAYeC3UIxnLDVdD6phydZ/FOQluffiNDjzteCCv - oyOUatqt8WB+oND6LToHp028g1YUYLHG6mur0dPdcHOVXLSmUDuZ1HDz1nDuYvIVKjB/MpH9 - aW9XeaQ6ZFIlZVPwuuvI2brR+ThH7Gv27GL/o8qFdC300VQfoTZ+rKPGDE8K1cI906BL4kiw - x9z0oiDcE96QCz+B0vsjc9mGaA1jgAxlXWsCAwEAAaOCAhcwggITMB0GA1UdDgQWBBSPuCGP - BuUKtwKn2W3I8M3Ngo9/FzAfBgNVHSMEGDAWgBTfSoAX5mqekXLkYS2M9Mg/I43iozBVBgNV - HR8ETjBMMEqgSKBGhkRyc3luYzovL2xvY2FsaG9zdDo0NDAwL3Rlc3RiZWQvUklSLzEvMzBx - QUYtWnFucEZ5NUdFdGpQVElQeU9ONHFNLmNybDBFBggrBgEFBQcBAQQ5MDcwNQYIKwYBBQUH - MAKGKXJzeW5jOi8vbG9jYWxob3N0OjQ0MDAvdGVzdGJlZC9XT01CQVQuY2VyMBgGA1UdIAEB - /wQOMAwwCgYIKwYBBQUHDgIwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwgZsG - CCsGAQUFBwELBIGOMIGLMDQGCCsGAQUFBzAFhihyc3luYzovL2xvY2FsaG9zdDo0NDAwL3Rl - c3RiZWQvUklSL1IwLzEvMFMGCCsGAQUFBzAKhkdyc3luYzovL2xvY2FsaG9zdDo0NDAwL3Rl - c3RiZWQvUklSL1IwLzEvajdnaGp3YmxDcmNDcDlsdHlQRE56WUtQZnhjLm1uZjAaBggrBgEF - BQcBCAEB/wQLMAmgBzAFAgMA/BUwPgYIKwYBBQUHAQcBAf8ELzAtMCsEAgABMCUDAwAKAzAO - AwUAwAACAQMFAcAAAiAwDgMFAsAAAiwDBQDAAAJkMA0GCSqGSIb3DQEBCwUAA4IBAQCEhuH7 - jtI2PJY6+zwv306vmCuXhtu9Lr2mmRw2ZErB8EMcb5xypMrNqMoKeu14K2x4a4RPJkK4yATh - M81FPNRsU5mM0acIRnAPtxjHvPME7PHN2w2nGLASRsZmaa+b8A7SSOxVcFURazENztppsolH - eTpm0cpLItK7mNpudUg1JGuFo94VLf1MnE2EqARG1vTsNhel/SM/UvOArCCOBvf0Gz7kSuup - DSZ7qx+LiDmtEsLdbGNQBiYPbLrDk41PHrxdx28qIj7ejZkRzNFw/3pi8/XK281h8zeHoFVu - 6ghRPy5dbOA4akX/KG6b8XIx0iwPYdLiDbdWFbtTdPcXBauY - </certificate> - </msg> - - <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <certificate action="publish" uri="rsync://wombat.invalid/testbed/RIR/1/j7ghjwblCrcCp9ltyPDNzYKPfxc.cer"/> - </msg> + <msg xmlns="http://www.hactrn.net/uris/rpki/publication-spec/" type="query" version="3"> + <!-- Zero or more PDUs --> + </msg> + + <msg xmlns="http://www.hactrn.net/uris/rpki/publication-spec/" type="reply" version="3"> + <!-- Zero or more PDUs --> + </msg> + + <msg xmlns="http://www.hactrn.net/uris/rpki/publication-spec/" type="query" version="3"> + <publish uri="rsync://wombat.example/Alice/blCrcCp9ltyPDNzYKPfxc.cer"> + MIIE+jCCA+KgAwIBAgIBDTANBgkqhkiG9w0BAQsFADAzMTEwLwYDVQQDEyhE + RjRBODAxN0U2NkE5RTkxNzJFNDYxMkQ4Q0Y0QzgzRjIzOERFMkEzMB4XDTA4 + MDUyMjE4MDUxMloXDTA4MDUyNDE3NTQ1M1owMzExMC8GA1UEAxMoOEZCODIx + OEYwNkU1MEFCNzAyQTdEOTZEQzhGMENEQ0Q4MjhGN0YxNzCCASIwDQYJKoZI + hvcNAQEBBQADggEPADCCAQoCggEBAMeziKp0k5nP7v6SZoNsXIMQYRgNtC6F + r/9Xm/1yQHomiPqHUk47rHhGojYiK5AhkrwoYhkH4UjJl2iwklDYczXuaBU3 + F5qrKlZ4aZnjIxdlP7+hktVpeApL6yuJTUAYeC3UIxnLDVdD6phydZ/FOQlu + ffiNDjzteCCvoyOUatqt8WB+oND6LToHp028g1YUYLHG6mur0dPdcHOVXLSm + UDuZ1HDz1nDuYvIVKjB/MpH9aW9XeaQ6ZFIlZVPwuuvI2brR+ThH7Gv27GL/ + o8qFdC300VQfoTZ+rKPGDE8K1cI906BL4kiwx9z0oiDcE96QCz+B0vsjc9mG + aA1jgAxlXWsCAwEAAaOCAhcwggITMB0GA1UdDgQWBBSPuCGPBuUKtwKn2W3I + 8M3Ngo9/FzAfBgNVHSMEGDAWgBTfSoAX5mqekXLkYS2M9Mg/I43iozBVBgNV + HR8ETjBMMEqgSKBGhkRyc3luYzovL2xvY2FsaG9zdDo0NDAwL3Rlc3RiZWQv + UklSLzEvMzBxQUYtWnFucEZ5NUdFdGpQVElQeU9ONHFNLmNybDBFBggrBgEF + BQcBAQQ5MDcwNQYIKwYBBQUHMAKGKXJzeW5jOi8vbG9jYWxob3N0OjQ0MDAv + dGVzdGJlZC9XT01CQVQuY2VyMBgGA1UdIAEB/wQOMAwwCgYIKwYBBQUHDgIw + DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwgZsGCCsGAQUFBwEL + BIGOMIGLMDQGCCsGAQUFBzAFhihyc3luYzovL2xvY2FsaG9zdDo0NDAwL3Rl + c3RiZWQvUklSL1IwLzEvMFMGCCsGAQUFBzAKhkdyc3luYzovL2xvY2FsaG9z + dDo0NDAwL3Rlc3RiZWQvUklSL1IwLzEvajdnaGp3YmxDcmNDcDlsdHlQRE56 + WUtQZnhjLm1uZjAaBggrBgEFBQcBCAEB/wQLMAmgBzAFAgMA/BUwPgYIKwYB + BQUHAQcBAf8ELzAtMCsEAgABMCUDAwAKAzAOAwUAwAACAQMFAcAAAiAwDgMF + AsAAAiwDBQDAAAJkMA0GCSqGSIb3DQEBCwUAA4IBAQCEhuH7jtI2PJY6+zwv + 306vmCuXhtu9Lr2mmRw2ZErB8EMcb5xypMrNqMoKeu14K2x4a4RPJkK4yATh + M81FPNRsU5mM0acIRnAPtxjHvPME7PHN2w2nGLASRsZmaa+b8A7SSOxVcFUR + azENztppsolHeTpm0cpLItK7mNpudUg1JGuFo94VLf1MnE2EqARG1vTsNhel + /SM/UvOArCCOBvf0Gz7kSuupDSZ7qx+LiDmtEsLdbGNQBiYPbLrDk41PHrxd + x28qIj7ejZkRzNFw/3pi8/XK281h8zeHoFVu6ghRPy5dbOA4akX/KG6b8XIx + 0iwPYdLiDbdWFbtTdPcXBauY + </publish> + </msg> + + <msg xmlns="http://www.hactrn.net/uris/rpki/publication-spec/" type="reply" version="3"> + <publish uri="rsync://wombat.example/Alice/blCrcCp9ltyPDNzYKPfxc.cer"/> + </msg> + + <msg xmlns="http://www.hactrn.net/uris/rpki/publication-spec/" type="reply" version="3"> + <report_error error_code="your_hair_is_on_fire"> + Shampooing with sterno again, are we? + </report_error> + </msg> + + <msg xmlns="http://www.hactrn.net/uris/rpki/publication-spec/" type="reply" version="3"> + <report_error error_code="your_hair_is_on_fire"/> + </msg> + + <msg xmlns="http://www.hactrn.net/uris/rpki/publication-spec/" type="query" version="3"> + <withdraw uri="rsync://wombat.example/Alice/blCrcCp9ltyPDNzYKPfxc.cer" hash="deadf00d"/> + </msg> + + <msg xmlns="http://www.hactrn.net/uris/rpki/publication-spec/" type="reply" version="3"> + <withdraw uri="rsync://wombat.example/Alice/blCrcCp9ltyPDNzYKPfxc.cer"/> + </msg> - <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <certificate action="withdraw" uri="rsync://wombat.invalid/testbed/RIR/1/j7ghjwblCrcCp9ltyPDNzYKPfxc.cer"/> - </msg> - - <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <certificate action="withdraw" uri="rsync://wombat.invalid/testbed/RIR/1/j7ghjwblCrcCp9ltyPDNzYKPfxc.cer"/> - </msg> - - <!-- === --> - - <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <crl action="publish" uri="rsync://wombat.invalid/testbed/RIR/1/30qAF-ZqnpFy5GEtjPTIPyON4qM.crl"> - MIIBwzCBrAIBATANBgkqhkiG9w0BAQsFADAzMTEwLwYDVQQDEyhERjRBODAxN0U2NkE5RTkx - NzJFNDYxMkQ4Q0Y0QzgzRjIzOERFMkEzFw0wODA1MjIxODA0MTZaFw0wODA1MjIxODA1MTZa - MBQwEgIBAhcNMDgwNTIyMTc1ODQwWqAvMC0wHwYDVR0jBBgwFoAU30qAF+ZqnpFy5GEtjPTI - PyON4qMwCgYDVR0UBAMCAQYwDQYJKoZIhvcNAQELBQADggEBAKkM0Fb/pJpHVHWZyjp4wojH - W2KkvA/DFtBiz3moxocSnkDVP3QI19uVvqdC6nH3hJyFmsAMwULR0f1XU/V4j+X+FqYEl6Nv - p8zAEPIB4r8xbEFs7udRwXRAjkJmOQbv9aomF2i+d7jpTFVJxShZWOgsoGEhIy/aktKQrOIR - c4ZDrXpQwXVj2Y7+cGVfQ4gvnPOdlyLcnNovoegazATvA3EcidBNPWRg7XTCz0LVBEB7JgPd - nNyXRg35HdMEHBl7U9uUQJXP7S02oaQ1ehNDMfaJPgBBpQtAnM1lIzJfevd9+e4ywGsRpxAV - 8wxTXSPd1jwuKtS0kwrgsrQ8Ya85xUE= - </crl> - </msg> - - <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <crl action="publish" uri="rsync://wombat.invalid/testbed/RIR/1/30qAF-ZqnpFy5GEtjPTIPyON4qM.crl"/> - </msg> - - <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <crl action="withdraw" uri="rsync://wombat.invalid/testbed/RIR/1/30qAF-ZqnpFy5GEtjPTIPyON4qM.crl"/> - </msg> - - <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <crl action="withdraw" uri="rsync://wombat.invalid/testbed/RIR/1/30qAF-ZqnpFy5GEtjPTIPyON4qM.crl"/> - </msg> - - <!-- === --> - - <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <manifest action="publish" uri="rsync://wombat.invalid/testbed/RIR/R0/1/j7ghjwblCrcCp9ltyPDNzYKPfxc.mft"> - MIIHCgYJKoZIhvcNAQcCoIIG+zCCBvcCAQMxDTALBglghkgBZQMEAgEwggEeBgsqhkiG9w0B - CRABGqCCAQ0EggEJMIIBBQIBEhgPMjAwODA1MjIxODA1MTVaGA8yMDA4MDUyMjE4MDYxNVoG - CWCGSAFlAwQCATCB0jBEFh9ZbTVUTzRJYnlDb0pNZ3E2R2o4dG41Mng5U0UuY2VyAyEA4L8Z - WMyuhOx+o6kUfsRR++QjSaRaATy4UOeVtjvZVqYwRBYfWnRxbjB3NEVFbU9hclAzQmd1SUY3 - MDhhNTM4LmNlcgMhAGQI1gYJotxWmwzcmpLNFZJ656uWOjcPYANlbNz80xm8MEQWH2xxa1Vx - RHEwMDBESW9ZVjlybXdLTGdrN2F6by5jZXIDIQB7jRAEpkPvc4s4PX9vDvnTifj3BIE145FO - 1ne2kEejVqCCBBEwggQNMIIC9aADAgECAgEFMA0GCSqGSIb3DQEBCwUAMDMxMTAvBgNVBAMT - KDhGQjgyMThGMDZFNTBBQjcwMkE3RDk2REM4RjBDRENEODI4RjdGMTcwHhcNMDgwNTIyMTc1 - NzQ5WhcNMDgwNTI0MTc1NDUzWjAzMTEwLwYDVQQDEyhERkRBMjMyMUJENEVCMDNFQTE1RkUy - N0NGRkRGMEFGRkU1QjBFNjY4MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2/Gk - AHW5pDqye0+TvUp7sl0rVgmTfeHpVp18ypxvuovogVJgkjEtBEikfaFU0646wYD6JM6IJFJX - lWLWd7bVmhkWViKuZL0VmT2wpUToNHCLUGUQUVVX8R7oSHFdTArv2AqH+6yt0LmczDH1y2M6 - 2Tgkz9wZ9ryyuPx3VX4PkHzUMlkGFICj1fvyXkcAu8jBaxR9UME1c413TPaMi6lMh1HUmtVN - LJMP5+/SnwEAW/Z3dPClCFIgQXK3nAKPVzAIwADEiqhK7cSchhO7ikI1CVt0XzG4n7oaILc3 - Hq/DAxyiutw5GlkUlKPri2YJzJ3+H4P+TveSa/b02fVA5csm/QIDAQABo4IBKjCCASYwHQYD - VR0OBBYEFN/aIyG9TrA+oV/ifP/fCv/lsOZoMB8GA1UdIwQYMBaAFI+4IY8G5Qq3AqfZbcjw - zc2Cj38XMFgGA1UdHwRRME8wTaBLoEmGR3JzeW5jOi8vbG9jYWxob3N0OjQ0MDAvdGVzdGJl - ZC9SSVIvUjAvMS9qN2doandibENyY0NwOWx0eVBETnpZS1BmeGMuY3JsMGAGCCsGAQUFBwEB - BFQwUjBQBggrBgEFBQcwAoZEcnN5bmM6Ly9sb2NhbGhvc3Q6NDQwMC90ZXN0YmVkL1JJUi8x - L2o3Z2hqd2JsQ3JjQ3A5bHR5UEROellLUGZ4Yy5jZXIwGAYDVR0gAQH/BA4wDDAKBggrBgEF - BQcOAjAOBgNVHQ8BAf8EBAMCB4AwDQYJKoZIhvcNAQELBQADggEBADpsE9HfgVTgmX1WeJTE - fm87CXuOoGH85RFiAngSt5kR4gYCyadklOZ7Eta+ERUZVu4tcKO6sJOTuHPfVrAvR0VpgH+j - PvXboYWSfwJdi00BC28ScrVM2zarA7B10+J6Oq8tbFlAyVBkrbuPet/axmndBtGWhrBTynGl - nc/5L371Lxy6CrOYqXO0Qx3SrOKaailAe3zTIpHQeACqnPdL00zIBw/hVy/VNaH1wy+FmhAz - TsmsQUrMyovJcu/ry5w0KHlP8BTnqfykikCWR+Lw0VQHmpJGAbtrmsOeIbfLY1zl7A81lDAl - AG/ZH1DUdDOUIXMLHWur+D2rwjp7RL16LHYxggGqMIIBpgIBA4AU39ojIb1OsD6hX+J8/98K - /+Ww5mgwCwYJYIZIAWUDBAIBoGswGgYJKoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEaMBwGCSqG - SIb3DQEJBTEPFw0wODA1MjIxODA1MTVaMC8GCSqGSIb3DQEJBDEiBCBj/GjEQw3LgKPf5DTz - 8eu1fcp6/cQjqqne6ZqFkF42azANBgkqhkiG9w0BAQEFAASCAQBOY0uHNMwy/o1nFANSgha5 - PZxt8fz+wTrbeomCb+lxqQKq1clcSiQORVGc8NmqC8sS5OR3eTw/3qnK9yPHxz2UQ4hn1pBa - +Zy5veM61qMaXCw6w98EyNcvUfA1AkezAjkabfHQDs3o4Ezh49thXXyRcBoF+O6Lmi+LZbT2 - 4jvfFbaXW9zsb6/DaoDkeHnlk+YYgfSP4wOnkK5uqxtDW8QpMPq3GGdIp0oJDkzEdj7VsWIL - 9JP2mxxL8fTPVUyAPOmURYwYDXqhke2O9eVDiCYhrEfB8/84Rint4Cj8n5aCujnAtqtwxHpD - 0NRYO/V1MjhG+ARy1vRH1Dm0r92RBam3 - </manifest> - </msg> - - <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <manifest action="publish" uri="rsync://wombat.invalid/testbed/RIR/R0/1/j7ghjwblCrcCp9ltyPDNzYKPfxc.mft"/> - </msg> - - <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <manifest action="withdraw" uri="rsync://wombat.invalid/testbed/RIR/R0/1/j7ghjwblCrcCp9ltyPDNzYKPfxc.mft"/> - </msg> - - <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <manifest action="withdraw" uri="rsync://wombat.invalid/testbed/RIR/R0/1/j7ghjwblCrcCp9ltyPDNzYKPfxc.mft"/> - </msg> - - <!-- === --> - - <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <roa action="publish" uri="rsync://wombat.invalid/testbed/RIR/R0/1/lqkUqDq000DIoYV9rmwKLgk7azo.roa"> - MIIGmwYJKoZIhvcNAQcCoIIGjDCCBogCAQMxDTALBglghkgBZQMEAgEwKgYLKoZIhvcNAQkQ - ARigGwQZMBcCAgKaMBEwDwQCAAEwCTAHAwUACgMALKCCBJgwggSUMIIDfKADAgECAgEJMA0G - CSqGSIb3DQEBCwUAMDMxMTAvBgNVBAMTKDhGQjgyMThGMDZFNTBBQjcwMkE3RDk2REM4RjBD - RENEODI4RjdGMTcwHhcNMDgwNTIyMTc1ODI0WhcNMDgwNTI0MTc1NDUzWjAzMTEwLwYDVQQD - Eyg5NkE5MTRBODNBQjREMzQwQzhBMTg1N0RBRTZDMEEyRTA5M0I2QjNBMIIBIjANBgkqhkiG - 9w0BAQEFAAOCAQ8AMIIBCgKCAQEApoK50BjW5bcF4gsdaYhndtVADZvQk3RCsvuqDElF6uLi - 9BYQq/NHyDOIMyJtvCmzjdv3Y135n1sNO7YvssqHlt7dMfCQTD5ND1GpFnQLdWP7stWM5AbO - nJV6+PtDITUA/QHOli7Do0YCUgR6G+1QJsMu0DK+TRSzBJ6WP7WIYOBOOg3y/NKc1rkWhS1Q - dcQepbHgQYZHzzpjNDR6+oYVuhuUEWx1P6O4pv/p+tpE0SDua7jBjMywIYHkPQBecf2IX1RU - WNojB9dJlnRx5YUUneP2SvF2MrmdDbclgzwhf6alqD2OjiMuoBOG8yeTKcuhzCMnrFAklbst - 6x3Rnq9BswIDAQABo4IBsTCCAa0wHQYDVR0OBBYEFJapFKg6tNNAyKGFfa5sCi4JO2s6MB8G - A1UdIwQYMBaAFI+4IY8G5Qq3AqfZbcjwzc2Cj38XMFgGA1UdHwRRME8wTaBLoEmGR3JzeW5j - Oi8vbG9jYWxob3N0OjQ0MDAvdGVzdGJlZC9SSVIvUjAvMS9qN2doandibENyY0NwOWx0eVBE - TnpZS1BmeGMuY3JsMGAGCCsGAQUFBwEBBFQwUjBQBggrBgEFBQcwAoZEcnN5bmM6Ly9sb2Nh - bGhvc3Q6NDQwMC90ZXN0YmVkL1JJUi8xL2o3Z2hqd2JsQ3JjQ3A5bHR5UEROellLUGZ4Yy5j - ZXIwGAYDVR0gAQH/BA4wDDAKBggrBgEFBQcOAjAOBgNVHQ8BAf8EBAMCB4AwYwYIKwYBBQUH - AQsEVzBVMFMGCCsGAQUFBzALhkdyc3luYzovL2xvY2FsaG9zdDo0NDAwL3Rlc3RiZWQvUklS - L1IwLzEvbHFrVXFEcTAwMERJb1lWOXJtd0tMZ2s3YXpvLnJvYTAgBggrBgEFBQcBBwEB/wQR - MA8wDQQCAAEwBwMFAAoDACwwDQYJKoZIhvcNAQELBQADggEBAL8iHwsyGOYhhIf3nVuL361y - TOJSP8SR0mtQLHULPl+GkYk+5MRNWtL8ucTXFvniYJtOCXEGGEIO9eDXvkQIXQSz/qbF9URQ - fuf38ghRza257syVhal6UHTgCFYuRIO9CUjcU1vkWUxH05BBIHlYdtlIQbAG/mRsCPCEgSmG - bbQaomGlUOqmJMlKxLLcoAtz2vDrwVotgHyfS5h2mgINFjnlLcNLTci+sfs7/aQAkDYx7K98 - se/ZlMorvGkFNhHoOTcGIrWkYsfkbTygVwWRm278PaB3o4449Kvsg/gb8BZeHXRs68cr5Mcf - jP7Q6jeypjTgDBnwb1yzoJIKWszFuSgxggGqMIIBpgIBA4AUlqkUqDq000DIoYV9rmwKLgk7 - azowCwYJYIZIAWUDBAIBoGswGgYJKoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEYMBwGCSqGSIb3 - DQEJBTEPFw0wODA1MjIxNzU4MjRaMC8GCSqGSIb3DQEJBDEiBCDCyf9v9Wed515TRp2WwnyM - 1rk6dB///X+aqIym2e9jdTANBgkqhkiG9w0BAQEFAASCAQAFvzrHeRPW+wn4WSyoyBEq0zKS - Cyh5tu1qTR0NHs6Rr/p8Pk81P1HQLND/U+znJZKLWlO2niEHUXPIicPDYchbj8ApH9VxKA+1 - lCWllOzFAsYyZFr3/VNs9pVp2eT4F9eEYBrBVDSNrD72MMTlWm1T5MEXqltTJJOCKzUEX96x - 91iW6A+4erop7S8hpCnxqkTin4bFVreqYcGc4CC4bh+L9pPqJnURcEk7Qeu/WEHQBm38voB4 - S11qRZNrJMQ99oiJR7hXDIBm66HjGqoUL2gPCfpgJEVVnM9pVv2k889z4eTTck2Qj54gga2W - Xkvw4Je420aDx88s9T2+PqXcbZ4g - </roa> - </msg> - - <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <roa action="publish" uri="rsync://wombat.invalid/testbed/RIR/R0/1/lqkUqDq000DIoYV9rmwKLgk7azo.roa"/> - </msg> - - <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <roa action="withdraw" uri="rsync://wombat.invalid/testbed/RIR/R0/1/lqkUqDq000DIoYV9rmwKLgk7azo.roa"/> - </msg> - - <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <roa action="withdraw" uri="rsync://wombat.invalid/testbed/RIR/R0/1/lqkUqDq000DIoYV9rmwKLgk7azo.roa"/> - </msg> - - <!-- === --> - - <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <report_error error_code="your_hair_is_on_fire">text string</report_error> - </msg> - - <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <report_error error_code="your_hair_is_on_fire"/> - </msg> </completely_gratuitous_wrapper_element_to_let_me_run_this_through_xmllint> diff --git a/ca/tests/rrdp-samples.xml b/ca/tests/rrdp-samples.xml new file mode 100644 index 00000000..966d9887 --- /dev/null +++ b/ca/tests/rrdp-samples.xml @@ -0,0 +1,81 @@ +<!-- -*- SGML -*- + - $Id$ + - + - This is a collection of sample RRDP PDU samples to use as test + - cases for the RRDP RelaxNG schema. + - + - Need to figure out whose copyright should be on these examples. + - BSD in any case so makes little practical difference, just need to + - be sure we give proper credit. Might be RIPE, might be IETF + - Trust, might be us for derivative work. Slap ours on for the + - moment, fix when we figure this out. + - + - Copyright (C) 2014 Dragon Research Labs ("DRL") + - + - Permission to use, copy, modify, and distribute this software for any + - purpose with or without fee is hereby granted, provided that the above + - copyright notice and this permission notice appear in all copies. + - + - THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH + - REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + - AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, + - INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + - LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + - OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + - PERFORMANCE OF THIS SOFTWARE. + --> + +<completely_gratuitous_wrapper_element_to_let_me_run_this_through_xmllint> + + <!-- Notification file: lists current snapshots and deltas --> + + <notification xmlns="http://www.ripe.net/rpki/rrdp" version="1" session_id="d9f6dc91-0394-40b9-9663-66aef4bb623a" serial="185"> + <snapshot uri="http://host.example/d9f6dc91-0394-40b9-9663-66aeb623a/snapshot/202.xml" hash="279b79fd8389e20585f26735ee70e0e4d4b8af23bb2e2e611c70e92d2433edea"/> + <delta serial="183" uri="http://host.example/d9f6c91-0394-40b9-9663-66aeb623a/deltas/183.xml" hash="a2d56ec180f2dde2a46bf90565932e25829b852a0b43107d5de6e41394c29100"/> + <delta serial="184" uri="http://host.example/d9f6c91-0394-40b9-9663-66aeb623a/deltas/184.xml" hash="a2d56ec180f2dde2a46b2e0565932e25829b852a0b43107d5de6e41394c29200"/> + <delta serial="185" uri="http://host.example/d9f6c91-0394-40b9-9663-66aeb623a/deltas/185.xml" hash="a2d56ec180f2dde2a46b2e0565932e25829b852a0b43107d5de6e41394c29201"/> + </notification> + + <!-- Snapshot segment: think DNS AXFR --> + + <snapshot version="1" xmlns="http://www.ripe.net/rpki/rrdp" session_id="d9f6dc91-0394-40b9-9663-66aef4bb623a" serial="1"> + <publish uri="http://host.example/foo/bar/cer1.cer"> + MIIE+jCCA+KgAwIBAgIBDTANBgkqhkiG9w0BAQsFADAzMTEwLwYDVQQD + jRBODAxN0U2NkE5RTkxNzJFNDYxMkQ4Q0Y0QzgzRjIzOERFMkEzMB4XE + h8zeHoFVu6ghRPy5dbOA4akX/KG6b8XIx0iwPYdLiDbdWFbtTdPcXBau + </publish> + <publish uri="http://host.example/foo/bar/cer2.cer"> + MIIE+jCCA+KgAwIBAgIBDTANBgkqhkiG9w0BAQsFADAzMTEwLwYDVQQD + h8zeHoFVu6ghRPy5dbOA4akX/KG6b8XIx0iwPYdLiDbdWFbtTdPcXBau + jRBODAxN0U2NkE5RTkxNzJFNDYxMkQ4Q0Y0QzgzRjIzOERFMkEzMB4XD + </publish> + <publish uri="http://host.example/foo/bar/cer3.cer"> + MIIE+jCCA+KgAwIBAgIBDTANBgkqhkiG9w0BAQsFADAzMTEwLwYDVQQD + h8zeHoFVu6ghRPy5dbOA4akX/KG6b8XIx0iwPYdLiDbdWFbtTdPcXBau + jRBODAxN0U2NkE5RTkxNzJFNDYxMkQ4Q0Y0QzgzRjIzOERFMkEzMB4XD + </publish> + </snapshot> + + <!-- Delta segment: think DNS IXFR --> + + <delta version="1" xmlns="http://www.ripe.net/rpki/rrdp" session_id="d9f6dc91-0394-40b9-9663-66aef4bb623a" serial="3"> + <publish uri="http://host.example/foo/bar/cer1.cer"> + MIIE+jCCA+KgAwIBAgIBDTANBgkqhkiG9w0BAQsFADAzMTEw + jRBODAxN0U2NkE5RTkxNzJFNDYxMkQ4Q0Y0QzgzRjIzOERFM + h8zeHoFVu6ghRPy5dbOA4akX/KG6b8XIx0iwPYdLiDbdWFbt + </publish> + <withdraw uri="http://host.example/foo/bar/cer1.cer" hash="deadf00d"/> + <publish uri="http://host.example/foo/bar/cer2.cer"> + MIIE+jCCA+KgAwIBAgIBDTANBgkqhkiG9w0BAQsFADAzMTEw + h8zeHoFVu6ghRPy5dbOA4akX/KG6b8XIx0iwPYdLiDbdWFbt + jRBODAxN0U2NkE5RTkxNzJFNDYxMkQ4Q0Y0QzgzRjIzOERFM + </publish> + <publish uri="http://host.example/foo/bar/cer3.cer" hash="deadf00d"> + MIIE+jCCA+KgAwIBAgIBDTANBgkqhkiG9w0BAQsFADAzMTEw + h8zeHoFVu6ghRPy5dbOA4akX/KG6b8XIx0iwPYdLiDbdWFbt + jRBODAxN0U2NkE5RTkxNzJFNDYxMkQ4Q0Y0QzgzRjIzOERFM + </publish> + <withdraw uri="http://host.example/foo/bar/cer4.cer" hash="deadf00d"/> + </delta> + +</completely_gratuitous_wrapper_element_to_let_me_run_this_through_xmllint> diff --git a/ca/tests/smoketest.py b/ca/tests/smoketest.py index 32f11cc3..f52ce984 100644 --- a/ca/tests/smoketest.py +++ b/ca/tests/smoketest.py @@ -47,7 +47,7 @@ import rpki.http import rpki.log import rpki.left_right import rpki.config -import rpki.publication +import rpki.publication_control import rpki.async from rpki.mysql_import import MySQLdb @@ -68,7 +68,7 @@ parser.add_argument("yaml_file", type = argparse.FileType("r"), help = "YAML description of test network") args = parser.parse_args() -cfg = rpki.config.parser(args.config, "smoketest", allow_missing = True) +cfg = rpki.config.parser(set_filename = args.config, section = "smoketest", allow_missing = True) # Load the YAML script early, so we can report errors ASAP @@ -80,6 +80,7 @@ def allocate_port(): """ Allocate a TCP port number. """ + global base_port p = base_port base_port += 1 @@ -219,8 +220,8 @@ def main(): for a in db: a.setup_bpki_certs() - setup_publication(pubd_sql) - setup_rootd(db.root, y.get("rootd", {})) + setup_publication(pubd_sql, db.root.irdb_db_name) + setup_rootd(db.root, y.get("rootd", {}), db) setup_rsyncd() setup_rcynic() @@ -232,11 +233,13 @@ def main(): try: logger.info("Starting rootd") - rootd_process = subprocess.Popen((prog_python, prog_rootd, "--foreground", "--log-stdout", "--log-level", "debug", "--config", rootd_name + ".conf")) + rootd_process = subprocess.Popen((prog_python, prog_rootd, "--foreground", "--log-stdout", "--log-level", "debug"), + env = dict(os.environ, RPKI_CONF = rootd_name + ".conf")) logger.info("Starting pubd") - pubd_process = subprocess.Popen((prog_python, prog_pubd, "--foreground", "--log-stdout", "--log-level", "debug", "--config", pubd_name + ".conf") + - (("-p", pubd_name + ".prof") if args.profile else ())) + pubd_process = subprocess.Popen((prog_python, prog_pubd, "--foreground", "--log-stdout", "--log-level", "debug") + + (("-p", pubd_name + ".prof") if args.profile else ()), + env = dict(os.environ, RPKI_CONF = pubd_name + ".conf")) logger.info("Starting rsyncd") rsyncd_process = subprocess.Popen((prog_rsyncd, "--daemon", "--no-detach", "--config", rsyncd_name + ".conf")) @@ -249,15 +252,17 @@ def main(): # the code until final exit is all closures. def start(): - rpki.async.iterator(db.engines, create_rpki_objects, created_rpki_objects) + rpki.async.iterator(db.engines, create_rpki_objects, create_pubd_objects) def create_rpki_objects(iterator, a): a.create_rpki_objects(iterator) - def created_rpki_objects(): - - # Set pubd's BPKI CRL - set_pubd_crl(yaml_loop) + def create_pubd_objects(): + call_pubd([rpki.publication_control.client_elt.make_pdu(action = "create", + client_handle = db.root.client_handle + "-" + rootd_name, + base_uri = rootd_sia, + bpki_cert = cross_certify(rootd_name + "-TA", pubd_name + "-TA"))], + cb = lambda ignored: yaml_loop()) def yaml_loop(): @@ -324,6 +329,7 @@ def cmd_sleep(cb, interval): """ Set an alarm, then wait for it to go off. """ + howlong = rpki.sundial.timedelta.parse(interval) logger.info("Sleeping %r", howlong) rpki.async.timer(cb).set(howlong) @@ -332,6 +338,7 @@ def cmd_shell(cb, *cmd): """ Run a shell command. """ + cmd = " ".join(cmd) status = subprocess.call(cmd, shell = True) logger.info("Shell command returned status %d", status) @@ -341,6 +348,7 @@ def cmd_echo(cb, *words): """ Echo some text to the log. """ + logger.info(" ".join(words)) cb() @@ -498,6 +506,7 @@ class allocation_db(list): """ Print content of the database. """ + for a in self: print a @@ -518,6 +527,7 @@ class allocation(object): """ Initialize one entity and insert it into the database. """ + db.append(self) self.name = yaml["name"] self.parent = parent @@ -554,6 +564,7 @@ class allocation(object): """ Compute the transitive resource closure. """ + resources = self.base for kid in self.kids: resources |= kid.closure() @@ -708,6 +719,7 @@ class allocation(object): """ Set the engine number for this entity. """ + self.irdb_db_name = "irdb%d" % n self.irdb_port = allocate_port() self.rpki_db_name = "rpki%d" % n @@ -717,6 +729,7 @@ class allocation(object): """ Get rpki port to use for this entity. """ + if self.is_hosted: assert self.hosted_by.rpki_port is not None return self.hosted_by.rpki_port @@ -728,6 +741,7 @@ class allocation(object): """ Create BPKI certificates for this entity. """ + logger.info("Constructing BPKI keys and certs for %s", self.name) setup_bpki_cert_chain(name = self.name, ee = ("RPKI", "IRDB", "IRBE"), @@ -741,15 +755,16 @@ class allocation(object): """ Write config files for this entity. """ + logger.info("Writing config files for %s", self.name) assert self.rpki_port is not None - d = { "my_name" : self.name, - "irdb_db_name" : self.irdb_db_name, - "irdb_db_pass" : irdb_db_pass, - "irdb_port" : self.irdb_port, - "rpki_db_name" : self.rpki_db_name, - "rpki_db_pass" : rpki_db_pass, - "rpki_port" : self.rpki_port } + d = dict(my_name = self.name, + irdb_db_name = self.irdb_db_name, + irdb_db_pass = irdb_db_pass, + irdb_port = self.irdb_port, + rpki_db_name = self.rpki_db_name, + rpki_db_pass = rpki_db_pass, + rpki_port = self.rpki_port) f = open(self.name + ".conf", "w") f.write(conf_fmt_1 % d) for line in self.extra_conf: @@ -760,6 +775,7 @@ class allocation(object): """ Set up this entity's IRDB. """ + logger.info("Setting up MySQL for %s", self.name) db = MySQLdb.connect(user = "rpki", db = self.rpki_db_name, passwd = rpki_db_pass, conv = sql_conversions) @@ -794,6 +810,7 @@ class allocation(object): once during setup, then do it again every time we apply a delta to this entity. """ + logger.info("Updating MySQL data for IRDB %s", self.name) db = MySQLdb.connect(user = "irdb", db = self.irdb_db_name, passwd = irdb_db_pass, conv = sql_conversions) @@ -847,15 +864,20 @@ class allocation(object): """ Run daemons for this entity. """ + logger.info("Running daemons for %s", self.name) - self.rpkid_process = subprocess.Popen((prog_python, prog_rpkid, "--foreground", "--log-stdout", "--log-level", "debug", "--config", self.name + ".conf") + - (("--profile", self.name + ".prof") if args.profile else ())) - self.irdbd_process = subprocess.Popen((prog_python, prog_irdbd, "--foreground", "--log-stdout", "--log-level", "debug", "--config", self.name + ".conf")) + env = dict(os.environ, RPKI_CONF = self.name + ".conf") + self.rpkid_process = subprocess.Popen((prog_python, prog_rpkid, "--foreground", "--log-stdout", "--log-level", "debug") + + (("--profile", self.name + ".prof") if args.profile else ()), + env = env) + self.irdbd_process = subprocess.Popen((prog_python, prog_irdbd, "--foreground", "--log-stdout", "--log-level", "debug"), + env = env) def kill_daemons(self): """ Kill daemons for this entity. """ + # pylint: disable=E1103 for proc, name in ((self.rpkid_process, "rpkid"), (self.irdbd_process, "irdbd")): @@ -889,7 +911,7 @@ class allocation(object): assert self.rpki_port is not None q_msg = rpki.left_right.msg.query(*pdus) - q_cms = rpki.left_right.cms_msg() + q_cms = rpki.left_right.cms_msg_saxify() q_der = q_cms.wrap(q_msg, self.irbe_key, self.irbe_cert) q_url = "http://localhost:%d/left-right" % self.rpki_port @@ -897,7 +919,7 @@ class allocation(object): def done(r_der): logger.info("Callback from rpkid %s", self.name) - r_cms = rpki.left_right.cms_msg(DER = r_der) + r_cms = rpki.left_right.cms_msg_saxify(DER = r_der) r_msg = r_cms.unwrap((self.rpkid_ta, self.rpkid_cert)) self.last_cms_time = r_cms.check_replay(self.last_cms_time, q_url) logger.debug(r_cms.pretty_print_content()) @@ -925,45 +947,7 @@ class allocation(object): certificant = self.name + "-SELF" else: certifier = self.name + "-SELF" - certfile = certifier + "-" + certificant + ".cer" - - logger.info("Cross certifying %s into %s's BPKI (%s)", certificant, certifier, certfile) - - child = rpki.x509.X509(Auto_file = certificant + ".cer") - parent = rpki.x509.X509(Auto_file = certifier + ".cer") - keypair = rpki.x509.RSA(Auto_file = certifier + ".key") - serial_file = certifier + ".srl" - - now = rpki.sundial.now() - notAfter = now + rpki.sundial.timedelta(days = 30) - - try: - f = open(serial_file, "r") - serial = f.read() - f.close() - serial = int(serial.splitlines()[0], 16) - except IOError: - serial = 1 - - x = parent.bpki_cross_certify( - keypair = keypair, - source_cert = child, - serial = serial, - notAfter = notAfter, - now = now) - - f = open(serial_file, "w") - f.write("%02x\n" % (serial + 1)) - f.close() - - f = open(certfile, "w") - f.write(x.get_PEM()) - f.close() - - logger.debug("Cross certified %s:", certfile) - logger.debug(" Issuer %s [%s]", x.getIssuer(), x.hAKI()) - logger.debug(" Subject %s [%s]", x.getSubject(), x.hSKI()) - return x + return cross_certify(certificant, certifier) def create_rpki_objects(self, cb): """ @@ -982,13 +966,11 @@ class allocation(object): selves = [self] + self.hosts - for i, s in enumerate(selves): - logger.info("Creating RPKI objects for [%d] %s", i, s.name) - rpkid_pdus = [] pubd_pdus = [] - for s in selves: + for i, s in enumerate(selves): + logger.info("Creating RPKI objects for [%d] %s", i, s.name) rpkid_pdus.append(rpki.left_right.self_elt.make_pdu( action = "create", @@ -1005,7 +987,7 @@ class allocation(object): bsc_handle = "b", generate_keypair = True)) - pubd_pdus.append(rpki.publication.client_elt.make_pdu( + pubd_pdus.append(rpki.publication_control.client_elt.make_pdu( action = "create", client_handle = s.client_handle, base_uri = s.sia_base, @@ -1036,7 +1018,7 @@ class allocation(object): bsc_handle = "b", repository_handle = "r", sia_base = s.sia_base, - bpki_cms_cert = rootd_cert, + bpki_cert = rootd_cert, sender_name = s.name, recipient_name = "rootd", peer_contact_uri = "http://localhost:%s/" % rootd_port)) @@ -1048,7 +1030,7 @@ class allocation(object): bsc_handle = "b", repository_handle = "r", sia_base = s.sia_base, - bpki_cms_cert = s.cross_certify(s.parent.name + "-SELF"), + bpki_cert = s.cross_certify(s.parent.name + "-SELF"), sender_name = s.name, recipient_name = s.parent.name, peer_contact_uri = "http://localhost:%s/up-down/%s/%s" % (s.parent.get_rpki_port(), @@ -1126,18 +1108,6 @@ class allocation(object): self.cross_certify(self.parent.name + "-SELF") self.cross_certify(parent_host + "-TA") - logger.info("Writing leaf YAML for %s", self.name) - f = open(self.name + ".yaml", "w") - f.write(yaml_fmt_1 % { - "parent_name" : self.parent.name, - "parent_host" : parent_host, - "my_name" : self.name, - "http_port" : self.parent.get_rpki_port(), - "class_name" : 2 if self.parent.is_hosted else 1, - "sia" : self.sia_base, - "ski" : ski }) - f.close() - def run_cron(self, cb): """ Trigger cron run for this engine. @@ -1174,20 +1144,22 @@ def setup_bpki_cert_chain(name, ee = (), ca = ()): """ Build a set of BPKI certificates. """ + s = "exec >/dev/null 2>&1\n" #s = "set -x\n" for kind in ("TA",) + ee + ca: - d = { "name" : name, - "kind" : kind, - "ca" : "false" if kind in ee else "true", - "openssl" : prog_openssl } + d = dict(name = name, + kind = kind, + ca = "false" if kind in ee else "true", + openssl = prog_openssl) f = open("%(name)s-%(kind)s.conf" % d, "w") f.write(bpki_cert_fmt_1 % d) f.close() if not os.path.exists("%(name)s-%(kind)s.key" % d): s += bpki_cert_fmt_2 % d s += bpki_cert_fmt_3 % d - d = { "name" : name, "openssl" : prog_openssl } + d = dict(name = name, + openssl = prog_openssl) s += bpki_cert_fmt_4 % d for kind in ee + ca: d["kind"] = kind @@ -1197,19 +1169,24 @@ def setup_bpki_cert_chain(name, ee = (), ca = ()): s += bpki_cert_fmt_6 % d subprocess.check_call(s, shell = True) -def setup_rootd(rpkid, rootd_yaml): +def setup_rootd(rpkid, rootd_yaml, db): """ Write the config files for rootd. """ + rpkid.cross_certify(rootd_name + "-TA", reverse = True) + cross_certify(pubd_name + "-TA", rootd_name + "-TA") logger.info("Writing config files for %s", rootd_name) - d = { "rootd_name" : rootd_name, - "rootd_port" : rootd_port, - "rpkid_name" : rpkid.name, - "rootd_sia" : rootd_sia, - "rsyncd_dir" : rsyncd_dir, - "openssl" : prog_openssl, - "lifetime" : rootd_yaml.get("lifetime", "30d") } + d = dict(rootd_name = rootd_name, + rootd_port = rootd_port, + rpkid_name = rpkid.name, + pubd_name = pubd_name, + rootd_sia = rootd_sia, + rsyncd_dir = rsyncd_dir, + openssl = prog_openssl, + lifetime = rootd_yaml.get("lifetime", "30d"), + pubd_port = pubd_port, + rootd_handle = db.root.client_handle + "-" + rootd_name) f = open(rootd_name + ".conf", "w") f.write(rootd_fmt_1 % d) f.close() @@ -1224,10 +1201,11 @@ def setup_rcynic(): """ Write the config file for rcynic. """ + logger.info("Config file for rcynic") - d = { "rcynic_name" : rcynic_name, - "rootd_name" : rootd_name, - "rootd_sia" : rootd_sia } + d = dict(rcynic_name = rcynic_name, + rootd_name = rootd_name, + rootd_sia = rootd_sia) f = open(rcynic_name + ".conf", "w") f.write(rcynic_fmt_1 % d) f.close() @@ -1236,19 +1214,21 @@ def setup_rsyncd(): """ Write the config file for rsyncd. """ + logger.info("Config file for rsyncd") - d = { "rsyncd_name" : rsyncd_name, - "rsyncd_port" : rsyncd_port, - "rsyncd_module" : rsyncd_module, - "rsyncd_dir" : rsyncd_dir } + d = dict(rsyncd_name = rsyncd_name, + rsyncd_port = rsyncd_port, + rsyncd_module = rsyncd_module, + rsyncd_dir = rsyncd_dir) f = open(rsyncd_name + ".conf", "w") f.write(rsyncd_fmt_1 % d) f.close() -def setup_publication(pubd_sql): +def setup_publication(pubd_sql, irdb_db_name): """ Set up publication daemon. """ + logger.info("Configure publication daemon") publication_dir = os.getcwd() + "/publication" assert rootd_sia.startswith("rsync://") @@ -1268,12 +1248,14 @@ def setup_publication(pubd_sql): if "DROP TABLE IF EXISTS" not in sql.upper(): raise db.close() - d = { "pubd_name" : pubd_name, - "pubd_port" : pubd_port, - "pubd_db_name" : pubd_db_name, - "pubd_db_user" : pubd_db_user, - "pubd_db_pass" : pubd_db_pass, - "pubd_dir" : rsyncd_dir } + d = dict(pubd_name = pubd_name, + pubd_port = pubd_port, + pubd_db_name = pubd_db_name, + pubd_db_user = pubd_db_user, + pubd_db_pass = pubd_db_pass, + pubd_dir = rsyncd_dir, + irdb_db_name = irdb_db_name, + irdb_db_pass = irdb_db_pass) f = open(pubd_name + ".conf", "w") f.write(pubd_fmt_1 % d) f.close() @@ -1288,12 +1270,13 @@ def setup_publication(pubd_sql): def call_pubd(pdus, cb): """ - Send a publication message to publication daemon and return the - response. + Send a publication control message to publication daemon and return + the response. """ + logger.info("Calling pubd") - q_msg = rpki.publication.msg.query(*pdus) - q_cms = rpki.publication.cms_msg() + q_msg = rpki.publication_control.msg.query(*pdus) + q_cms = rpki.publication_control.cms_msg_saxify() q_der = q_cms.wrap(q_msg, pubd_irbe_key, pubd_irbe_cert) q_url = "http://localhost:%d/control" % pubd_port @@ -1301,13 +1284,13 @@ def call_pubd(pdus, cb): def call_pubd_cb(r_der): global pubd_last_cms_time - r_cms = rpki.publication.cms_msg(DER = r_der) + r_cms = rpki.publication_control.cms_msg_saxify(DER = r_der) r_msg = r_cms.unwrap((pubd_ta, pubd_pubd_cert)) pubd_last_cms_time = r_cms.check_replay(pubd_last_cms_time, q_url) logger.debug(r_cms.pretty_print_content()) assert r_msg.is_reply for r_pdu in r_msg: - assert not isinstance(r_pdu, rpki.publication.report_error_elt) + r_pdu.raise_if_error() cb(r_msg) def call_pubd_eb(e): @@ -1319,15 +1302,47 @@ def call_pubd(pdus, cb): callback = call_pubd_cb, errback = call_pubd_eb) -def set_pubd_crl(cb): + +def cross_certify(certificant, certifier): """ - Whack publication daemon's bpki_crl. This must be configured before - publication daemon starts talking to its clients, and must be - updated whenever we update the CRL. + Cross-certify and return the resulting certificate. """ - logger.info("Setting pubd's BPKI CRL") - crl = rpki.x509.CRL(Auto_file = pubd_name + "-TA.crl") - call_pubd([rpki.publication.config_elt.make_pdu(action = "set", bpki_crl = crl)], cb = lambda ignored: cb()) + + certfile = certifier + "-" + certificant + ".cer" + + logger.info("Cross certifying %s into %s's BPKI (%s)", certificant, certifier, certfile) + + child = rpki.x509.X509(Auto_file = certificant + ".cer") + parent = rpki.x509.X509(Auto_file = certifier + ".cer") + keypair = rpki.x509.RSA(Auto_file = certifier + ".key") + serial_file = certifier + ".srl" + + now = rpki.sundial.now() + notAfter = now + rpki.sundial.timedelta(days = 30) + + try: + with open(serial_file, "r") as f: + serial = int(f.read().splitlines()[0], 16) + except IOError: + serial = 1 + + x = parent.bpki_cross_certify( + keypair = keypair, + source_cert = child, + serial = serial, + notAfter = notAfter, + now = now) + + with open(serial_file, "w") as f: + f.write("%02x\n" % (serial + 1)) + + with open(certfile, "w") as f: + f.write(x.get_PEM()) + + logger.debug("Cross certified %s:", certfile) + logger.debug(" Issuer %s [%s]", x.getIssuer(), x.hAKI()) + logger.debug(" Subject %s [%s]", x.getSubject(), x.hSKI()) + return x last_rcynic_run = None @@ -1335,6 +1350,7 @@ def run_rcynic(): """ Run rcynic to see whether what was published makes sense. """ + logger.info("Running rcynic") env = os.environ.copy() env["TZ"] = "" @@ -1350,6 +1366,7 @@ def mangle_sql(filename): """ Mangle an SQL file into a sequence of SQL statements. """ + words = [] f = open(filename) for line in f: @@ -1432,88 +1449,57 @@ bpki_cert_fmt_6 = ''' && \ -config %(name)s-%(kind)s.conf \ ''' -yaml_fmt_1 = '''--- -version: 1 -posturl: http://localhost:%(http_port)s/up-down/%(parent_name)s/%(my_name)s -recipient-id: "%(parent_name)s" -sender-id: "%(my_name)s" - -cms-cert-file: %(my_name)s-RPKI.cer -cms-key-file: %(my_name)s-RPKI.key -cms-ca-cert-file: %(my_name)s-TA.cer -cms-crl-file: %(my_name)s-TA.crl -cms-ca-certs-file: - - %(my_name)s-TA-%(parent_name)s-SELF.cer - -ssl-cert-file: %(my_name)s-RPKI.cer -ssl-key-file: %(my_name)s-RPKI.key -ssl-ca-cert-file: %(my_name)s-TA.cer -ssl-ca-certs-file: - - %(my_name)s-TA-%(parent_host)s-TA.cer - -# We're cheating here by hardwiring the class name - -requests: - list: - type: list - issue: - type: issue - class: %(class_name)s - sia: - - %(sia)s - cert-request-key-file: %(my_name)s.key - revoke: - type: revoke - class: %(class_name)s - ski: %(ski)s -''' - conf_fmt_1 = '''\ [irdbd] -startup-message = This is %(my_name)s irdbd +startup-message = This is %(my_name)s irdbd -sql-database = %(irdb_db_name)s -sql-username = irdb -sql-password = %(irdb_db_pass)s -bpki-ta = %(my_name)s-TA.cer -rpkid-cert = %(my_name)s-RPKI.cer -irdbd-cert = %(my_name)s-IRDB.cer -irdbd-key = %(my_name)s-IRDB.key -http-url = http://localhost:%(irdb_port)d/ -enable_tracebacks = yes +sql-database = %(irdb_db_name)s +sql-username = irdb +sql-password = %(irdb_db_pass)s +bpki-ta = %(my_name)s-TA.cer +rpkid-cert = %(my_name)s-RPKI.cer +irdbd-cert = %(my_name)s-IRDB.cer +irdbd-key = %(my_name)s-IRDB.key +http-url = http://localhost:%(irdb_port)d/ +enable_tracebacks = yes [irbe_cli] -rpkid-bpki-ta = %(my_name)s-TA.cer -rpkid-cert = %(my_name)s-RPKI.cer -rpkid-irbe-cert = %(my_name)s-IRBE.cer -rpkid-irbe-key = %(my_name)s-IRBE.key -rpkid-url = http://localhost:%(rpki_port)d/left-right -enable_tracebacks = yes +rpkid-bpki-ta = %(my_name)s-TA.cer +rpkid-cert = %(my_name)s-RPKI.cer +rpkid-irbe-cert = %(my_name)s-IRBE.cer +rpkid-irbe-key = %(my_name)s-IRBE.key +rpkid-url = http://localhost:%(rpki_port)d/left-right +enable_tracebacks = yes [rpkid] -startup-message = This is %(my_name)s rpkid +startup-message = This is %(my_name)s rpkid -sql-database = %(rpki_db_name)s -sql-username = rpki -sql-password = %(rpki_db_pass)s +sql-database = %(rpki_db_name)s +sql-username = rpki +sql-password = %(rpki_db_pass)s -bpki-ta = %(my_name)s-TA.cer -rpkid-key = %(my_name)s-RPKI.key -rpkid-cert = %(my_name)s-RPKI.cer -irdb-cert = %(my_name)s-IRDB.cer -irbe-cert = %(my_name)s-IRBE.cer +bpki-ta = %(my_name)s-TA.cer +rpkid-key = %(my_name)s-RPKI.key +rpkid-cert = %(my_name)s-RPKI.cer +irdb-cert = %(my_name)s-IRDB.cer +irbe-cert = %(my_name)s-IRBE.cer -irdb-url = http://localhost:%(irdb_port)d/ +irdb-url = http://localhost:%(irdb_port)d/ -server-host = localhost -server-port = %(rpki_port)d +server-host = localhost +server-port = %(rpki_port)d + +use-internal-cron = false +enable_tracebacks = yes -use-internal-cron = false -enable_tracebacks = yes +[myrpki] +start_rpkid = yes +start_irdbd = yes +start_pubd = no ''' rootd_fmt_1 = '''\ @@ -1525,24 +1511,28 @@ rootd-bpki-cert = %(rootd_name)s-RPKI.cer rootd-bpki-key = %(rootd_name)s-RPKI.key rootd-bpki-crl = %(rootd_name)s-TA.crl child-bpki-cert = %(rootd_name)s-TA-%(rpkid_name)s-SELF.cer +pubd-bpki-cert = %(rootd_name)s-TA-%(pubd_name)s-TA.cer server-port = %(rootd_port)s -rpki-root-dir = %(rsyncd_dir)sroot -rpki-base-uri = %(rootd_sia)sroot/ -rpki-root-cert-uri = %(rootd_sia)sroot.cer +rpki-class-name = trunk + +pubd-contact-uri = http://localhost:%(pubd_port)d/client/%(rootd_handle)s -rpki-root-key = root.key -rpki-root-cert = root.cer +rpki-root-cert-file = root.cer +rpki-root-cert-uri = %(rootd_sia)sroot.cer +rpki-root-key-file = root.key -rpki-subject-pkcs10 = %(rootd_name)s.subject.pkcs10 +rpki-subject-cert-file = trunk.cer +rpki-subject-cert-uri = %(rootd_sia)sroot/trunk.cer +rpki-subject-pkcs10-file= trunk.p10 rpki-subject-lifetime = %(lifetime)s -rpki-root-crl = root.crl -rpki-root-manifest = root.mft +rpki-root-crl-file = root.crl +rpki-root-crl-uri = %(rootd_sia)sroot/root.crl -rpki-class-name = trunk -rpki-subject-cert = trunk.cer +rpki-root-manifest-file = root.mft +rpki-root-manifest-uri = %(rootd_sia)sroot/root.mft include-bpki-crl = yes enable_tracebacks = yes @@ -1579,7 +1569,7 @@ certificatePolicies = critical, @rpki_certificate_policy [rpki_certificate_policy] -policyIdentifier = 1.3.6.1.5.5.7.14.2 +policyIdentifier = 1.3.6.1.5.5.7.14.2 ''' rootd_fmt_2 = '''\ @@ -1602,8 +1592,7 @@ awk '!/-----(BEGIN|END)/' >>%(rootd_name)s.tal && -outform DER \ -extfile %(rootd_name)s.conf \ -extensions req_x509_rpki_ext \ - -signkey root.key && -ln -f root.cer %(rsyncd_dir)s + -signkey root.key ''' rcynic_fmt_1 = '''\ @@ -1636,6 +1625,7 @@ sql-database = %(pubd_db_name)s sql-username = %(pubd_db_user)s sql-password = %(pubd_db_pass)s bpki-ta = %(pubd_name)s-TA.cer +pubd-crl = %(pubd_name)s-TA.crl pubd-cert = %(pubd_name)s-PUBD.cer pubd-key = %(pubd_name)s-PUBD.key irbe-cert = %(pubd_name)s-IRBE.cer @@ -1643,6 +1633,17 @@ server-host = localhost server-port = %(pubd_port)d publication-base = %(pubd_dir)s enable_tracebacks = yes + +[irdbd] + +sql-database = %(irdb_db_name)s +sql-username = irdb +sql-password = %(irdb_db_pass)s + +[myrpki] +start_rpkid = no +start_irdbd = no +start_pubd = yes ''' main() diff --git a/ca/tests/sql-cleaner.py b/ca/tests/sql-cleaner.py index ca88d456..369a68ea 100644 --- a/ca/tests/sql-cleaner.py +++ b/ca/tests/sql-cleaner.py @@ -22,18 +22,21 @@ import rpki.config import rpki.sql_schemas from rpki.mysql_import import MySQLdb -cfg = rpki.config.parser(None, "yamltest", allow_missing = True) +cfg = rpki.config.parser(section = "yamltest", allow_missing = True) for name in ("rpkid", "irdbd", "pubd"): username = cfg.get("%s_sql_username" % name, name[:4]) password = cfg.get("%s_sql_password" % name, "fnord") + # All of this schema creation stuff will go away once we're on Django ORM. + # For the moment, a quick kludge for testing. schema = [] - for line in getattr(rpki.sql_schemas, name, "").splitlines(): - schema.extend(line.partition("--")[0].split()) - schema = " ".join(schema).strip(";").split(";") - schema = [statement.strip() for statement in schema if statement and "DROP TABLE" not in statement] + if name == "rpkid": + for line in getattr(rpki.sql_schemas, name, "").splitlines(): + schema.extend(line.partition("--")[0].split()) + schema = " ".join(schema).strip(";").split(";") + schema = [statement.strip() for statement in schema if statement and "DROP TABLE" not in statement] db = MySQLdb.connect(user = username, passwd = password) cur = db.cursor() diff --git a/ca/tests/sql-dumper.py b/ca/tests/sql-dumper.py index 19cc1b34..d0fe3489 100644 --- a/ca/tests/sql-dumper.py +++ b/ca/tests/sql-dumper.py @@ -22,7 +22,7 @@ import subprocess import rpki.config from rpki.mysql_import import MySQLdb -cfg = rpki.config.parser(None, "yamltest", allow_missing = True) +cfg = rpki.config.parser(section = "yamltest", allow_missing = True) for name in ("rpkid", "irdbd", "pubd"): diff --git a/ca/tests/test-rrdp.py b/ca/tests/test-rrdp.py new file mode 100755 index 00000000..1a9db929 --- /dev/null +++ b/ca/tests/test-rrdp.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# $Id$ +# +# Copyright (C) 2013 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +Proof-of-concept test driver for RRDP code. Still fairly kludgy in places. +""" + +import os +import sys +import glob +import time +import signal +import textwrap +import argparse +import subprocess + +parser = argparse.ArgumentParser(description = __doc__) +parser.add_argument("--use-smoketest", action = "store_true") +parser.add_argument("--yaml-file", default = "smoketest.2.yaml") +parser.add_argument("--delay", type = int, default = 30) +parser.add_argument("--exhaustive", action = "store_true") +parser.add_argument("--skip-daemons", action = "store_true") +parser.add_argument("--dry-run", action = "store_true") +args = parser.parse_args() + +def log(msg): + sys.stdout.write(msg + "\n") + sys.stdout.flush() + +def run(*argv): + log("Running: " + " ".join(argv)) + if not args.dry_run: + subprocess.check_call(argv) + +def dataglob(pattern): + return glob.iglob(os.path.join(("smoketest.dir" if args.use_smoketest else "yamltest.dir/RIR"), pattern)) + +def snapshot_to_serial(fn): + return int(os.path.splitext(os.path.basename(fn))[0]) + +def delta_to_serial(fn): + return int(os.path.splitext(os.path.basename(fn))[0]) + +top = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "..", "..")) + +rrdp_test_tool = os.path.join(top, "potpourri/rrdp-test-tool") +rcynic = os.path.join(top, "rp/rcynic/rcynic") +rcynic_text = os.path.join(top, "rp/rcynic/rcynic-text") + +with open("rcynic-rrdp.conf", "w") as f: + f.write(textwrap.dedent('''# Automatically generated for RRDP tests, do not edit. + [rcynic] + xml-summary = rcynic.xml + jitter = 0 + use-links = yes + use-syslog = no + use-stderr = yes + log-level = log_debug + run-rsync = no + ''')) + if args.use_smoketest: + f.write("trust-anchor = smoketest.dir/root.cer\n") + else: + f.write("trust-anchor = yamltest.dir/RIR/publication/RIR-root/root.cer\n") + +if args.skip_daemons: + log("--skip-daemons specified, so running neither smoketest nor yamltest") +elif args.use_smoketest: + run("python", "smoketest.py", args.yaml_file) +else: + run("python", "sql-cleaner.py") + class GotSIGUSR1(Exception): + pass + def handle_sigusr1(signum, frame): + raise GotSIGUSR1 + old_sigusr1 = signal.signal(signal.SIGUSR1, handle_sigusr1) + cmd = ("python", "yamltest.py", args.yaml_file, "--notify-when-startup-complete", str(os.getpid())) + log("Running: " + " ".join(cmd)) + yamltest = subprocess.Popen(cmd) + log("Waiting for SIGUSR1 from yamltest") + try: + while True: + signal.pause() + except GotSIGUSR1: + signal.signal(signal.SIGUSR1, old_sigusr1) + log("Sleeping %s" % args.delay) + time.sleep(args.delay) + yamltest.terminate() + +snapshots = dict((snapshot_to_serial(fn), fn) for fn in dataglob("rrdp-publication/*/snapshot/*.xml")) +deltas = dict((delta_to_serial(fn), fn) for fn in dataglob("rrdp-publication/*/deltas/*.xml")) + +for snapshot in sorted(snapshots): + + time.sleep(1) + run("rm", "-rf", "rcynic-data") + run(rrdp_test_tool, snapshots[snapshot]) + run(rcynic, "-c", "rcynic-rrdp.conf") + run(rcynic_text, "rcynic.xml") + + for delta in sorted(deltas): + if delta > snapshot: + time.sleep(1) + run(rrdp_test_tool, deltas[delta]) + run(rcynic, "-c", "rcynic-rrdp.conf") + run(rcynic_text, "rcynic.xml") + + if not args.exhaustive: + break diff --git a/ca/tests/testpoke.py b/ca/tests/testpoke.py index c28ed397..68f967b9 100644 --- a/ca/tests/testpoke.py +++ b/ca/tests/testpoke.py @@ -74,9 +74,9 @@ def get_PEM_chain(name, cert = None): if cert is not None: chain.append(cert) if name in yaml_data: - chain.extend([rpki.x509.X509(PEM = x) for x in yaml_data[name]]) + chain.extend(rpki.x509.X509(PEM = x) for x in yaml_data[name]) elif name + "-file" in yaml_data: - chain.extend([rpki.x509.X509(PEM_file = x) for x in yaml_data[name + "-file"]]) + chain.extend(rpki.x509.X509(PEM_file = x) for x in yaml_data[name + "-file"]) return chain def query_up_down(q_pdu): @@ -84,11 +84,11 @@ def query_up_down(q_pdu): payload = q_pdu, sender = yaml_data["sender-id"], recipient = yaml_data["recipient-id"]) - q_der = rpki.up_down.cms_msg().wrap(q_msg, cms_key, cms_certs, cms_crl) + q_der = rpki.up_down.cms_msg_saxify().wrap(q_msg, cms_key, cms_certs, cms_crl) def done(r_der): global last_cms_timestamp - r_cms = rpki.up_down.cms_msg(DER = r_der) + r_cms = rpki.up_down.cms_msg_saxify(DER = r_der) r_msg = r_cms.unwrap([cms_ta] + cms_ca_certs) last_cms_timestamp = r_cms.check_replay(last_cms_timestamp) print r_cms.pretty_print_content() diff --git a/ca/tests/xml-parse-test.py b/ca/tests/xml-parse-test.py index 5ea25492..90c80775 100644 --- a/ca/tests/xml-parse-test.py +++ b/ca/tests/xml-parse-test.py @@ -28,30 +28,34 @@ # OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR # PERFORMANCE OF THIS SOFTWARE. -import glob, lxml.etree, lxml.sax -import rpki.up_down, rpki.left_right, rpki.publication, rpki.relaxng +import glob +import lxml.etree +import rpki.up_down +import rpki.left_right +import rpki.publication +import rpki.publication_control +import rpki.relaxng verbose = False -def test(fileglob, rng, sax_handler, encoding, tester = None): +def test(fileglob, rng, parser, encoding, tester = None): files = glob.glob(fileglob) files.sort() for f in files: print "<!--", f, "-->" - handler = sax_handler() elt_in = lxml.etree.parse(f).getroot() if verbose: print "<!-- Input -->" print lxml.etree.tostring(elt_in, pretty_print = True, encoding = encoding, xml_declaration = True) rng.assertValid(elt_in) - lxml.sax.saxify(elt_in, handler) - elt_out = handler.result.toXML() + parsed = parser.fromXML(elt_in) + elt_out = parsed.toXML() if verbose: print "<!-- Output -->" print lxml.etree.tostring(elt_out, pretty_print = True, encoding = encoding, xml_declaration = True) rng.assertValid(elt_out) if tester: - tester(elt_in, elt_out, handler.result) + tester(elt_in, elt_out, parsed) if verbose: print @@ -79,41 +83,47 @@ def lr_tester(elt_in, elt_out, msg): (obj.signing_cert_crl, "Signing certificate CRL"))) # (obj.pkcs10_request, "PKCS #10 request") if isinstance(obj, rpki.left_right.parent_elt): - pprint(((obj.bpki_cms_cert, "CMS certificate"), - (obj.bpki_cms_glue, "CMS glue"))) + pprint(((obj.bpki_cert, "BPKI certificate"), + (obj.bpki_glue, "BPKI glue"))) if isinstance(obj, (rpki.left_right.child_elt, rpki.left_right.repository_elt)): - pprint(((obj.bpki_cert, "Certificate"), - (obj.bpki_glue, "Glue"))) + pprint(((obj.bpki_cert, "BPKI certificate"), + (obj.bpki_glue, "BPKI glue"))) def pp_tester(elt_in, elt_out, msg): assert isinstance(msg, rpki.publication.msg) for obj in msg: - if isinstance(obj, rpki.publication.client_elt): + if isinstance(obj, rpki.publication.publish_elt): + pprint(((obj.payload, "Publish object"),)) + if isinstance(obj, rpki.publication.withdraw_elt): + pprint(((None, "Withdraw object"),)) + +def pc_tester(elt_in, elt_out, msg): + assert isinstance(msg, rpki.publication_control.msg) + for obj in msg: + if isinstance(obj, rpki.publication_control.client_elt): pprint(((obj.bpki_cert, "BPKI cert"), (obj.bpki_glue, "BPKI glue"))) - if isinstance(obj, rpki.publication.certificate_elt): - pprint(((obj.payload, "RPKI cert"),)) - if isinstance(obj, rpki.publication.crl_elt): - pprint(((obj.payload, "RPKI CRL"),)) - if isinstance(obj, rpki.publication.manifest_elt): - pprint(((obj.payload, "RPKI manifest"),)) - if isinstance(obj, rpki.publication.roa_elt): - pprint(((obj.payload, "ROA"),)) test(fileglob = "up-down-protocol-samples/*.xml", rng = rpki.relaxng.up_down, - sax_handler = rpki.up_down.sax_handler, + parser = rpki.up_down.msg, encoding = "utf-8", tester = ud_tester) test(fileglob = "left-right-protocol-samples/*.xml", rng = rpki.relaxng.left_right, - sax_handler = rpki.left_right.sax_handler, + parser = rpki.left_right.msg, encoding = "us-ascii", tester = lr_tester) test(fileglob = "publication-protocol-samples/*.xml", rng = rpki.relaxng.publication, - sax_handler = rpki.publication.sax_handler, + parser = rpki.publication.msg, encoding = "us-ascii", tester = pp_tester) + +test(fileglob = "publication-control-protocol-samples/*.xml", + rng = rpki.relaxng.publication_control, + parser = rpki.publication_control.msg, + encoding = "us-ascii", + tester = pc_tester) diff --git a/ca/tests/yamlconf.py b/ca/tests/yamlconf.py index f1073c92..13456377 100644 --- a/ca/tests/yamlconf.py +++ b/ca/tests/yamlconf.py @@ -125,7 +125,7 @@ class router_cert(object): def __init__(self, asn, router_id): self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) self.router_id = router_id - self.keypair = rpki.x509.ECDSA.generate(self.ecparams()) + self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True) self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) self.gski = self.pkcs10.gSKI() @@ -491,16 +491,18 @@ class allocation(object): def syncdb(self): import django.core.management assert not self.is_hosted - django.core.management.call_command("syncdb", - database = self.irdb_name, - load_initial_data = False, - interactive = False, - verbosity = 0) + django.core.management.call_command( + "syncdb", + verbosity = 0, + database = self.irdb_name, + migrate = True, + load_initial_data = False, + interactive = False) def hire_zookeeper(self): assert not self.is_hosted self._zoo = rpki.irdb.Zookeeper( - cfg = rpki.config.parser(self.path("rpki.conf")), + cfg = rpki.config.parser(filename = self.path("rpki.conf")), logstream = None if quiet else sys.stdout) @property @@ -520,7 +522,7 @@ class allocation(object): root_uri = "rsync://%s/rpki/" % self.rsync_server - root_sia = (root_uri, root_uri + "root.mft", None) + root_sia = (root_uri, root_uri + "root.mft", None, rpki.publication.rrdp_sia_uri_kludge) root_cert = rpki.x509.X509.self_certify( keypair = root_key, @@ -530,15 +532,15 @@ class allocation(object): notAfter = rpki.sundial.now() + rpki.sundial.timedelta(days = 365), resources = root_resources) - with open(self.path("publication.root", "root.cer"), "wb") as f: + with open(self.path("root.cer"), "wb") as f: f.write(root_cert.get_DER()) with open(self.path("root.key"), "wb") as f: f.write(root_key.get_DER()) with open(cleanpath(test_dir, "root.tal"), "w") as f: - f.write("rsync://%s/root/root.cer\n\n%s" % ( - self.rsync_server, root_key.get_public().get_Base64())) + f.write("rsync://%s/root/root.cer\n\n" % self.rsync_server) + f.write(root_key.get_public().get_Base64()) def mkdir(self, *path): path = self.path(*path) @@ -681,7 +683,7 @@ def main(): # passwords: this is mostly so that I can show a complete working # example without publishing my own server's passwords. - cfg = rpki.config.parser(args.config, "yamlconf", allow_missing = True) + cfg = rpki.config.parser(set_filename = args.config, section = "yamlconf", allow_missing = True) try: cfg.set_global_flags() except: @@ -755,9 +757,13 @@ def body(): pre_django_sql_setup(set(d.irdb_name for d in db if not d.is_hosted)) # Now ready for fun with multiple databases in Django! - + # # https://docs.djangoproject.com/en/1.4/topics/db/multi-db/ # https://docs.djangoproject.com/en/1.4/topics/db/sql/ + # + # This program's use of the ORM is sufficiently different that it's + # not worth straining to use rpki.django_settings, so we just use + # Django's settings API directly. database_template = { "ENGINE" : "django.db.backends.mysql", @@ -767,25 +773,20 @@ def body(): "PORT" : "", "OPTIONS" : { "init_command": "SET storage_engine=INNODB" }} - databases = dict((d.irdb_name, - dict(database_template, NAME = d.irdb_name)) + databases = dict((d.irdb_name, dict(database_template, NAME = d.irdb_name)) for d in db if not d.is_hosted) databases["default"] = databases[db.root.irdb_name] import django + django.setup() from django.conf import settings settings.configure( - DATABASES = databases, + DATABASES = databases, DATABASE_ROUTERS = ["rpki.irdb.router.DBContextRouter"], - MIDDLEWARE_CLASSES = (), - INSTALLED_APPS = ("rpki.irdb",)) - - if django.VERSION >= (1, 7): # API change, feh - from django.apps import apps - apps.populate(settings.INSTALLED_APPS) + INSTALLED_APPS = ["rpki.irdb"]) import rpki.irdb diff --git a/ca/tests/yamltest.py b/ca/tests/yamltest.py index 0932049b..71a83c14 100644 --- a/ca/tests/yamltest.py +++ b/ca/tests/yamltest.py @@ -43,6 +43,7 @@ import re import os import logging import argparse +import webbrowser import sys import yaml import signal @@ -67,19 +68,21 @@ def cleanpath(*names): """ Construct normalized pathnames. """ + return os.path.normpath(os.path.join(*names)) # Pathnames for various things we need this_dir = os.getcwd() test_dir = cleanpath(this_dir, "yamltest.dir") -rpkid_dir = cleanpath(this_dir, "..") +ca_dir = cleanpath(this_dir, "..") -prog_rpkic = cleanpath(rpkid_dir, "rpkic") -prog_rpkid = cleanpath(rpkid_dir, "rpkid") -prog_irdbd = cleanpath(rpkid_dir, "irdbd") -prog_pubd = cleanpath(rpkid_dir, "pubd") -prog_rootd = cleanpath(rpkid_dir, "rootd") +prog_rpkic = cleanpath(ca_dir, "rpkic") +prog_rpkid = cleanpath(ca_dir, "rpkid") +prog_irdbd = cleanpath(ca_dir, "irdbd") +prog_pubd = cleanpath(ca_dir, "pubd") +prog_rootd = cleanpath(ca_dir, "rootd") +prog_rpki_manage = cleanpath(ca_dir, "rpki-manage") class roa_request(object): """ @@ -110,6 +113,7 @@ class roa_request(object): """ Parse a ROA request from YAML format. """ + return cls(y.get("asn"), y.get("ipv4"), y.get("ipv6")) @@ -129,7 +133,7 @@ class router_cert(object): def __init__(self, asn, router_id): self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) self.router_id = router_id - self.keypair = rpki.x509.ECDSA.generate(self.ecparams()) + self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True) self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) self.gski = self.pkcs10.gSKI() @@ -154,7 +158,7 @@ class allocation_db(list): def __init__(self, yaml): list.__init__(self) self.root = allocation(yaml, self) - assert self.root.is_root + assert self.root.is_root and not any(a.is_root for a in self if a is not self.root) and self[0] is self.root if self.root.crl_interval is None: self.root.crl_interval = 60 * 60 if self.root.regen_margin is None: @@ -180,6 +184,7 @@ class allocation_db(list): """ Show contents of allocation database. """ + for a in self: a.dump() @@ -203,6 +208,7 @@ class allocation(object): pubd_port = -1 rsync_port = -1 rootd_port = -1 + rrdp_port = -1 rpkic_counter = 0L @classmethod @@ -210,6 +216,7 @@ class allocation(object): """ Allocate a TCP port. """ + cls.base_port += 1 return cls.base_port @@ -221,6 +228,7 @@ class allocation(object): Allocate an engine number, mostly used to construct MySQL database names. """ + cls.base_engine += 1 return cls.base_engine @@ -267,6 +275,7 @@ class allocation(object): if self.runs_pubd: self.pubd_port = self.allocate_port() self.rsync_port = self.allocate_port() + self.rrdp_port = self.allocate_port() if self.is_root: self.rootd_port = self.allocate_port() @@ -275,6 +284,7 @@ class allocation(object): Compute resource closure of this node and its children, to avoid a lot of tedious (and error-prone) duplication in the YAML file. """ + resources = self.base for kid in self.kids: resources |= kid.closure() @@ -285,6 +295,7 @@ class allocation(object): """ Show content of this allocation node. """ + print str(self) def __str__(self): @@ -309,6 +320,7 @@ class allocation(object): """ Is this the root node? """ + return self.parent is None @property @@ -316,6 +328,7 @@ class allocation(object): """ Is this entity hosted? """ + return self.hosted_by is not None @property @@ -323,18 +336,21 @@ class allocation(object): """ Does this entity run a pubd? """ + return self.is_root or not (self.is_hosted or only_one_pubd) def path(self, *names): """ Construct pathnames in this entity's test directory. """ + return cleanpath(test_dir, self.host.name, *names) def csvout(self, fn): """ Open and log a CSV output file. """ + path = self.path(fn) print "Writing", path return rpki.csv_utils.csv_writer(path) @@ -343,6 +359,7 @@ class allocation(object): """ Construct service URL for this node's parent. """ + return "http://localhost:%d/up-down/%s/%s" % (self.parent.host.rpkid_port, self.parent.name, self.name) @@ -351,12 +368,12 @@ class allocation(object): """ Write Autonomous System Numbers CSV file. """ + fn = "%s.asns.csv" % d.name if not args.skip_config: - f = self.csvout(fn) - for k in self.kids: - f.writerows((k.name, a) for a in k.resources.asn) - f.close() + with self.csvout(fn) as f: + for k in self.kids: + f.writerows((k.name, a) for a in k.resources.asn) if not args.stop_after_config: self.run_rpkic("load_asns", fn) @@ -364,12 +381,12 @@ class allocation(object): """ Write prefixes CSV file. """ + fn = "%s.prefixes.csv" % d.name if not args.skip_config: - f = self.csvout(fn) - for k in self.kids: - f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) - f.close() + with self.csvout(fn) as f: + for k in self.kids: + f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) if not args.stop_after_config: self.run_rpkic("load_prefixes", fn) @@ -377,13 +394,13 @@ class allocation(object): """ Write ROA CSV file. """ + fn = "%s.roas.csv" % d.name if not args.skip_config: - f = self.csvout(fn) - for g1, r in enumerate(self.roa_requests): - f.writerows((p, r.asn, "G%08d%08d" % (g1, g2)) - for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ()))) - f.close() + with self.csvout(fn) as f: + for g1, r in enumerate(self.roa_requests): + f.writerows((p, r.asn, "G%08d%08d" % (g1, g2)) + for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ()))) if not args.stop_after_config: self.run_rpkic("load_roa_requests", fn) @@ -391,17 +408,14 @@ class allocation(object): """ Write Ghostbusters vCard file. """ + if self.ghostbusters: fn = "%s.ghostbusters.vcard" % d.name if not args.skip_config: path = self.path(fn) print "Writing", path - f = open(path, "w") - for i, g in enumerate(self.ghostbusters): - if i: - f.write("\n") - f.write(g) - f.close() + with open(path, "w") as f: + f.write("\n".join(self.ghostbusters)) if not args.stop_after_config: self.run_rpkic("load_ghostbuster_requests", fn) @@ -409,6 +423,7 @@ class allocation(object): """ Write EE certificates (router certificates, etc). """ + if self.router_certs: fn = "%s.routercerts.xml" % d.name if not args.skip_config: @@ -431,15 +446,16 @@ class allocation(object): if not args.skip_config and args.store_router_private_keys: path = self.path("%s.routercerts.keys" % d.name) print "Writing", path - with open(path, "w") as f: - for r in self.router_certs: - f.write(r.keypair.get_PEM()) + with open(path, "w") as f: + for r in self.router_certs: + f.write(r.keypair.get_PEM()) @property def pubd(self): """ Walk up tree until we find somebody who runs pubd. """ + s = self while not s.runs_pubd: s = s.parent @@ -450,6 +466,7 @@ class allocation(object): """ Work out what pubd configure_publication_client will call us. """ + path = [] s = self if not args.flat_publication: @@ -469,46 +486,46 @@ class allocation(object): """ r = dict( - handle = self.name, - run_rpkid = str(not self.is_hosted), - run_pubd = str(self.runs_pubd), - run_rootd = str(self.is_root), - irdbd_sql_database = "irdb%d" % self.engine, - irdbd_sql_username = "irdb", - rpkid_sql_database = "rpki%d" % self.engine, - rpkid_sql_username = "rpki", - rpkid_server_host = "localhost", - rpkid_server_port = str(self.rpkid_port), - irdbd_server_host = "localhost", - irdbd_server_port = str(self.irdbd_port), - rootd_server_port = str(self.rootd_port), - pubd_sql_database = "pubd%d" % self.engine, - pubd_sql_username = "pubd", - pubd_server_host = "localhost", - pubd_server_port = str(self.pubd.pubd_port), - publication_rsync_server = "localhost:%s" % self.pubd.rsync_port, - bpki_servers_directory = self.path(), - publication_base_directory = self.path("publication"), - shared_sql_password = "fnord") + handle = self.name, + run_rpkid = str(not self.is_hosted), + run_pubd = str(self.runs_pubd), + run_rootd = str(self.is_root), + irdbd_sql_database = "irdb%d" % self.engine, + irdbd_sql_username = "irdb", + rpkid_sql_database = "rpki%d" % self.engine, + rpkid_sql_username = "rpki", + rpkid_server_host = "localhost", + rpkid_server_port = str(self.rpkid_port), + irdbd_server_host = "localhost", + irdbd_server_port = str(self.irdbd_port), + rootd_server_port = str(self.rootd_port), + pubd_sql_database = "pubd%d" % self.engine, + pubd_sql_username = "pubd", + pubd_server_host = "localhost", + pubd_server_port = str(self.pubd.pubd_port), + publication_rsync_server = "localhost:%s" % self.pubd.rsync_port, + publication_rrdp_notification_uri = "http://localhost:%s/rrdp/notify.xml" % self.pubd.rrdp_port, + bpki_servers_directory = self.path(), + publication_base_directory = self.path("publication"), + rrdp_publication_base_directory = self.path("rrdp-publication"), + shared_sql_password = "fnord") r.update(config_overrides) - f = open(self.path("rpki.conf"), "w") - f.write("# Automatically generated, do not edit\n") - print "Writing", f.name - - section = None - for line in open(cleanpath(rpkid_dir, "examples/rpki.conf")): - m = section_regexp.match(line) - if m: - section = m.group(1) - m = variable_regexp.match(line) - option = m.group(1) if m and section == "myrpki" else None - if option and option in r: - line = "%s = %s\n" % (option, r[option]) - f.write(line) + with open(self.path("rpki.conf"), "w") as f: + f.write("# Automatically generated, do not edit\n") + print "Writing", f.name - f.close() + section = None + for line in open(cleanpath(ca_dir, "examples/rpki.conf")): + m = section_regexp.match(line) + if m: + section = m.group(1) + m = variable_regexp.match(line) + option = m.group(1) if m and section == "myrpki" else None + if option and option in r: + line = "%s = %s\n" % (option, r[option]) + f.write(line) def dump_rsyncd(self): """ @@ -516,25 +533,24 @@ class allocation(object): """ if self.runs_pubd: - f = open(self.path("rsyncd.conf"), "w") - print "Writing", f.name - f.writelines(s + "\n" for s in - ("# Automatically generated, do not edit", - "port = %d" % self.rsync_port, - "address = localhost", - "[rpki]", - "log file = rsyncd.log", - "read only = yes", - "use chroot = no", - "path = %s" % self.path("publication"), - "comment = RPKI test", - "[root]", - "log file = rsyncd_root.log", - "read only = yes", - "use chroot = no", - "path = %s" % self.path("publication.root"), - "comment = RPKI test root")) - f.close() + with open(self.path("rsyncd.conf"), "w") as f: + print "Writing", f.name + f.writelines(s + "\n" for s in + ("# Automatically generated, do not edit", + "port = %d" % self.rsync_port, + "address = localhost", + "[rpki]", + "log file = rsyncd.log", + "read only = yes", + "use chroot = no", + "path = %s" % self.path("publication"), + "comment = RPKI test", + "[root]", + "log file = rsyncd_root.log", + "read only = yes", + "use chroot = no", + "path = %s" % self.path("publication.root"), + "comment = RPKI test root")) @classmethod def next_rpkic_counter(cls): @@ -545,65 +561,133 @@ class allocation(object): """ Run rpkic for this entity. """ - cmd = [prog_rpkic, "-i", self.name, "-c", self.path("rpki.conf")] + + cmd = [prog_rpkic, "-i", self.name] if args.profile: cmd.append("--profile") cmd.append(self.path("rpkic.%s.prof" % rpki.sundial.now())) cmd.extend(str(a) for a in argv if a is not None) print 'Running "%s"' % " ".join(cmd) - env = os.environ.copy() - env["YAMLTEST_RPKIC_COUNTER"] = self.next_rpkic_counter() + env = dict(os.environ, + YAMLTEST_RPKIC_COUNTER = self.next_rpkic_counter(), + RPKI_CONF = self.path("rpki.conf")) subprocess.check_call(cmd, cwd = self.host.path(), env = env) + def syncdb(self): + """ + Run whatever Django ORM commands are necessary to set up the + database this week. + """ + + # Fork a sub-process for each syncdb/migrate run, because it's + # easier than figuring out how to change Django settings after + # initialization. + + def sync_settings(settings, verbosity = 1): + + if verbosity > 0: + print "Running Django setup for", self.name + + pid = os.fork() + + if pid == 0: + logging.getLogger().setLevel(logging.WARNING) + + os.environ.update(RPKI_CONF = self.path("rpki.conf"), + DJANGO_SETTINGS_MODULE = "rpki.django_settings." + settings) + + import django + django.setup() + + import django.core.management + django.core.management.call_command("migrate", verbosity = verbosity, no_color = True, + load_initial_data = False, interactive = False) + + if settings in ("gui", "irdb"): + from django.contrib.auth.models import User + User.objects.create_superuser("root", "root@example.org", "fnord") + + sys.exit(0) + + elif os.waitpid(pid, 0)[1]: + raise RuntimeError("Django setup failed for %s %s" % (self.name, settings)) + + for settings in ("rpkid", "pubd", "gui"): + sync_settings(settings) + def run_python_daemon(self, prog): """ Start a Python daemon and return a subprocess.Popen object representing the running daemon. """ + basename = os.path.splitext(os.path.basename(prog))[0] cmd = [prog, "--foreground", "--log-level", "debug", - "--log-file", self.path(basename + ".log"), - "--config", self.path("rpki.conf")] + "--log-file", self.path(basename + ".log")] if args.profile and basename != "rootd": cmd.extend(( "--profile", self.path(basename + ".prof"))) - p = subprocess.Popen(cmd, cwd = self.path()) - print 'Running %s for %s: pid %d process %r' % (" ".join(cmd), self.name, p.pid, p) + env = dict(os.environ, RPKI_CONF = self.path("rpki.conf")) + p = subprocess.Popen(cmd, cwd = self.path(), env = env) + print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p) return p def run_rpkid(self): """ Run rpkid. """ + return self.run_python_daemon(prog_rpkid) def run_irdbd(self): """ Run irdbd. """ + return self.run_python_daemon(prog_irdbd) def run_pubd(self): """ Run pubd. """ + return self.run_python_daemon(prog_pubd) def run_rootd(self): """ Run rootd. """ + return self.run_python_daemon(prog_rootd) def run_rsyncd(self): """ Run rsyncd. """ + p = subprocess.Popen(("rsync", "--daemon", "--no-detach", "--config", "rsyncd.conf"), cwd = self.path()) print "Running rsyncd for %s: pid %d process %r" % (self.name, p.pid, p) return p + def run_gui(self): + """ + Start an instance of the RPKI GUI under the Django test server and + return a subprocess.Popen object representing the running daemon. + """ + + port = 8000 + self.engine + cmd = (prog_rpki_manage, "runserver", str(port)) + env = dict(os.environ, + RPKI_CONF = self.path("rpki.conf"), + RPKI_DJANGO_DEBUG = "yes", + ALLOW_PLAIN_HTTP_FOR_TESTING = "I solemnly swear that I am not running this in production") + p = subprocess.Popen(cmd, cwd = self.path(), env = env, + stdout = open(self.path("gui.log"), "w"), stderr = subprocess.STDOUT) + print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p) + return p + + def create_root_certificate(db_root): print "Creating rootd RPKI root certificate" @@ -615,9 +699,11 @@ def create_root_certificate(db_root): root_key = rpki.x509.RSA.generate(quiet = True) - root_uri = "rsync://localhost:%d/rpki/" % db_root.pubd.rsync_port + root_uri = "rsync://localhost:%d/rpki/%s-root/root" % (db_root.pubd.rsync_port, db_root.name) + + rrdp_uri = "http://localhost:%s/rrdp/notify.xml" % db.root.pubd.rrdp_port - root_sia = (root_uri, root_uri + "root.mft", None) + root_sia = (root_uri + "/", root_uri + "/root.mft", None, rrdp_uri) root_cert = rpki.x509.X509.self_certify( keypair = root_key, @@ -627,22 +713,21 @@ def create_root_certificate(db_root): notAfter = rpki.sundial.now() + rpki.sundial.timedelta(days = 365), resources = root_resources) - f = open(db_root.path("publication.root/root.cer"), "wb") - f.write(root_cert.get_DER()) - f.close() + with open(db_root.path("root.cer"), "wb") as f: + f.write(root_cert.get_DER()) - f = open(db_root.path("root.key"), "wb") - f.write(root_key.get_DER()) - f.close() + with open(db_root.path("root.key"), "wb") as f: + f.write(root_key.get_DER()) - f = open(os.path.join(test_dir, "root.tal"), "w") - f.write("rsync://localhost:%d/root/root.cer\n\n" % db_root.pubd.rsync_port) - f.write(root_key.get_public().get_Base64()) - f.close() + with open(os.path.join(test_dir, "root.tal"), "w") as f: + f.write(root_uri + ".cer\n\n") + f.write(root_key.get_public().get_Base64()) +logger = logging.getLogger(__name__) -os.environ["TZ"] = "UTC" +os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings.irdb", + TZ = "UTC") time.tzset() parser = argparse.ArgumentParser(description = __doc__) @@ -662,6 +747,12 @@ parser.add_argument("--synchronize", action = "store_true", help = "synchronize IRDB with daemons") parser.add_argument("--profile", action = "store_true", help = "enable profiling") +parser.add_argument("-g", "--run_gui", action = "store_true", + help = "enable GUI using django-admin runserver") +parser.add_argument("--browser", action = "store_true", + help = "create web browser tabs for GUI") +parser.add_argument("--notify-when-startup-complete", type = int, + help = "send SIGUSR1 to this process when startup is complete") parser.add_argument("--store-router-private-keys", action = "store_true", help = "write generate router private keys to disk") parser.add_argument("yaml_file", type = argparse.FileType("r"), @@ -680,7 +771,7 @@ try: # passwords: this is mostly so that I can show a complete working # example without publishing my own server's passwords. - cfg = rpki.config.parser(args.config, "yamltest", allow_missing = True) + cfg = rpki.config.parser(set_filename = args.config, section = "yamltest", allow_missing = True) only_one_pubd = cfg.getboolean("only_one_pubd", True) allocation.base_port = cfg.getint("base_port", 4400) @@ -721,6 +812,7 @@ try: for d in db: if not d.is_hosted: + print "Initializing", d.name os.makedirs(d.path()) d.dump_conf() if d.runs_pubd: @@ -728,7 +820,9 @@ try: d.dump_rsyncd() if d.is_root: os.makedirs(d.path("publication.root")) + d.syncdb() d.run_rpkic("initialize_server_bpki") + print # Initialize resource holding BPKI and generate self-descriptor # for each entity. @@ -766,6 +860,8 @@ try: if d.runs_pubd: progs.append(d.run_pubd()) progs.append(d.run_rsyncd()) + if args.run_gui: + progs.append(d.run_gui()) if args.synchronize or not args.skip_config: @@ -834,9 +930,27 @@ try: d.dump_ghostbusters() d.dump_router_certificates() + if args.run_gui: + print + print 'GUI user "root", password "fnord"' + for d in db: + if not d.is_hosted: + url = "http://127.0.0.1:%d/rpki/" % (8000 + d.engine) + print "GUI URL", url, "for", d.name + if args.browser: + if d is db.root: + webbrowser.open_new(url) + else: + webbrowser.open_new_tab(url) + time.sleep(2) + # Wait until something terminates. if not args.stop_after_config or args.keep_going: + if args.notify_when_startup_complete: + print + print "Sending SIGUSR1 to process", args.notify_when_startup_complete + os.kill(args.notify_when_startup_complete, signal.SIGUSR1) print print "Waiting for daemons to exit" signal.signal(signal.SIGCHLD, lambda *dont_care: None) @@ -882,3 +996,7 @@ try: finally: if args.pidfile is not None: os.unlink(args.pidfile) + +# Local Variables: +# indent-tabs-mode: nil +# End: @@ -170,6 +170,10 @@ static int NID_rpkiManifest; static int NID_signedObject; #endif +#ifndef NID_rpkiNotify +static int NID_rpkiNotify; +#endif + static const struct { int *nid; const char *oid; @@ -182,7 +186,11 @@ static const struct { #endif #ifndef NID_signedObject - {&NID_signedObject, "1.3.6.1.5.5.7.48.11", "id-ad-signedObject", "Signed Object"} + {&NID_signedObject, "1.3.6.1.5.5.7.48.11", "id-ad-signedObject", "Signed Object"}, +#endif + +#ifndef NID_rpkiNotify + {&NID_rpkiNotify, "1.3.6.1.5.5.7.48.13", "id-ad-rpkiNotify", "RPKI RRDP Notification"}, #endif }; @@ -1295,8 +1303,8 @@ extension_set_basic_constraints(X509_EXTENSIONS **exts, PyObject *args) #define EXTENSION_GET_SIA__DOC__ \ "If there is no SIA extension, this method returns None.\n" \ "\n" \ - "Otherwise, it returns a tuple containing three values:\n" \ - "caRepository URIs, rpkiManifest URIs, and signedObject URIs.\n" \ + "Otherwise, it returns a tuple containing four values:\n" \ + "caRepository URIs, rpkiManifest URIs, signedObject, and rpkiNotify URIs.\n" \ "Each of these values is a tuple of strings, representing an ordered\n" \ "sequence of URIs. Any or all of these sequences may be empty.\n" \ "\n" \ @@ -1310,9 +1318,11 @@ extension_get_sia(X509_EXTENSIONS **exts) PyObject *result_caRepository = NULL; PyObject *result_rpkiManifest = NULL; PyObject *result_signedObject = NULL; + PyObject *result_rpkiNotify = NULL; int n_caRepository = 0; int n_rpkiManifest = 0; int n_signedObject = 0; + int n_rpkiNotify = 0; const char *uri; PyObject *obj; int i, nid; @@ -1334,26 +1344,23 @@ extension_get_sia(X509_EXTENSIONS **exts) if (a->location->type != GEN_URI) continue; nid = OBJ_obj2nid(a->method); - if (nid == NID_caRepository) { + if (nid == NID_caRepository) n_caRepository++; - continue; - } - if (nid == NID_rpkiManifest) { + else if (nid == NID_rpkiManifest) n_rpkiManifest++; - continue; - } - if (nid == NID_signedObject) { + else if (nid == NID_signedObject) n_signedObject++; - continue; - } + else if (nid == NID_rpkiNotify) + n_rpkiNotify++; } if (((result_caRepository = PyTuple_New(n_caRepository)) == NULL) || ((result_rpkiManifest = PyTuple_New(n_rpkiManifest)) == NULL) || - ((result_signedObject = PyTuple_New(n_signedObject)) == NULL)) + ((result_signedObject = PyTuple_New(n_signedObject)) == NULL) || + ((result_rpkiNotify = PyTuple_New(n_rpkiNotify)) == NULL)) goto error; - n_caRepository = n_rpkiManifest = n_signedObject = 0; + n_caRepository = n_rpkiManifest = n_signedObject = n_rpkiNotify = 0; for (i = 0; i < sk_ACCESS_DESCRIPTION_num(ext); i++) { ACCESS_DESCRIPTION *a = sk_ACCESS_DESCRIPTION_value(ext, i); @@ -1379,24 +1386,32 @@ extension_get_sia(X509_EXTENSIONS **exts) PyTuple_SET_ITEM(result_signedObject, n_signedObject++, obj); continue; } + if (nid == NID_rpkiNotify) { + if ((obj = PyString_FromString(uri)) == NULL) + goto error; + PyTuple_SET_ITEM(result_rpkiNotify, n_rpkiNotify++, obj); + continue; + } } - result = Py_BuildValue("(OOO)", + result = Py_BuildValue("(OOOO)", result_caRepository, result_rpkiManifest, - result_signedObject); + result_signedObject, + result_rpkiNotify); error: AUTHORITY_INFO_ACCESS_free(ext); Py_XDECREF(result_caRepository); Py_XDECREF(result_rpkiManifest); Py_XDECREF(result_signedObject); + Py_XDECREF(result_rpkiNotify); return result; } #define EXTENSION_SET_SIA__DOC__ \ - "This method Takes three arguments:\n" \ - "\"caRepository\", \"rpkiManifest\", and \"signedObject\".\n" \ + "This method takes four arguments: \"caRepository\"\n," \ + "\"rpkiManifest\", \"signedObject\", and \"rpkiNotify\".\n" \ "Each of these should be an iterable which returns URIs.\n" \ "\n" \ "None is acceptable as an alternate way of specifying an empty\n" \ @@ -1405,11 +1420,12 @@ extension_get_sia(X509_EXTENSIONS **exts) static PyObject * extension_set_sia(X509_EXTENSIONS **exts, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"caRepository", "rpkiManifest", "signedObject", NULL}; + static char *kwlist[] = {"caRepository", "rpkiManifest", "signedObject", "rpkiNotify", NULL}; AUTHORITY_INFO_ACCESS *ext = NULL; PyObject *caRepository = Py_None; PyObject *rpkiManifest = Py_None; PyObject *signedObject = Py_None; + PyObject *rpkiNotify = Py_None; PyObject *iterator = NULL; ASN1_OBJECT *oid = NULL; PyObject **pobj = NULL; @@ -1424,8 +1440,8 @@ extension_set_sia(X509_EXTENSIONS **exts, PyObject *args, PyObject *kwds) if (!exts) goto error; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOO", kwlist, - &caRepository, &rpkiManifest, &signedObject)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOOO", kwlist, + &caRepository, &rpkiManifest, &signedObject, &rpkiNotify)) goto error; if ((ext = AUTHORITY_INFO_ACCESS_new()) == NULL) @@ -1437,11 +1453,12 @@ extension_set_sia(X509_EXTENSIONS **exts, PyObject *args, PyObject *kwds) * single URI as an abbreviation for a collection containing one URI. */ - for (i = 0; i < 3; i++) { + for (i = 0; i < 4; i++) { switch (i) { case 0: pobj = &caRepository; nid = NID_caRepository; break; case 1: pobj = &rpkiManifest; nid = NID_rpkiManifest; break; case 2: pobj = &signedObject; nid = NID_signedObject; break; + case 3: pobj = &rpkiNotify; nid = NID_rpkiNotify; break; } if (*pobj == Py_None) @@ -1461,7 +1478,8 @@ extension_set_sia(X509_EXTENSIONS **exts, PyObject *args, PyObject *kwds) if ((a = ACCESS_DESCRIPTION_new()) == NULL || (a->method = OBJ_dup(oid)) == NULL || (a->location->d.uniformResourceIdentifier = ASN1_IA5STRING_new()) == NULL || - !ASN1_OCTET_STRING_set(a->location->d.uniformResourceIdentifier, (unsigned char *) uri, urilen)) + !ASN1_OCTET_STRING_set(a->location->d.uniformResourceIdentifier, + (unsigned char *) uri, urilen)) lose_no_memory(); a->location->type = GEN_URI; diff --git a/h/rpki/sk_roa.h b/h/rpki/sk_roa.h index 13036955..7423f8ff 100644 --- a/h/rpki/sk_roa.h +++ b/h/rpki/sk_roa.h @@ -1,6 +1,6 @@ /* * Automatically generated, do not edit. - * Generator $Id: defstack.py 4878 2012-11-15 22:13:53Z sra $ + * Generator $Id: defstack.py 5784 2014-04-10 22:56:47Z sra $ */ #ifndef __RPKI_ROA_H__DEFSTACK_H__ diff --git a/potpourri/django-legacy-database.README b/potpourri/django-legacy-database.README new file mode 100644 index 00000000..41a3b911 --- /dev/null +++ b/potpourri/django-legacy-database.README @@ -0,0 +1,4 @@ +Snapshot of work in progress on converting our existing databases into +Django using South 1.0 migrations. This will probably need rewriting +to address changes in how we deal with Django settings and multiple +databases, this snapshot is just to get it into the subversion archive. diff --git a/potpourri/django-legacy-database.tar.xz b/potpourri/django-legacy-database.tar.xz Binary files differnew file mode 100644 index 00000000..762dde7d --- /dev/null +++ b/potpourri/django-legacy-database.tar.xz diff --git a/potpourri/rrdp-fetch-from-tal b/potpourri/rrdp-fetch-from-tal new file mode 100755 index 00000000..08d245dd --- /dev/null +++ b/potpourri/rrdp-fetch-from-tal @@ -0,0 +1,229 @@ +#!/usr/bin/env python +# $Id$ +# +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +Fetch RPKI data using RRDP starting from a TAL. + +Work in progress, don't be too surprised by anything this does or +doesn't do. +""" + +import rpki.relaxng +import rpki.x509 +import lxml.etree +import argparse +import urlparse +import urllib2 +import sys +import os + + +class Tags(object): + def __init__(self, *tags): + for tag in tags: + setattr(self, tag, rpki.relaxng.rrdp.xmlns + tag) + +tags = Tags("notification", "delta", "snapshot", "publish", "withdraw") + + +class RSyncHandler(urllib2.BaseHandler): + """ + Jam support for rsync:// URIs into urllib2 framework. + Very basic, probably not paranoid enough. + """ + + _n = 0 + + def rsync_open(self, req): + import subprocess, mimetools + u = req.get_full_url() + if u.endswith("/"): + raise urllib2.URLError("rsync directory URI not allowed") + t = "/tmp/rrdp-fetch-from-tal.%d.%d" % (os.getpid(), self._n) + self._n += 1 + subprocess.check_call(("rsync", u, t)) + h = mimetools.Message(open("/dev/null")) + h["Content-type"] = "text/plain" + h["Content-length"] = str(os.stat(t).st_size) + f = open(t, "rb") + os.unlink(t) + return urllib2.addinfourl(f, h, u) + +urllib2.install_opener(urllib2.build_opener(RSyncHandler)) + + +class main(object): + + def __init__(self): + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument("--rcynic-tree", default = "rcynic-data/unauthenticated", + help = "directory tree in which to write extracted RPKI objects") + parser.add_argument("--serial-filename", # default handled later + help = "file name in which to store RRDP serial number") + parser.add_argument("tal", help = "trust anchor locator") + self.args = parser.parse_args() + if not os.path.isdir(self.args.rcynic_tree): + os.makedirs(self.args.rcynic_tree) + self.urls = set() + self.ta = self.ta_fetch() + url = self.ta.get_sia_rrdp_notify() + if url is None: + sys.exit("Couldn't get RRDP URI from trust anchor") + self.rrdp_fetch(url) + self.write_ta() + + def rrdp_fetch(self, url): + if url in self.urls: + print "Already fetched %s, skipping" % url + return + self.urls.add(url) + xml = lxml.etree.ElementTree(file = urllib2.urlopen(url)).getroot() + rpki.relaxng.rrdp.assertValid(xml) + if xml.tag[len(rpki.relaxng.rrdp.xmlns):] != "notification": + sys.exit("Expected notification at %s, found %s" % (url, xml.tag)) + self.prettyprint_notification(xml) + + # We should be checking session_id here, but we're not storing it yet + + old_serial = self.get_serial() + new_serial = int(xml.get("serial")) + deltas = dict((int(elt.get("serial")), elt) + for elt in xml.iterchildren(tags.delta)) + if old_serial == 0 or not all(serial + 1 in deltas + for serial in xrange(old_serial, new_serial)): + return self.snapshot_fetch(xml.iterchildren(tags.snapshot).next()) + for serial in sorted(deltas): + if serial > old_serial: + self.delta_fetch(deltas[serial]) + + def prettyprint_notification(self, xml): + print "Notification version %s session %s serial %s" % ( + xml.get("version"), xml.get("session_id"), xml.get("serial")) + elt = xml.iterchildren(tags.snapshot).next() + print " Snapshot URI %s hash %s" % ( + elt.get("uri"), elt.get("hash")) + for elt in xml.iterchildren(tags.delta): + print " Delta %6s URI %s hash %s" % ( + elt.get("serial"), elt.get("uri"), elt.get("hash")) + + def ta_fetch(self): + with open(self.args.tal, "r") as f: + tal = f.read() + uris, key = tal.split("\n\n", 2) + key = rpki.x509.PublicKey(Base64 = key) + for uri in uris.split(): + ta = rpki.x509.X509(DER = urllib2.urlopen(uri).read()) + if ta.getPublicKey() == key: + return ta + print "TAL key mismatch for certificate", url + sys.exit("Could not fetch trust anchor") + + @property + def serial_filename(self): + return self.args.serial_filename or os.path.join(self.args.rcynic_tree, "serial") + + def get_serial(self): + try: + with open(self.serial_filename, "r") as f: + return int(f.read().strip()) + except: + return 0 + + def set_serial(self, value): + with open(self.serial_filename, "w") as f: + f.write("%s\n" % value) + + def uri_to_filename(self, uri): + assert uri.startswith("rsync://") + return os.path.join(self.args.rcynic_tree, uri[len("rsync://"):]) + + def add_obj(self, uri, obj): + fn = self.uri_to_filename(uri) + dn = os.path.dirname(fn) + if not os.path.isdir(dn): + os.makedirs(dn) + with open(fn, "wb") as f: + f.write(obj) + + def del_obj(self, uri, hash): + fn = self.uri_to_filename(uri) + with open(fn, "rb") as f: + if hash.lower() != rpki.x509.sha256(f.read()).encode("hex"): + raise RuntimeError("Hash mismatch for URI %s" % uri) + os.unlink(fn) + dn = os.path.dirname(fn) + while True: + try: + os.rmdir(dn) + except OSError: + break + else: + dn = os.path.dirname(dn) + + def xml_fetch(self, elt): + url = elt.get("uri") + hash = elt.get("hash").lower() + print "Fetching", url + text = urllib2.urlopen(url).read() + h = rpki.x509.sha256(text).encode("hex") + if h != hash: + sys.exit("Bad hash for %s: expected %s got %s" % (url, hash, h)) + xml = lxml.etree.XML(text) + rpki.relaxng.rrdp.schema.assertValid(xml) + return xml + + def snapshot_fetch(self, xml): + xml = self.xml_fetch(xml) + print "Unpacking snapshot version %s session %s serial %6s" % ( + xml.get("version"), xml.get("session_id"), xml.get("serial")) + for elt in xml.iterchildren(tags.publish): + print " ", elt.get("uri") + self.add_obj(elt.get("uri"), elt.text.decode("base64")) + self.set_serial(xml.get("serial")) + + def delta_fetch(self, xml): + xml = self.xml_fetch(xml) + old_serial = int(self.get_serial()) + new_serial = int(xml.get("serial")) + print "Unpacking deltas version %s session %s serial %s" % ( + xml.get("version"), xml.get("session_id"), new_serial) + if old_serial != new_serial - 1: + raise RuntimeError("Can't apply deltas: old serial %s new serial %s" % (old_serial, new_serial)) + for i, elt in enumerate(xml.iterchildren(tags.withdraw)): + uri = elt.get("uri") + hash = elt.get("hash") + print " %3d withdraw URI %s hash %s" % (i, uri, hash) + self.del_obj(uri, hash) + for i, elt in enumerate(xml.iterchildren(tags.publish)): + uri = elt.get("uri") + hash = elt.get("hash", None) + print " %3d publish URI %s hash %s" % (i, uri, hash) + if hash is not None: + self.del_obj(uri, hash) + self.add_obj(elt.get("uri"), elt.text.decode("base64")) + self.set_serial(new_serial) + + def write_ta(self): + der = self.ta.get_DER() + fn = rpki.x509.sha256(der).encode("hex") + ".cer" + if not os.path.exists(fn): + print "Writing", fn + with open(fn, "wb") as f: + f.write(der) + +if __name__ == "__main__": + main() diff --git a/potpourri/rrdp-fetch.py b/potpourri/rrdp-fetch.py new file mode 100755 index 00000000..469c0c9f --- /dev/null +++ b/potpourri/rrdp-fetch.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# $Id$ +# +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +Fetch an RRDP notifcation file and follow all the links. Should be +merged into rrdp-test-tool eventually, but one thing at a time. +""" + +from urllib2 import urlopen +from lxml.etree import ElementTree, XML +from socket import getfqdn +from rpki.x509 import sha256 +from rpki.relaxng import rrdp +from urlparse import urlparse +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter + +class BadHash(Exception): + "Calculated hash value doesn't match expected hash value." + +def fetch(elt): + uri = elt.get("uri") + hash = elt.get("hash").lower() + print "Fetching", uri + + text = urlopen(uri).read() + h = sha256(text).encode("hex") + if h != hash: + raise BadHash("Bad hash for %s: expected %s got %s" % (uri, hash, h)) + + xml = XML(text) + rrdp.schema.assertValid(xml) + + u = urlparse(uri) + fn = u.netloc + u.path + + return elt, xml, fn + +parser = ArgumentParser(description = __doc__, formatter_class = ArgumentDefaultsHelpFormatter) +parser.add_argument("uri", nargs = "?", + default = "http://" + getfqdn() + "/rrdp/updates.xml", + help = "RRDP notification file to fetch") +args = parser.parse_args() + +updates = ElementTree(file = urlopen(args.uri)) +rrdp.schema.assertValid(updates) + +snapshot = fetch(updates.find(rrdp.xmlns + "snapshot")) + +deltas = [fetch(elt) for elt in updates.findall(rrdp.xmlns + "delta")] + +print updates +print snapshot +for delta in deltas: + print delta diff --git a/potpourri/rrdp-test-tool b/potpourri/rrdp-test-tool new file mode 100755 index 00000000..ccf17960 --- /dev/null +++ b/potpourri/rrdp-test-tool @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# $Id$ +# +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +Test tool for prototype RRDP implementation. Eventually some of this +code will likely be refactored into more user-friendly form, but for +the moment this just does whatever insane thing I need to do this week +for testing. +""" + +import rpki.relaxng +import rpki.x509 +import lxml.etree +import argparse +import os + +class Tags(object): + def __init__(self, *tags): + for tag in tags: + setattr(self, tag, rpki.relaxng.rrdp.xmlns + tag) + +tags = Tags("notification", "delta", "snapshot", "publish", "withdraw") + +class main(object): + + def __init__(self): + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument("--rcynic-tree", default = "rcynic-data/unauthenticated", + help = "directory tree in which to write extracted RPKI objects") + parser.add_argument("--serial-filename", + help = "file name in which to store RRDP serial number") + parser.add_argument("rrdp_file", nargs = "+", + help = "RRDP snapshot or deltas file") + self.args = parser.parse_args() + if not os.path.isdir(self.args.rcynic_tree): + os.makedirs(self.args.rcynic_tree) + for rrdp_file in self.args.rrdp_file: + xml = lxml.etree.ElementTree(file = rrdp_file).getroot() + rpki.relaxng.rrdp.assertValid(xml) + getattr(self, "handle_" + xml.tag[len(rpki.relaxng.rrdp.xmlns):])(xml) + + @property + def serial_filename(self): + return self.args.serial_filename or os.path.join(self.args.rcynic_tree, "serial") + + def get_serial(self): + with open(self.serial_filename, "r") as f: + return f.read().strip() + + def set_serial(self, value): + with open(self.serial_filename, "w") as f: + f.write("%s\n" % value) + + def handle_notification(self, xml): + print "Notification version %s session %s serial %s" % ( + xml.get("version"), xml.get("session_id"), xml.get("serial")) + assert xml[0].tag == tags.snapshot + print " Snapshot URI %s hash %s" % ( + xml[0].get("uri"), xml[0].get("hash")) + for i, elt in enumerate(xml.iterchildren(tags.delta)): + print " Delta %3d serial %6s URI %s hash %s" % ( + i, elt.get("serial"), elt.get("uri"), elt.get("hash")) + + def uri_to_filename(self, uri): + assert uri.startswith("rsync://") + return os.path.join(self.args.rcynic_tree, uri[len("rsync://"):]) + + def add_obj(self, uri, obj): + fn = self.uri_to_filename(uri) + dn = os.path.dirname(fn) + if not os.path.isdir(dn): + os.makedirs(dn) + with open(fn, "wb") as f: + f.write(obj) + + def del_obj(self, uri, hash): + fn = self.uri_to_filename(uri) + with open(fn, "rb") as f: + if hash.lower() != rpki.x509.sha256(f.read()).encode("hex"): + raise RuntimeError("Hash mismatch for URI %s" % uri) + os.unlink(fn) + dn = os.path.dirname(fn) + while True: + try: + os.rmdir(dn) + except OSError: + break + else: + dn = os.path.dirname(dn) + + def handle_snapshot(self, xml): + print "Unpacking snapshot version %s session %s serial %6s" % ( + xml.get("version"), xml.get("session_id"), xml.get("serial")) + for elt in xml.iterchildren(tags.publish): + print " ", elt.get("uri") + self.add_obj(elt.get("uri"), elt.text.decode("base64")) + self.set_serial(xml.get("serial")) + + def handle_delta(self, xml): + old_serial = int(self.get_serial()) + new_serial = int(xml.get("serial")) + print "Unpacking deltas version %s session %s serial %s" % ( + xml.get("version"), xml.get("session_id"), new_serial) + if old_serial != new_serial - 1: + raise RuntimeError("Can't apply deltas: old serial %s new serial %s" % (old_serial, new_serial)) + for i, elt in enumerate(xml.iterchildren(tags.withdraw)): + uri = elt.get("uri") + hash = elt.get("hash") + print " %3d withdraw URI %s hash %s" % (i, uri, hash) + self.del_obj(uri, hash) + for i, elt in enumerate(xml.iterchildren(tags.publish)): + uri = elt.get("uri") + hash = elt.get("hash", None) + print " %3d publish URI %s hash %s" % (i, uri, hash) + if hash is not None: + self.del_obj(uri, hash) + self.add_obj(elt.get("uri"), elt.text.decode("base64")) + self.set_serial(new_serial) + +if __name__ == "__main__": + main() diff --git a/potpourri/upgrade-add-ghostbusters.py b/potpourri/upgrade-add-ghostbusters.py index a8c8a92b..2548487c 100644 --- a/potpourri/upgrade-add-ghostbusters.py +++ b/potpourri/upgrade-add-ghostbusters.py @@ -43,7 +43,7 @@ for o, a in opts: if o in ("-c", "--config"): cfg_file = a -cfg = rpki.config.parser(cfg_file, "myrpki") +cfg = rpki.config.parser(filename = cfg_file, section = "myrpki") fix("irdbd", """ CREATE TABLE ghostbuster_request ( diff --git a/rp/rcynic/rcynic.c b/rp/rcynic/rcynic.c index d0da40f5..36c1950f 100644 --- a/rp/rcynic/rcynic.c +++ b/rp/rcynic/rcynic.c @@ -3190,7 +3190,7 @@ static int extract_access_uri(rcynic_ctx_t *rc, if (OBJ_obj2nid(a->method) != nid) continue; ++*count; - if (!relevant((char *) a->location->d.uniformResourceIdentifier->data)) + if (relevant && !relevant((char *) a->location->d.uniformResourceIdentifier->data)) continue; if (sizeof(result->s) <= a->location->d.uniformResourceIdentifier->length) log_validation_status(rc, uri, uri_too_long, generation); @@ -3707,7 +3707,7 @@ static int check_x509(rcynic_ctx_t *rc, int n_caIssuers = 0; ex_count--; if (!extract_access_uri(rc, uri, generation, aia, NID_ad_ca_issuers, - &certinfo->aia, &n_caIssuers, is_rsync) || + &certinfo->aia, &n_caIssuers, NULL) || !certinfo->aia.s[0] || sk_ACCESS_DESCRIPTION_num(aia) != n_caIssuers) { log_validation_status(rc, uri, malformed_aia_extension, generation); diff --git a/rp/utils/find_roa b/rp/utils/find_roa index 4cfcccac..2b537bf4 100755 --- a/rp/utils/find_roa +++ b/rp/utils/find_roa @@ -61,7 +61,7 @@ class Prefix(object): if self.prefix & ((1 << (self.prefix.bits - self.length)) - 1) != 0: raise ValueError - def matches(self, roa): + def matches(self, roa): # pylint: disable=W0621 return any(self.prefix == prefix and self.length == length and (not args.match_maxlength or @@ -71,13 +71,13 @@ class Prefix(object): for prefix, length, maxlength in roa.prefixes) -class ROA(rpki.POW.ROA): +class ROA(rpki.POW.ROA): # pylint: disable=W0232 """ Aspects of a ROA that we care about. """ @classmethod - def parse(cls, fn): + def parse(cls, fn): # pylint: disable=W0621 assert fn.startswith(args.rcynic_dir) self = cls.derReadFile(fn) self.fn = fn @@ -112,7 +112,7 @@ class ROA(rpki.POW.ROA): def show_expiration(self): print self x = self.certs()[0] - fn = self.fn + fn = self.fn # pylint: disable=W0621 uri = self.uri while uri is not None: name = fn if args.show_filenames else uri diff --git a/rp/utils/print_roa b/rp/utils/print_roa index d5db0c3c..c96a7c66 100755 --- a/rp/utils/print_roa +++ b/rp/utils/print_roa @@ -24,14 +24,14 @@ signature. import argparse import rpki.POW -class ROA(rpki.POW.ROA): +class ROA(rpki.POW.ROA): # pylint: disable=W0232 @staticmethod - def _format_prefix(prefix): - if prefix[2] is None or prefix[1] == prefix[2]: - return "%s/%d" % (prefix[0], prefix[1]) + def _format_prefix(p): + if p[2] in (None, p[1]): + return "%s/%d" % (p[0], p[1]) else: - return "%s/%d-%d" % (prefix[0], prefix[1], prefix[2]) + return "%s/%d-%d" % (p[0], p[1], p[2]) def parse(self): self.extractWithoutVerifying() @@ -58,12 +58,12 @@ for roa in args.roas: print "asID: ", roa.getASID() if roa.v4_prefixes: print " addressFamily:", 1 - for p in roa.v4_prefixes: - print " IPAddress:", p + for prefix in roa.v4_prefixes: + print " IPAddress:", prefix if roa.v6_prefixes: print " addressFamily:", 2 - for p in roa.v6_prefixes: - print " IPAddress:", p + for prefix in roa.v6_prefixes: + print " IPAddress:", prefix if args.cms: print roa.pprint() for cer in roa.certs(): diff --git a/rp/utils/print_rpki_manifest b/rp/utils/print_rpki_manifest index 5ebc6356..ce9b25ea 100755 --- a/rp/utils/print_rpki_manifest +++ b/rp/utils/print_rpki_manifest @@ -39,8 +39,8 @@ for mft in args.manifests: print "nextUpdate: ", mft.getNextUpdate() print "fileHashAlg: ", rpki.oids.oid2name(mft.getAlgorithm()) for i, fah in enumerate(mft.getFiles()): - name, hash = fah - print "fileList[%3d]: %s %s" % (i, ":".join(("%02X" % ord(h) for h in hash)), name) + name, obj_hash = fah + print "fileList[%3d]: %s %s" % (i, ":".join(("%02X" % ord(h) for h in obj_hash)), name) if args.cms: print mft.pprint() for cer in mft.certs(): diff --git a/rp/utils/scan_roas b/rp/utils/scan_roas index a1b64f01..4f3dc7f0 100755 --- a/rp/utils/scan_roas +++ b/rp/utils/scan_roas @@ -31,10 +31,10 @@ def check_dir(d): raise argparse.ArgumentTypeError("%r is not a directory" % d) return d -class ROA(rpki.POW.ROA): +class ROA(rpki.POW.ROA): # pylint: disable=W0232 @classmethod - def parse(cls, fn): + def parse(cls, fn): # pylint: disable=W0621 self = cls.derReadFile(fn) self.extractWithoutVerifying() return self diff --git a/rp/utils/uri b/rp/utils/uri index e72d5e0d..df6e710b 100755 --- a/rp/utils/uri +++ b/rp/utils/uri @@ -30,13 +30,19 @@ import rpki.POW class Certificate(object): @staticmethod - def first_rsync(uris): + def first_whatever(uris, prefix): if uris is not None: for uri in uris: - if uri.startswith("rsync://"): + if uri.startswith(prefix): return uri return None + def first_rsync(self, uris): + return self.first_whatever(uris, "rsync://") + + def first_http(self, uris): + return self.first_whatever(uris, "http://") + def __init__(self, fn): try: x = rpki.POW.X509.derReadFile(fn) @@ -47,13 +53,14 @@ class Certificate(object): x = cms.certs()[0] except: raise ValueError - sia = x.getSIA() or (None, None, None) + sia = x.getSIA() or (None, None, None, None) self.fn = fn self.uris = ( ("AIA:caIssuers", self.first_rsync(x.getAIA())), ("SIA:caRepository", self.first_rsync(sia[0])), ("SIA:rpkiManifest", self.first_rsync(sia[1])), ("SIA:signedObject", self.first_rsync(sia[2])), + ("SIA:rpkiNotify", self.first_http(sia[3])), ("CRLDP", self.first_rsync(x.getCRLDP()))) def __str__(self): diff --git a/rpki/adns.py b/rpki/adns.py index 968684b5..018bb7cf 100644 --- a/rpki/adns.py +++ b/rpki/adns.py @@ -88,6 +88,7 @@ class dispatcher(asyncore.dispatcher): """ Receive a packet, hand it off to query class callback. """ + wire, from_address = self.recvfrom(self.bufsize) self.cb(self.af, from_address[0], from_address[1], wire) @@ -95,18 +96,21 @@ class dispatcher(asyncore.dispatcher): """ Pass errors to query class errback. """ + self.eb(sys.exc_info()[1]) def handle_connect(self): """ Quietly ignore UDP "connection" events. """ + pass def writable(self): """ We don't need to hear about UDP socket becoming writable. """ + return False @@ -138,6 +142,7 @@ class query(object): query; if we find an answer there, just return it. Otherwise start the network query. """ + if resolver.cache: answer = resolver.cache.get((self.qname, self.qtype, self.qclass)) else: @@ -161,6 +166,7 @@ class query(object): Outer loop. If we haven't got a response yet and still have nameservers to check, start inner loop. Otherwise, we're done. """ + self.timer.cancel() if self.response is None and self.nameservers: self.iterator = rpki.async.iterator(self.nameservers[:], self.loop2, self.done2) @@ -172,6 +178,7 @@ class query(object): Inner loop. Send query to next nameserver in our list, unless we've hit the overall timeout for this query. """ + self.timer.cancel() try: timeout = resolver._compute_timeout(self.start) @@ -191,6 +198,7 @@ class query(object): """ No answer from nameserver, move on to next one (inner loop). """ + self.response = None self.iterator() @@ -200,6 +208,7 @@ class query(object): error, handle as if we've timed out on this nameserver; otherwise, pass error back to caller. """ + self.timer.cancel() if isinstance(e, socket.error): self.response = None @@ -215,6 +224,7 @@ class query(object): we're done, otherwise handle error appropriately and move on to next nameserver. """ + sender = (af, dns.inet.inet_pton(af, from_host)) if from_port != resolver.port or sender not in self.nameservers: return @@ -240,6 +250,7 @@ class query(object): while before starting the cycle again, unless we've hit the timeout threshold for the whole query. """ + if self.response is None and self.nameservers: try: delay = rpki.sundial.timedelta(seconds = min(resolver._compute_timeout(self.start), self.backoff)) @@ -256,6 +267,7 @@ class query(object): """ Shut down our timer and sockets. """ + self.timer.cancel() for s in self.sockets.itervalues(): s.close() @@ -264,6 +276,7 @@ class query(object): """ Something bad happened. Clean up, then pass error back to caller. """ + self.cleanup() self.eb(self, e) @@ -273,6 +286,7 @@ class query(object): pass it back to caller; if we got an error, pass the appropriate exception back to caller. """ + self.cleanup() try: if not self.nameservers: diff --git a/rpki/async.py b/rpki/async.py index 75b4b656..74143bd1 100644 --- a/rpki/async.py +++ b/rpki/async.py @@ -131,6 +131,7 @@ class timer(object): """ Debug logging. """ + if self.gc_debug: bt = traceback.extract_stack(limit = 3) logger.debug("%s from %s:%d", msg, bt[0][0], bt[0][1]) @@ -140,6 +141,7 @@ class timer(object): Set a timer. Argument can be a datetime, to specify an absolute time, or a timedelta, to specify an offset time. """ + if self.gc_debug: self.trace("Setting %r to %r" % (self, when)) if isinstance(when, rpki.sundial.timedelta): @@ -162,6 +164,7 @@ class timer(object): """ Cancel a timer, if it was set. """ + if self.gc_debug: self.trace("Canceling %r" % self) try: @@ -174,6 +177,7 @@ class timer(object): """ Test whether this timer is currently set. """ + return self in timer_queue def set_handler(self, handler): @@ -184,12 +188,14 @@ class timer(object): bound method to an object in a class representing a network connection). """ + self.handler = handler def set_errback(self, errback): """ Set a timer's errback. Like set_handler(), for errbacks. """ + self.errback = errback @classmethod @@ -202,6 +208,7 @@ class timer(object): called, so that even if new events keep getting scheduled, we'll return to the I/O loop reasonably quickly. """ + now = rpki.sundial.now() while timer_queue and now >= timer_queue[0].when: t = timer_queue.pop(0) @@ -233,6 +240,7 @@ class timer(object): the same units (argh!), and we're not doing anything that hair-triggered, so rounding up is simplest. """ + if not timer_queue: return None now = rpki.sundial.now() @@ -251,6 +259,7 @@ class timer(object): queue content, but this way we can notify subclasses that provide their own cancel() method. """ + while timer_queue: timer_queue.pop(0).cancel() @@ -258,12 +267,14 @@ def _raiseExitNow(signum, frame): """ Signal handler for event_loop(). """ + raise ExitNow def exit_event_loop(): """ Force exit from event_loop(). """ + raise ExitNow def event_defer(handler, delay = rpki.sundial.timedelta(seconds = 0)): @@ -271,6 +282,7 @@ def event_defer(handler, delay = rpki.sundial.timedelta(seconds = 0)): Use a near-term (default: zero interval) timer to schedule an event to run after letting the I/O system have a turn. """ + timer(handler).set(delay) ## @var debug_event_timing @@ -282,6 +294,7 @@ def event_loop(catch_signals = (signal.SIGINT, signal.SIGTERM)): """ Replacement for asyncore.loop(), adding timer and signal support. """ + old_signal_handlers = {} while True: save_sigs = len(old_signal_handlers) == 0 @@ -323,71 +336,6 @@ def event_loop(catch_signals = (signal.SIGINT, signal.SIGTERM)): for sig in old_signal_handlers: signal.signal(sig, old_signal_handlers[sig]) -class sync_wrapper(object): - """ - Synchronous wrapper around asynchronous functions. Running in - asynchronous mode at all times makes sense for event-driven daemons, - but is kind of tedious for simple scripts, hence this wrapper. - - The wrapped function should take at least two arguments: a callback - function and an errback function. If any arguments are passed to - the wrapper, they will be passed as additional arguments to the - wrapped function. - """ - - res = None - err = None - fin = False - - def __init__(self, func, disable_signal_handlers = False): - self.func = func - self.disable_signal_handlers = disable_signal_handlers - - def cb(self, res = None): - """ - Wrapped code has requested normal termination. Store result, and - exit the event loop. - """ - self.res = res - self.fin = True - logger.debug("%r callback with result %r", self, self.res) - raise ExitNow - - def eb(self, err): - """ - Wrapped code raised an exception. Store exception data, then exit - the event loop. - """ - exc_info = sys.exc_info() - self.err = exc_info if exc_info[1] is err else err - self.fin = True - logger.debug("%r errback with exception %r", self, self.err) - raise ExitNow - - def __call__(self, *args, **kwargs): - - def thunk(): - try: - self.func(self.cb, self.eb, *args, **kwargs) - except ExitNow: - raise - except Exception, e: - self.eb(e) - - event_defer(thunk) - if self.disable_signal_handlers: - event_loop(catch_signals = ()) - else: - event_loop() - if not self.fin: - logger.warning("%r event_loop terminated without callback or errback", self) - if self.err is None: - return self.res - elif isinstance(self.err, tuple): - raise self.err[0], self.err[1], self.err[2] - else: - raise self.err - class gc_summary(object): """ Periodic summary of GC state, for tracking down memory bloat. @@ -405,6 +353,7 @@ class gc_summary(object): """ Collect and log GC state for this period, reset timer. """ + logger.debug("gc_summary: Running gc.collect()") gc.collect() logger.debug("gc_summary: Summarizing (threshold %d)", self.threshold) diff --git a/rpki/config.py b/rpki/config.py index 253e56cf..0be0d1a0 100644 --- a/rpki/config.py +++ b/rpki/config.py @@ -32,23 +32,16 @@ logger = logging.getLogger(__name__) ## @var default_filename # Default name of config file if caller doesn't specify one explictly. -default_filename = "rpki.conf" - -## @var default_dirname -# Default name of directory to check for global config file, or None -# if no global config file. Autoconf-generated code may set this to a -# non-None value during script startup. - try: import rpki.autoconf - default_dirname = rpki.autoconf.sysconfdir + default_filename = os.path.join(rpki.autoconf.sysconfdir, "rpki.conf") except ImportError: - default_dirname = None + default_filename = None -## @var default_envname +## @var rpki_conf_envname # Name of environment variable containing config file name. -default_envname = "RPKI_CONF" +rpki_conf_envname = "RPKI_CONF" class parser(object): """ @@ -61,44 +54,48 @@ class parser(object): get-methods with default values and default section name. - If no filename is given to the constructor (filename = None), we - check for an environment variable naming the config file, then we - check for a default filename in the current directory, then finally - we check for a global config file if autoconf provided a directory - name to check. + If no filename is given to the constructor (filename and + set_filename both None), we check for an environment variable naming + the config file, then finally we check for a global config file if + autoconf provided a directory name to check. + + NB: Programs which accept a configuration filename on the command + lines should pass that filename using set_filename so that we can + set the magic environment variable. Constraints from some external + libraries (principally Django) sometimes require library code to + look things up in the configuration file without the knowledge of + the controlling program, but setting the environment variable + insures that everybody's reading from the same script, as it were. """ - def __init__(self, filename = None, section = None, allow_missing = False): + # Odd keyword-only calling sequence is a defense against old code + # that thinks it knows how __init__() handles positional arguments. + + def __init__(self, **kwargs): + section = kwargs.pop("section", None) + allow_missing = kwargs.pop("allow_missing", False) + set_filename = kwargs.pop("set_filename", None) + filename = kwargs.pop("filename", set_filename) + + assert not kwargs, "Unexpected keyword arguments: " + ", ".join("%s = %r" % kv for kv in kwargs.iteritems()) + + if set_filename is not None: + os.environ[rpki_conf_envname] = set_filename self.cfg = ConfigParser.RawConfigParser() self.default_section = section - filenames = [] - if filename is not None: - filenames.append(filename) - else: - if default_envname in os.environ: - filenames.append(os.environ[default_envname]) - filenames.append(default_filename) - if default_dirname is not None: - filenames.append("%s/%s" % (default_dirname, default_filename)) + self.filename = filename or os.getenv(rpki_conf_envname) or default_filename - f = fn = None + try: + with open(self.filename, "r") as f: + self.cfg.readfp(f) + except IOError: + if allow_missing: + self.filename = None + else: + raise - for fn in filenames: - try: - f = open(fn) - break - except IOError: - f = None - - if f is not None: - self.filename = fn - self.cfg.readfp(f, fn) - elif allow_missing: - self.filename = None - else: - raise def has_section(self, section): """ @@ -107,6 +104,7 @@ class parser(object): return self.cfg.has_section(section) + def has_option(self, option, section = None): """ Test whether an option exists. @@ -116,6 +114,7 @@ class parser(object): section = self.default_section return self.cfg.has_option(section, option) + def multiget(self, option, section = None): """ Parse OpenSSL-style foo.0, foo.1, ... subscripted options. @@ -134,6 +133,7 @@ class parser(object): for option in matches: yield self.cfg.get(section, option) + _regexp = re.compile("\\${(.*?)::(.*?)}") def _repl(self, m): @@ -141,16 +141,19 @@ class parser(object): Replacement function for indirect variable substitution. This is intended for use with re.subn(). """ + section, option = m.group(1, 2) if section == "ENV": return os.getenv(option, "") else: return self.cfg.get(section, option) + def get(self, option, default = None, section = None): """ Get an option, perhaps with a default value. """ + if section is None: section = self.default_section if default is not None and not self.cfg.has_option(section, option): @@ -161,10 +164,12 @@ class parser(object): if not modified: return val + def getboolean(self, option, default = None, section = None): """ Get a boolean option, perhaps with a default value. """ + v = self.get(option, default, section) if isinstance(v, str): v = v.lower() @@ -173,18 +178,23 @@ class parser(object): v = self.cfg._boolean_states[v] return v + def getint(self, option, default = None, section = None): """ Get an integer option, perhaps with a default value. """ + return int(self.get(option, default, section)) + def getlong(self, option, default = None, section = None): """ Get a long integer option, perhaps with a default value. """ + return long(self.get(option, default, section)) + def set_global_flags(self): """ Consolidated control for all the little global control flags @@ -302,7 +312,7 @@ class parser(object): rpki.x509.generate_insecure_debug_only_rsa_key = rpki.x509.insecure_debug_only_rsa_key_generator(*self.get("insecure-debug-only-rsa-key-db").split()) except ConfigParser.NoOptionError: pass - except: + except: # pylint: disable=W0702 logger.warning("insecure-debug-only-rsa-key-db configured but initialization failed, check for corrupted database file") try: diff --git a/rpki/csv_utils.py b/rpki/csv_utils.py index 9ba04a02..9034e96b 100644 --- a/rpki/csv_utils.py +++ b/rpki/csv_utils.py @@ -99,6 +99,7 @@ class csv_writer(object): """ Close this writer. """ + if self.file is not None: self.file.close() self.file = None @@ -109,4 +110,5 @@ class csv_writer(object): """ Fake inheritance from whatever object csv.writer deigns to give us. """ + return getattr(self.writer, attr) diff --git a/rpki/django_settings/__init__.py b/rpki/django_settings/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/rpki/django_settings/__init__.py diff --git a/rpki/django_settings/common.py b/rpki/django_settings/common.py new file mode 100644 index 00000000..d410d984 --- /dev/null +++ b/rpki/django_settings/common.py @@ -0,0 +1,75 @@ +# $Id$ + +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +This module contains common configuration settings for Django libraries. + +Most of our CA code uses at least the Django ORM; the web interface +uses a lot more of Django. We also want to handle all normal user +configuration via rpki.conf, so some of the code here is just pulling +settings from rpki.conf and stuffing them into the form Django wants. +""" + +__version__ = "$Id$" + +import os +import rpki.config +import rpki.autoconf + +# Some configuration, including SQL authorization, comes from rpki.conf. +cfg = rpki.config.parser() + + +# Do -not- turn on DEBUG here except for short-lived tests, otherwise +# long-running programs like irdbd will eventually run out of memory +# and crash. This is also why this is controlled by an environment +# variable rather than by an rpki.conf setting: just because we want +# debugging enabled in the GUI doesn't mean we want it in irdb. +# +# If you must enable debugging, you may need to add code that uses +# django.db.reset_queries() to clear the query list manually, but it's +# probably better just to run with debugging disabled, since that's +# the expectation for production code. +# +# https://docs.djangoproject.com/en/dev/faq/models/#why-is-django-leaking-memory + +if os.getenv("RPKI_DJANGO_DEBUG") == "yes": + DEBUG = True + + +# Database configuration is handled in the modules that import this +# one, as it differs from program to program. We tried using a Django +# "database router" here, and it sort of worked, but it was a bit +# fragile, tedious to use, and generally more complex than we need, +# because any given program is only going to be using one database. + + +# Apps are also handled by the modules that import this one, now that +# we don't require South. + + +# Silence whining about MIDDLEWARE_CLASSES + +MIDDLEWARE_CLASSES = () + +# That would be it if we just need the ORM, but Django throws a hissy +# fit if SECRET_KEY isn't set, whether we use it for anything or not. +# +# Make this unique, and don't share it with anybody. +if cfg.has_option("secret-key", section = "web_portal"): + SECRET_KEY = cfg.get("secret-key", section = "web_portal") +else: + SECRET_KEY = os.urandom(66).encode("hex") diff --git a/rpki/django_settings/gui.py b/rpki/django_settings/gui.py new file mode 100644 index 00000000..4e9ac0f3 --- /dev/null +++ b/rpki/django_settings/gui.py @@ -0,0 +1,156 @@ +# $Id$ + +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +This module contains GUI-specific configuration settings for Django libraries. +""" + +# Pull in the irdb configuration, which in turn pulls in the common configuration. + +from .irdb import * # pylint: disable=W0401 + +__version__ = "$Id$" + +import socket + +# GUI uses the IRDB database configuration, so we don't need to set +# anything here. + +# Where to put static files. +STATIC_ROOT = rpki.autoconf.datarootdir + "/rpki/media" + +# Must end with a slash! +STATIC_URL = "/media/" + +# Where to email server errors. +ADMINS = (("Administrator", "root@localhost"),) + +LOGGING = { + "version": 1, + "formatters": { + "verbose": { + # see http://docs.python.org/2.7/library/logging.html#logging.LogRecord + "format": "%(levelname)s %(asctime)s %(name)s %(message)s" + }, + }, + "handlers": { + "stderr": { + "class": "logging.StreamHandler", + "level": "DEBUG", + "formatter": "verbose", + }, + "mail_admins": { + "level": "ERROR", + "class": "django.utils.log.AdminEmailHandler", + }, + }, + "loggers": { + "django": { + "level": "ERROR", + "handlers": ["stderr", "mail_admins"], + }, + "rpki.gui": { + "level": "WARNING", + "handlers": ["stderr"], + }, + }, +} + +def select_tz(): + "Find a supported timezone that looks like UTC" + for tz in ("UTC", "GMT", "Etc/UTC", "Etc/GMT"): + if os.path.exists("/usr/share/zoneinfo/" + tz): + return tz + # Can't determine the proper timezone, fall back to UTC and let Django + # report the error to the user. + return "UTC" + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# although not all choices may be available on all operating systems. +# If running in a Windows environment this must be set to the same as your +# system time zone. +TIME_ZONE = select_tz() + +# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts +# for details on why you might need this. +def get_allowed_hosts(): + allowed_hosts = set(cfg.multiget("allowed-hosts", section = "web_portal")) + allowed_hosts.add(socket.getfqdn()) + allowed_hosts.add("127.0.0.1") + allowed_hosts.add("::1") + try: + import netifaces + for interface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(interface) + for af in (netifaces.AF_INET, netifaces.AF_INET6): + if af in addresses: + for address in addresses[af]: + if "addr" in address: + allowed_hosts.add(address["addr"]) + except ImportError: + pass + return list(allowed_hosts) + +ALLOWED_HOSTS = get_allowed_hosts() + +# List of callables that know how to import templates from various sources. +TEMPLATE_LOADERS = ( + "django.template.loaders.filesystem.Loader", + "django.template.loaders.app_directories.Loader", + "django.template.loaders.eggs.Loader" +) + +MIDDLEWARE_CLASSES = ( + "django.middleware.common.CommonMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware" +) + +ROOT_URLCONF = "rpki.gui.urls" + +INSTALLED_APPS.extend(( + "django.contrib.auth", + #"django.contrib.admin", + #"django.contrib.admindocs", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.staticfiles", + "rpki.gui.app", + "rpki.gui.cacheview", + "rpki.gui.routeview", +)) + +TEMPLATE_CONTEXT_PROCESSORS = ( + "django.contrib.auth.context_processors.auth", + "django.core.context_processors.debug", + "django.core.context_processors.i18n", + "django.core.context_processors.media", + "django.contrib.messages.context_processors.messages", + "django.core.context_processors.request", + "django.core.context_processors.static" +) + +# Allow local site to override any setting above -- but if there's +# anything that local sites routinely need to modify, please consider +# putting that configuration into rpki.conf and just adding code here +# to read that configuration. +try: + from local_settings import * # pylint: disable=W0401,F0401 +except ImportError: + pass diff --git a/rpki/django_settings/irdb.py b/rpki/django_settings/irdb.py new file mode 100644 index 00000000..11e7417a --- /dev/null +++ b/rpki/django_settings/irdb.py @@ -0,0 +1,49 @@ +# $Id$ + +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +This module contains configuration settings for Django libraries. All +of the back-end programs (rpkic, irdbd, etc) use this configuration; +the GUI code also uses this but adds a bunch of other stuff, thus has +its own settings file. +""" + +from .common import * # pylint: disable=W0401 + +__version__ = "$Id$" + + +# Database configuration. + +DATABASES = dict( + default = dict(ENGINE = "django.db.backends.mysql", + NAME = cfg.get("sql-database", section = "irdbd"), + USER = cfg.get("sql-username", section = "irdbd"), + PASSWORD = cfg.get("sql-password", section = "irdbd"))) + +# Apps. + +INSTALLED_APPS = ["rpki.irdb"] + + +# Allow local site to override any setting above -- but if there's +# anything that local sites routinely need to modify, please consider +# putting that configuration into rpki.conf and just adding code here +# to read that configuration. +try: + from local_settings import * # pylint: disable=W0401,F0401 +except ImportError: + pass diff --git a/rpki/django_settings/pubd.py b/rpki/django_settings/pubd.py new file mode 100644 index 00000000..7ae533e7 --- /dev/null +++ b/rpki/django_settings/pubd.py @@ -0,0 +1,48 @@ +# $Id$ + +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +This module contains configuration settings for Django libraries for +the pubd program. +""" + +from .common import * # pylint: disable=W0401 + +__version__ = "$Id$" + + +# Database configuration. + +DATABASES = dict( + default = dict(ENGINE = "django.db.backends.mysql", + NAME = cfg.get("sql-database", section = "pubd"), + USER = cfg.get("sql-username", section = "pubd"), + PASSWORD = cfg.get("sql-password", section = "pubd"))) + + +# Apps. + +INSTALLED_APPS = ["rpki.pubdb"] + + +# Allow local site to override any setting above -- but if there's +# anything that local sites routinely need to modify, please consider +# putting that configuration into rpki.conf and just adding code here +# to read that configuration. +try: + from local_settings import * # pylint: disable=W0401,F0401 +except ImportError: + pass diff --git a/rpki/django_settings/rpkid.py b/rpki/django_settings/rpkid.py new file mode 100644 index 00000000..a2aa9401 --- /dev/null +++ b/rpki/django_settings/rpkid.py @@ -0,0 +1,48 @@ +# $Id$ + +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +This module contains configuration settings for Django libraries for +the rpkid program. +""" + +from .common import * # pylint: disable=W0401 + +__version__ = "$Id$" + + +# Database configuration. + +DATABASES = dict( + default = dict(ENGINE = "django.db.backends.mysql", + NAME = cfg.get("sql-database", section = "rpkid"), + USER = cfg.get("sql-username", section = "rpkid"), + PASSWORD = cfg.get("sql-password", section = "rpkid"))) + + +# Apps. + +INSTALLED_APPS = ["rpki.rpkidb"] + + +# Allow local site to override any setting above -- but if there's +# anything that local sites routinely need to modify, please consider +# putting that configuration into rpki.conf and just adding code here +# to read that configuration. +try: + from local_settings import * # pylint: disable=W0401,F0401 +except ImportError: + pass diff --git a/rpki/exceptions.py b/rpki/exceptions.py index 504c6f28..3ca8bd81 100644 --- a/rpki/exceptions.py +++ b/rpki/exceptions.py @@ -288,6 +288,16 @@ class NoObjectAtURI(RPKI_Exception): No object published at specified URI. """ +class ExistingObjectAtURI(RPKI_Exception): + """ + An object has already been published at specified URI. + """ + +class DifferentObjectAtURI(RPKI_Exception): + """ + An object with a different hash exists at specified URI. + """ + class CMSContentNotSet(RPKI_Exception): """ Inner content of a CMS_object has not been set. If object is known @@ -365,3 +375,8 @@ class WrongEKU(RPKI_Exception): """ Extended Key Usage extension does not match profile. """ + +class UnexpectedUpDownResponse(RPKI_Exception): + """ + Up-down message is not of the expected type. + """ diff --git a/rpki/fields.py b/rpki/fields.py new file mode 100644 index 00000000..4a826f4e --- /dev/null +++ b/rpki/fields.py @@ -0,0 +1,190 @@ +# $Id$ +# +# Copyright (C) 2013--2014 Dragon Research Labs ("DRL") +# Portions copyright (C) 2011--2012 Internet Systems Consortium ("ISC") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notices and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL +# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR +# ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL +# DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA +# OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +Common Django ORM field classes. + +Many of these are complex ASN.1 DER objects stored as SQL BLOBs, since +the only sane text representation would just be the Base64 encoding of +the DER and thus would add no value. +""" + +import logging + +from django.db import models + +import rpki.x509 +import rpki.sundial + +logger = logging.getLogger(__name__) + + +class EnumField(models.PositiveSmallIntegerField): + """ + An enumeration type that uses strings in Python and small integers + in SQL. + """ + + description = "An enumeration type" + + __metaclass__ = models.SubfieldBase + + def __init__(self, *args, **kwargs): + if isinstance(kwargs.get("choices"), (tuple, list)) and isinstance(kwargs["choices"][0], (str, unicode)): + kwargs["choices"] = tuple(enumerate(kwargs["choices"], 1)) + models.PositiveSmallIntegerField.__init__(self, *args, **kwargs) + self.enum_i2s = dict(self.flatchoices) + self.enum_s2i = dict((v, k) for k, v in self.flatchoices) + + def to_python(self, value): + return self.enum_i2s.get(value, value) + + def get_prep_value(self, value): + return self.enum_s2i.get(value, value) + + +class SundialField(models.DateTimeField): + """ + A field type for our customized datetime objects. + """ + __metaclass__ = models.SubfieldBase + + description = "A datetime type using our customized datetime objects" + + def to_python(self, value): + if isinstance(value, rpki.sundial.pydatetime.datetime): + return rpki.sundial.datetime.from_datetime( + models.DateTimeField.to_python(self, value)) + else: + return value + + def get_prep_value(self, value): + if isinstance(value, rpki.sundial.datetime): + return value.to_datetime() + else: + return value + + +class BlobField(models.Field): + """ + Basic BLOB field, no type conversion, just an opaque byte string. + + "BLOB" = "Binary Large OBject". Most SQL implementations seem to + have such a thing, but support appears to predate standardization, + so they all do it slightly differently and we have to cope. + + In PostgreSQL, BLOBs are called "bytea". + + In MySQL, there are different sizes of BLOBs and one must pick the + right one to avoid data truncation. RPKI manifests and CRLs can be + longer than 65535 octets, so in MySQL the only safe BLOB type for + general use is "LONGBLOB". + + SQLite...is not like the other children: data types are more like + guidelines than actual rules. But "BLOB" works. + + For anything else, we just use "BLOB" and hope for the best. + + NB: This field type predates Django 1.6's BinaryField. Probably + this should be retired in favor of BinaryField, but I'd have to + figure out what that does to field types that derive from this one. + """ + + __metaclass__ = models.SubfieldBase + description = "Raw BLOB type without ASN.1 encoding/decoding" + + def __init__(self, *args, **kwargs): + self.blob_type = kwargs.pop("blob_type", None) + kwargs["serialize"] = False + kwargs["blank"] = True + kwargs["default"] = None + models.Field.__init__(self, *args, **kwargs) + + def db_type(self, connection): + if self.blob_type is not None: + return self.blob_type + elif connection.settings_dict['ENGINE'] == "django.db.backends.mysql": + return "LONGBLOB" + elif connection.settings_dict['ENGINE'] == "django.db.backends.posgresql": + return "bytea" + else: + return "BLOB" + + +# For reasons which now escape me, I had a few fields in the old +# hand-coded SQL which used MySQL type BINARY(20) to hold SKIs. +# Presumably this was so that I could then use those SKIs in indexes +# and searches, but apparently I never got around to that part. +# +# SKIs probably would be better stored as hex strings anyway, so not +# bothering with a separate binary type model for this. Deal with +# this if and when it ever becomes an issue. + + +class DERField(BlobField): + """ + Field class for DER objects. These are derived from BLOBs, but with + automatic translation between ASN.1 and Python types. + + DERField itself is an abstract class, concrete field classes are + derived from it. + """ + + __metaclass__ = models.SubfieldBase + + def to_python(self, value): + assert value is None or isinstance(value, (self.rpki_type, str)) + if isinstance(value, str): + return self.rpki_type(DER = value) + else: + return value + + def get_prep_value(self, value): + assert value is None or isinstance(value, (self.rpki_type, str)) + if isinstance(value, self.rpki_type): + return value.get_DER() + else: + return value + +class CertificateField(DERField): + description = "X.509 certificate" + rpki_type = rpki.x509.X509 + +class KeyField(DERField): + description = "RSA keypair" + rpki_type = rpki.x509.RSA + +class CRLField(DERField): + description = "Certificate Revocation List" + rpki_type = rpki.x509.CRL + +class PKCS10Field(DERField): + description = "PKCS #10 certificate request" + rpki_type = rpki.x509.PKCS10 + +class ManifestField(DERField): + description = "RPKI Manifest" + rpki_type = rpki.x509.SignedManifest + +class ROAField(DERField): + description = "ROA" + rpki_type = rpki.x509.ROA + +class GhostbusterField(DERField): + description = "Ghostbuster Record" + rpki_type = rpki.x509.Ghostbuster diff --git a/rpki/gui/app/check_expired.py b/rpki/gui/app/check_expired.py index a084af79..62292e66 100644 --- a/rpki/gui/app/check_expired.py +++ b/rpki/gui/app/check_expired.py @@ -25,9 +25,10 @@ from rpki.gui.cacheview.models import Cert from rpki.gui.app.models import Conf, ResourceCert, Timestamp, Alert from rpki.gui.app.glue import list_received_resources from rpki.irdb import Zookeeper -from rpki.left_right import report_error_elt, list_published_objects_elt from rpki.x509 import X509 +from rpki.left_right import version, nsmap, tag_msg, tag_list_published_objects +from lxml.etree import Element, SubElement from django.core.mail import send_mail logger = logging.getLogger(__name__) @@ -41,8 +42,8 @@ def check_cert(handle, p, errs): The displayed object name defaults to the class name, but can be overridden using the `object_name` argument. - """ + t = p.certificate.getNotAfter() if t <= expire_time: e = 'expired' if t <= now else 'will expire' @@ -102,30 +103,26 @@ def check_expire(conf, errs): def check_child_certs(conf, errs): """Fetch the list of published objects from rpkid, and inspect the issued resource certs (uri ending in .cer). - """ + z = Zookeeper(handle=conf.handle) - req = list_published_objects_elt.make_pdu(action="list", - tag="list_published_objects", - self_handle=conf.handle) + req = Element(tag_msg, nsmap=nsmap, type="query", version=version) + SubElement(req, tag_list_published_objects, + tag="list_published_objects", self_handle=conf.handle) pdus = z.call_rpkid(req) for pdu in pdus: - if isinstance(pdu, report_error_elt): - logger.error("rpkid reported an error: %s", pdu.error_code) - elif isinstance(pdu, list_published_objects_elt): - if pdu.uri.endswith('.cer'): - cert = X509() - cert.set(Base64=pdu.obj) - t = cert.getNotAfter() - if t <= expire_time: - e = 'expired' if t <= now else 'will expire' - errs.write("%(handle)s's rescert for Child %(child)s %(expire)s on %(date)s uri=%(uri)s subject=%(subject)s\n" % { - 'handle': conf.handle, - 'child': pdu.child_handle, - 'uri': pdu.uri, - 'subject': cert.getSubject(), - 'expire': e, - 'date': t}) + if pdu.get("uri").endswith('.cer'): + cert = X509(Base64=pdu.text) + t = cert.getNotAfter() + if t <= expire_time: + e = 'expired' if t <= now else 'will expire' + errs.write("%(handle)s's rescert for Child %(child)s %(expire)s on %(date)s uri=%(uri)s subject=%(subject)s\n" % { + 'handle': conf.handle, + 'child': pdu.get("child_handle"), + 'uri': pdu.get("uri"), + 'subject': cert.getSubject(), + 'expire': e, + 'date': t}) class NetworkError(Exception): @@ -139,8 +136,8 @@ def notify_expired(expire_days=14, from_email=None): expire_days: the number of days ahead of today to warn from_email: set the From: address for the email - """ + global expire_time # so i don't have to pass it around global now diff --git a/rpki/gui/app/forms.py b/rpki/gui/app/forms.py index a1214297..306b8dce 100644 --- a/rpki/gui/app/forms.py +++ b/rpki/gui/app/forms.py @@ -193,7 +193,7 @@ def ROARequestFormFactory(conf): 'class': 'span1' }) ) - protect_children = forms.BooleanField(required=False) + protect_children = forms.BooleanField(required=False) def __init__(self, *args, **kwargs): kwargs['auto_id'] = False diff --git a/rpki/gui/app/glue.py b/rpki/gui/app/glue.py index a2dddb51..bfade6d8 100644 --- a/rpki/gui/app/glue.py +++ b/rpki/gui/app/glue.py @@ -16,7 +16,6 @@ """ This file contains code that interfaces between the django views implementing the portal gui and the rpki.* modules. - """ from __future__ import with_statement @@ -28,17 +27,19 @@ from datetime import datetime from rpki.resource_set import (resource_set_as, resource_set_ipv4, resource_set_ipv6, resource_range_ipv4, resource_range_ipv6) -from rpki.left_right import list_received_resources_elt, report_error_elt from rpki.irdb.zookeeper import Zookeeper from rpki.gui.app import models from rpki.exceptions import BadIPResource +from rpki.left_right import nsmap, version, tag_msg, tag_list_received_resources +from lxml.etree import Element, SubElement from django.contrib.auth.models import User from django.db.transaction import commit_on_success def ghostbuster_to_vcard(gbr): """Convert a GhostbusterRequest object into a vCard object.""" + import vobject vcard = vobject.vCard() @@ -66,18 +67,6 @@ def ghostbuster_to_vcard(gbr): return vcard.serialize() -class LeftRightError(Exception): - """Class for wrapping report_error_elt errors from Zookeeper.call_rpkid(). - - It expects a single argument, which is the associated report_error_elt instance.""" - - def __str__(self): - return 'Error occurred while communicating with rpkid: handle=%s code=%s text=%s' % ( - self.args[0].self_handle, - self.args[0].error_code, - self.args[0].error_text) - - @commit_on_success def list_received_resources(log, conf): """ @@ -86,11 +75,12 @@ def list_received_resources(log, conf): The semantics are to clear the entire table and populate with the list of certs received. Other models should not reference the table directly with foreign keys. - """ z = Zookeeper(handle=conf.handle, disable_signal_handlers=True) - pdus = z.call_rpkid(list_received_resources_elt.make_pdu(self_handle=conf.handle)) + req = Element(tag_msg, nsmap=nsmap, type="query", version=version) + SubElement(req, tag_list_received_resources, self_handle=conf.handle) + pdus = z.call_rpkid(req) # pdus is sometimes None (see https://trac.rpki.net/ticket/681) if pdus is None: print >>log, 'error: call_rpkid() returned None for handle %s when fetching received resources' % conf.handle @@ -99,34 +89,27 @@ def list_received_resources(log, conf): models.ResourceCert.objects.filter(conf=conf).delete() for pdu in pdus: - if isinstance(pdu, report_error_elt): - # this will cause the db to be rolled back so the above delete() - # won't clobber existing resources - raise LeftRightError(pdu) - elif isinstance(pdu, list_received_resources_elt): - if pdu.parent_handle != conf.handle: - parent = models.Parent.objects.get(issuer=conf, - handle=pdu.parent_handle) - else: - # root cert, self-signed - parent = None - - not_before = datetime.strptime(pdu.notBefore, "%Y-%m-%dT%H:%M:%SZ") - not_after = datetime.strptime(pdu.notAfter, "%Y-%m-%dT%H:%M:%SZ") - - cert = models.ResourceCert.objects.create( - conf=conf, parent=parent, not_before=not_before, - not_after=not_after, uri=pdu.uri) - - for asn in resource_set_as(pdu.asn): - cert.asn_ranges.create(min=asn.min, max=asn.max) - - for rng in resource_set_ipv4(pdu.ipv4): - cert.address_ranges.create(prefix_min=rng.min, - prefix_max=rng.max) - - for rng in resource_set_ipv6(pdu.ipv6): - cert.address_ranges_v6.create(prefix_min=rng.min, - prefix_max=rng.max) + if pdu.get("parent_handle") != conf.handle: + parent = models.Parent.objects.get(issuer=conf, + handle=pdu.get("parent_handle")) else: - print >>log, "error: unexpected pdu from rpkid type=%s" % type(pdu) + # root cert, self-signed + parent = None + + not_before = datetime.strptime(pdu.get("notBefore"), "%Y-%m-%dT%H:%M:%SZ") + not_after = datetime.strptime(pdu.get("notAfter"), "%Y-%m-%dT%H:%M:%SZ") + + cert = models.ResourceCert.objects.create( + conf=conf, parent=parent, not_before=not_before, + not_after=not_after, uri=pdu.get("uri")) + + for asn in resource_set_as(pdu.get("asn")): + cert.asn_ranges.create(min=asn.min, max=asn.max) + + for rng in resource_set_ipv4(pdu.get("ipv4")): + cert.address_ranges.create(prefix_min=rng.min, + prefix_max=rng.max) + + for rng in resource_set_ipv6(pdu.get("ipv6")): + cert.address_ranges_v6.create(prefix_min=rng.min, + prefix_max=rng.max) diff --git a/rpki/gui/app/models.py b/rpki/gui/app/models.py index 40bdbe2c..c49e6d43 100644 --- a/rpki/gui/app/models.py +++ b/rpki/gui/app/models.py @@ -26,7 +26,6 @@ import rpki.irdb.models import rpki.gui.models import rpki.gui.routeview.models import rpki.oids -from south.modelsinspector import add_introspection_rules class TelephoneField(models.CharField): @@ -35,7 +34,7 @@ class TelephoneField(models.CharField): kwargs['max_length'] = 40 models.CharField.__init__(self, **kwargs) -add_introspection_rules([], [r'^rpki\.gui\.app\.models\.TelephoneField']) + class Parent(rpki.irdb.models.Parent): diff --git a/rpki/gui/app/range_list.py b/rpki/gui/app/range_list.py index 21fd1f29..5cb4f5e4 100755 --- a/rpki/gui/app/range_list.py +++ b/rpki/gui/app/range_list.py @@ -70,6 +70,7 @@ class RangeList(list): def difference(self, other): """Return a RangeList object which contains ranges in this object which are not in "other".""" + it = iter(other) try: @@ -85,6 +86,7 @@ class RangeList(list): def V(v): """convert the integer value to the appropriate type for this range""" + return x.__class__.datum_type(v) try: diff --git a/rpki/gui/app/migrations/0001_initial.py b/rpki/gui/app/south_migrations/0001_initial.py index 80877901..80877901 100644 --- a/rpki/gui/app/migrations/0001_initial.py +++ b/rpki/gui/app/south_migrations/0001_initial.py diff --git a/rpki/gui/app/migrations/0002_auto__add_field_resourcecert_conf.py b/rpki/gui/app/south_migrations/0002_auto__add_field_resourcecert_conf.py index d3326f90..d3326f90 100644 --- a/rpki/gui/app/migrations/0002_auto__add_field_resourcecert_conf.py +++ b/rpki/gui/app/south_migrations/0002_auto__add_field_resourcecert_conf.py diff --git a/rpki/gui/app/migrations/0003_set_conf_from_parent.py b/rpki/gui/app/south_migrations/0003_set_conf_from_parent.py index a90a11cc..a90a11cc 100644 --- a/rpki/gui/app/migrations/0003_set_conf_from_parent.py +++ b/rpki/gui/app/south_migrations/0003_set_conf_from_parent.py diff --git a/rpki/gui/app/migrations/0004_auto__chg_field_resourcecert_conf.py b/rpki/gui/app/south_migrations/0004_auto__chg_field_resourcecert_conf.py index a236ad4a..a236ad4a 100644 --- a/rpki/gui/app/migrations/0004_auto__chg_field_resourcecert_conf.py +++ b/rpki/gui/app/south_migrations/0004_auto__chg_field_resourcecert_conf.py diff --git a/rpki/gui/app/migrations/0005_auto__chg_field_resourcecert_parent.py b/rpki/gui/app/south_migrations/0005_auto__chg_field_resourcecert_parent.py index 11e9c814..11e9c814 100644 --- a/rpki/gui/app/migrations/0005_auto__chg_field_resourcecert_parent.py +++ b/rpki/gui/app/south_migrations/0005_auto__chg_field_resourcecert_parent.py diff --git a/rpki/gui/app/migrations/0006_add_conf_acl.py b/rpki/gui/app/south_migrations/0006_add_conf_acl.py index 88fe8171..88fe8171 100644 --- a/rpki/gui/app/migrations/0006_add_conf_acl.py +++ b/rpki/gui/app/south_migrations/0006_add_conf_acl.py diff --git a/rpki/gui/app/migrations/0007_default_acls.py b/rpki/gui/app/south_migrations/0007_default_acls.py index 40656d0f..40656d0f 100644 --- a/rpki/gui/app/migrations/0007_default_acls.py +++ b/rpki/gui/app/south_migrations/0007_default_acls.py diff --git a/rpki/gui/app/migrations/0008_add_alerts.py b/rpki/gui/app/south_migrations/0008_add_alerts.py index 77af68d2..77af68d2 100644 --- a/rpki/gui/app/migrations/0008_add_alerts.py +++ b/rpki/gui/app/south_migrations/0008_add_alerts.py diff --git a/rpki/gui/app/south_migrations/__init__.py b/rpki/gui/app/south_migrations/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/rpki/gui/app/south_migrations/__init__.py diff --git a/rpki/gui/app/views.py b/rpki/gui/app/views.py index bf152f8e..d9b3fee3 100644 --- a/rpki/gui/app/views.py +++ b/rpki/gui/app/views.py @@ -1239,22 +1239,13 @@ def resource_holder_create(request): zk_child = Zookeeper(handle=handle, logstream=log) identity_xml = zk_child.initialize_resource_bpki() if parent: - # FIXME etree_wrapper should allow us to deal with file objects - t = NamedTemporaryFile(delete=False) - t.close() - - identity_xml.save(t.name) zk_parent = Zookeeper(handle=parent.handle, logstream=log) - parent_response, _ = zk_parent.configure_child(t.name) - parent_response.save(t.name) + parent_response, _ = zk_parent.configure_child(identity_xml) zk_parent.synchronize_ca() - repo_req, _ = zk_child.configure_parent(t.name) - repo_req.save(t.name) - repo_resp, _ = zk_parent.configure_publication_client(t.name) - repo_resp.save(t.name) + repo_req, _ = zk_child.configure_parent(parent_response) + repo_resp, _ = zk_parent.configure_publication_client(repo_req) zk_parent.synchronize_pubd() - zk_child.configure_repository(t.name) - os.remove(t.name) + zk_child.configure_repository(repo_resp) zk_child.synchronize_ca() return redirect(resource_holder_list) else: diff --git a/rpki/gui/cacheview/models.py b/rpki/gui/cacheview/models.py index c3ee8421..08acfa2d 100644 --- a/rpki/gui/cacheview/models.py +++ b/rpki/gui/cacheview/models.py @@ -58,6 +58,7 @@ class ValidationLabel(models.Model): Represents a specific error condition defined in the rcynic XML output file. """ + label = models.CharField(max_length=79, db_index=True, unique=True) status = models.CharField(max_length=255) kind = models.PositiveSmallIntegerField(choices=kinds) @@ -70,6 +71,7 @@ class RepositoryObject(models.Model): """ Represents a globally unique RPKI repository object, specified by its URI. """ + uri = models.URLField(unique=True, db_index=True) generations = list(enumerate(('current', 'backup'))) @@ -89,6 +91,7 @@ class SignedObject(models.Model): The signing certificate is ommitted here in order to give a proper value for the 'related_name' attribute. """ + repo = models.ForeignKey(RepositoryObject, related_name='cert', unique=True) # on-disk file modification time @@ -108,6 +111,7 @@ class SignedObject(models.Model): """ convert the local timestamp to UTC and convert to a datetime object """ + return datetime.utcfromtimestamp(self.mtime + time.timezone) def status_id(self): @@ -116,6 +120,7 @@ class SignedObject(models.Model): The selector is chosen based on the current generation only. If there is any bad status, return bad, else if there are any warn status, return warn, else return good. """ + for x in reversed(kinds): if self.repo.statuses.filter(generation=generations_dict['current'], status__kind=x[0]): return x[1] @@ -129,6 +134,7 @@ class Cert(SignedObject): """ Object representing a resource certificate. """ + addresses = models.ManyToManyField(AddressRange, related_name='certs') addresses_v6 = models.ManyToManyField(AddressRangeV6, related_name='certs') asns = models.ManyToManyField(ASRange, related_name='certs') @@ -141,6 +147,7 @@ class Cert(SignedObject): def get_cert_chain(self): """Return a list containing the complete certificate chain for this certificate.""" + cert = self x = [cert] while cert != cert.issuer: @@ -180,6 +187,7 @@ class ROAPrefixV4(ROAPrefix, rpki.gui.models.PrefixV4): @property def routes(self): """return all routes covered by this roa prefix""" + return RouteOrigin.objects.filter(prefix_min__gte=self.prefix_min, prefix_max__lte=self.prefix_max) diff --git a/rpki/gui/cacheview/tests.py b/rpki/gui/cacheview/tests.py index 2247054b..daca07bf 100644 --- a/rpki/gui/cacheview/tests.py +++ b/rpki/gui/cacheview/tests.py @@ -12,6 +12,7 @@ class SimpleTest(TestCase): """ Tests that 1 + 1 always equals 2. """ + self.failUnlessEqual(1 + 1, 2) __test__ = {"doctest": """ diff --git a/rpki/gui/cacheview/util.py b/rpki/gui/cacheview/util.py index 9e8748bf..21430091 100644 --- a/rpki/gui/cacheview/util.py +++ b/rpki/gui/cacheview/util.py @@ -32,6 +32,7 @@ from django.db import transaction import django.db.models import rpki +import rpki.left_right import rpki.gui.app.timestamp from rpki.gui.app.models import Conf, Alert from rpki.gui.cacheview import models @@ -39,6 +40,8 @@ from rpki.rcynic import rcynic_xml_iterator, label_iterator from rpki.sundial import datetime from rpki.irdb.zookeeper import Zookeeper +from lxml.etree import Element, SubElement + logger = logging.getLogger(__name__) @@ -310,23 +313,26 @@ def fetch_published_objects(): """Query rpkid for all objects published by local users, and look up the current validation status of each object. The validation status is used later to send alerts for objects which have transitioned to invalid. - """ + logger.info('querying for published objects') handles = [conf.handle for conf in Conf.objects.all()] - req = [rpki.left_right.list_published_objects_elt.make_pdu(action='list', self_handle=h, tag=h) for h in handles] + q_msg = Element(rpki.left_right.tag_msg, nsmap = rpki.left_right.nsmap, + type = "query", version = rpki.left_right.version) + for h in handles: + SubElement(q_msg, rpki.left_right.tag_list_published_objects, action="list", self_handle=h, tag=h) z = Zookeeper() - pdus = z.call_rpkid(*req) - for pdu in pdus: - if isinstance(pdu, rpki.left_right.list_published_objects_elt): + r_msg = z.call_rpkid(q_msg) + for r_pdu in r_msg: + if r_pdu.tag == rpki.left_right.tag_list_published_objects: # Look up the object in the rcynic cache - qs = models.RepositoryObject.objects.filter(uri=pdu.uri) + qs = models.RepositoryObject.objects.filter(uri=r_pdu.get("uri")) if qs: # get the current validity state valid = qs[0].statuses.filter(status=object_accepted).exists() - uris[pdu.uri] = (pdu.self_handle, valid, False, None) - logger.debug('adding ' + pdu.uri) + uris[r_pdu.get("uri")] = (r_pdu.get("self_handle"), valid, False, None) + logger.debug('adding %s', r_pdu.get("uri")) else: # this object is not in the cache. it was either published # recently, or disappared previously. if it disappeared @@ -334,8 +340,8 @@ def fetch_published_objects(): # omit the uri from the list since we are interested only in # objects which were valid and are no longer valid pass - elif isinstance(pdu, rpki.left_right.report_error_elt): - logging.error('rpkid reported an error: %s', pdu.error_code) + elif r_pdu.tag == rpki.left_right.tag_report_error: + logging.error('rpkid reported an error: %s', r_pdu.get("error_code")) class Handle(object): @@ -353,7 +359,6 @@ class Handle(object): def notify_invalid(): """Send email alerts to the addresses registered in ghostbuster records for any invalid objects that were published by users of this system. - """ logger.info('sending notifications for invalid objects') diff --git a/rpki/gui/cacheview/views.py b/rpki/gui/cacheview/views.py index 94870eb2..451c0d1e 100644 --- a/rpki/gui/cacheview/views.py +++ b/rpki/gui/cacheview/views.py @@ -29,6 +29,7 @@ def cert_chain(obj): """ returns an iterator covering all certs from the root cert down to the EE. """ + chain = [obj] while obj != obj.issuer: obj = obj.issuer diff --git a/rpki/gui/default_settings.py b/rpki/gui/default_settings.py deleted file mode 100644 index 02987bb8..00000000 --- a/rpki/gui/default_settings.py +++ /dev/null @@ -1,182 +0,0 @@ -""" -This module contains static configuration settings for the web portal. -""" - -__version__ = '$Id$' - -import os -import random -import string -import socket - -import rpki.config -import rpki.autoconf - -# Where to put static files. -STATIC_ROOT = rpki.autoconf.datarootdir + '/rpki/media' - -# Must end with a slash! -STATIC_URL = '/media/' - -# Where to email server errors. -ADMINS = (('Administrator', 'root@localhost'),) - -LOGGING = { - 'version': 1, - 'formatters': { - 'verbose': { - # see http://docs.python.org/2.7/library/logging.html#logging.LogRecord - 'format': '%(levelname)s %(asctime)s %(name)s %(message)s' - }, - }, - 'handlers': { - 'stderr': { - 'class': 'logging.StreamHandler', - 'level': 'DEBUG', - 'formatter': 'verbose', - }, - 'mail_admins': { - 'level': 'ERROR', - 'class': 'django.utils.log.AdminEmailHandler', - }, - }, - 'loggers': { - 'rpki.async': { - # enabled for tracking https://trac.rpki.net/ticket/681 - # need to change this to WARNING once ticket is closed - 'level': 'DEBUG', - }, - # The Django default LOGGING configuration disables propagate on these - # two loggers. Re-enable propagate so they will hit our root logger. - 'django.request': { - 'propagate': True, - }, - 'django.security': { - 'propagate': True, - }, - }, - 'root': { - 'level': 'WARNING', - 'handlers': ['stderr', 'mail_admins'], - }, -} - -# Load the SQL authentication bits from the system rpki.conf. -rpki_config = rpki.config.parser(section='web_portal') - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': rpki_config.get('sql-database'), - 'USER': rpki_config.get('sql-username'), - 'PASSWORD': rpki_config.get('sql-password'), - - # Ensure the default storage engine is InnoDB since we need - # foreign key support. The Django documentation suggests - # removing this after the syncdb is performed as an optimization, - # but there isn't an easy way to do this automatically. - - 'OPTIONS': { - 'init_command': 'SET storage_engine=INNODB', - } - } -} - - -def select_tz(): - "Find a supported timezone that looks like UTC" - for tz in ('UTC', 'GMT', 'Etc/UTC', 'Etc/GMT'): - if os.path.exists('/usr/share/zoneinfo/' + tz): - return tz - # Can't determine the proper timezone, fall back to UTC and let Django - # report the error to the user. - return 'UTC' - -# Local time zone for this installation. Choices can be found here: -# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name -# although not all choices may be available on all operating systems. -# If running in a Windows environment this must be set to the same as your -# system time zone. -TIME_ZONE = select_tz() - -def get_secret_key(): - """Retrieve the secret-key value from rpki.conf or generate a random value - if it is not present.""" - d = string.letters + string.digits - val = ''.join([random.choice(d) for _ in range(50)]) - return rpki_config.get('secret-key', val) - -# Make this unique, and don't share it with anybody. -SECRET_KEY = get_secret_key() - -# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts -# for details on why you might need this. -def get_allowed_hosts(): - allowed_hosts = set(rpki_config.multiget("allowed-hosts")) - allowed_hosts.add(socket.getfqdn()) - try: - import netifaces - for interface in netifaces.interfaces(): - addresses = netifaces.ifaddresses(interface) - for af in (netifaces.AF_INET, netifaces.AF_INET6): - if af in addresses: - for address in addresses[af]: - if "addr" in address: - allowed_hosts.add(address["addr"]) - except ImportError: - pass - return list(allowed_hosts) - -ALLOWED_HOSTS = get_allowed_hosts() - -DOWNLOAD_DIRECTORY = rpki_config.get('download-directory', '/var/tmp') - -# List of callables that know how to import templates from various sources. -TEMPLATE_LOADERS = ( - 'django.template.loaders.filesystem.Loader', - 'django.template.loaders.app_directories.Loader', - 'django.template.loaders.eggs.Loader' -) - -MIDDLEWARE_CLASSES = ( - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware' -) - -ROOT_URLCONF = 'rpki.gui.urls' - -INSTALLED_APPS = ( - 'django.contrib.auth', - #'django.contrib.admin', - #'django.contrib.admindocs', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.staticfiles', - 'rpki.irdb', - 'rpki.gui.app', - 'rpki.gui.cacheview', - 'rpki.gui.routeview', - 'south', -) - -TEMPLATE_CONTEXT_PROCESSORS = ( - "django.contrib.auth.context_processors.auth", - "django.core.context_processors.debug", - "django.core.context_processors.i18n", - "django.core.context_processors.media", - "django.contrib.messages.context_processors.messages", - "django.core.context_processors.request", - "django.core.context_processors.static" -) - -# Allow local site to override any setting above -- but if there's -# anything that local sites routinely need to modify, please consider -# putting that configuration into rpki.conf and just adding code here -# to read that configuration. -try: - from local_settings import * -except: - pass diff --git a/rpki/gui/models.py b/rpki/gui/models.py index 184383c0..ced14926 100644 --- a/rpki/gui/models.py +++ b/rpki/gui/models.py @@ -22,7 +22,6 @@ from django.db import models import rpki.resource_set import rpki.POW -from south.modelsinspector import add_introspection_rules class IPv6AddressField(models.Field): @@ -42,8 +41,8 @@ class IPv6AddressField(models.Field): """ Note that we add a custom conversion to encode long values as hex strings in SQL statements. See settings.get_conv() for details. - """ + return value.toBytes() @@ -63,14 +62,6 @@ class IPv4AddressField(models.Field): def get_db_prep_value(self, value, connection, prepared): return long(value) -add_introspection_rules( - [ - ([IPv4AddressField, IPv6AddressField], [], {}) - ], - [r'^rpki\.gui\.models\.IPv4AddressField', - r'^rpki\.gui\.models\.IPv6AddressField'] -) - class Prefix(models.Model): """Common implementation for models with an IP address range. @@ -82,6 +73,7 @@ class Prefix(models.Model): """ Returns the prefix as a rpki.resource_set.resource_range_ip object. """ + return self.range_cls(self.prefix_min, self.prefix_max) @property @@ -96,6 +88,7 @@ class Prefix(models.Model): def __unicode__(self): """This method may be overridden by subclasses. The default implementation calls get_prefix_display(). """ + return self.get_prefix_display() class Meta: diff --git a/rpki/gui/routeview/api.py b/rpki/gui/routeview/api.py index cf699c9a..b4ff297a 100644 --- a/rpki/gui/routeview/api.py +++ b/rpki/gui/routeview/api.py @@ -29,8 +29,8 @@ def route_list(request): By default, only returns up to 10 matching routes, but the client may request a different limit with the 'count=' query string parameter. - """ + hard_limit = 100 if request.method == 'GET' and 'prefix__in' in request.GET: diff --git a/rpki/gui/routeview/util.py b/rpki/gui/routeview/util.py index 1340e9fa..77ff04c7 100644 --- a/rpki/gui/routeview/util.py +++ b/rpki/gui/routeview/util.py @@ -215,8 +215,8 @@ def import_routeviews_dump(filename=DEFAULT_URL, filetype='text'): filename [optional]: the full path to the downloaded file to parse filetype [optional]: 'text' or 'mrt' - """ + start_time = time.time() tmpname = None diff --git a/rpki/gui/script_util.py b/rpki/gui/script_util.py index 43a53bc6..31e40821 100644 --- a/rpki/gui/script_util.py +++ b/rpki/gui/script_util.py @@ -16,13 +16,6 @@ This module contains utility functions for use in standalone scripts. """ -import django - -from django.conf import settings - -from rpki import config -from rpki import autoconf - __version__ = '$Id$' @@ -30,21 +23,8 @@ def setup(): """ Configure Django enough to use the ORM. """ - cfg = config.parser(section='web_portal') - # INSTALLED_APPS doesn't seem necessary so long as you are only accessing - # existing tables. - settings.configure( - DATABASES={ - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': cfg.get('sql-database'), - 'USER': cfg.get('sql-username'), - 'PASSWORD': cfg.get('sql-password'), - } - }, - MIDDLEWARE_CLASSES = (), - DOWNLOAD_DIRECTORY = cfg.get('download-directory', '/var/tmp'), - ) - if django.VERSION >= (1, 7): - from django.apps import apps - apps.populate(settings.INSTALLED_APPS) + + import os + + # If this doesn't work, try changing it to "rpki.django_settings.gui". + os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings.irdb") diff --git a/rpki/http.py b/rpki/http.py index 71239c7f..16ed0453 100644 --- a/rpki/http.py +++ b/rpki/http.py @@ -113,6 +113,7 @@ def supported_address_families(enable_ipv6): IP address families on which servers should listen, and to consider when selecting addresses for client connections. """ + if enable_ipv6 and have_ipv6: return (socket.AF_INET, socket.AF_INET6) else: @@ -122,6 +123,7 @@ def localhost_addrinfo(): """ Return pseudo-getaddrinfo results for localhost. """ + result = [(socket.AF_INET, "127.0.0.1")] if enable_ipv6_clients and have_ipv6: result.append((socket.AF_INET6, "::1")) @@ -145,6 +147,7 @@ class http_message(object): Clean up (some of) the horrible messes that HTTP allows in its headers. """ + if headers is None: headers = () if self.headers is None else self.headers.items() translate_underscore = True @@ -167,6 +170,7 @@ class http_message(object): """ Parse and normalize an incoming HTTP message. """ + self = cls() headers = headers.split("\r\n") self.parse_first_line(*headers.pop(0).split(None, 2)) @@ -181,6 +185,7 @@ class http_message(object): """ Format an outgoing HTTP message. """ + s = self.format_first_line() if self.body is not None: assert isinstance(self.body, str) @@ -199,6 +204,7 @@ class http_message(object): """ Parse HTTP version, raise an exception if we can't. """ + if version[:5] != "HTTP/": raise rpki.exceptions.HTTPBadVersion("Couldn't parse version %s" % version) self.version = tuple(int(i) for i in version[5:].split(".")) @@ -208,6 +214,7 @@ class http_message(object): """ Figure out whether this HTTP message encourages a persistent connection. """ + c = self.headers.get("Connection") if self.version == (1, 1): return c is None or "close" not in c.lower() @@ -234,6 +241,7 @@ class http_request(http_message): """ Parse first line of HTTP request message. """ + self.parse_version(version) self.cmd = cmd self.path = path @@ -243,6 +251,7 @@ class http_request(http_message): Format first line of HTTP request message, and set up the User-Agent header. """ + self.headers.setdefault("User-Agent", self.software_name) return "%s %s HTTP/%d.%d\r\n" % (self.cmd, self.path, self.version[0], self.version[1]) @@ -263,6 +272,7 @@ class http_response(http_message): """ Parse first line of HTTP response message. """ + self.parse_version(version) self.code = int(code) self.reason = reason @@ -272,6 +282,7 @@ class http_response(http_message): Format first line of HTTP response message, and set up Date and Server headers. """ + self.headers.setdefault("Date", time.strftime("%a, %d %b %Y %T GMT")) self.headers.setdefault("Server", self.software_name) return "HTTP/%d.%d %s %s\r\n" % (self.version[0], self.version[1], self.code, self.reason) @@ -320,6 +331,7 @@ class http_stream(asynchat.async_chat): """ (Re)start HTTP message parser, reset timer. """ + assert not self.buffer self.chunk_handler = None self.set_terminator("\r\n\r\n") @@ -331,6 +343,7 @@ class http_stream(asynchat.async_chat): stream's timeout value if we're doing timeouts, otherwise clear it. """ + if self.timeout is not None: self.logger.debug("Setting timeout %s", self.timeout) self.timer.set(self.timeout) @@ -342,6 +355,7 @@ class http_stream(asynchat.async_chat): """ Buffer incoming data from asynchat. """ + self.buffer.append(data) self.update_timeout() @@ -349,6 +363,7 @@ class http_stream(asynchat.async_chat): """ Consume data buffered from asynchat. """ + val = "".join(self.buffer) self.buffer = [] return val @@ -370,6 +385,7 @@ class http_stream(asynchat.async_chat): separate mechanisms (chunked, content-length, TCP close) is going to tell us how to find the end of the message body. """ + self.update_timeout() if self.chunk_handler: self.chunk_handler() @@ -393,6 +409,7 @@ class http_stream(asynchat.async_chat): stream up to read it; otherwise, this is the last chunk, so start the process of exiting the chunk decoder. """ + n = int(self.get_buffer().partition(";")[0], 16) self.logger.debug("Chunk length %s", n) if n: @@ -408,6 +425,7 @@ class http_stream(asynchat.async_chat): body of a chunked message (sic). Save it, and prepare to move on to the next chunk. """ + self.logger.debug("Chunk body") self.msg.body += self.buffer self.buffer = [] @@ -419,6 +437,7 @@ class http_stream(asynchat.async_chat): Consume the CRLF that terminates a chunk, reinitialize chunk decoder to be ready for the next chunk. """ + self.logger.debug("Chunk CRLF") s = self.get_buffer() assert s == "", "%r: Expected chunk CRLF, got '%s'" % (self, s) @@ -429,6 +448,7 @@ class http_stream(asynchat.async_chat): Consume chunk trailer, which should be empty, then (finally!) exit the chunk decoder and hand complete message off to the application. """ + self.logger.debug("Chunk trailer") s = self.get_buffer() assert s == "", "%r: Expected end of chunk trailers, got '%s'" % (self, s) @@ -439,6 +459,7 @@ class http_stream(asynchat.async_chat): """ Hand normal (not chunked) message off to the application. """ + self.msg.body = self.get_buffer() self.handle_message() @@ -448,6 +469,7 @@ class http_stream(asynchat.async_chat): whether it's one we should just pass along, otherwise log a stack trace and close the stream. """ + self.timer.cancel() etype = sys.exc_info()[0] if etype in (SystemExit, rpki.async.ExitNow): @@ -460,6 +482,7 @@ class http_stream(asynchat.async_chat): """ Inactivity timer expired, close connection with prejudice. """ + self.logger.debug("Timeout, closing") self.close() @@ -468,6 +491,7 @@ class http_stream(asynchat.async_chat): Wrapper around asynchat connection close handler, so that we can log the event, cancel timer, and so forth. """ + self.logger.debug("Close event in HTTP stream handler") self.timer.cancel() asynchat.async_chat.handle_close(self) @@ -499,12 +523,14 @@ class http_server(http_stream): Content-Length header (that is: this message will be the last one in this server stream). No special action required. """ + self.handle_message() def find_handler(self, path): """ Helper method to search self.handlers. """ + for h in self.handlers: if path.startswith(h[0]): return h[1], h[2] if len(h) > 2 else (default_content_type,) @@ -517,6 +543,7 @@ class http_server(http_stream): Content-Type, look for a handler, and if everything looks right, pass the message body, path, and a reply callback to the handler. """ + self.logger.debug("Received request %r", self.msg) if not self.msg.persistent: self.expect_close = True @@ -544,12 +571,14 @@ class http_server(http_stream): """ Send an error response to this request. """ + self.send_message(code = code, reason = reason) def send_reply(self, code, body = None, reason = "OK"): """ Send a reply to this request. """ + self.send_message(code = code, body = body, reason = reason) def send_message(self, code, reason = "OK", body = None): @@ -559,6 +588,7 @@ class http_server(http_stream): listen for next message; otherwise, queue up a close event for this stream so it will shut down once the reply has been sent. """ + self.logger.debug("Sending response %s %s", code, reason) if code >= 400: self.expect_close = True @@ -614,6 +644,7 @@ class http_listener(asyncore.dispatcher): Asyncore says we have an incoming connection, spawn an http_server stream for it and pass along all of our handler data. """ + try: res = self.accept() if res is None: @@ -630,6 +661,7 @@ class http_listener(asyncore.dispatcher): """ Asyncore signaled an error, pass it along or log it. """ + if sys.exc_info()[0] in (SystemExit, rpki.async.ExitNow): raise self.logger.exception("Error in HTTP listener") @@ -665,6 +697,7 @@ class http_client(http_stream): """ Create socket and request a connection. """ + if not use_adns: self.logger.debug("Not using ADNS") self.gotaddrinfo([(socket.AF_INET, self.host)]) @@ -681,12 +714,14 @@ class http_client(http_stream): Handle DNS lookup errors. For now, just whack the connection. Undoubtedly we should do something better with diagnostics here. """ + self.handle_error() def gotaddrinfo(self, addrinfo): """ Got address data from DNS, create socket and request connection. """ + try: self.af, self.address = random.choice(addrinfo) self.logger.debug("Connecting to AF %s host %s port %s addr %s", self.af, self.host, self.port, self.address) @@ -704,6 +739,7 @@ class http_client(http_stream): """ Asyncore says socket has connected. """ + self.logger.debug("Socket connected") self.set_state("idle") assert self.queue.client is self @@ -713,6 +749,7 @@ class http_client(http_stream): """ Set HTTP client connection state. """ + self.logger.debug("State transition %s => %s", self.state, state) self.state = state @@ -723,12 +760,14 @@ class http_client(http_stream): in this server stream). In this case we want to read until we reach the end of the data stream. """ + self.set_terminator(None) def send_request(self, msg): """ Queue up request message and kickstart connection. """ + self.logger.debug("Sending request %r", msg) assert self.state == "idle", "%r: state should be idle, is %s" % (self, self.state) self.set_state("request-sent") @@ -785,6 +824,7 @@ class http_client(http_stream): message now; if we were waiting for the response to a request we sent, signal the error. """ + http_stream.handle_close(self) self.logger.debug("State %s", self.state) if self.get_terminator() is None: @@ -799,6 +839,7 @@ class http_client(http_stream): Connection idle timer has expired. Shut down connection in any case, noisily if we weren't idle. """ + bad = self.state not in ("idle", "closing") if bad: self.logger.warning("Timeout while in state %s", self.state) @@ -816,6 +857,7 @@ class http_client(http_stream): Asyncore says something threw an exception. Log it, then shut down the connection and pass back the exception. """ + eclass, edata = sys.exc_info()[0:2] self.logger.warning("Error on HTTP client connection %s:%s %s %s", self.host, self.port, eclass, edata) http_stream.handle_error(self) @@ -843,6 +885,7 @@ class http_queue(object): """ Append http_request object(s) to this queue. """ + self.logger.debug("Adding requests %r", requests) self.queue.extend(requests) @@ -855,6 +898,7 @@ class http_queue(object): exception, or timeout) for the query currently in progress will call this method when it's time to kick out the next query. """ + try: if self.client is None: self.client = http_client(self, self.hostport) @@ -874,6 +918,7 @@ class http_queue(object): """ Kick out the next query in this queue, if any. """ + if self.queue: self.client.send_request(self.queue[0]) @@ -884,6 +929,7 @@ class http_queue(object): handling of what otherwise would be a nasty set of race conditions. """ + if client_ is self.client: self.logger.debug("Detaching client %r", client_) self.client = None @@ -1008,51 +1054,3 @@ def server(handlers, port, host = ""): http_listener(addrinfo = a, handlers = handlers) rpki.async.event_loop() - -class caller(object): - """ - Handle client-side mechanics for protocols based on HTTP, CMS, and - rpki.xml_utils. Calling sequence is intended to nest within - rpki.async.sync_wrapper. - """ - - debug = False - - def __init__(self, proto, client_key, client_cert, server_ta, server_cert, url, debug = None): - self.proto = proto - self.client_key = client_key - self.client_cert = client_cert - self.server_ta = server_ta - self.server_cert = server_cert - self.url = url - self.cms_timestamp = None - if debug is not None: - self.debug = debug - - def __call__(self, cb, eb, *pdus): - - def done(r_der): - """ - Handle CMS-wrapped XML response message. - """ - try: - r_cms = self.proto.cms_msg(DER = r_der) - r_msg = r_cms.unwrap((self.server_ta, self.server_cert)) - self.cms_timestamp = r_cms.check_replay(self.cms_timestamp, self.url) - if self.debug: - print "<!-- Reply -->" - print r_cms.pretty_print_content() - cb(r_msg) - except (rpki.async.ExitNow, SystemExit): - raise - except Exception, e: - eb(e) - - q_msg = self.proto.msg.query(*pdus) - q_cms = self.proto.cms_msg() - q_der = q_cms.wrap(q_msg, self.client_key, self.client_cert) - if self.debug: - print "<!-- Query -->" - print q_cms.pretty_print_content() - - client(url = self.url, msg = q_der, callback = done, errback = eb) diff --git a/rpki/http_simple.py b/rpki/http_simple.py new file mode 100644 index 00000000..ee9cac35 --- /dev/null +++ b/rpki/http_simple.py @@ -0,0 +1,136 @@ +# $Id$ +# +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +HTTP using Python standard libraries, for RPKI programs that don't +need the full-blown rpki.http asynchronous code. +""" + +import logging +import httplib +import urlparse +import BaseHTTPServer + +logger = logging.getLogger(__name__) + + +default_content_type = "application/x-rpki" + + +class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + """ + HTTP request handler simple RPKI servers. + """ + + def do_POST(self): + try: + content_type = self.headers.get("Content-Type") + content_length = self.headers.get("Content-Length") + for handler_path, handler, handler_content_type in self.rpki_handlers: + if self.path.startswith(handler_path) and content_type in handler_content_type: + return handler(self, + self.rfile.read() + if content_length is None else + self.rfile.read(int(content_length))) + self.send_error(404, "No handler for path %s" % self.path) + except Exception, e: + logger.exception("Unhandled exception") + self.send_error(501, "Unhandled exception %s" % e) + + def send_cms_response(self, der): + self.send_response(200) + self.send_header("Content-Type", default_content_type) + self.send_header("Content-Length", str(len(der))) + self.end_headers() + self.wfile.write(der) + + def log_message(self, *args): + logger.info(*args, extra = dict(context = "%s:%s" % self.client_address)) + + def send_error(self, code, message = None): + # BaseHTTPRequestHandler.send_error() generates HTML error messages, + # which we don't want, so we override the method to suppress this. + self.send_response(code, message) + self.send_header("Content-Type", default_content_type) + self.send_header("Connection", "close") + self.end_headers() + + +def server(handlers, port, host = ""): + """ + Run an HTTP server and wait (forever) for connections. + """ + + if isinstance(handlers, (tuple, list)): + handlers = tuple(h[:3] if len(h) > 2 else (h[0], h[1], default_content_type) + for h in handlers) + else: + handlers = (("/", handlers, default_content_type),) + + class RequestHandler(HTTPRequestHandler): + rpki_handlers = handlers + + BaseHTTPServer.HTTPServer((host, port), RequestHandler).serve_forever() + + +class BadURL(Exception): + "Bad contact URL" + +class RequestFailed(Exception): + "HTTP returned failure" + +class BadContentType(Exception): + "Wrong HTTP Content-Type" + + +def client(proto_cms_msg, client_key, client_cert, server_ta, server_cert, url, q_msg, + debug = False, replay_track = None, client_crl = None, content_type = default_content_type): + """ + Issue single a query and return the response, handling all the CMS and XML goo. + """ + + u = urlparse.urlparse(url) + + if u.scheme not in ("", "http") or u.username or u.password or u.params or u.query or u.fragment: + raise BadURL("Unusable URL %s", url) + + q_cms = proto_cms_msg() + q_der = q_cms.wrap(q_msg, client_key, client_cert, client_crl) + + if debug: + debug.write("<!-- Query -->\n" + q_cms.pretty_print_content() + "\n") + + http = httplib.HTTPConnection(u.hostname, u.port or httplib.HTTP_PORT) + http.request("POST", u.path, q_der, {"Content-Type" : content_type}) + r = http.getresponse() + + if r.status != 200: + raise RequestFailed("HTTP request failed with status %r reason %r" % (r.status, r.reason)) + + if r.getheader("Content-Type") != content_type: + raise BadContentType("HTTP Content-Type %r, expected %r" % (r.getheader("Content-Type"), content_type)) + + r_der = r.read() + r_cms = proto_cms_msg(DER = r_der) + r_msg = r_cms.unwrap((server_ta, server_cert)) + + if replay_track is not None: + replay_track.cms_timestamp = r_cms.check_replay(replay_track.cms_timestamp, url) + + if debug: + debug.write("<!-- Reply -->\n" + r_cms.pretty_print_content() + "\n") + + return r_msg diff --git a/rpki/ipaddrs.py b/rpki/ipaddrs.py index 68b2d27d..25eefd0d 100644 --- a/rpki/ipaddrs.py +++ b/rpki/ipaddrs.py @@ -61,6 +61,7 @@ class v4addr(long): """ Construct a v4addr object. """ + if isinstance(x, unicode): x = x.encode("ascii") if isinstance(x, str): @@ -72,6 +73,7 @@ class v4addr(long): """ Convert a v4addr object to a raw byte string. """ + return struct.pack("!I", long(self)) @classmethod @@ -79,12 +81,14 @@ class v4addr(long): """ Convert from a raw byte string to a v4addr object. """ + return cls(struct.unpack("!I", x)[0]) def __str__(self): """ Convert a v4addr object to string format. """ + return socket.inet_ntop(socket.AF_INET, self.to_bytes()) class v6addr(long): @@ -101,6 +105,7 @@ class v6addr(long): """ Construct a v6addr object. """ + if isinstance(x, unicode): x = x.encode("ascii") if isinstance(x, str): @@ -112,6 +117,7 @@ class v6addr(long): """ Convert a v6addr object to a raw byte string. """ + return struct.pack("!QQ", long(self) >> 64, long(self) & 0xFFFFFFFFFFFFFFFF) @classmethod @@ -119,6 +125,7 @@ class v6addr(long): """ Convert from a raw byte string to a v6addr object. """ + x = struct.unpack("!QQ", x) return cls((x[0] << 64) | x[1]) @@ -126,12 +133,14 @@ class v6addr(long): """ Convert a v6addr object to string format. """ + return socket.inet_ntop(socket.AF_INET6, self.to_bytes()) def parse(s): """ Parse a string as either an IPv4 or IPv6 address, and return object of appropriate class. """ + if isinstance(s, unicode): s = s.encode("ascii") return v6addr(s) if ":" in s else v4addr(s) diff --git a/rpki/irdb/__init__.py b/rpki/irdb/__init__.py index 7f3b880e..25dedfe3 100644 --- a/rpki/irdb/__init__.py +++ b/rpki/irdb/__init__.py @@ -21,6 +21,5 @@ Python package, so humor it. # pylint: disable=W0401 -from rpki.irdb.models import * from rpki.irdb.zookeeper import Zookeeper from rpki.irdb.router import DBContextRouter, database diff --git a/rpki/irdb/migrations/0001_initial.py b/rpki/irdb/migrations/0001_initial.py new file mode 100644 index 00000000..d10b62d3 --- /dev/null +++ b/rpki/irdb/migrations/0001_initial.py @@ -0,0 +1,375 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import rpki.irdb.models +import rpki.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='BSC', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('certificate', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('handle', rpki.irdb.models.HandleField(max_length=120)), + ('pkcs10', rpki.fields.PKCS10Field(default=None, serialize=False, blank=True)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Child', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('certificate', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('handle', rpki.irdb.models.HandleField(max_length=120)), + ('ta', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('valid_until', rpki.fields.SundialField()), + ('name', models.TextField(null=True, blank=True)), + ], + ), + migrations.CreateModel( + name='ChildASN', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('start_as', models.BigIntegerField()), + ('end_as', models.BigIntegerField()), + ('child', models.ForeignKey(related_name='asns', to='irdb.Child')), + ], + ), + migrations.CreateModel( + name='ChildNet', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('start_ip', models.CharField(max_length=40)), + ('end_ip', models.CharField(max_length=40)), + ('version', rpki.fields.EnumField(choices=[(4, b'IPv4'), (6, b'IPv6')])), + ('child', models.ForeignKey(related_name='address_ranges', to='irdb.Child')), + ], + ), + migrations.CreateModel( + name='Client', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('certificate', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('handle', rpki.irdb.models.HandleField(max_length=120)), + ('ta', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('sia_base', models.TextField()), + ('parent_handle', rpki.irdb.models.HandleField(max_length=120)), + ], + ), + migrations.CreateModel( + name='EECertificateRequest', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('valid_until', rpki.fields.SundialField()), + ('pkcs10', rpki.fields.PKCS10Field(default=None, serialize=False, blank=True)), + ('gski', models.CharField(max_length=27)), + ('cn', models.CharField(max_length=64)), + ('sn', models.CharField(max_length=64)), + ('eku', models.TextField(null=True)), + ], + ), + migrations.CreateModel( + name='EECertificateRequestASN', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('start_as', models.BigIntegerField()), + ('end_as', models.BigIntegerField()), + ('ee_certificate_request', models.ForeignKey(related_name='asns', to='irdb.EECertificateRequest')), + ], + ), + migrations.CreateModel( + name='EECertificateRequestNet', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('start_ip', models.CharField(max_length=40)), + ('end_ip', models.CharField(max_length=40)), + ('version', rpki.fields.EnumField(choices=[(4, b'IPv4'), (6, b'IPv6')])), + ('ee_certificate_request', models.ForeignKey(related_name='address_ranges', to='irdb.EECertificateRequest')), + ], + ), + migrations.CreateModel( + name='GhostbusterRequest', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('vcard', models.TextField()), + ], + ), + migrations.CreateModel( + name='HostedCA', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('certificate', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ], + ), + migrations.CreateModel( + name='Referral', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('certificate', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('private_key', rpki.fields.KeyField(default=None, serialize=False, blank=True)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Repository', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('certificate', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('handle', rpki.irdb.models.HandleField(max_length=120)), + ('ta', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('client_handle', rpki.irdb.models.HandleField(max_length=120)), + ('service_uri', models.CharField(max_length=255)), + ('sia_base', models.TextField()), + ], + ), + migrations.CreateModel( + name='ResourceHolderCA', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('certificate', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('private_key', rpki.fields.KeyField(default=None, serialize=False, blank=True)), + ('latest_crl', rpki.fields.CRLField(default=None, serialize=False, blank=True)), + ('next_serial', models.BigIntegerField(default=1)), + ('next_crl_number', models.BigIntegerField(default=1)), + ('last_crl_update', rpki.fields.SundialField()), + ('next_crl_update', rpki.fields.SundialField()), + ('handle', rpki.irdb.models.HandleField(unique=True, max_length=120)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='ResourceHolderRevocation', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('serial', models.BigIntegerField()), + ('revoked', rpki.fields.SundialField()), + ('expires', rpki.fields.SundialField()), + ('issuer', models.ForeignKey(related_name='revocations', to='irdb.ResourceHolderCA')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='ROARequest', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('asn', models.BigIntegerField()), + ('issuer', models.ForeignKey(related_name='roa_requests', to='irdb.ResourceHolderCA')), + ], + ), + migrations.CreateModel( + name='ROARequestPrefix', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('version', rpki.fields.EnumField(choices=[(4, b'IPv4'), (6, b'IPv6')])), + ('prefix', models.CharField(max_length=40)), + ('prefixlen', models.PositiveSmallIntegerField()), + ('max_prefixlen', models.PositiveSmallIntegerField()), + ('roa_request', models.ForeignKey(related_name='prefixes', to='irdb.ROARequest')), + ], + ), + migrations.CreateModel( + name='ServerCA', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('certificate', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('private_key', rpki.fields.KeyField(default=None, serialize=False, blank=True)), + ('latest_crl', rpki.fields.CRLField(default=None, serialize=False, blank=True)), + ('next_serial', models.BigIntegerField(default=1)), + ('next_crl_number', models.BigIntegerField(default=1)), + ('last_crl_update', rpki.fields.SundialField()), + ('next_crl_update', rpki.fields.SundialField()), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='ServerEE', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('certificate', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('private_key', rpki.fields.KeyField(default=None, serialize=False, blank=True)), + ('purpose', rpki.fields.EnumField(choices=[(1, b'rpkid'), (2, b'pubd'), (3, b'irdbd'), (4, b'irbe')])), + ('issuer', models.ForeignKey(related_name='ee_certificates', to='irdb.ServerCA')), + ], + ), + migrations.CreateModel( + name='ServerRevocation', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('serial', models.BigIntegerField()), + ('revoked', rpki.fields.SundialField()), + ('expires', rpki.fields.SundialField()), + ('issuer', models.ForeignKey(related_name='revocations', to='irdb.ServerCA')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Turtle', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('service_uri', models.CharField(max_length=255)), + ], + ), + migrations.CreateModel( + name='Parent', + fields=[ + ('turtle_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='irdb.Turtle')), + ('certificate', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('handle', rpki.irdb.models.HandleField(max_length=120)), + ('ta', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('parent_handle', rpki.irdb.models.HandleField(max_length=120)), + ('child_handle', rpki.irdb.models.HandleField(max_length=120)), + ('repository_type', rpki.fields.EnumField(choices=[(1, b'none'), (2, b'offer'), (3, b'referral')])), + ('referrer', rpki.irdb.models.HandleField(max_length=120, null=True, blank=True)), + ('referral_authorization', rpki.irdb.models.SignedReferralField(default=None, serialize=False, null=True, blank=True)), + ('issuer', models.ForeignKey(related_name='parents', to='irdb.ResourceHolderCA')), + ], + bases=('irdb.turtle', models.Model), + ), + migrations.CreateModel( + name='Rootd', + fields=[ + ('turtle_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='irdb.Turtle')), + ('certificate', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('private_key', rpki.fields.KeyField(default=None, serialize=False, blank=True)), + ('issuer', models.OneToOneField(related_name='rootd', to='irdb.ResourceHolderCA')), + ], + options={ + 'abstract': False, + }, + bases=('irdb.turtle', models.Model), + ), + migrations.AddField( + model_name='repository', + name='issuer', + field=models.ForeignKey(related_name='repositories', to='irdb.ResourceHolderCA'), + ), + migrations.AddField( + model_name='repository', + name='turtle', + field=models.OneToOneField(related_name='repository', to='irdb.Turtle'), + ), + migrations.AddField( + model_name='referral', + name='issuer', + field=models.OneToOneField(related_name='referral_certificate', to='irdb.ResourceHolderCA'), + ), + migrations.AddField( + model_name='hostedca', + name='hosted', + field=models.OneToOneField(related_name='hosted_by', to='irdb.ResourceHolderCA'), + ), + migrations.AddField( + model_name='hostedca', + name='issuer', + field=models.ForeignKey(to='irdb.ServerCA'), + ), + migrations.AddField( + model_name='ghostbusterrequest', + name='issuer', + field=models.ForeignKey(related_name='ghostbuster_requests', to='irdb.ResourceHolderCA'), + ), + migrations.AddField( + model_name='eecertificaterequest', + name='issuer', + field=models.ForeignKey(related_name='ee_certificate_requests', to='irdb.ResourceHolderCA'), + ), + migrations.AddField( + model_name='client', + name='issuer', + field=models.ForeignKey(related_name='clients', to='irdb.ServerCA'), + ), + migrations.AddField( + model_name='child', + name='issuer', + field=models.ForeignKey(related_name='children', to='irdb.ResourceHolderCA'), + ), + migrations.AddField( + model_name='bsc', + name='issuer', + field=models.ForeignKey(related_name='bscs', to='irdb.ResourceHolderCA'), + ), + migrations.AlterUniqueTogether( + name='serverrevocation', + unique_together=set([('issuer', 'serial')]), + ), + migrations.AlterUniqueTogether( + name='serveree', + unique_together=set([('issuer', 'purpose')]), + ), + migrations.AlterUniqueTogether( + name='roarequestprefix', + unique_together=set([('roa_request', 'version', 'prefix', 'prefixlen', 'max_prefixlen')]), + ), + migrations.AlterUniqueTogether( + name='resourceholderrevocation', + unique_together=set([('issuer', 'serial')]), + ), + migrations.AlterUniqueTogether( + name='repository', + unique_together=set([('issuer', 'handle')]), + ), + migrations.AlterUniqueTogether( + name='hostedca', + unique_together=set([('issuer', 'hosted')]), + ), + migrations.AddField( + model_name='ghostbusterrequest', + name='parent', + field=models.ForeignKey(related_name='ghostbuster_requests', to='irdb.Parent', null=True), + ), + migrations.AlterUniqueTogether( + name='eecertificaterequestnet', + unique_together=set([('ee_certificate_request', 'start_ip', 'end_ip', 'version')]), + ), + migrations.AlterUniqueTogether( + name='eecertificaterequestasn', + unique_together=set([('ee_certificate_request', 'start_as', 'end_as')]), + ), + migrations.AlterUniqueTogether( + name='eecertificaterequest', + unique_together=set([('issuer', 'gski')]), + ), + migrations.AlterUniqueTogether( + name='client', + unique_together=set([('issuer', 'handle')]), + ), + migrations.AlterUniqueTogether( + name='childnet', + unique_together=set([('child', 'start_ip', 'end_ip', 'version')]), + ), + migrations.AlterUniqueTogether( + name='childasn', + unique_together=set([('child', 'start_as', 'end_as')]), + ), + migrations.AlterUniqueTogether( + name='child', + unique_together=set([('issuer', 'handle')]), + ), + migrations.AlterUniqueTogether( + name='bsc', + unique_together=set([('issuer', 'handle')]), + ), + migrations.AlterUniqueTogether( + name='parent', + unique_together=set([('issuer', 'handle')]), + ), + ] diff --git a/rpki/irdb/migrations/0002_remove_client_parent_handle.py b/rpki/irdb/migrations/0002_remove_client_parent_handle.py new file mode 100644 index 00000000..f86d165d --- /dev/null +++ b/rpki/irdb/migrations/0002_remove_client_parent_handle.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('irdb', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='client', + name='parent_handle', + ), + ] diff --git a/rpki/irdb/migrations/0003_repository_rrdp_notification_uri.py b/rpki/irdb/migrations/0003_repository_rrdp_notification_uri.py new file mode 100644 index 00000000..1e0e43c2 --- /dev/null +++ b/rpki/irdb/migrations/0003_repository_rrdp_notification_uri.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('irdb', '0002_remove_client_parent_handle'), + ] + + operations = [ + migrations.AddField( + model_name='repository', + name='rrdp_notification_uri', + field=models.TextField(null=True), + ), + ] diff --git a/rpki/irdb/migrations/__init__.py b/rpki/irdb/migrations/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/rpki/irdb/migrations/__init__.py diff --git a/rpki/irdb/models.py b/rpki/irdb/models.py index 6fa48c59..0911d7aa 100644 --- a/rpki/irdb/models.py +++ b/rpki/irdb/models.py @@ -32,7 +32,8 @@ import rpki.sundial import rpki.resource_set import socket import rpki.POW -from south.modelsinspector import add_introspection_rules + +from rpki.fields import EnumField, SundialField, CertificateField, DERField, KeyField, CRLField, PKCS10Field ## @var ip_version_choices # Choice argument for fields implementing IP version numbers. @@ -61,11 +62,11 @@ ee_certificate_lifetime = rpki.sundial.timedelta(days = 60) ### -# Field types +# Field classes class HandleField(django.db.models.CharField): """ - A handle field type. + A handle field class. Replace this with SlugField? """ description = 'A "handle" in one of the RPKI protocols' @@ -74,99 +75,6 @@ class HandleField(django.db.models.CharField): kwargs["max_length"] = 120 django.db.models.CharField.__init__(self, *args, **kwargs) -class EnumField(django.db.models.PositiveSmallIntegerField): - """ - An enumeration type that uses strings in Python and small integers - in SQL. - """ - - description = "An enumeration type" - - __metaclass__ = django.db.models.SubfieldBase - - def __init__(self, *args, **kwargs): - if isinstance(kwargs.get("choices"), (tuple, list)) and isinstance(kwargs["choices"][0], str): - kwargs["choices"] = tuple(enumerate(kwargs["choices"], 1)) - django.db.models.PositiveSmallIntegerField.__init__(self, *args, **kwargs) - self.enum_i2s = dict(self.flatchoices) - self.enum_s2i = dict((v, k) for k, v in self.flatchoices) - - def to_python(self, value): - return self.enum_i2s.get(value, value) - - def get_prep_value(self, value): - return self.enum_s2i.get(value, value) - -class SundialField(django.db.models.DateTimeField): - """ - A field type for our customized datetime objects. - """ - __metaclass__ = django.db.models.SubfieldBase - - description = "A datetime type using our customized datetime objects" - - def to_python(self, value): - if isinstance(value, rpki.sundial.pydatetime.datetime): - return rpki.sundial.datetime.from_datetime( - django.db.models.DateTimeField.to_python(self, value)) - else: - return value - - def get_prep_value(self, value): - if isinstance(value, rpki.sundial.datetime): - return value.to_datetime() - else: - return value - - -class DERField(django.db.models.Field): - """ - Field types for DER objects. - """ - - __metaclass__ = django.db.models.SubfieldBase - - def __init__(self, *args, **kwargs): - kwargs["serialize"] = False - kwargs["blank"] = True - kwargs["default"] = None - django.db.models.Field.__init__(self, *args, **kwargs) - - def db_type(self, connection): - if connection.settings_dict['ENGINE'] == "django.db.backends.posgresql": - return "bytea" - else: - return "BLOB" - - def to_python(self, value): - assert value is None or isinstance(value, (self.rpki_type, str)) - if isinstance(value, str): - return self.rpki_type(DER = value) - else: - return value - - def get_prep_value(self, value): - assert value is None or isinstance(value, (self.rpki_type, str)) - if isinstance(value, self.rpki_type): - return value.get_DER() - else: - return value - -class CertificateField(DERField): - description = "X.509 certificate" - rpki_type = rpki.x509.X509 - -class RSAKeyField(DERField): - description = "RSA keypair" - rpki_type = rpki.x509.RSA - -class CRLField(DERField): - description = "Certificate Revocation List" - rpki_type = rpki.x509.CRL - -class PKCS10Field(DERField): - description = "PKCS #10 certificate request" - rpki_type = rpki.x509.PKCS10 class SignedReferralField(DERField): description = "CMS signed object containing XML" @@ -231,7 +139,7 @@ class ResourceHolderEEManager(CertificateManager): class CA(django.db.models.Model): certificate = CertificateField() - private_key = RSAKeyField() + private_key = KeyField() latest_crl = CRLField() # Might want to bring these into line with what rpkid does. Current @@ -391,7 +299,7 @@ class ResourceHolderRevocation(Revocation): issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "revocations") class EECertificate(Certificate): - private_key = RSAKeyField() + private_key = KeyField() class Meta: abstract = True @@ -497,12 +405,12 @@ class Child(CrossCertification, ResourceSet): name = django.db.models.TextField(null = True, blank = True) def _select_resource_bag(self): - child_asn = rpki.irdb.ChildASN.objects.raw(""" + child_asn = rpki.irdb.models.ChildASN.objects.raw(""" SELECT * FROM irdb_childasn WHERE child_id = %s """, [self.id]) - child_net = list(rpki.irdb.ChildNet.objects.raw(""" + child_net = list(rpki.irdb.models.ChildNet.objects.raw(""" SELECT * FROM irdb_childnet WHERE child_id = %s @@ -542,7 +450,7 @@ class ROARequest(django.db.models.Model): @property def roa_prefix_bag(self): - prefixes = list(rpki.irdb.ROARequestPrefix.objects.raw(""" + prefixes = list(rpki.irdb.models.ROARequestPrefix.objects.raw(""" SELECT * FROM irdb_roarequestprefix WHERE roa_request_id = %s @@ -588,12 +496,12 @@ class EECertificateRequest(ResourceSet): eku = django.db.models.TextField(null = True) def _select_resource_bag(self): - ee_asn = rpki.irdb.EECertificateRequestASN.objects.raw(""" + ee_asn = rpki.irdb.models.EECertificateRequestASN.objects.raw(""" SELECT * FROM irdb_eecertificaterequestasn WHERE ee_certificate_request_id = %s """, [self.id]) - ee_net = rpki.irdb.EECertificateRequestNet.objects.raw(""" + ee_net = rpki.irdb.models.EECertificateRequestNet.objects.raw(""" SELECT * FROM irdb_eecertificaterequestnet WHERE ee_certificate_request_id = %s @@ -620,6 +528,7 @@ class Repository(CrossCertification): client_handle = HandleField() service_uri = django.db.models.CharField(max_length = 255) sia_base = django.db.models.TextField() + rrdp_notification_uri = django.db.models.TextField(null = True) turtle = django.db.models.OneToOneField(Turtle, related_name = "repository") # This shouldn't be necessary @@ -629,18 +538,7 @@ class Repository(CrossCertification): class Client(CrossCertification): issuer = django.db.models.ForeignKey(ServerCA, related_name = "clients") sia_base = django.db.models.TextField() - parent_handle = HandleField() # This shouldn't be necessary class Meta: unique_together = ("issuer", "handle") - -# for Django South -- these are just simple subclasses -add_introspection_rules([], - (r'^rpki\.irdb\.models\.CertificateField', - r'^rpki\.irdb\.models\.CRLField', - r'^rpki\.irdb\.models\.EnumField', - r'^rpki\.irdb\.models\.HandleField', - r'^rpki\.irdb\.models\.RSAKeyField', - r'^rpki\.irdb\.models\.SignedReferralField', - r'^rpki\.irdb\.models\.SundialField')) diff --git a/rpki/irdb/router.py b/rpki/irdb/router.py index 97e3d0b7..0aaf53ce 100644 --- a/rpki/irdb/router.py +++ b/rpki/irdb/router.py @@ -58,7 +58,7 @@ class DBContextRouter(object): else: return None - def allow_syncdb(self, db, model): + def allow_migrate(self, db, model): if db == self._database and model._meta.app_label == self._app: return True else: diff --git a/rpki/irdb/zookeeper.py b/rpki/irdb/zookeeper.py index 3fba99f9..4b4a2f46 100644 --- a/rpki/irdb/zookeeper.py +++ b/rpki/irdb/zookeeper.py @@ -24,17 +24,18 @@ Management code for the IRDB. import os import copy -import types + import rpki.config import rpki.sundial import rpki.oids -import rpki.http +import rpki.http_simple import rpki.resource_set import rpki.relaxng import rpki.left_right import rpki.x509 -import rpki.async import rpki.irdb +import rpki.publication_control + import django.db.transaction from lxml.etree import (Element, SubElement, ElementTree, @@ -42,12 +43,11 @@ from lxml.etree import (Element, SubElement, ElementTree, from rpki.csv_utils import csv_reader -# XML namespace and protocol version for OOB setup protocol. The name -# is historical and may change before we propose this as the basis for -# a standard. +# XML namespace and protocol version for OOB setup protocol. -myrpki_xmlns = rpki.relaxng.myrpki.xmlns -myrpki_version = rpki.relaxng.myrpki.version +oob_xmlns = rpki.relaxng.oob_setup.xmlns +oob_nsmap = rpki.relaxng.oob_setup.nsmap +oob_version = rpki.relaxng.oob_setup.version # XML namespace and protocol version for router certificate requests. # We probably ought to be pulling this sort of thing from the schema, @@ -56,8 +56,28 @@ myrpki_version = rpki.relaxng.myrpki.version # I'm ready to rewrite the rpki.relaxng code. routercert_xmlns = rpki.relaxng.router_certificate.xmlns +routercert_nsmap = rpki.relaxng.router_certificate.nsmap routercert_version = rpki.relaxng.router_certificate.version +# XML tags for elements in the above + +tag_oob_authorization = oob_xmlns + "authorization" +tag_oob_child_bpki_ta = oob_xmlns + "child_bpki_ta" +tag_oob_child_request = oob_xmlns + "child_request" +tag_oob_error = oob_xmlns + "error" +tag_oob_offer = oob_xmlns + "offer" +tag_oob_parent_bpki_ta = oob_xmlns + "parent_bpki_ta" +tag_oob_parent_response = oob_xmlns + "parent_response" +tag_oob_publisher_bpki_ta = oob_xmlns + "publisher_bpki_ta" +tag_oob_publisher_request = oob_xmlns + "publisher_request" +tag_oob_referral = oob_xmlns + "referral" +tag_oob_repository_bpki_ta = oob_xmlns + "repository_bpki_ta" +tag_oob_repository_response = oob_xmlns + "repository_response" + +tag_router_certificate_request = routercert_xmlns + "router_certificate_request" + +# Configuration file section names + myrpki_section = "myrpki" irdbd_section = "irdbd" rpkid_section = "rpkid" @@ -72,6 +92,7 @@ class CouldntTalkToDaemon(Exception): "Couldn't talk to daemon." class BadXMLMessage(Exception): "Bad XML message." class PastExpiration(Exception): "Expiration date has already passed." class CantRunRootd(Exception): "Can't run rootd." +class CouldntFindRepoParent(Exception): "Couldn't find repository's parent." def B64Element(e, tag, obj, **kwargs): @@ -128,19 +149,19 @@ class PEM_writer(object): self.wrote.add(filename) -def etree_read(filename): +def etree_read(filename_or_etree_wrapper, schema = rpki.relaxng.oob_setup): """ Read an etree from a file, verifying then stripping XML namespace - cruft. + cruft. As a convenience, we also accept an etree_wrapper object in + place of a filename, in which case we deepcopy the etree directly + from the etree_wrapper and there's no need for a file. """ - e = ElementTree(file = filename).getroot() - rpki.relaxng.myrpki.assertValid(e) - for i in e.getiterator(): - if i.tag.startswith(myrpki_xmlns): - i.tag = i.tag[len(myrpki_xmlns):] - else: - raise BadXMLMessage("XML tag %r is not in namespace %r" % (i.tag, myrpki_xmlns[1:-1])) + if isinstance(filename_or_etree_wrapper, etree_wrapper): + e = copy.deepcopy(filename_or_etree_wrapper.etree) + else: + e = ElementTree(file = filename_or_etree_wrapper).getroot() + schema.assertValid(e) return e @@ -148,20 +169,14 @@ class etree_wrapper(object): """ Wrapper for ETree objects so we can return them as function results without requiring the caller to understand much about them. - """ - def __init__(self, e, msg = None, debug = False): + def __init__(self, e, msg = None, debug = False, schema = rpki.relaxng.oob_setup): self.msg = msg e = copy.deepcopy(e) - e.set("version", myrpki_version) - for i in e.getiterator(): - if i.tag[0] != "{": - i.tag = myrpki_xmlns + i.tag - assert i.tag.startswith(myrpki_xmlns) if debug: print ElementToString(e) - rpki.relaxng.myrpki.assertValid(e) + schema.assertValid(e) self.etree = e def __str__(self): @@ -189,9 +204,9 @@ class etree_wrapper(object): class Zookeeper(object): ## @var show_xml - # Whether to show XML for debugging + # If not None, a file-like object to which to prettyprint XML, for debugging. - show_xml = False + show_xml = None def __init__(self, cfg = None, handle = None, logstream = None, disable_signal_handlers = False): @@ -259,7 +274,7 @@ class Zookeeper(object): if self.handle is None: raise HandleNotSet - return rpki.irdb.ResourceHolderCA.objects.get(handle = self.handle) + return rpki.irdb.models.ResourceHolderCA.objects.get(handle = self.handle) @property @@ -268,10 +283,10 @@ class Zookeeper(object): Get ServerCA object. """ - return rpki.irdb.ServerCA.objects.get() + return rpki.irdb.models.ServerCA.objects.get() - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def initialize_server_bpki(self): """ Initialize server BPKI portion of an RPKI installation. Reads the @@ -280,18 +295,18 @@ class Zookeeper(object): """ if self.run_rpkid or self.run_pubd: - server_ca, created = rpki.irdb.ServerCA.objects.get_or_certify() - rpki.irdb.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "irbe") + server_ca, created = rpki.irdb.models.ServerCA.objects.get_or_certify() + rpki.irdb.models.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "irbe") if self.run_rpkid: - rpki.irdb.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "rpkid") - rpki.irdb.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "irdbd") + rpki.irdb.models.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "rpkid") + rpki.irdb.models.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "irdbd") if self.run_pubd: - rpki.irdb.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "pubd") + rpki.irdb.models.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "pubd") - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def initialize_resource_bpki(self): """ Initialize the resource-holding BPKI for an RPKI installation. @@ -305,7 +320,7 @@ class Zookeeper(object): resource-holding BPKI idenity if needed. """ - resource_ca, created = rpki.irdb.ResourceHolderCA.objects.get_or_certify(handle = self.handle) + resource_ca, created = rpki.irdb.models.ResourceHolderCA.objects.get_or_certify(handle = self.handle) return self.generate_identity() @@ -325,12 +340,13 @@ class Zookeeper(object): easier for the GUI this way. """ - e = Element("identity", handle = self.handle) - B64Element(e, "bpki_ta", self.resource_ca.certificate) + e = Element(tag_oob_child_request, nsmap = oob_nsmap, version = oob_version, + child_handle = self.handle) + B64Element(e, tag_oob_child_bpki_ta, self.resource_ca.certificate) return etree_wrapper(e, msg = 'This is the "identity" file you will need to send to your parent') - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def delete_self(self): """ Delete the ResourceHolderCA object corresponding to the current handle. @@ -349,14 +365,15 @@ class Zookeeper(object): self.log("No such ResourceHolderCA \"%s\"" % self.handle) - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def configure_rootd(self): assert self.run_rpkid and self.run_pubd and self.run_rootd - rpki.irdb.Rootd.objects.get_or_certify( + rpki.irdb.models.Rootd.objects.get_or_certify( issuer = self.resource_ca, - service_uri = "http://localhost:%s/" % self.cfg.get("rootd_server_port", section = myrpki_section)) + service_uri = "http://localhost:%s/" % self.cfg.get("rootd_server_port", + section = myrpki_section)) return self.generate_rootd_repository_offer() @@ -367,17 +384,14 @@ class Zookeeper(object): configure_rootd() because that's easier for the GUI. """ - # The following assumes we'll set up the respository manually. - # Not sure this is a reasonable assumption, particularly if we - # ever fix rootd to use the publication protocol. - try: self.resource_ca.repositories.get(handle = self.handle) return None - except rpki.irdb.Repository.DoesNotExist: - e = Element("repository", type = "offer", handle = self.handle, parent_handle = self.handle) - B64Element(e, "bpki_client_ta", self.resource_ca.certificate) + except rpki.irdb.models.Repository.DoesNotExist: + e = Element(tag_oob_publisher_request, nsmap = oob_nsmap, version = oob_version, + publisher_handle = self.handle) + B64Element(e, tag_oob_publisher_bpki_ta, self.resource_ca.certificate) return etree_wrapper(e, msg = 'This is the "repository offer" file for you to use if you want to publish in your own repository') @@ -409,19 +423,19 @@ class Zookeeper(object): if self.run_rootd: try: - rootd = rpki.irdb.ResourceHolderCA.objects.get(handle = self.handle).rootd + rootd = rpki.irdb.models.ResourceHolderCA.objects.get(handle = self.handle).rootd writer(self.cfg.get("bpki-ta", section = rootd_section), self.server_ca.certificate) writer(self.cfg.get("rootd-bpki-crl", section = rootd_section), self.server_ca.latest_crl) writer(self.cfg.get("rootd-bpki-key", section = rootd_section), rootd.private_key) writer(self.cfg.get("rootd-bpki-cert", section = rootd_section), rootd.certificate) writer(self.cfg.get("child-bpki-cert", section = rootd_section), rootd.issuer.certificate) - except rpki.irdb.ResourceHolderCA.DoesNotExist: + except rpki.irdb.models.ResourceHolderCA.DoesNotExist: self.log("rootd enabled but resource holding entity not yet configured, skipping rootd setup") - except rpki.irdb.Rootd.DoesNotExist: + except rpki.irdb.models.Rootd.DoesNotExist: self.log("rootd enabled but not yet configured, skipping rootd setup") - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def update_bpki(self): """ Update BPKI certificates. Assumes an existing RPKI installation. @@ -435,17 +449,17 @@ class Zookeeper(object): Most likely this should be run under cron. """ - for model in (rpki.irdb.ServerCA, - rpki.irdb.ResourceHolderCA, - rpki.irdb.ServerEE, - rpki.irdb.Referral, - rpki.irdb.Rootd, - rpki.irdb.HostedCA, - rpki.irdb.BSC, - rpki.irdb.Child, - rpki.irdb.Parent, - rpki.irdb.Client, - rpki.irdb.Repository): + for model in (rpki.irdb.models.ServerCA, + rpki.irdb.models.ResourceHolderCA, + rpki.irdb.models.ServerEE, + rpki.irdb.models.Referral, + rpki.irdb.models.Rootd, + rpki.irdb.models.HostedCA, + rpki.irdb.models.BSC, + rpki.irdb.models.Child, + rpki.irdb.models.Parent, + rpki.irdb.models.Client, + rpki.irdb.models.Repository): for obj in model.objects.all(): self.log("Regenerating BPKI certificate %s" % obj.certificate.getSubject()) obj.avow() @@ -455,13 +469,33 @@ class Zookeeper(object): self.server_ca.generate_crl() self.server_ca.save() - for ca in rpki.irdb.ResourceHolderCA.objects.all(): + for ca in rpki.irdb.models.ResourceHolderCA.objects.all(): self.log("Regenerating BPKI CRL for Resource Holder %s" % ca.handle) ca.generate_crl() ca.save() - @django.db.transaction.commit_on_success + @staticmethod + def _compose_left_right_query(): + """ + Compose top level element of a left-right query. + """ + + return Element(rpki.left_right.tag_msg, nsmap = rpki.left_right.nsmap, + type = "query", version = rpki.left_right.version) + + + @staticmethod + def _compose_publication_control_query(): + """ + Compose top level element of a publication-control query. + """ + + return Element(rpki.publication_control.tag_msg, nsmap = rpki.publication_control.nsmap, + type = "query", version = rpki.publication_control.version) + + + @django.db.transaction.atomic def synchronize_bpki(self): """ Synchronize BPKI updates. This is separate from .update_bpki() @@ -472,85 +506,71 @@ class Zookeeper(object): """ if self.run_rpkid: - updates = [] - - updates.extend( - rpki.left_right.self_elt.make_pdu( - action = "set", - tag = "%s__self" % ca.handle, - self_handle = ca.handle, - bpki_cert = ca.certificate) - for ca in rpki.irdb.ResourceHolderCA.objects.all()) - - updates.extend( - rpki.left_right.bsc_elt.make_pdu( - action = "set", - tag = "%s__bsc__%s" % (bsc.issuer.handle, bsc.handle), - self_handle = bsc.issuer.handle, - bsc_handle = bsc.handle, - signing_cert = bsc.certificate, - signing_cert_crl = bsc.issuer.latest_crl) - for bsc in rpki.irdb.BSC.objects.all()) - - updates.extend( - rpki.left_right.repository_elt.make_pdu( - action = "set", - tag = "%s__repository__%s" % (repository.issuer.handle, repository.handle), - self_handle = repository.issuer.handle, - repository_handle = repository.handle, - bpki_cert = repository.certificate) - for repository in rpki.irdb.Repository.objects.all()) - - updates.extend( - rpki.left_right.parent_elt.make_pdu( - action = "set", - tag = "%s__parent__%s" % (parent.issuer.handle, parent.handle), - self_handle = parent.issuer.handle, - parent_handle = parent.handle, - bpki_cms_cert = parent.certificate) - for parent in rpki.irdb.Parent.objects.all()) - - updates.extend( - rpki.left_right.parent_elt.make_pdu( - action = "set", - tag = "%s__rootd" % rootd.issuer.handle, - self_handle = rootd.issuer.handle, - parent_handle = rootd.issuer.handle, - bpki_cms_cert = rootd.certificate) - for rootd in rpki.irdb.Rootd.objects.all()) - - updates.extend( - rpki.left_right.child_elt.make_pdu( - action = "set", - tag = "%s__child__%s" % (child.issuer.handle, child.handle), - self_handle = child.issuer.handle, - child_handle = child.handle, - bpki_cert = child.certificate) - for child in rpki.irdb.Child.objects.all()) - - if updates: - self.check_error_report(self.call_rpkid(updates)) + q_msg = self._compose_left_right_query() + + for ca in rpki.irdb.models.ResourceHolderCA.objects.all(): + q_pdu = SubElement(q_msg, rpki.left_right.tag_self, + action = "set", + tag = "%s__self" % ca.handle, + self_handle = ca.handle) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = ca.certificate.get_Base64() + + for bsc in rpki.irdb.models.BSC.objects.all(): + q_pdu = SubElement(q_msg, rpki.left_right.tag_bsc, + action = "set", + tag = "%s__bsc__%s" % (bsc.issuer.handle, bsc.handle), + self_handle = bsc.issuer.handle, + bsc_handle = bsc.handle) + SubElement(q_pdu, rpki.left_right.tag_signing_cert).text = bsc.certificate.get_Base64() + SubElement(q_pdu, rpki.left_right.tag_signing_cert_crl).text = bsc.issuer.latest_crl.get_Base64() + + for repository in rpki.irdb.models.Repository.objects.all(): + q_pdu = SubElement(q_msg, rpki.left_right.tag_repository, + action = "set", + tag = "%s__repository__%s" % (repository.issuer.handle, repository.handle), + self_handle = repository.issuer.handle, + repository_handle = repository.handle) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = repository.certificate.get_Base64() + + for parent in rpki.irdb.models.Parent.objects.all(): + q_pdu = SubElement(q_msg, rpki.left_right.tag_parent, + action = "set", + tag = "%s__parent__%s" % (parent.issuer.handle, parent.handle), + self_handle = parent.issuer.handle, + parent_handle = parent.handle) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = parent.certificate.get_Base64() + + for rootd in rpki.irdb.models.Rootd.objects.all(): + q_pdu = SubElement(q_msg, rpki.left_right.tag_parent, + action = "set", + tag = "%s__rootd" % rootd.issuer.handle, + self_handle = rootd.issuer.handle, + parent_handle = rootd.issuer.handle) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = rootd.certificate.get_Base64() + + for child in rpki.irdb.models.Child.objects.all(): + q_pdu = SubElement(q_msg, rpki.left_right.tag_child, + action = "set", + tag = "%s__child__%s" % (child.issuer.handle, child.handle), + self_handle = child.issuer.handle, + child_handle = child.handle) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = child.certificate.get_Base64() + + if len(q_msg) > 0: + self.call_rpkid(q_msg) if self.run_pubd: - updates = [] - - updates.append( - rpki.publication.config_elt.make_pdu( - action = "set", - bpki_crl = self.server_ca.latest_crl)) + q_msg = self._compose_publication_control_query() - updates.extend( - rpki.publication.client_elt.make_pdu( - action = "set", - client_handle = client.handle, - bpki_cert = client.certificate) - for client in self.server_ca.clients.all()) + for client in self.server_ca.clients.all(): + q_pdu = SubElement(q_msg, rpki.publication_control.tag_client, action = "set", client_handle = client.handle) + SubElement(q_pdu, rpki.publication_control.tag_bpki_cert).text = client.certificate.get_Base64() - if updates: - self.check_error_report(self.call_pubd(updates)) + if len(q_msg) > 0: + self.call_pubd(q_msg) - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def configure_child(self, filename, child_handle = None, valid_until = None): """ Configure a new child of this RPKI entity, given the child's XML @@ -561,10 +581,10 @@ class Zookeeper(object): data and up-down protocol service URI. """ - c = etree_read(filename) + x = etree_read(filename) if child_handle is None: - child_handle = c.get("handle") + child_handle = x.get("child_handle") if valid_until is None: valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 365) @@ -573,18 +593,18 @@ class Zookeeper(object): if valid_until < rpki.sundial.now(): raise PastExpiration("Specified new expiration time %s has passed" % valid_until) - self.log("Child calls itself %r, we call it %r" % (c.get("handle"), child_handle)) + self.log("Child calls itself %r, we call it %r" % (x.get("child_handle"), child_handle)) - child, created = rpki.irdb.Child.objects.get_or_certify( + child, created = rpki.irdb.models.Child.objects.get_or_certify( issuer = self.resource_ca, handle = child_handle, - ta = rpki.x509.X509(Base64 = c.findtext("bpki_ta")), + ta = rpki.x509.X509(Base64 = x.findtext(tag_oob_child_bpki_ta)), valid_until = valid_until) return self.generate_parental_response(child), child_handle - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def generate_parental_response(self, child): """ Generate parental response XML. Broken out of .configure_child() @@ -596,43 +616,41 @@ class Zookeeper(object): self.cfg.get("rpkid_server_port", section = myrpki_section), self.handle, child.handle) - e = Element("parent", parent_handle = self.handle, child_handle = child.handle, - service_uri = service_uri, valid_until = str(child.valid_until)) - B64Element(e, "bpki_resource_ta", self.resource_ca.certificate) - B64Element(e, "bpki_child_ta", child.ta) + e = Element(tag_oob_parent_response, nsmap = oob_nsmap, version = oob_version, + service_uri = service_uri, + child_handle = child.handle, + parent_handle = self.handle) + B64Element(e, tag_oob_parent_bpki_ta, self.resource_ca.certificate) try: if self.default_repository: repo = self.resource_ca.repositories.get(handle = self.default_repository) else: repo = self.resource_ca.repositories.get() - except rpki.irdb.Repository.DoesNotExist: + except rpki.irdb.models.Repository.DoesNotExist: repo = None if repo is None: self.log("Couldn't find any usable repositories, not giving referral") elif repo.handle == self.handle: - SubElement(e, "repository", type = "offer") + SubElement(e, tag_oob_offer) else: proposed_sia_base = repo.sia_base + child.handle + "/" - referral_cert, created = rpki.irdb.Referral.objects.get_or_certify(issuer = self.resource_ca) + referral_cert, created = rpki.irdb.models.Referral.objects.get_or_certify(issuer = self.resource_ca) auth = rpki.x509.SignedReferral() - auth.set_content(B64Element(None, myrpki_xmlns + "referral", child.ta, - version = myrpki_version, + auth.set_content(B64Element(None, tag_oob_authorization, child.ta, + nsmap = oob_nsmap, version = oob_version, authorized_sia_base = proposed_sia_base)) auth.schema_check() auth.sign(referral_cert.private_key, referral_cert.certificate, self.resource_ca.latest_crl) - - r = SubElement(e, "repository", type = "referral") - B64Element(r, "authorization", auth, referrer = repo.client_handle) - SubElement(r, "contact_info") + B64Element(e, tag_oob_referral, auth, referrer = repo.client_handle) return etree_wrapper(e, msg = "Send this file back to the child you just configured") - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def delete_child(self, child_handle): """ Delete a child of this RPKI entity. @@ -641,7 +659,7 @@ class Zookeeper(object): self.resource_ca.children.get(handle = child_handle).delete() - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def configure_parent(self, filename, parent_handle = None): """ Configure a new parent of this RPKI entity, given the output of @@ -654,35 +672,39 @@ class Zookeeper(object): the user wants to avail herself of the referral or offer. """ - p = etree_read(filename) + x = etree_read(filename) if parent_handle is None: - parent_handle = p.get("parent_handle") + parent_handle = x.get("parent_handle") - r = p.find("repository") + offer = x.find(tag_oob_offer) + referral = x.find(tag_oob_referral) - repository_type = "none" - referrer = None - referral_authorization = None + if offer is not None: + repository_type = "offer" + referrer = None + referral_authorization = None - if r is not None: - repository_type = r.get("type") + elif referral is not None: + repository_type = "referral" + referrer = referral.get("referrer") + referral_authorization = rpki.x509.SignedReferral(Base64 = referral.text) - if repository_type == "referral": - a = r.find("authorization") - referrer = a.get("referrer") - referral_authorization = rpki.x509.SignedReferral(Base64 = a.text) + else: + repository_type = "none" + referrer = None + referral_authorization = None - self.log("Parent calls itself %r, we call it %r" % (p.get("parent_handle"), parent_handle)) - self.log("Parent calls us %r" % p.get("child_handle")) + self.log("Parent calls itself %r, we call it %r" % (x.get("parent_handle"), parent_handle)) + self.log("Parent calls us %r" % x.get("child_handle")) - parent, created = rpki.irdb.Parent.objects.get_or_certify( + parent, created = rpki.irdb.models.Parent.objects.get_or_certify( issuer = self.resource_ca, handle = parent_handle, - child_handle = p.get("child_handle"), - parent_handle = p.get("parent_handle"), - service_uri = p.get("service_uri"), - ta = rpki.x509.X509(Base64 = p.findtext("bpki_resource_ta")), + child_handle = x.get("child_handle"), + parent_handle = x.get("parent_handle"), + service_uri = x.get("service_uri"), + ta = rpki.x509.X509(Base64 = x.findtext(tag_oob_parent_bpki_ta)), repository_type = repository_type, referrer = referrer, referral_authorization = referral_authorization) @@ -695,16 +717,17 @@ class Zookeeper(object): Generate repository request for a given parent. """ - e = Element("repository", handle = self.handle, - parent_handle = parent.handle, type = parent.repository_type) + e = Element(tag_oob_publisher_request, nsmap = oob_nsmap, version = oob_version, + publisher_handle = self.handle) + B64Element(e, tag_oob_publisher_bpki_ta, self.resource_ca.certificate) if parent.repository_type == "referral": - B64Element(e, "authorization", parent.referral_authorization, referrer = parent.referrer) - SubElement(e, "contact_info") - B64Element(e, "bpki_client_ta", self.resource_ca.certificate) + B64Element(e, tag_oob_referral, parent.referral_authorization, + referrer = parent.referrer) + return etree_wrapper(e, msg = "This is the file to send to the repository operator") - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def delete_parent(self, parent_handle): """ Delete a parent of this RPKI entity. @@ -713,7 +736,7 @@ class Zookeeper(object): self.resource_ca.parents.get(handle = parent_handle).delete() - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def delete_rootd(self): """ Delete rootd associated with this RPKI entity. @@ -722,7 +745,7 @@ class Zookeeper(object): self.resource_ca.rootd.delete() - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def configure_publication_client(self, filename, sia_base = None, flat = False): """ Configure publication server to know about a new client, given the @@ -732,65 +755,67 @@ class Zookeeper(object): and service URI. """ - client = etree_read(filename) + x = etree_read(filename) + + client_ta = rpki.x509.X509(Base64 = x.findtext(tag_oob_publisher_bpki_ta)) - client_ta = rpki.x509.X509(Base64 = client.findtext("bpki_client_ta")) + referral = x.find(tag_oob_referral) + + default_sia_base = "rsync://{self.rsync_server}/{self.rsync_module}/{handle}/".format( + self = self, handle = x.get("publisher_handle")) if sia_base is None and flat: self.log("Flat publication structure forced, homing client at top-level") - sia_base = "rsync://%s/%s/%s/" % (self.rsync_server, self.rsync_module, client.get("handle")) + sia_base = default_sia_base - if sia_base is None and client.get("type") == "referral": + if sia_base is None and referral is not None: self.log("This looks like a referral, checking") try: - auth = client.find("authorization") - referrer = self.server_ca.clients.get(handle = auth.get("referrer")) - referral_cms = rpki.x509.SignedReferral(Base64 = auth.text) - referral_xml = referral_cms.unwrap(ta = (referrer.certificate, self.server_ca.certificate)) - if rpki.x509.X509(Base64 = referral_xml.text) != client_ta: + referrer = referral.get("referrer") + referrer = self.server_ca.clients.get(handle = referrer) + referral = rpki.x509.SignedReferral(Base64 = referral.text) + referral = referral.unwrap(ta = (referrer.certificate, self.server_ca.certificate)) + if rpki.x509.X509(Base64 = referral.text) != client_ta: raise BadXMLMessage("Referral trust anchor does not match") - sia_base = referral_xml.get("authorized_sia_base") - except rpki.irdb.Client.DoesNotExist: - self.log("We have no record of the client (%s) alleged to have made this referral" % auth.get("referrer")) + sia_base = referral.get("authorized_sia_base") + except rpki.irdb.models.Client.DoesNotExist: + self.log("We have no record of the client ({}) alleged to have made this referral".format(referrer)) - if sia_base is None and client.get("type") == "offer": - self.log("This looks like an offer, checking") + if sia_base is None and referral is None: + self.log("This might be an offer, checking") try: - parent = rpki.irdb.ResourceHolderCA.objects.get(children__ta__exact = client_ta) + parent = rpki.irdb.models.ResourceHolderCA.objects.get(children__ta__exact = client_ta) if "/" in parent.repositories.get(ta = self.server_ca.certificate).client_handle: self.log("Client's parent is not top-level, this is not a valid offer") else: self.log("Found client and its parent, nesting") - sia_base = "rsync://%s/%s/%s/%s/" % (self.rsync_server, self.rsync_module, - parent.handle, client.get("handle")) - except rpki.irdb.Repository.DoesNotExist: + sia_base = "rsync://{self.rsync_server}/{self.rsync_module}/{parent_handle}/{client_handle}/".format( + self = self, parent_handle = parent.handle, client_handle = x.get("publisher_handle")) + except rpki.irdb.models.Repository.DoesNotExist: self.log("Found client's parent, but repository isn't set, this shouldn't happen!") - except rpki.irdb.ResourceHolderCA.DoesNotExist: + except rpki.irdb.models.ResourceHolderCA.DoesNotExist: try: - rpki.irdb.Rootd.objects.get(issuer__certificate__exact = client_ta) - except rpki.irdb.Rootd.DoesNotExist: - self.log("We don't host this client's parent, so we didn't make this offer") - else: + rpki.irdb.models.Rootd.objects.get(issuer__certificate__exact = client_ta) self.log("This client's parent is rootd") + sia_base = default_sia_base + except rpki.irdb.models.Rootd.DoesNotExist: + self.log("We don't host this client's parent, so we didn't make an offer") if sia_base is None: - self.log("Don't know where to nest this client, defaulting to top-level") - sia_base = "rsync://%s/%s/%s/" % (self.rsync_server, self.rsync_module, client.get("handle")) + self.log("Don't know where else to nest this client, so defaulting to top-level") + sia_base = default_sia_base if not sia_base.startswith("rsync://"): raise BadXMLMessage("Malformed sia_base parameter %r, should start with 'rsync://'" % sia_base) client_handle = "/".join(sia_base.rstrip("/").split("/")[4:]) - parent_handle = client.get("parent_handle") + self.log("Client calls itself %r, we call it %r" % ( + x.get("publisher_handle"), client_handle)) - self.log("Client calls itself %r, we call it %r" % (client.get("handle"), client_handle)) - self.log("Client says its parent handle is %r" % parent_handle) - - client, created = rpki.irdb.Client.objects.get_or_certify( + client, created = rpki.irdb.models.Client.objects.get_or_certify( issuer = self.server_ca, handle = client_handle, - parent_handle = parent_handle, ta = client_ta, sia_base = sia_base) @@ -802,24 +827,27 @@ class Zookeeper(object): Generate repository response XML to a given client. """ - service_uri = "http://%s:%s/client/%s" % ( - self.cfg.get("pubd_server_host", section = myrpki_section), - self.cfg.get("pubd_server_port", section = myrpki_section), - client.handle) - - e = Element("repository", type = "confirmed", - client_handle = client.handle, - parent_handle = client.parent_handle, - sia_base = client.sia_base, - service_uri = service_uri) - - B64Element(e, "bpki_server_ta", self.server_ca.certificate) - B64Element(e, "bpki_client_ta", client.ta) - SubElement(e, "contact_info").text = self.pubd_contact_info + service_uri = "http://{host}:{port}/client/{handle}".format( + host = self.cfg.get("pubd_server_host", section = myrpki_section), + port = self.cfg.get("pubd_server_port", section = myrpki_section), + handle = client.handle) + + rrdp_uri = self.cfg.get("publication_rrdp_notification_uri", section = myrpki_section, + default = "") or None + + e = Element(tag_oob_repository_response, nsmap = oob_nsmap, version = oob_version, + service_uri = service_uri, + publisher_handle = client.handle, + sia_base = client.sia_base) + + if rrdp_uri is not None: + e.set("rrdp_notification_uri", rrdp_uri) + + B64Element(e, tag_oob_repository_bpki_ta, self.server_ca.certificate) return etree_wrapper(e, msg = "Send this file back to the publication client you just configured") - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def delete_publication_client(self, client_handle): """ Delete a publication client of this RPKI entity. @@ -828,7 +856,7 @@ class Zookeeper(object): self.server_ca.clients.get(handle = client_handle).delete() - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def configure_repository(self, filename, parent_handle = None): """ Configure a publication repository for this RPKI entity, given the @@ -838,35 +866,56 @@ class Zookeeper(object): corresponding parent data in our local database. """ - r = etree_read(filename) + x = etree_read(filename) - if parent_handle is None: - parent_handle = r.get("parent_handle") + self.log("Repository calls us %r" % (x.get("client_handle"))) - self.log("Repository calls us %r" % (r.get("client_handle"))) - self.log("Repository response associated with parent_handle %r" % parent_handle) + if parent_handle is not None: + self.log("Explicit parent_handle given") + try: + if parent_handle == self.handle: + turtle = self.resource_ca.rootd + else: + turtle = self.resource_ca.parents.get(handle = parent_handle) + except (rpki.irdb.models.Parent.DoesNotExist, rpki.irdb.models.Rootd.DoesNotExist): + self.log("Could not find parent %r in our database" % parent_handle) + raise CouldntFindRepoParent - try: - if parent_handle == self.handle: - turtle = self.resource_ca.rootd + else: + turtles = [] + for parent in self.resource_ca.parents.all(): + try: + _ = parent.repository + except rpki.irdb.models.Repository.DoesNotExist: + turtles.append(parent) + try: + _ = self.resource_ca.rootd.repository + except rpki.irdb.models.Repository.DoesNotExist: + turtles.append(self.resource_ca.rootd) + except rpki.irdb.models.Rootd.DoesNotExist: + pass + if len(turtles) != 1: + self.log("No explicit parent_handle given and unable to guess") + raise CouldntFindRepoParent + turtle = turtles[0] + if isinstance(turtle, rpki.irdb.models.Rootd): + parent_handle = self.handle else: - turtle = self.resource_ca.parents.get(handle = parent_handle) + parent_handle = turtle.handle + self.log("No explicit parent_handle given, guessing parent {}".format(parent_handle)) - except (rpki.irdb.Parent.DoesNotExist, rpki.irdb.Rootd.DoesNotExist): - self.log("Could not find parent %r in our database" % parent_handle) - - else: - rpki.irdb.Repository.objects.get_or_certify( - issuer = self.resource_ca, - handle = parent_handle, - client_handle = r.get("client_handle"), - service_uri = r.get("service_uri"), - sia_base = r.get("sia_base"), - ta = rpki.x509.X509(Base64 = r.findtext("bpki_server_ta")), - turtle = turtle) + rpki.irdb.models.Repository.objects.get_or_certify( + issuer = self.resource_ca, + handle = parent_handle, + client_handle = x.get("publisher_handle"), + service_uri = x.get("service_uri"), + sia_base = x.get("sia_base"), + rrdp_notification_uri = x.get("rrdp_notification_uri"), + ta = rpki.x509.X509(Base64 = x.findtext(tag_oob_repository_bpki_ta)), + turtle = turtle) - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def delete_repository(self, repository_handle): """ Delete a repository of this RPKI entity. @@ -875,7 +924,7 @@ class Zookeeper(object): self.resource_ca.repositories.get(handle = repository_handle).delete() - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def renew_children(self, child_handle, valid_until = None): """ Update validity period for one child entity or, if child_handle is @@ -901,7 +950,7 @@ class Zookeeper(object): child.save() - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def load_prefixes(self, filename, ignore_missing_children = False): """ Whack IRDB to match prefixes.csv. @@ -923,25 +972,25 @@ class Zookeeper(object): for handle, prefixes in grouped.iteritems(): try: child = self.resource_ca.children.get(handle = handle) - except rpki.irdb.Child.DoesNotExist: + except rpki.irdb.models.Child.DoesNotExist: if not ignore_missing_children: raise else: for prefix in rset(",".join(prefixes)): - obj, created = rpki.irdb.ChildNet.objects.get_or_create( + obj, created = rpki.irdb.models.ChildNet.objects.get_or_create( child = child, start_ip = str(prefix.min), end_ip = str(prefix.max), version = version) primary_keys.append(obj.pk) - q = rpki.irdb.ChildNet.objects + q = rpki.irdb.models.ChildNet.objects q = q.filter(child__issuer__exact = self.resource_ca) q = q.exclude(pk__in = primary_keys) q.delete() - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def load_asns(self, filename, ignore_missing_children = False): """ Whack IRDB to match asns.csv. @@ -959,24 +1008,24 @@ class Zookeeper(object): for handle, asns in grouped.iteritems(): try: child = self.resource_ca.children.get(handle = handle) - except rpki.irdb.Child.DoesNotExist: + except rpki.irdb.models.Child.DoesNotExist: if not ignore_missing_children: raise else: for asn in rpki.resource_set.resource_set_as(",".join(asns)): - obj, created = rpki.irdb.ChildASN.objects.get_or_create( + obj, created = rpki.irdb.models.ChildASN.objects.get_or_create( child = child, start_as = str(asn.min), end_as = str(asn.max)) primary_keys.append(obj.pk) - q = rpki.irdb.ChildASN.objects + q = rpki.irdb.models.ChildASN.objects q = q.filter(child__issuer__exact = self.resource_ca) q = q.exclude(pk__in = primary_keys) q.delete() - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def load_roa_requests(self, filename): """ Whack IRDB to match roa.csv. @@ -1017,7 +1066,7 @@ class Zookeeper(object): max_prefixlen = int(p.max_prefixlen)) - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def load_ghostbuster_requests(self, filename, parent = None): """ Whack IRDB to match ghostbusters.vcard. @@ -1038,14 +1087,9 @@ class Zookeeper(object): vcard = [] - def call_rpkid(self, *pdus): + def call_rpkid(self, q_msg, suppress_error_check = False): """ Issue a call to rpkid, return result. - - Implementation is a little silly, constructs a wrapper object, - invokes it once, then throws it away. Hard to do better without - rewriting a bit of the HTTP code, as we want to be sure we're - using the current BPKI certificate and key objects. """ url = "http://%s:%s/left-right" % ( @@ -1055,23 +1099,28 @@ class Zookeeper(object): rpkid = self.server_ca.ee_certificates.get(purpose = "rpkid") irbe = self.server_ca.ee_certificates.get(purpose = "irbe") - if len(pdus) == 1 and isinstance(pdus[0], types.GeneratorType): - pdus = tuple(pdus[0]) - elif len(pdus) == 1 and isinstance(pdus[0], (tuple, list)): - pdus = pdus[0] + r_msg = rpki.http_simple.client( + proto_cms_msg = rpki.left_right.cms_msg, + client_key = irbe.private_key, + client_cert = irbe.certificate, + server_ta = self.server_ca.certificate, + server_cert = rpkid.certificate, + url = url, + q_msg = q_msg, + debug = self.show_xml) + + if not suppress_error_check: + self.check_error_report(r_msg) + return r_msg - call_rpkid = rpki.async.sync_wrapper( - disable_signal_handlers = self.disable_signal_handlers, - func = rpki.http.caller( - proto = rpki.left_right, - client_key = irbe.private_key, - client_cert = irbe.certificate, - server_ta = self.server_ca.certificate, - server_cert = rpkid.certificate, - url = url, - debug = self.show_xml)) - return call_rpkid(*pdus) + def _rpkid_self_control(self, *bools): + assert all(isinstance(b, str) for b in bools) + q_msg = self._compose_left_right_query() + q_pdu = SubElement(q_msg, rpki.left_right.tag_self, action = "set", self_handle = self.handle) + for b in bools: + q_pdu.set(b, "yes") + return self.call_rpkid(q_msg) def run_rpkid_now(self): @@ -1083,8 +1132,7 @@ class Zookeeper(object): to force the object to be immediately issued. """ - self.call_rpkid(rpki.left_right.self_elt.make_pdu( - action = "set", self_handle = self.handle, run_now = "yes")) + return self._rpkid_self_control("run_now") def publish_world_now(self): @@ -1092,8 +1140,7 @@ class Zookeeper(object): Poke rpkid to (re)publish everything for the current handle. """ - self.call_rpkid(rpki.left_right.self_elt.make_pdu( - action = "set", self_handle = self.handle, publish_world_now = "yes")) + return self._rpkid_self_control("publish_world_now") def reissue(self): @@ -1101,8 +1148,8 @@ class Zookeeper(object): Poke rpkid to reissue everything for the current handle. """ - self.call_rpkid(rpki.left_right.self_elt.make_pdu( - action = "set", self_handle = self.handle, reissue = "yes")) + return self._rpkid_self_control("reissue") + def rekey(self): """ @@ -1110,8 +1157,7 @@ class Zookeeper(object): handle. """ - self.call_rpkid(rpki.left_right.self_elt.make_pdu( - action = "set", self_handle = self.handle, rekey = "yes")) + return self._rpkid_self_control("rekey") def revoke(self): @@ -1119,8 +1165,7 @@ class Zookeeper(object): Poke rpkid to revoke old RPKI keys for the current handle. """ - self.call_rpkid(rpki.left_right.self_elt.make_pdu( - action = "set", self_handle = self.handle, revoke = "yes")) + return self._rpkid_self_control("revoke") def revoke_forgotten(self): @@ -1128,8 +1173,7 @@ class Zookeeper(object): Poke rpkid to revoke old forgotten RPKI keys for the current handle. """ - self.call_rpkid(rpki.left_right.self_elt.make_pdu( - action = "set", self_handle = self.handle, revoke_forgotten = "yes")) + return self._rpkid_self_control("revoke_forgotten") def clear_all_sql_cms_replay_protection(self): @@ -1137,27 +1181,27 @@ class Zookeeper(object): Tell rpkid and pubd to clear replay protection for all SQL-based entities. This is a fairly blunt instrument, but as we don't expect this to be necessary except in the case of gross - misconfiguration, it should suffice + misconfiguration, it should suffice. """ - self.call_rpkid(rpki.left_right.self_elt.make_pdu(action = "set", self_handle = ca.handle, - clear_replay_protection = "yes") - for ca in rpki.irdb.ResourceHolderCA.objects.all()) + if self.run_rpkid: + q_msg = self._compose_left_right_query() + for ca in rpki.irdb.models.ResourceHolderCA.objects.all(): + SubElement(q_msg, rpki.left_right.tag_self, action = "set", + self_handle = ca.handle, clear_replay_protection = "yes") + self.call_rpkid(q_msg) + if self.run_pubd: - self.call_pubd(rpki.publication.client_elt.make_pdu(action = "set", - client_handle = client.handle, - clear_replay_protection = "yes") - for client in self.server_ca.clients.all()) + q_msg = self._compose_publication_control_query() + for client in self.server_ca.clients.all(): + SubElement(q_msg, rpki.publication_control.tag_client, action = "set", + client_handle = client.handle, clear_reply_protection = "yes") + self.call_pubd(q_msg) - def call_pubd(self, *pdus): + def call_pubd(self, q_msg): """ Issue a call to pubd, return result. - - Implementation is a little silly, constructs a wrapper object, - invokes it once, then throws it away. Hard to do better without - rewriting a bit of the HTTP code, as we want to be sure we're - using the current BPKI certificate and key objects. """ url = "http://%s:%s/control" % ( @@ -1167,45 +1211,42 @@ class Zookeeper(object): pubd = self.server_ca.ee_certificates.get(purpose = "pubd") irbe = self.server_ca.ee_certificates.get(purpose = "irbe") - if len(pdus) == 1 and isinstance(pdus[0], types.GeneratorType): - pdus = tuple(pdus[0]) - elif len(pdus) == 1 and isinstance(pdus[0], (tuple, list)): - pdus = pdus[0] + r_msg = rpki.http_simple.client( + proto_cms_msg = rpki.publication_control.cms_msg, + client_key = irbe.private_key, + client_cert = irbe.certificate, + server_ta = self.server_ca.certificate, + server_cert = pubd.certificate, + url = url, + q_msg = q_msg, + debug = self.show_xml) - call_pubd = rpki.async.sync_wrapper( - disable_signal_handlers = self.disable_signal_handlers, - func = rpki.http.caller( - proto = rpki.publication, - client_key = irbe.private_key, - client_cert = irbe.certificate, - server_ta = self.server_ca.certificate, - server_cert = pubd.certificate, - url = url, - debug = self.show_xml)) + self.check_error_report(r_msg) + return r_msg - return call_pubd(*pdus) - - def check_error_report(self, pdus): + def check_error_report(self, r_msg): """ Check a response from rpkid or pubd for error_report PDUs, log and throw exceptions as needed. """ - if any(isinstance(pdu, (rpki.left_right.report_error_elt, rpki.publication.report_error_elt)) for pdu in pdus): - for pdu in pdus: - if isinstance(pdu, rpki.left_right.report_error_elt): - self.log("rpkid reported failure: %s" % pdu.error_code) - elif isinstance(pdu, rpki.publication.report_error_elt): - self.log("pubd reported failure: %s" % pdu.error_code) + if any(r_pdu.tag in (rpki.left_right.tag_report_error, + rpki.publication_control.tag_report_error) + for r_pdu in r_msg): + for r_pdu in r_msg: + if r_pdu.tag == rpki.left_right.tag_report_error: + self.log("rpkid reported failure: %s" % r_pdu.get("error_code")) + elif r_pdu.tag == rpki.publication_control.tag_report_error: + self.log("pubd reported failure: %s" % r_pdu.get("error_code")) else: continue - if pdu.error_text: - self.log(pdu.error_text) + if r_pdu.text: + self.log(r_pdu.text) raise CouldntTalkToDaemon - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def synchronize(self, *handles_to_poke): """ Configure RPKI daemons with the data built up by the other @@ -1217,13 +1258,13 @@ class Zookeeper(object): <self run_now="yes"/> operation. """ - for ca in rpki.irdb.ResourceHolderCA.objects.all(): + for ca in rpki.irdb.models.ResourceHolderCA.objects.all(): self.synchronize_rpkid_one_ca_core(ca, ca.handle in handles_to_poke) self.synchronize_pubd_core() self.synchronize_rpkid_deleted_core() - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def synchronize_ca(self, ca = None, poke = False): """ Synchronize one CA. Most commands which modify a CA should call @@ -1235,7 +1276,7 @@ class Zookeeper(object): self.synchronize_rpkid_one_ca_core(ca, poke) - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def synchronize_deleted_ca(self): """ Delete CAs which are present in rpkid's database but not in the @@ -1245,7 +1286,7 @@ class Zookeeper(object): self.synchronize_rpkid_deleted_core() - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def synchronize_pubd(self): """ Synchronize pubd. Most commands which modify pubd should call this. @@ -1293,94 +1334,103 @@ class Zookeeper(object): # See what rpkid already has on file for this entity. - rpkid_reply = self.call_rpkid( - rpki.left_right.self_elt.make_pdu( action = "get", tag = "self", self_handle = ca.handle), - rpki.left_right.bsc_elt.make_pdu( action = "list", tag = "bsc", self_handle = ca.handle), - rpki.left_right.repository_elt.make_pdu(action = "list", tag = "repository", self_handle = ca.handle), - rpki.left_right.parent_elt.make_pdu( action = "list", tag = "parent", self_handle = ca.handle), - rpki.left_right.child_elt.make_pdu( action = "list", tag = "child", self_handle = ca.handle)) - - self_pdu = rpkid_reply[0] - bsc_pdus = dict((x.bsc_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.bsc_elt)) - repository_pdus = dict((x.repository_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.repository_elt)) - parent_pdus = dict((x.parent_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.parent_elt)) - child_pdus = dict((x.child_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.child_elt)) + q_msg = self._compose_left_right_query() + SubElement(q_msg, rpki.left_right.tag_self, action = "get", self_handle = ca.handle) + SubElement(q_msg, rpki.left_right.tag_bsc, action = "list", self_handle = ca.handle) + SubElement(q_msg, rpki.left_right.tag_repository, action = "list", self_handle = ca.handle) + SubElement(q_msg, rpki.left_right.tag_parent, action = "list", self_handle = ca.handle) + SubElement(q_msg, rpki.left_right.tag_child, action = "list", self_handle = ca.handle) - rpkid_query = [] + r_msg = self.call_rpkid(q_msg, suppress_error_check = True) - self_cert, created = rpki.irdb.HostedCA.objects.get_or_certify( + if r_msg[0].tag == rpki.left_right.tag_self: + self.check_error_report(r_msg) + self_pdu = r_msg[0] + else: + self_pdu = None + + bsc_pdus = dict((r_pdu.get("bsc_handle"), r_pdu) + for r_pdu in r_msg + if r_pdu.tag == rpki.left_right.tag_bsc) + repository_pdus = dict((r_pdu.get("repository_handle"), r_pdu) + for r_pdu in r_msg + if r_pdu.tag == rpki.left_right.tag_repository) + parent_pdus = dict((r_pdu.get("parent_handle"), r_pdu) + for r_pdu in r_msg + if r_pdu.tag == rpki.left_right.tag_parent) + child_pdus = dict((r_pdu.get("child_handle"), r_pdu) + for r_pdu in r_msg + if r_pdu.tag == rpki.left_right.tag_child) + + q_msg = self._compose_left_right_query() + + self_cert, created = rpki.irdb.models.HostedCA.objects.get_or_certify( issuer = self.server_ca, hosted = ca) # There should be exactly one <self/> object per hosted entity, by definition - if (isinstance(self_pdu, rpki.left_right.report_error_elt) or - self_pdu.crl_interval != self_crl_interval or - self_pdu.regen_margin != self_regen_margin or - self_pdu.bpki_cert != self_cert.certificate): - rpkid_query.append(rpki.left_right.self_elt.make_pdu( - action = "create" if isinstance(self_pdu, rpki.left_right.report_error_elt) else "set", - tag = "self", - self_handle = ca.handle, - bpki_cert = ca.certificate, - crl_interval = self_crl_interval, - regen_margin = self_regen_margin)) + if (self_pdu is None or + self_pdu.get("crl_interval") != str(self_crl_interval) or + self_pdu.get("regen_margin") != str(self_regen_margin) or + self_pdu.findtext(rpki.left_right.tag_bpki_cert, "").decode("base64") != self_cert.certificate.get_DER()): + q_pdu = SubElement(q_msg, rpki.left_right.tag_self, + action = "create" if self_pdu is None else "set", + tag = "self", + self_handle = ca.handle, + crl_interval = str(self_crl_interval), + regen_margin = str(self_regen_margin)) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = ca.certificate.get_Base64() # In general we only need one <bsc/> per <self/>. BSC objects # are a little unusual in that the keypair and PKCS #10 - # subelement is generated by rpkid, so complete setup requires + # subelement are generated by rpkid, so complete setup requires # two round trips. bsc_pdu = bsc_pdus.pop(bsc_handle, None) - if bsc_pdu is None: - rpkid_query.append(rpki.left_right.bsc_elt.make_pdu( - action = "create", - tag = "bsc", - self_handle = ca.handle, - bsc_handle = bsc_handle, - generate_keypair = "yes")) - - elif bsc_pdu.pkcs10_request is None: - rpkid_query.append(rpki.left_right.bsc_elt.make_pdu( - action = "set", - tag = "bsc", - self_handle = ca.handle, - bsc_handle = bsc_handle, - generate_keypair = "yes")) - - rpkid_query.extend(rpki.left_right.bsc_elt.make_pdu( - action = "destroy", self_handle = ca.handle, bsc_handle = b) for b in bsc_pdus) + if bsc_pdu is None or bsc_pdu.find(rpki.left_right.tag_pkcs10_request) is None: + SubElement(q_msg, rpki.left_right.tag_bsc, + action = "create" if bsc_pdu is None else "set", + tag = "bsc", + self_handle = ca.handle, + bsc_handle = bsc_handle, + generate_keypair = "yes") + + for bsc_handle in bsc_pdus: + SubElement(q_msg, rpki.left_right.tag_bsc, + action = "destroy", self_handle = ca.handle, bsc_handle = bsc_handle) # If we've already got actions queued up, run them now, so we # can finish setting up the BSC before anything tries to use it. - if rpkid_query: - rpkid_query.append(rpki.left_right.bsc_elt.make_pdu(action = "list", tag = "bsc", self_handle = ca.handle)) - rpkid_reply = self.call_rpkid(rpkid_query) - bsc_pdus = dict((x.bsc_handle, x) - for x in rpkid_reply - if isinstance(x, rpki.left_right.bsc_elt) and x.action == "list") + if len(q_msg) > 0: + SubElement(q_msg, rpki.left_right.tag_bsc, action = "list", tag = "bsc", self_handle = ca.handle) + r_msg = self.call_rpkid(q_msg) + bsc_pdus = dict((r_pdu.get("bsc_handle"), r_pdu) + for r_pdu in r_msg + if r_pdu.tag == rpki.left_right.tag_bsc and r_pdu.get("action") == "list") bsc_pdu = bsc_pdus.pop(bsc_handle, None) - self.check_error_report(rpkid_reply) - rpkid_query = [] + q_msg = self._compose_left_right_query() - assert bsc_pdu.pkcs10_request is not None + bsc_pkcs10 = bsc_pdu.find(rpki.left_right.tag_pkcs10_request) + assert bsc_pkcs10 is not None - bsc, created = rpki.irdb.BSC.objects.get_or_certify( + bsc, created = rpki.irdb.models.BSC.objects.get_or_certify( issuer = ca, handle = bsc_handle, - pkcs10 = bsc_pdu.pkcs10_request) - - if bsc_pdu.signing_cert != bsc.certificate or bsc_pdu.signing_cert_crl != ca.latest_crl: - rpkid_query.append(rpki.left_right.bsc_elt.make_pdu( - action = "set", - tag = "bsc", - self_handle = ca.handle, - bsc_handle = bsc_handle, - signing_cert = bsc.certificate, - signing_cert_crl = ca.latest_crl)) + pkcs10 = rpki.x509.PKCS10(Base64 = bsc_pkcs10.text)) + + if (bsc_pdu.findtext(rpki.left_right.tag_signing_cert, "").decode("base64") != bsc.certificate.get_DER() or + bsc_pdu.findtext(rpki.left_right.tag_signing_cert_crl, "").decode("base64") != ca.latest_crl.get_DER()): + q_pdu = SubElement(q_msg, rpki.left_right.tag_bsc, + action = "set", + tag = "bsc", + self_handle = ca.handle, + bsc_handle = bsc_handle) + SubElement(q_pdu, rpki.left_right.tag_signing_cert).text = bsc.certificate.get_Base64() + SubElement(q_pdu, rpki.left_right.tag_signing_cert_crl).text = ca.latest_crl.get_Base64() # At present we need one <repository/> per <parent/>, not because # rpkid requires that, but because pubd does. pubd probably should @@ -1393,20 +1443,24 @@ class Zookeeper(object): repository_pdu = repository_pdus.pop(repository.handle, None) if (repository_pdu is None or - repository_pdu.bsc_handle != bsc_handle or - repository_pdu.peer_contact_uri != repository.service_uri or - repository_pdu.bpki_cert != repository.certificate): - rpkid_query.append(rpki.left_right.repository_elt.make_pdu( - action = "create" if repository_pdu is None else "set", - tag = repository.handle, - self_handle = ca.handle, - repository_handle = repository.handle, - bsc_handle = bsc_handle, - peer_contact_uri = repository.service_uri, - bpki_cert = repository.certificate)) - - rpkid_query.extend(rpki.left_right.repository_elt.make_pdu( - action = "destroy", self_handle = ca.handle, repository_handle = r) for r in repository_pdus) + repository_pdu.get("bsc_handle") != bsc_handle or + repository_pdu.get("peer_contact_uri") != repository.service_uri or + repository_pdu.get("rrdp_notification_uri") != repository.rrdp_notification_uri or + repository_pdu.findtext(rpki.left_right.tag_bpki_cert, "").decode("base64") != repository.certificate.get_DER()): + q_pdu = SubElement(q_msg, rpki.left_right.tag_repository, + action = "create" if repository_pdu is None else "set", + tag = repository.handle, + self_handle = ca.handle, + repository_handle = repository.handle, + bsc_handle = bsc_handle, + peer_contact_uri = repository.service_uri) + if repository.rrdp_notification_uri: + q_pdu.set("rrdp_notification_uri", repository.rrdp_notification_uri) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = repository.certificate.get_Base64() + + for repository_handle in repository_pdus: + SubElement(q_msg, rpki.left_right.tag_repository, action = "destroy", + self_handle = ca.handle, repository_handle = repository_handle) # <parent/> setup code currently assumes 1:1 mapping between # <repository/> and <parent/>, and further assumes that the handles @@ -1419,31 +1473,30 @@ class Zookeeper(object): for parent in ca.parents.all(): try: - parent_pdu = parent_pdus.pop(parent.handle, None) if (parent_pdu is None or - parent_pdu.bsc_handle != bsc_handle or - parent_pdu.repository_handle != parent.handle or - parent_pdu.peer_contact_uri != parent.service_uri or - parent_pdu.sia_base != parent.repository.sia_base or - parent_pdu.sender_name != parent.child_handle or - parent_pdu.recipient_name != parent.parent_handle or - parent_pdu.bpki_cms_cert != parent.certificate): - rpkid_query.append(rpki.left_right.parent_elt.make_pdu( - action = "create" if parent_pdu is None else "set", - tag = parent.handle, - self_handle = ca.handle, - parent_handle = parent.handle, - bsc_handle = bsc_handle, - repository_handle = parent.handle, - peer_contact_uri = parent.service_uri, - sia_base = parent.repository.sia_base, - sender_name = parent.child_handle, - recipient_name = parent.parent_handle, - bpki_cms_cert = parent.certificate)) - - except rpki.irdb.Repository.DoesNotExist: + parent_pdu.get("bsc_handle") != bsc_handle or + parent_pdu.get("repository_handle") != parent.handle or + parent_pdu.get("peer_contact_uri") != parent.service_uri or + parent_pdu.get("sia_base") != parent.repository.sia_base or + parent_pdu.get("sender_name") != parent.child_handle or + parent_pdu.get("recipient_name") != parent.parent_handle or + parent_pdu.findtext(rpki.left_right.tag_bpki_cert, "").decode("base64") != parent.certificate.get_DER()): + q_pdu = SubElement(q_msg, rpki.left_right.tag_parent, + action = "create" if parent_pdu is None else "set", + tag = parent.handle, + self_handle = ca.handle, + parent_handle = parent.handle, + bsc_handle = bsc_handle, + repository_handle = parent.handle, + peer_contact_uri = parent.service_uri, + sia_base = parent.repository.sia_base, + sender_name = parent.child_handle, + recipient_name = parent.parent_handle) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = parent.certificate.get_Base64() + + except rpki.irdb.models.Repository.DoesNotExist: pass try: @@ -1451,31 +1504,32 @@ class Zookeeper(object): parent_pdu = parent_pdus.pop(ca.handle, None) if (parent_pdu is None or - parent_pdu.bsc_handle != bsc_handle or - parent_pdu.repository_handle != ca.handle or - parent_pdu.peer_contact_uri != ca.rootd.service_uri or - parent_pdu.sia_base != ca.rootd.repository.sia_base or - parent_pdu.sender_name != ca.handle or - parent_pdu.recipient_name != ca.handle or - parent_pdu.bpki_cms_cert != ca.rootd.certificate): - rpkid_query.append(rpki.left_right.parent_elt.make_pdu( - action = "create" if parent_pdu is None else "set", - tag = ca.handle, - self_handle = ca.handle, - parent_handle = ca.handle, - bsc_handle = bsc_handle, - repository_handle = ca.handle, - peer_contact_uri = ca.rootd.service_uri, - sia_base = ca.rootd.repository.sia_base, - sender_name = ca.handle, - recipient_name = ca.handle, - bpki_cms_cert = ca.rootd.certificate)) - - except rpki.irdb.Rootd.DoesNotExist: + parent_pdu.get("bsc_handle") != bsc_handle or + parent_pdu.get("repository_handle") != ca.handle or + parent_pdu.get("peer_contact_uri") != ca.rootd.service_uri or + parent_pdu.get("sia_base") != ca.rootd.repository.sia_base or + parent_pdu.get("sender_name") != ca.handle or + parent_pdu.get("recipient_name") != ca.handle or + parent_pdu.findtext(rpki.left_right.tag_bpki_cert).decode("base64") != ca.rootd.certificate.get_DER()): + q_pdu = SubElement(q_msg, rpki.left_right.tag_parent, + action = "create" if parent_pdu is None else "set", + tag = ca.handle, + self_handle = ca.handle, + parent_handle = ca.handle, + bsc_handle = bsc_handle, + repository_handle = ca.handle, + peer_contact_uri = ca.rootd.service_uri, + sia_base = ca.rootd.repository.sia_base, + sender_name = ca.handle, + recipient_name = ca.handle) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = ca.rootd.certificate.get_Base64() + + except rpki.irdb.models.Rootd.DoesNotExist: pass - rpkid_query.extend(rpki.left_right.parent_elt.make_pdu( - action = "destroy", self_handle = ca.handle, parent_handle = p) for p in parent_pdus) + for parent_handle in parent_pdus: + SubElement(q_msg, rpki.left_right.tag_parent, action = "destroy", + self_handle = ca.handle, parent_handle = parent_handle) # Children are simpler than parents, because they call us, so no URL # to construct and figuring out what certificate to use is their @@ -1486,33 +1540,29 @@ class Zookeeper(object): child_pdu = child_pdus.pop(child.handle, None) if (child_pdu is None or - child_pdu.bsc_handle != bsc_handle or - child_pdu.bpki_cert != child.certificate): - rpkid_query.append(rpki.left_right.child_elt.make_pdu( - action = "create" if child_pdu is None else "set", - tag = child.handle, - self_handle = ca.handle, - child_handle = child.handle, - bsc_handle = bsc_handle, - bpki_cert = child.certificate)) - - rpkid_query.extend(rpki.left_right.child_elt.make_pdu( - action = "destroy", self_handle = ca.handle, child_handle = c) for c in child_pdus) + child_pdu.get("bsc_handle") != bsc_handle or + child_pdu.findtext(rpki.left_right.tag_bpki_cert).decode("base64") != child.certificate.get_DER()): + q_pdu = SubElement(q_msg, rpki.left_right.tag_child, + action = "create" if child_pdu is None else "set", + tag = child.handle, + self_handle = ca.handle, + child_handle = child.handle, + bsc_handle = bsc_handle) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = child.certificate.get_Base64() + + for child_handle in child_pdus: + SubElement(q_msg, rpki.left_right.tag_child, action = "destroy", + self_handle = ca.handle, child_handle = child_handle) # If caller wants us to poke rpkid, add that to the very end of the message if poke: - rpkid_query.append(rpki.left_right.self_elt.make_pdu( - action = "set", self_handle = ca.handle, run_now = "yes")) + SubElement(q_msg, rpki.left_right.tag_self, action = "set", self_handle = ca.handle, run_now = "yes") - # If we changed anything, ship updates off to rpkid + # If we changed anything, ship updates off to rpkid. - if rpkid_query: - rpkid_reply = self.call_rpkid(rpkid_query) - bsc_pdus = dict((x.bsc_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.bsc_elt)) - if bsc_handle in bsc_pdus and bsc_pdus[bsc_handle].pkcs10_request: - bsc_req = bsc_pdus[bsc_handle].pkcs10_request - self.check_error_report(rpkid_reply) + if len(q_msg) > 0: + self.call_rpkid(q_msg) def synchronize_pubd_core(self): @@ -1532,43 +1582,57 @@ class Zookeeper(object): if not self.run_pubd: return - # Make sure that pubd's BPKI CRL is up to date. - - self.call_pubd(rpki.publication.config_elt.make_pdu( - action = "set", - bpki_crl = self.server_ca.latest_crl)) - # See what pubd already has on file - pubd_reply = self.call_pubd(rpki.publication.client_elt.make_pdu(action = "list")) - client_pdus = dict((x.client_handle, x) for x in pubd_reply if isinstance(x, rpki.publication.client_elt)) - pubd_query = [] + q_msg = self._compose_publication_control_query() + SubElement(q_msg, rpki.publication_control.tag_client, action = "list") + r_msg = self.call_pubd(q_msg) + client_pdus = dict((r_pdu.get("client_handle"), r_pdu) + for r_pdu in r_msg) # Check all clients + q_msg = self._compose_publication_control_query() + for client in self.server_ca.clients.all(): client_pdu = client_pdus.pop(client.handle, None) if (client_pdu is None or - client_pdu.base_uri != client.sia_base or - client_pdu.bpki_cert != client.certificate): - pubd_query.append(rpki.publication.client_elt.make_pdu( - action = "create" if client_pdu is None else "set", - client_handle = client.handle, - bpki_cert = client.certificate, - base_uri = client.sia_base)) + client_pdu.get("base_uri") != client.sia_base or + client_pdu.findtext(rpki.publication_control.tag_bpki_cert, "").decode("base64") != client.certificate.get_DER()): + q_pdu = SubElement(q_msg, rpki.publication_control.tag_client, + action = "create" if client_pdu is None else "set", + client_handle = client.handle, + base_uri = client.sia_base) + SubElement(q_pdu, rpki.publication_control.tag_bpki_cert).text = client.certificate.get_Base64() + + # rootd instances are also a weird sort of client + + for rootd in rpki.irdb.models.Rootd.objects.all(): + + client_handle = rootd.issuer.handle + "-root" + client_pdu = client_pdus.pop(client_handle, None) + sia_base = "rsync://%s/%s/%s/" % (self.rsync_server, self.rsync_module, client_handle) + + if (client_pdu is None or + client_pdu.get("base_uri") != sia_base or + client_pdu.findtext(rpki.publication_control.tag_bpki_cert, "").decode("base64") != rootd.issuer.certificate.get_DER()): + q_pdu = SubElement(q_msg, rpki.publication_control.tag_client, + action = "create" if client_pdu is None else "set", + client_handle = client_handle, + base_uri = sia_base) + SubElement(q_pdu, rpki.publication_control.tag_bpki_cert).text = rootd.issuer.certificate.get_Base64() # Delete any unknown clients - pubd_query.extend(rpki.publication.client_elt.make_pdu( - action = "destroy", client_handle = p) for p in client_pdus) + for client_handle in client_pdus: + SubElement(q_msg, rpki.publication_control.tag_client, action = "destroy", client_handle = client_handle) # If we changed anything, ship updates off to pubd - if pubd_query: - pubd_reply = self.call_pubd(pubd_query) - self.check_error_report(pubd_reply) + if len(q_msg) > 0: + self.call_pubd(q_msg) def synchronize_rpkid_deleted_core(self): @@ -1579,22 +1643,23 @@ class Zookeeper(object): inside a Django commit wrapper. """ - rpkid_reply = self.call_rpkid(rpki.left_right.self_elt.make_pdu(action = "list")) - self.check_error_report(rpkid_reply) + q_msg = self._compose_left_right_query() + SubElement(q_msg, rpki.left_right.tag_self, action = "list") + self.call_rpkid(q_msg) - self_handles = set(s.self_handle for s in rpkid_reply) - ca_handles = set(ca.handle for ca in rpki.irdb.ResourceHolderCA.objects.all()) + self_handles = set(s.get("self_handle") for s in q_msg) + ca_handles = set(ca.handle for ca in rpki.irdb.models.ResourceHolderCA.objects.all()) assert ca_handles <= self_handles - rpkid_query = [rpki.left_right.self_elt.make_pdu(action = "destroy", self_handle = handle) - for handle in (self_handles - ca_handles)] + q_msg = self._compose_left_right_query() + for handle in (self_handles - ca_handles): + SubElement(q_msg, rpki.left_right.tag_self, action = "destroy", self_handle = handle) - if rpkid_query: - rpkid_reply = self.call_rpkid(rpkid_query) - self.check_error_report(rpkid_reply) + if len(q_msg) > 0: + self.call_rpkid(q_msg) - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def add_ee_certificate_request(self, pkcs10, resources): """ Check a PKCS #10 request to see if it complies with the @@ -1619,7 +1684,7 @@ class Zookeeper(object): ee_request.address_ranges.create(start_ip = str(r.min), end_ip = str(r.max), version = 6) - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def add_router_certificate_request(self, router_certificate_request_xml, valid_until = None): """ Read XML file containing one or more router certificate requests, @@ -1631,16 +1696,15 @@ class Zookeeper(object): router-ID supplied in the XML. """ - xml = ElementTree(file = router_certificate_request_xml).getroot() - rpki.relaxng.router_certificate.assertValid(xml) + x = etree_read(router_certificate_request_xml, schema = rpki.relaxng.router_certificate) - for req in xml.getiterator(routercert_xmlns + "router_certificate_request"): + for x in x.getiterator(tag_router_certificate_request): - pkcs10 = rpki.x509.PKCS10(Base64 = req.text) - router_id = long(req.get("router_id")) - asns = rpki.resource_set.resource_set_as(req.get("asn")) + pkcs10 = rpki.x509.PKCS10(Base64 = x.text) + router_id = long(x.get("router_id")) + asns = rpki.resource_set.resource_set_as(x.get("asn")) if not valid_until: - valid_until = req.get("valid_until") + valid_until = x.get("valid_until") if valid_until and isinstance(valid_until, (str, unicode)): valid_until = rpki.sundial.datetime.fromXMLtime(valid_until) @@ -1667,7 +1731,7 @@ class Zookeeper(object): ee_request.asns.create(start_as = str(r.min), end_as = str(r.max)) - @django.db.transaction.commit_on_success + @django.db.transaction.atomic def delete_router_certificate_request(self, gski): """ Delete a router certificate request from this RPKI entity. diff --git a/rpki/irdbd.py b/rpki/irdbd.py index ae08b6fb..d58128e5 100644 --- a/rpki/irdbd.py +++ b/rpki/irdbd.py @@ -25,8 +25,7 @@ import os import time import logging import argparse -import urlparse -import rpki.http +import rpki.http_simple import rpki.config import rpki.resource_set import rpki.relaxng @@ -36,120 +35,114 @@ import rpki.log import rpki.x509 import rpki.daemonize +from lxml.etree import Element, SubElement + logger = logging.getLogger(__name__) class main(object): def handle_list_resources(self, q_pdu, r_msg): - child = rpki.irdb.Child.objects.get( - issuer__handle__exact = q_pdu.self_handle, - handle = q_pdu.child_handle) + self_handle = q_pdu.get("self_handle") + child_handle = q_pdu.get("child_handle") + child = rpki.irdb.models.Child.objects.get( + issuer__handle__exact = self_handle, + handle = child_handle) resources = child.resource_bag - r_pdu = rpki.left_right.list_resources_elt() - r_pdu.tag = q_pdu.tag - r_pdu.self_handle = q_pdu.self_handle - r_pdu.child_handle = q_pdu.child_handle - r_pdu.valid_until = child.valid_until.strftime("%Y-%m-%dT%H:%M:%SZ") - r_pdu.asn = resources.asn - r_pdu.ipv4 = resources.v4 - r_pdu.ipv6 = resources.v6 - r_msg.append(r_pdu) + r_pdu = SubElement(r_msg, rpki.left_right.tag_list_resources, self_handle = self_handle, child_handle = child_handle, + valid_until = child.valid_until.strftime("%Y-%m-%dT%H:%M:%SZ")) + for k, v in (("asn", resources.asn), + ("ipv4", resources.v4), + ("ipv6", resources.v6), + ("tag", q_pdu.get("tag"))): + if v: + r_pdu.set(k, str(v)) def handle_list_roa_requests(self, q_pdu, r_msg): - for request in rpki.irdb.ROARequest.objects.raw(""" + self_handle = q_pdu.get("self_handle") + for request in rpki.irdb.models.ROARequest.objects.raw(""" SELECT irdb_roarequest.* FROM irdb_roarequest, irdb_resourceholderca WHERE irdb_roarequest.issuer_id = irdb_resourceholderca.id AND irdb_resourceholderca.handle = %s - """, [q_pdu.self_handle]): + """, [self_handle]): prefix_bag = request.roa_prefix_bag - r_pdu = rpki.left_right.list_roa_requests_elt() - r_pdu.tag = q_pdu.tag - r_pdu.self_handle = q_pdu.self_handle - r_pdu.asn = request.asn - r_pdu.ipv4 = prefix_bag.v4 - r_pdu.ipv6 = prefix_bag.v6 - r_msg.append(r_pdu) + r_pdu = SubElement(r_msg, rpki.left_right.tag_list_roa_requests, self_handle = self_handle, asn = str(request.asn)) + for k, v in (("ipv4", prefix_bag.v4), + ("ipv6", prefix_bag.v6), + ("tag", q_pdu.get("tag"))): + if v: + r_pdu.set(k, str(v)) def handle_list_ghostbuster_requests(self, q_pdu, r_msg): - ghostbusters = rpki.irdb.GhostbusterRequest.objects.filter( - issuer__handle__exact = q_pdu.self_handle, - parent__handle__exact = q_pdu.parent_handle) + self_handle = q_pdu.get("self_handle") + parent_handle = q_pdu.get("parent_handle") + ghostbusters = rpki.irdb.models.GhostbusterRequest.objects.filter( + issuer__handle__exact = self_handle, + parent__handle__exact = parent_handle) if ghostbusters.count() == 0: - ghostbusters = rpki.irdb.GhostbusterRequest.objects.filter( - issuer__handle__exact = q_pdu.self_handle, + ghostbusters = rpki.irdb.models.GhostbusterRequest.objects.filter( + issuer__handle__exact = self_handle, parent = None) for ghostbuster in ghostbusters: - r_pdu = rpki.left_right.list_ghostbuster_requests_elt() - r_pdu.tag = q_pdu.tag - r_pdu.self_handle = q_pdu.self_handle - r_pdu.parent_handle = q_pdu.parent_handle - r_pdu.vcard = ghostbuster.vcard - r_msg.append(r_pdu) + r_pdu = SubElement(r_msg, q_pdu.tag, self_handle = self_handle, parent_handle = parent_handle) + if q_pdu.get("tag"): + r_pdu.set("tag", q_pdu.get("tag")) + r_pdu.text = ghostbuster.vcard def handle_list_ee_certificate_requests(self, q_pdu, r_msg): - for ee_req in rpki.irdb.EECertificateRequest.objects.filter(issuer__handle__exact = q_pdu.self_handle): + self_handle = q_pdu.get("self_handle") + for ee_req in rpki.irdb.models.EECertificateRequest.objects.filter(issuer__handle__exact = self_handle): resources = ee_req.resource_bag - r_pdu = rpki.left_right.list_ee_certificate_requests_elt() - r_pdu.tag = q_pdu.tag - r_pdu.self_handle = q_pdu.self_handle - r_pdu.gski = ee_req.gski - r_pdu.valid_until = ee_req.valid_until.strftime("%Y-%m-%dT%H:%M:%SZ") - r_pdu.asn = resources.asn - r_pdu.ipv4 = resources.v4 - r_pdu.ipv6 = resources.v6 - r_pdu.cn = ee_req.cn - r_pdu.sn = ee_req.sn - r_pdu.eku = ee_req.eku - r_pdu.pkcs10 = ee_req.pkcs10 - r_msg.append(r_pdu) - - def handler(self, query, path, cb): + r_pdu = SubElement(r_msg, q_pdu.tag, self_handle = self_handle, gski = ee_req.gski, + valid_until = ee_req.valid_until.strftime("%Y-%m-%dT%H:%M:%SZ"), + cn = ee_req.cn, sn = ee_req.sn) + for k, v in (("asn", resources.asn), + ("ipv4", resources.v4), + ("ipv6", resources.v6), + ("eku", ee_req.eku), + ("tag", q_pdu.get("tag"))): + if v: + r_pdu.set(k, str(v)) + SubElement(r_pdu, rpki.left_right.tag_pkcs10).text = ee_req.pkcs10.get_Base64() + + def handler(self, request, q_der): try: - q_pdu = None - r_msg = rpki.left_right.msg.reply() from django.db import connection connection.cursor() # Reconnect to mysqld if necessary self.start_new_transaction() - serverCA = rpki.irdb.ServerCA.objects.get() + serverCA = rpki.irdb.models.ServerCA.objects.get() rpkid = serverCA.ee_certificates.get(purpose = "rpkid") + irdbd = serverCA.ee_certificates.get(purpose = "irdbd") + q_cms = rpki.left_right.cms_msg(DER = q_der) + q_msg = q_cms.unwrap((serverCA.certificate, rpkid.certificate)) + self.cms_timestamp = q_cms.check_replay(self.cms_timestamp, request.path) + if q_msg.get("type") != "query": + raise rpki.exceptions.BadQuery("Message type is %s, expected query" % q_msg.get("type")) + r_msg = Element(rpki.left_right.tag_msg, nsmap = rpki.left_right.nsmap, + type = "reply", version = rpki.left_right.version) try: - q_cms = rpki.left_right.cms_msg(DER = query) - q_msg = q_cms.unwrap((serverCA.certificate, rpkid.certificate)) - self.cms_timestamp = q_cms.check_replay(self.cms_timestamp, path) - if not isinstance(q_msg, rpki.left_right.msg) or not q_msg.is_query(): - raise rpki.exceptions.BadQuery("Unexpected %r PDU" % q_msg) for q_pdu in q_msg: - self.dispatch(q_pdu, r_msg) - except (rpki.async.ExitNow, SystemExit): - raise + getattr(self, "handle_" + q_pdu.tag[len(rpki.left_right.xmlns):])(q_pdu, r_msg) + except Exception, e: - logger.exception("Exception while handling HTTP request") - if q_pdu is None: - r_msg.append(rpki.left_right.report_error_elt.from_exception(e)) - else: - r_msg.append(rpki.left_right.report_error_elt.from_exception(e, q_pdu.self_handle, q_pdu.tag)) - irdbd = serverCA.ee_certificates.get(purpose = "irdbd") - cb(200, body = rpki.left_right.cms_msg().wrap(r_msg, irdbd.private_key, irdbd.certificate)) - except (rpki.async.ExitNow, SystemExit): - raise + logger.exception("Exception processing PDU %r", q_pdu) + r_pdu = SubElement(r_msg, rpki.left_right.tag_report_error, error_code = e.__class__.__name__) + r_pdu.text = str(e) + if q_pdu.get("tag") is not None: + r_pdu.set("tag", q_pdu.get("tag")) + + request.send_cms_response(rpki.left_right.cms_msg().wrap(r_msg, irdbd.private_key, irdbd.certificate)) + except Exception, e: logger.exception("Unhandled exception while processing HTTP request") - cb(500, reason = "Unhandled exception %s: %s" % (e.__class__.__name__, e)) - - def dispatch(self, q_pdu, r_msg): - try: - handler = self.dispatch_vector[type(q_pdu)] - except KeyError: - raise rpki.exceptions.BadQuery("Unexpected %r PDU" % q_pdu) - else: - handler(q_pdu, r_msg) + request.send_error(500, "Unhandled exception %s: %s" % (e.__class__.__name__, e)) def __init__(self, **kwargs): global rpki # pylint: disable=W0602 - os.environ["TZ"] = "UTC" + os.environ.update(TZ = "UTC", + DJANGO_SETTINGS_MODULE = "rpki.django_settings.irdb") time.tzset() parser = argparse.ArgumentParser(description = __doc__) @@ -166,7 +159,7 @@ class main(object): rpki.log.init("irdbd", args) - self.cfg = rpki.config.parser(args.config, "irdbd") + self.cfg = rpki.config.parser(set_filename = args.config, section = "irdbd") self.cfg.set_global_flags() if not args.foreground: @@ -185,45 +178,31 @@ class main(object): def main(self): - global rpki # pylint: disable=W0602 - - import django - - from django.conf import settings - startup_msg = self.cfg.get("startup-message", "") if startup_msg: logger.info(startup_msg) - # Do -not- turn on DEBUG here except for short-lived tests, - # otherwise irdbd will eventually run out of memory and crash. - # - # If you must enable debugging, use django.db.reset_queries() to - # clear the query list manually, but it's probably better just to - # run with debugging disabled, since that's the expectation for - # production code. - # - # https://docs.djangoproject.com/en/dev/faq/models/#why-is-django-leaking-memory - - settings.configure( - DATABASES = { - "default" : { - "ENGINE" : "django.db.backends.mysql", - "NAME" : self.cfg.get("sql-database"), - "USER" : self.cfg.get("sql-username"), - "PASSWORD" : self.cfg.get("sql-password"), - "HOST" : "", - "PORT" : "" }}, - INSTALLED_APPS = ("rpki.irdb",), - MIDDLEWARE_CLASSES = (), # API change, feh - ) - - if django.VERSION >= (1, 7): # API change, feh - from django.apps import apps - apps.populate(settings.INSTALLED_APPS) + # Now that we know which configuration file to use, it's OK to + # load modules that require Django's settings module. + + import django + django.setup() + global rpki # pylint: disable=W0602 import rpki.irdb # pylint: disable=W0621 + self.http_server_host = self.cfg.get("server-host", "") + self.http_server_port = self.cfg.getint("server-port") + + self.cms_timestamp = None + + rpki.http_simple.server( + host = self.http_server_host, + port = self.http_server_port, + handlers = self.handler) + + def start_new_transaction(self): + # Entirely too much fun with read-only access to transactional databases. # # http://stackoverflow.com/questions/3346124/how-do-i-force-django-to-ignore-any-caches-and-reload-data @@ -244,33 +223,7 @@ class main(object): # the transaction isolation snapshot. import django.db.transaction - self.start_new_transaction = django.db.transaction.commit_manually(django.db.transaction.commit) - self.dispatch_vector = { - rpki.left_right.list_resources_elt : self.handle_list_resources, - rpki.left_right.list_roa_requests_elt : self.handle_list_roa_requests, - rpki.left_right.list_ghostbuster_requests_elt : self.handle_list_ghostbuster_requests, - rpki.left_right.list_ee_certificate_requests_elt : self.handle_list_ee_certificate_requests} - - try: - self.http_server_host = self.cfg.get("server-host", "") - self.http_server_port = self.cfg.getint("server-port") - except: # pylint: disable=W0702 - # - # Backwards compatibility, remove this eventually. - # - u = urlparse.urlparse(self.cfg.get("http-url")) - if (u.scheme not in ("", "http") or - u.username is not None or - u.password is not None or - u.params or u.query or u.fragment): - raise - self.http_server_host = u.hostname - self.http_server_port = int(u.port) - - self.cms_timestamp = None - - rpki.http.server( - host = self.http_server_host, - port = self.http_server_port, - handlers = self.handler) + with django.db.transaction.atomic(): + #django.db.transaction.commit() + pass diff --git a/rpki/left_right.py b/rpki/left_right.py index c8b6d19b..7189f888 100644 --- a/rpki/left_right.py +++ b/rpki/left_right.py @@ -21,12 +21,13 @@ RPKI "left-right" protocol. """ +import base64 import logging -import rpki.resource_set +import collections + import rpki.x509 import rpki.sql import rpki.exceptions -import rpki.xml_utils import rpki.http import rpki.up_down import rpki.relaxng @@ -36,51 +37,203 @@ import rpki.publication import rpki.async import rpki.rpkid_tasks -logger = logging.getLogger(__name__) - -## @var enforce_strict_up_down_xml_sender -# Enforce strict checking of XML "sender" field in up-down protocol +from lxml.etree import Element, SubElement, tostring as ElementToString -enforce_strict_up_down_xml_sender = False - -class left_right_namespace(object): - """ - XML namespace parameters for left-right protocol. - """ +logger = logging.getLogger(__name__) - xmlns = rpki.relaxng.left_right.xmlns - nsmap = rpki.relaxng.left_right.nsmap -class data_elt(rpki.xml_utils.data_elt, rpki.sql.sql_persistent, left_right_namespace): +xmlns = rpki.relaxng.left_right.xmlns +nsmap = rpki.relaxng.left_right.nsmap +version = rpki.relaxng.left_right.version + +tag_bpki_cert = xmlns + "bpki_cert" +tag_bpki_glue = xmlns + "bpki_glue" +tag_bsc = xmlns + "bsc" +tag_child = xmlns + "child" +tag_list_ee_certificate_requests = xmlns + "list_ee_certificate_requests" +tag_list_ghostbuster_requests = xmlns + "list_ghostbuster_requests" +tag_list_published_objects = xmlns + "list_published_objects" +tag_list_received_resources = xmlns + "list_received_resources" +tag_list_resources = xmlns + "list_resources" +tag_list_roa_requests = xmlns + "list_roa_requests" +tag_msg = xmlns + "msg" +tag_parent = xmlns + "parent" +tag_pkcs10 = xmlns + "pkcs10" +tag_pkcs10_request = xmlns + "pkcs10_request" +tag_report_error = xmlns + "report_error" +tag_repository = xmlns + "repository" +tag_self = xmlns + "self" +tag_signing_cert = xmlns + "signing_cert" +tag_signing_cert_crl = xmlns + "signing_cert_crl" + + +class base_elt(rpki.sql.sql_persistent): """ - Virtual class for top-level left-right protocol data elements. + Virtual class for persistent left-right protocol elements. + These classes are being phased out in favor of Django ORM models. """ handles = () + attributes = () + elements = () + booleans = () self_id = None self_handle = None + def __str__(self): + return ElementToString(self.toXML(), pretty_print = True, encoding = "us-ascii") + + @classmethod + def fromXML(cls, elt): + self = cls() + for key in self.attributes: + val = elt.get(key, None) + if val is not None: + val = val.encode("ascii") + if isinstance(self.attributes, dict) and self.attributes[key] is not None: + val = self.attributes[key](val) + elif val.isdigit() and not key.endswith("_handle"): + val = long(val) + setattr(self, key, val) + for key in self.booleans: + setattr(self, key, elt.get(key, False)) + for b64 in elt: + assert b64.tag.startswith(xmlns) + setattr(self, b64.tag[len(xmlns):], self.elements[b64.tag](Base64 = b64.text)) + return self + + def toXML(self): + elt = Element(self.element_name, nsmap = nsmap) + for key in self.attributes: + val = getattr(self, key, None) + if val is not None: + elt.set(key, str(val)) + for key in self.booleans: + if getattr(self, key, False): + elt.set(key, "yes") + for name in self.elements: + value = getattr(self, name[len(xmlns):], None) + if value is not None and not value.empty(): + SubElement(elt, name, nsmap = nsmap).text = value.get_Base64() + return elt + + def make_reply(self, r_pdu = None): + if r_pdu is None: + r_pdu = self.__class__() + self.make_reply_clone_hook(r_pdu) + handle_name = self.element_name[len(xmlns):] + "_handle" + setattr(r_pdu, handle_name, getattr(self, handle_name, None)) + else: + self.make_reply_clone_hook(r_pdu) + for b in r_pdu.booleans: + setattr(r_pdu, b, False) + r_pdu.action = self.action + r_pdu.tag = self.tag + return r_pdu + + def serve_fetch_one(self): + """ + Find the object on which a get, set, or destroy method should + operate. + """ + + r = self.serve_fetch_one_maybe() + if r is None: + raise rpki.exceptions.NotFound + return r + + def serve_post_save_hook(self, q_pdu, r_pdu, cb, eb): + cb() + + def serve_create(self, r_msg, cb, eb): + r_pdu = self.make_reply() + def one(): + self.sql_store() + setattr(r_pdu, self.sql_template.index, getattr(self, self.sql_template.index)) + self.serve_post_save_hook(self, r_pdu, two, eb) + def two(): + r_msg.append(r_pdu) + cb() + oops = self.serve_fetch_one_maybe() + if oops is not None: + raise rpki.exceptions.DuplicateObject("Object already exists: %r[%r] %r[%r]" % ( + self, getattr(self, self.element_name[len(xmlns):] + "_handle"), + oops, getattr(oops, oops.element_name[len(xmlns):] + "_handle"))) + self.serve_pre_save_hook(self, r_pdu, one, eb) + + def serve_set(self, r_msg, cb, eb): + db_pdu = self.serve_fetch_one() + r_pdu = self.make_reply() + for a in db_pdu.sql_template.columns[1:]: + v = getattr(self, a, None) + if v is not None: + setattr(db_pdu, a, v) + db_pdu.sql_mark_dirty() + def one(): + db_pdu.sql_store() + db_pdu.serve_post_save_hook(self, r_pdu, two, eb) + def two(): + r_msg.append(r_pdu) + cb() + db_pdu.serve_pre_save_hook(self, r_pdu, one, eb) + + def serve_get(self, r_msg, cb, eb): + r_pdu = self.serve_fetch_one() + self.make_reply(r_pdu) + r_msg.append(r_pdu) + cb() + + def serve_list(self, r_msg, cb, eb): + for r_pdu in self.serve_fetch_all(): + self.make_reply(r_pdu) + r_msg.append(r_pdu) + cb() + + def serve_destroy_hook(self, cb, eb): + cb() + + def serve_destroy(self, r_msg, cb, eb): + def done(): + db_pdu.sql_delete() + r_msg.append(self.make_reply()) + cb() + db_pdu = self.serve_fetch_one() + db_pdu.serve_destroy_hook(done, eb) + + def serve_dispatch(self, r_msg, cb, eb): + # Transition hack: handle the .toXML() call for old handlers. + fake_r_msg = [] + def fake_convert(): + r_msg.extend(r_pdu.toXML() if isinstance(r_pdu, base_elt) else r_pdu + for r_pdu in fake_r_msg) + def fake_cb(): + fake_convert() + cb() + def fake_eb(e): + fake_convert() + eb(e) + method = getattr(self, "serve_" + self.action, None) + if method is None: + raise rpki.exceptions.BadQuery("Unexpected query: action %s" % self.action) + method(fake_r_msg, fake_cb, fake_eb) + + def unimplemented_control(self, *controls): + unimplemented = [x for x in controls if getattr(self, x, False)] + if unimplemented: + raise rpki.exceptions.NotImplementedYet("Unimplemented control %s" % ", ".join(unimplemented)) + @property @rpki.sql.cache_reference def self(self): - """ - Fetch self object to which this object links. - """ return self_elt.sql_fetch(self.gctx, self.self_id) @property @rpki.sql.cache_reference def bsc(self): - """ - Return BSC object to which this object links. - """ return bsc_elt.sql_fetch(self.gctx, self.bsc_id) def make_reply_clone_hook(self, r_pdu): - """ - Set handles when cloning, including _id -> _handle translation. - """ if r_pdu.self_handle is None: r_pdu.self_handle = self.self_handle for tag, elt in self.handles: @@ -94,36 +247,23 @@ class data_elt(rpki.xml_utils.data_elt, rpki.sql.sql_persistent, left_right_name @classmethod def serve_fetch_handle(cls, gctx, self_id, handle): - """ - Find an object based on its handle. - """ - return cls.sql_fetch_where1(gctx, cls.element_name + "_handle = %s AND self_id = %s", (handle, self_id)) + name = cls.element_name[len(xmlns):] + return cls.sql_fetch_where1(gctx, name + "_handle = %s AND self_id = %s", (handle, self_id)) def serve_fetch_one_maybe(self): - """ - Find the object on which a get, set, or destroy method should - operate, or which would conflict with a create method. - """ - where = "%s.%s_handle = %%s AND %s.self_id = self.self_id AND self.self_handle = %%s" % ((self.element_name,) * 3) - args = (getattr(self, self.element_name + "_handle"), self.self_handle) + name = self.element_name[len(xmlns):] + where = "%s.%s_handle = %%s AND %s.self_id = self.self_id AND self.self_handle = %%s" % (name, name, name) + args = (getattr(self, name + "_handle"), self.self_handle) + logger.debug(".serve_fetch_one_maybe() %s %s", args[0], args[1]) return self.sql_fetch_where1(self.gctx, where, args, "self") def serve_fetch_all(self): - """ - Find the objects on which a list method should operate. - """ - where = "%s.self_id = self.self_id and self.self_handle = %%s" % self.element_name + name = self.element_name[len(xmlns):] + where = "%s.self_id = self.self_id and self.self_handle = %%s" % name return self.sql_fetch_where(self.gctx, where, (self.self_handle,), "self") def serve_pre_save_hook(self, q_pdu, r_pdu, cb, eb): - """ - Hook to do _handle => _id translation before saving. - - self is always the object to be saved to SQL. For create - operations, self and q_pdu are be the same object; for set - operations, self is the pre-existing object from SQL and q_pdu is - the set request received from the the IRBE. - """ + # self is always the object to be saved to SQL. for tag, elt in self.handles: id_name = tag + "_id" if getattr(self, id_name, None) is None: @@ -133,17 +273,21 @@ class data_elt(rpki.xml_utils.data_elt, rpki.sql.sql_persistent, left_right_name setattr(self, id_name, getattr(x, id_name)) cb() -class self_elt(data_elt): + +class self_elt(base_elt): """ <self/> element. """ - element_name = "self" + element_name = xmlns + "self" attributes = ("action", "tag", "self_handle", "crl_interval", "regen_margin") - elements = ("bpki_cert", "bpki_glue") booleans = ("rekey", "reissue", "revoke", "run_now", "publish_world_now", "revoke_forgotten", "clear_replay_protection") + elements = collections.OrderedDict(( + (tag_bpki_cert, rpki.x509.X509), + (tag_bpki_glue, rpki.x509.X509))) + sql_template = rpki.sql.template( "self", "self_id", @@ -168,58 +312,33 @@ class self_elt(data_elt): @property def bscs(self): - """ - Fetch all BSC objects that link to this self object. - """ return bsc_elt.sql_fetch_where(self.gctx, "self_id = %s", (self.self_id,)) @property def repositories(self): - """ - Fetch all repository objects that link to this self object. - """ return repository_elt.sql_fetch_where(self.gctx, "self_id = %s", (self.self_id,)) @property def parents(self): - """ - Fetch all parent objects that link to this self object. - """ return parent_elt.sql_fetch_where(self.gctx, "self_id = %s", (self.self_id,)) @property def children(self): - """ - Fetch all child objects that link to this self object. - """ return child_elt.sql_fetch_where(self.gctx, "self_id = %s", (self.self_id,)) @property def roas(self): - """ - Fetch all ROA objects that link to this self object. - """ return rpki.rpkid.roa_obj.sql_fetch_where(self.gctx, "self_id = %s", (self.self_id,)) @property def ghostbusters(self): - """ - Fetch all Ghostbuster record objects that link to this self object. - """ return rpki.rpkid.ghostbuster_obj.sql_fetch_where(self.gctx, "self_id = %s", (self.self_id,)) @property def ee_certificates(self): - """ - Fetch all EE certificate objects that link to this self object. - """ return rpki.rpkid.ee_cert_obj.sql_fetch_where(self.gctx, "self_id = %s", (self.self_id,)) - def serve_post_save_hook(self, q_pdu, r_pdu, cb, eb): - """ - Extra server actions for self_elt. - """ actions = [] if q_pdu.rekey: actions.append(self.serve_rekey) @@ -240,96 +359,101 @@ class self_elt(data_elt): rpki.async.iterator(actions, loop, cb) def serve_rekey(self, cb, eb): - """ - Handle a left-right rekey action for this self. - """ def loop(iterator, parent): parent.serve_rekey(iterator, eb) rpki.async.iterator(self.parents, loop, cb) def serve_revoke(self, cb, eb): - """ - Handle a left-right revoke action for this self. - """ def loop(iterator, parent): parent.serve_revoke(iterator, eb) rpki.async.iterator(self.parents, loop, cb) def serve_reissue(self, cb, eb): - """ - Handle a left-right reissue action for this self. - """ def loop(iterator, parent): parent.serve_reissue(iterator, eb) rpki.async.iterator(self.parents, loop, cb) def serve_revoke_forgotten(self, cb, eb): - """ - Handle a left-right revoke_forgotten action for this self. - """ def loop(iterator, parent): parent.serve_revoke_forgotten(iterator, eb) rpki.async.iterator(self.parents, loop, cb) def serve_clear_replay_protection(self, cb, eb): - """ - Handle a left-right clear_replay_protection action for this self. - """ def loop(iterator, obj): obj.serve_clear_replay_protection(iterator, eb) rpki.async.iterator(self.parents + self.children + self.repositories, loop, cb) def serve_destroy_hook(self, cb, eb): - """ - Extra cleanup actions when destroying a self_elt. - """ def loop(iterator, parent): parent.delete(iterator) rpki.async.iterator(self.parents, loop, cb) def serve_publish_world_now(self, cb, eb): - """ - Handle a left-right publish_world_now action for this self. - - The publication stuff needs refactoring, right now publication is - interleaved with local operations in a way that forces far too - many bounces through the task system for any complex update. The - whole thing ought to be rewritten to queue up outgoing publication - PDUs and only send them when we're all done or when we need to - force publication at a particular point in a multi-phase operation. - - Once that reorganization has been done, this method should be - rewritten to reuse the low-level publish() methods that each - object will have...but we're not there yet. So, for now, we just - do this via brute force. Think of it as a trial version to see - whether we've identified everything that needs to be republished - for this operation. - """ + publisher = rpki.rpkid.publication_queue() + repositories = set() + objects = dict() def loop(iterator, parent): - q_msg = rpki.publication.msg.query() - for ca in parent.cas: - ca_detail = ca.active_ca_detail - if ca_detail is not None: - q_msg.append(rpki.publication.crl_elt.make_publish( - ca_detail.crl_uri, ca_detail.latest_crl)) - q_msg.append(rpki.publication.manifest_elt.make_publish( - ca_detail.manifest_uri, ca_detail.latest_manifest)) - q_msg.extend(rpki.publication.certificate_elt.make_publish( - c.uri, c.cert) for c in ca_detail.child_certs) - q_msg.extend(rpki.publication.roa_elt.make_publish( - r.uri, r.roa) for r in ca_detail.roas if r.roa is not None) - q_msg.extend(rpki.publication.ghostbuster_elt.make_publish( - g.uri, g.ghostbuster) for g in ca_detail.ghostbusters) - parent.repository.call_pubd(iterator, eb, q_msg) + repository = parent.repository + if repository.peer_contact_uri in repositories: + return iterator() + repositories.add(repository.peer_contact_uri) + q_msg = Element(rpki.publication.tag_msg, nsmap = rpki.publication.nsmap, + type = "query", version = rpki.publication.version) + SubElement(q_msg, rpki.publication.tag_list, tag = "list") + def list_handler(r_pdu): + rpki.publication.raise_if_error(r_pdu) + assert r_pdu.tag == rpki.publication.tag_list + assert r_pdu.get("uri") not in objects + objects[r_pdu.get("uri")] = (r_pdu.get("hash"), repository) + repository.call_pubd(iterator, eb, + q_msg, length_check = False, + handlers = dict(list = list_handler)) + + def reconcile(uri, obj, repository): + h, r = objects.pop(uri, (None, None)) + if h is not None: + assert r == repository + publisher.queue(uri = uri, new_obj = obj, old_hash = h, repository = repository) - rpki.async.iterator(self.parents, loop, cb) + def done(): + for parent in self.parents: + repository = parent.repository + for ca in parent.cas: + ca_detail = ca.active_ca_detail + if ca_detail is not None: + reconcile(uri = ca_detail.crl_uri, + obj = ca_detail.latest_crl, + repository = repository) + reconcile(uri = ca_detail.manifest_uri, + obj = ca_detail.latest_manifest, + repository = repository) + for c in ca_detail.child_certs: + reconcile(uri = c.uri, + obj = c.cert, + repository = repository) + for r in ca_detail.roas: + if r.roa is not None: + reconcile(uri = r.uri, + obj = r.roa, + repository = repository) + for g in ca_detail.ghostbusters: + reconcile(uri = g.uri, + obj = g.ghostbuster, + repository = repository) + for c in ca_detail.ee_certificates: + reconcile(uri = c.uri, + obj = c.cert, + repository = repository) + for u in objects: + h, r = objects[u] + publisher.queue(uri = u, old_hash = h, repository = r) + publisher.call_pubd(cb, eb) + + rpki.async.iterator(self.parents, loop, done) def serve_run_now(self, cb, eb): - """ - Handle a left-right run_now action for this self. - """ logger.debug("Forced immediate run of periodic actions for self %s[%d]", self.self_handle, self.self_id) completion = rpki.rpkid_tasks.CompletionHandler(cb) @@ -342,13 +466,11 @@ class self_elt(data_elt): Find the self object upon which a get, set, or destroy action should operate, or which would conflict with a create method. """ + return self.serve_fetch_handle(self.gctx, None, self.self_handle) @classmethod def serve_fetch_handle(cls, gctx, self_id, self_handle): - """ - Find a self object based on its self_handle. - """ return cls.sql_fetch_where1(gctx, "self_handle = %s", (self_handle,)) def serve_fetch_all(self): @@ -357,16 +479,12 @@ class self_elt(data_elt): This is different from the list action for all other objects, where list only works within a given self_id context. """ + return self.sql_fetch_all(self.gctx) def schedule_cron_tasks(self, completion): - """ - Schedule periodic tasks. - """ - if self.cron_tasks is None: self.cron_tasks = tuple(task(self) for task in rpki.rpkid_tasks.task_classes) - for task in self.cron_tasks: self.gctx.task_add(task) completion.register(task) @@ -392,16 +510,20 @@ class self_elt(data_elt): return results -class bsc_elt(data_elt): +class bsc_elt(base_elt): """ <bsc/> (Business Signing Context) element. """ - element_name = "bsc" + element_name = xmlns + "bsc" attributes = ("action", "tag", "self_handle", "bsc_handle", "key_type", "hash_alg", "key_length") - elements = ("signing_cert", "signing_cert_crl", "pkcs10_request") booleans = ("generate_keypair",) + elements = collections.OrderedDict(( + (tag_signing_cert, rpki.x509.X509), + (tag_signing_cert_crl, rpki.x509.CRL), + (tag_pkcs10_request, rpki.x509.PKCS10))) + sql_template = rpki.sql.template( "bsc", "bsc_id", @@ -425,47 +547,42 @@ class bsc_elt(data_elt): @property def repositories(self): - """ - Fetch all repository objects that link to this BSC object. - """ return repository_elt.sql_fetch_where(self.gctx, "bsc_id = %s", (self.bsc_id,)) @property def parents(self): - """ - Fetch all parent objects that link to this BSC object. - """ return parent_elt.sql_fetch_where(self.gctx, "bsc_id = %s", (self.bsc_id,)) @property def children(self): - """ - Fetch all child objects that link to this BSC object. - """ return child_elt.sql_fetch_where(self.gctx, "bsc_id = %s", (self.bsc_id,)) def serve_pre_save_hook(self, q_pdu, r_pdu, cb, eb): """ - Extra server actions for bsc_elt -- handle key generation. For - now this only allows RSA with SHA-256. + Extra server actions -- handle key generation, only RSA with SHA-256 for now. """ + if q_pdu.generate_keypair: assert q_pdu.key_type in (None, "rsa") and q_pdu.hash_alg in (None, "sha256") self.private_key_id = rpki.x509.RSA.generate(keylength = q_pdu.key_length or 2048) self.pkcs10_request = rpki.x509.PKCS10.create(keypair = self.private_key_id) r_pdu.pkcs10_request = self.pkcs10_request - data_elt.serve_pre_save_hook(self, q_pdu, r_pdu, cb, eb) + super(bsc_elt, self).serve_pre_save_hook(q_pdu, r_pdu, cb, eb) -class repository_elt(data_elt): + +class repository_elt(base_elt): """ <repository/> element. """ - element_name = "repository" - attributes = ("action", "tag", "self_handle", "repository_handle", "bsc_handle", "peer_contact_uri") - elements = ("bpki_cert", "bpki_glue") + element_name = xmlns + "repository" + attributes = ("action", "tag", "self_handle", "repository_handle", "bsc_handle", "peer_contact_uri", "rrdp_notification_uri") booleans = ("clear_replay_protection",) + elements = collections.OrderedDict(( + (tag_bpki_cert, rpki.x509.X509), + (tag_bpki_glue, rpki.x509.X509))) + sql_template = rpki.sql.template( "repository", "repository_id", @@ -473,6 +590,7 @@ class repository_elt(data_elt): "self_id", "bsc_id", "peer_contact_uri", + "rrdp_notification_uri", ("bpki_cert", rpki.x509.X509), ("bpki_glue", rpki.x509.X509), ("last_cms_timestamp", rpki.sundial.datetime)) @@ -483,21 +601,16 @@ class repository_elt(data_elt): bpki_cert = None bpki_glue = None last_cms_timestamp = None + rrdp_notification_uri = None def __repr__(self): return rpki.log.log_repr(self, self.repository_handle) @property def parents(self): - """ - Fetch all parent objects that link to this repository object. - """ return parent_elt.sql_fetch_where(self.gctx, "repository_id = %s", (self.repository_id,)) def serve_post_save_hook(self, q_pdu, r_pdu, cb, eb): - """ - Extra server actions for repository_elt. - """ actions = [] if q_pdu.clear_replay_protection: actions.append(self.serve_clear_replay_protection) @@ -506,45 +619,33 @@ class repository_elt(data_elt): rpki.async.iterator(actions, loop, cb) def serve_clear_replay_protection(self, cb, eb): - """ - Handle a left-right clear_replay_protection action for this repository. - """ self.last_cms_timestamp = None self.sql_mark_dirty() cb() - @staticmethod - def default_pubd_handler(pdu): - """ - Default handler for publication response PDUs. - """ - pdu.raise_if_error() - def call_pubd(self, callback, errback, q_msg, handlers = None): + def call_pubd(self, callback, errback, q_msg, handlers = {}, length_check = True): # pylint: disable=W0102 """ Send a message to publication daemon and return the response. As a convenience, attempting to send an empty message returns immediate success without sending anything. - Handlers is a dict of handler functions to process the response + handlers is a dict of handler functions to process the response PDUs. If the tag value in the response PDU appears in the dict, the associated handler is called to process the PDU. If no tag - matches, default_pubd_handler() is called. A handler value of - False suppresses calling of the default handler. + matches, a default handler is called to check for errors; a + handler value of False suppresses calling of the default handler. """ try: self.gctx.sql.sweep() - if not q_msg: + if len(q_msg) == 0: return callback() - if handlers is None: - handlers = {} - for q_pdu in q_msg: - logger.info("Sending %s %s to pubd", q_pdu.action, q_pdu.uri) + logger.info("Sending %r to pubd", q_pdu) bsc = self.bsc q_der = rpki.publication.cms_msg().wrap(q_msg, bsc.private_key_id, bsc.signing_cert, bsc.signing_cert_crl) @@ -557,11 +658,11 @@ class repository_elt(data_elt): r_msg = r_cms.unwrap(bpki_ta_path) r_cms.check_replay_sql(self, self.peer_contact_uri) for r_pdu in r_msg: - handler = handlers.get(r_pdu.tag, self.default_pubd_handler) + handler = handlers.get(r_pdu.get("tag"), rpki.publication.raise_if_error) if handler: logger.debug("Calling pubd handler %r", handler) handler(r_pdu) - if len(q_msg) != len(r_msg): + if length_check and len(q_msg) != len(r_msg): raise rpki.exceptions.BadPublicationReply("Wrong number of response PDUs from pubd: sent %r, got %r" % (q_msg, r_msg)) callback() except (rpki.async.ExitNow, SystemExit): @@ -581,17 +682,21 @@ class repository_elt(data_elt): except Exception, e: errback(e) -class parent_elt(data_elt): + +class parent_elt(base_elt): """ <parent/> element. """ - element_name = "parent" + element_name = xmlns + "parent" attributes = ("action", "tag", "self_handle", "parent_handle", "bsc_handle", "repository_handle", "peer_contact_uri", "sia_base", "sender_name", "recipient_name") - elements = ("bpki_cms_cert", "bpki_cms_glue") booleans = ("rekey", "reissue", "revoke", "revoke_forgotten", "clear_replay_protection") + elements = collections.OrderedDict(( + (tag_bpki_cert, rpki.x509.X509), + (tag_bpki_glue, rpki.x509.X509))) + sql_template = rpki.sql.template( "parent", "parent_id", @@ -603,16 +708,16 @@ class parent_elt(data_elt): "sia_base", "sender_name", "recipient_name", - ("bpki_cms_cert", rpki.x509.X509), - ("bpki_cms_glue", rpki.x509.X509), + ("bpki_cert", rpki.x509.X509), + ("bpki_glue", rpki.x509.X509), ("last_cms_timestamp", rpki.sundial.datetime)) handles = (("self", self_elt), ("bsc", bsc_elt), ("repository", repository_elt)) - bpki_cms_cert = None - bpki_cms_glue = None + bpki_cert = None + bpki_glue = None last_cms_timestamp = None def __repr__(self): @@ -621,22 +726,13 @@ class parent_elt(data_elt): @property @rpki.sql.cache_reference def repository(self): - """ - Fetch repository object to which this parent object links. - """ return repository_elt.sql_fetch(self.gctx, self.repository_id) @property def cas(self): - """ - Fetch all CA objects that link to this parent object. - """ return rpki.rpkid.ca_obj.sql_fetch_where(self.gctx, "parent_id = %s", (self.parent_id,)) def serve_post_save_hook(self, q_pdu, r_pdu, cb, eb): - """ - Extra server actions for parent_elt. - """ actions = [] if q_pdu.rekey: actions.append(self.serve_rekey) @@ -653,33 +749,21 @@ class parent_elt(data_elt): rpki.async.iterator(actions, loop, cb) def serve_rekey(self, cb, eb): - """ - Handle a left-right rekey action for this parent. - """ def loop(iterator, ca): ca.rekey(iterator, eb) rpki.async.iterator(self.cas, loop, cb) def serve_revoke(self, cb, eb): - """ - Handle a left-right revoke action for this parent. - """ def loop(iterator, ca): ca.revoke(cb = iterator, eb = eb) rpki.async.iterator(self.cas, loop, cb) def serve_reissue(self, cb, eb): - """ - Handle a left-right reissue action for this parent. - """ def loop(iterator, ca): ca.reissue(cb = iterator, eb = eb) rpki.async.iterator(self.cas, loop, cb) def serve_clear_replay_protection(self, cb, eb): - """ - Handle a left-right clear_replay_protection action for this parent. - """ self.last_cms_timestamp = None self.sql_mark_dirty() cb() @@ -696,10 +780,11 @@ class parent_elt(data_elt): """ def done(r_msg): - cb(dict((rc.class_name, set(c.cert.gSKI() for c in rc.certs)) - for rc in r_msg.payload.classes)) - - rpki.up_down.list_pdu.query(self, done, eb) + cb(dict((rc.get("class_name"), + set(rpki.x509.X509(Base64 = c.text).gSKI() + for c in rc.getiterator(rpki.up_down.tag_certificate))) + for rc in r_msg.getiterator(rpki.up_down.tag_class))) + self.up_down_list_query(done, eb) def revoke_skis(self, rc_name, skis_to_revoke, cb, eb): @@ -708,12 +793,10 @@ class parent_elt(data_elt): """ def loop(iterator, ski): + def revoked(r_pdu): + iterator() logger.debug("Asking parent %r to revoke class %r, SKI %s", self, rc_name, ski) - q_pdu = rpki.up_down.revoke_pdu() - q_pdu.class_name = rc_name - q_pdu.ski = ski - self.query_up_down(q_pdu, lambda r_pdu: iterator(), eb) - + self.up_down_revoke_query(rc_name, ski, revoked, eb) rpki.async.iterator(skis_to_revoke, loop, cb) @@ -775,17 +858,39 @@ class parent_elt(data_elt): def serve_destroy_hook(self, cb, eb): - """ - Extra server actions when destroying a parent_elt. - """ - self.delete(cb, delete_parent = False) - def query_up_down(self, q_pdu, cb, eb): - """ - Client code for sending one up-down query PDU to this parent. - """ + def _compose_up_down_query(self, query_type): + return Element(rpki.up_down.tag_message, nsmap = rpki.up_down.nsmap, version = rpki.up_down.version, + sender = self.sender_name, recipient = self.recipient_name, type = query_type) + + + def up_down_list_query(self, cb, eb): + q_msg = self._compose_up_down_query("list") + self.query_up_down(q_msg, cb, eb) + + + def up_down_issue_query(self, ca, ca_detail, cb, eb): + pkcs10 = rpki.x509.PKCS10.create( + keypair = ca_detail.private_key_id, + is_ca = True, + caRepository = ca.sia_uri, + rpkiManifest = ca_detail.manifest_uri, + rpkiNotify = ca.parent.repository.rrdp_notification_uri) + q_msg = self._compose_up_down_query("issue") + q_pdu = SubElement(q_msg, rpki.up_down.tag_request, class_name = ca.parent_resource_class) + q_pdu.text = pkcs10.get_Base64() + self.query_up_down(q_msg, cb, eb) + + + def up_down_revoke_query(self, class_name, ski, cb, eb): + q_msg = self._compose_up_down_query("revoke") + SubElement(q_msg, rpki.up_down.tag_key, class_name = class_name, ski = ski) + self.query_up_down(q_msg, cb, eb) + + + def query_up_down(self, q_msg, cb, eb): bsc = self.bsc if bsc is None: @@ -794,11 +899,6 @@ class parent_elt(data_elt): if bsc.signing_cert is None: raise rpki.exceptions.BSCNotReady("BSC %r[%s] is not yet usable" % (bsc.bsc_handle, bsc.bsc_id)) - q_msg = rpki.up_down.message_pdu.make_query( - payload = q_pdu, - sender = self.sender_name, - recipient = self.recipient_name) - q_der = rpki.up_down.cms_msg().wrap(q_msg, bsc.private_key_id, bsc.signing_cert, bsc.signing_cert_crl) @@ -809,10 +909,11 @@ class parent_elt(data_elt): r_msg = r_cms.unwrap((self.gctx.bpki_ta, self.self.bpki_cert, self.self.bpki_glue, - self.bpki_cms_cert, - self.bpki_cms_glue)) + self.bpki_cert, + self.bpki_glue)) r_cms.check_replay_sql(self, self.peer_contact_uri) - r_msg.payload.check_response() + rpki.up_down.check_response(r_msg, q_msg.get("type")) + except (SystemExit, rpki.async.ExitNow): raise except Exception, e: @@ -827,16 +928,20 @@ class parent_elt(data_elt): errback = eb, content_type = rpki.up_down.content_type) -class child_elt(data_elt): + +class child_elt(base_elt): """ <child/> element. """ - element_name = "child" + element_name = xmlns + "child" attributes = ("action", "tag", "self_handle", "child_handle", "bsc_handle") - elements = ("bpki_cert", "bpki_glue") booleans = ("reissue", "clear_replay_protection") + elements = collections.OrderedDict(( + (tag_bpki_cert, rpki.x509.X509), + (tag_bpki_glue, rpki.x509.X509))) + sql_template = rpki.sql.template( "child", "child_id", @@ -858,29 +963,17 @@ class child_elt(data_elt): return rpki.log.log_repr(self, self.child_handle) def fetch_child_certs(self, ca_detail = None, ski = None, unique = False): - """ - Fetch all child_cert objects that link to this child object. - """ return rpki.rpkid.child_cert_obj.fetch(self.gctx, self, ca_detail, ski, unique) @property def child_certs(self): - """ - Fetch all child_cert objects that link to this child object. - """ return self.fetch_child_certs() @property def parents(self): - """ - Fetch all parent objects that link to self object to which this child object links. - """ return parent_elt.sql_fetch_where(self.gctx, "self_id = %s", (self.self_id,)) def serve_post_save_hook(self, q_pdu, r_pdu, cb, eb): - """ - Extra server actions for child_elt. - """ actions = [] if q_pdu.reissue: actions.append(self.serve_reissue) @@ -891,26 +984,17 @@ class child_elt(data_elt): rpki.async.iterator(actions, loop, cb) def serve_reissue(self, cb, eb): - """ - Handle a left-right reissue action for this child. - """ publisher = rpki.rpkid.publication_queue() for child_cert in self.child_certs: child_cert.reissue(child_cert.ca_detail, publisher, force = True) publisher.call_pubd(cb, eb) def serve_clear_replay_protection(self, cb, eb): - """ - Handle a left-right clear_replay_protection action for this child. - """ self.last_cms_timestamp = None self.sql_mark_dirty() cb() def ca_from_class_name(self, class_name): - """ - Fetch the CA corresponding to an up-down class_name. - """ if not class_name.isdigit(): raise rpki.exceptions.BadClassNameSyntax("Bad class name %s" % class_name) ca = rpki.rpkid.ca_obj.sql_fetch(self.gctx, long(class_name)) @@ -924,368 +1008,185 @@ class child_elt(data_elt): return ca def serve_destroy_hook(self, cb, eb): - """ - Extra server actions when destroying a child_elt. - """ publisher = rpki.rpkid.publication_queue() for child_cert in self.child_certs: child_cert.revoke(publisher = publisher, generate_crl_and_manifest = True) publisher.call_pubd(cb, eb) - def serve_up_down(self, query, callback): - """ - Outer layer of server handling for one up-down PDU from this child. - """ - - bsc = self.bsc - if bsc is None: - raise rpki.exceptions.BSCNotFound("Could not find BSC %s" % self.bsc_id) - q_cms = rpki.up_down.cms_msg(DER = query) - q_msg = q_cms.unwrap((self.gctx.bpki_ta, - self.self.bpki_cert, - self.self.bpki_glue, - self.bpki_cert, - self.bpki_glue)) - q_cms.check_replay_sql(self, "child", self.child_handle) - q_msg.payload.gctx = self.gctx - if enforce_strict_up_down_xml_sender and q_msg.sender != self.child_handle: - raise rpki.exceptions.BadSender("Unexpected XML sender %s" % q_msg.sender) - self.gctx.sql.sweep() - - def done(r_msg): - # - # Exceptions from this point on are problematic, as we have no - # sane way of reporting errors in the error reporting mechanism. - # May require refactoring, ignore the issue for now. - # - reply = rpki.up_down.cms_msg().wrap(r_msg, bsc.private_key_id, - bsc.signing_cert, bsc.signing_cert_crl) - callback(reply) - - try: - q_msg.serve_top_level(self, done) - except (rpki.async.ExitNow, SystemExit): - raise - except rpki.exceptions.NoActiveCA, data: - done(q_msg.serve_error(data)) - except Exception, e: - logger.exception("Unhandled exception serving up-down request from %r", self) - done(q_msg.serve_error(e)) - -class list_resources_elt(rpki.xml_utils.base_elt, left_right_namespace): - """ - <list_resources/> element. - """ - - element_name = "list_resources" - attributes = ("self_handle", "tag", "child_handle", "valid_until", "asn", "ipv4", "ipv6") - valid_until = None - - def __repr__(self): - return rpki.log.log_repr(self, self.self_handle, self.child_handle, self.asn, self.ipv4, self.ipv6) - - def startElement(self, stack, name, attrs): - """ - Handle <list_resources/> element. This requires special handling - due to the data types of some of the attributes. - """ - assert name == "list_resources", "Unexpected name %s, stack %s" % (name, stack) - self.read_attrs(attrs) - if isinstance(self.valid_until, str): - self.valid_until = rpki.sundial.datetime.fromXMLtime(self.valid_until) - if self.asn is not None: - self.asn = rpki.resource_set.resource_set_as(self.asn) - if self.ipv4 is not None: - self.ipv4 = rpki.resource_set.resource_set_ipv4(self.ipv4) - if self.ipv6 is not None: - self.ipv6 = rpki.resource_set.resource_set_ipv6(self.ipv6) - - def toXML(self): - """ - Generate <list_resources/> element. This requires special - handling due to the data types of some of the attributes. - """ - elt = self.make_elt() - if isinstance(self.valid_until, int): - elt.set("valid_until", self.valid_until.toXMLtime()) - return elt - -class list_roa_requests_elt(rpki.xml_utils.base_elt, left_right_namespace): - """ - <list_roa_requests/> element. - """ - - element_name = "list_roa_requests" - attributes = ("self_handle", "tag", "asn", "ipv4", "ipv6") - - def startElement(self, stack, name, attrs): - """ - Handle <list_roa_requests/> element. This requires special handling - due to the data types of some of the attributes. - """ - assert name == "list_roa_requests", "Unexpected name %s, stack %s" % (name, stack) - self.read_attrs(attrs) - if self.ipv4 is not None: - self.ipv4 = rpki.resource_set.roa_prefix_set_ipv4(self.ipv4) - if self.ipv6 is not None: - self.ipv6 = rpki.resource_set.roa_prefix_set_ipv6(self.ipv6) - - def __repr__(self): - return rpki.log.log_repr(self, self.self_handle, self.asn, self.ipv4, self.ipv6) - -class list_ghostbuster_requests_elt(rpki.xml_utils.text_elt, left_right_namespace): - """ - <list_ghostbuster_requests/> element. - """ - - element_name = "list_ghostbuster_requests" - attributes = ("self_handle", "tag", "parent_handle") - text_attribute = "vcard" - - vcard = None - - def __repr__(self): - return rpki.log.log_repr(self, self.self_handle, self.parent_handle) - -class list_ee_certificate_requests_elt(rpki.xml_utils.base_elt, left_right_namespace): - """ - <list_ee_certificate_requests/> element. - """ - - element_name = "list_ee_certificate_requests" - attributes = ("self_handle", "tag", "gski", "valid_until", "asn", "ipv4", "ipv6", "cn", "sn", "eku") - elements = ("pkcs10",) - pkcs10 = None - valid_until = None - eku = None + def up_down_handle_list(self, q_msg, r_msg, callback, errback): - def __repr__(self): - return rpki.log.log_repr(self, self.self_handle, self.gski, self.cn, self.sn, self.asn, self.ipv4, self.ipv6) + def got_resources(irdb_resources): - def startElement(self, stack, name, attrs): - """ - Handle <list_ee_certificate_requests/> element. This requires special - handling due to the data types of some of the attributes. - """ - if name not in self.elements: - assert name == self.element_name, "Unexpected name %s, stack %s" % (name, stack) - self.read_attrs(attrs) - if isinstance(self.valid_until, str): - self.valid_until = rpki.sundial.datetime.fromXMLtime(self.valid_until) - if self.asn is not None: - self.asn = rpki.resource_set.resource_set_as(self.asn) - if self.ipv4 is not None: - self.ipv4 = rpki.resource_set.resource_set_ipv4(self.ipv4) - if self.ipv6 is not None: - self.ipv6 = rpki.resource_set.resource_set_ipv6(self.ipv6) - if self.eku is not None: - self.eku = self.eku.split(",") - - def endElement(self, stack, name, text): - """ - Handle <pkcs10/> sub-element. - """ - assert len(self.elements) == 1 - if name == self.elements[0]: - self.pkcs10 = rpki.x509.PKCS10(Base64 = text) - else: - assert name == self.element_name, "Unexpected name %s, stack %s" % (name, stack) - stack.pop() + if irdb_resources.valid_until < rpki.sundial.now(): + logger.debug("Child %s's resources expired %s", self.child_handle, irdb_resources.valid_until) + else: + for parent in self.parents: + for ca in parent.cas: + ca_detail = ca.active_ca_detail + if not ca_detail: + logger.debug("No active ca_detail, can't issue to %s", self.child_handle) + continue + resources = ca_detail.latest_ca_cert.get_3779resources() & irdb_resources + if resources.empty(): + logger.debug("No overlap between received resources and what child %s should get ([%s], [%s])", + self.child_handle, ca_detail.latest_ca_cert.get_3779resources(), irdb_resources) + continue + rc = SubElement(r_msg, rpki.up_down.tag_class, + class_name = str(ca.ca_id), + cert_url = ca_detail.ca_cert_uri, + resource_set_as = str(resources.asn), + resource_set_ipv4 = str(resources.v4), + resource_set_ipv6 = str(resources.v6), + resource_set_notafter = str(resources.valid_until)) + for child_cert in self.fetch_child_certs(ca_detail = ca_detail): + c = SubElement(rc, rpki.up_down.tag_certificate, cert_url = child_cert.uri) + c.text = child_cert.cert.get_Base64() + SubElement(rc, rpki.up_down.tag_issuer).text = ca_detail.latest_ca_cert.get_Base64() + callback() + + self.gctx.irdb_query_child_resources(self.self.self_handle, self.child_handle, got_resources, errback) + + + def up_down_handle_issue(self, q_msg, r_msg, callback, errback): + + def got_resources(irdb_resources): + + def done(): + rc = SubElement(r_msg, rpki.up_down.tag_class, + class_name = class_name, + cert_url = ca_detail.ca_cert_uri, + resource_set_as = str(resources.asn), + resource_set_ipv4 = str(resources.v4), + resource_set_ipv6 = str(resources.v6), + resource_set_notafter = str(resources.valid_until)) + c = SubElement(rc, rpki.up_down.tag_certificate, cert_url = child_cert.uri) + c.text = child_cert.cert.get_Base64() + SubElement(rc, rpki.up_down.tag_issuer).text = ca_detail.latest_ca_cert.get_Base64() + callback() + + if irdb_resources.valid_until < rpki.sundial.now(): + raise rpki.exceptions.IRDBExpired("IRDB entry for child %s expired %s" % ( + self.child_handle, irdb_resources.valid_until)) + + resources = irdb_resources & ca_detail.latest_ca_cert.get_3779resources() + resources.valid_until = irdb_resources.valid_until + req_key = pkcs10.getPublicKey() + req_sia = pkcs10.get_SIA() + child_cert = self.fetch_child_certs(ca_detail = ca_detail, ski = req_key.get_SKI(), unique = True) + + # Generate new cert or regenerate old one if necessary + + publisher = rpki.rpkid.publication_queue() + + if child_cert is None: + child_cert = ca_detail.issue( + ca = ca, + child = self, + subject_key = req_key, + sia = req_sia, + resources = resources, + publisher = publisher) + else: + child_cert = child_cert.reissue( + ca_detail = ca_detail, + sia = req_sia, + resources = resources, + publisher = publisher) - def toXML(self): - """ - Generate <list_ee_certificate_requests/> element. This requires special - handling due to the data types of some of the attributes. - """ - if isinstance(self.eku, (tuple, list)): - self.eku = ",".join(self.eku) - elt = self.make_elt() - for i in self.elements: - self.make_b64elt(elt, i, getattr(self, i, None)) - if isinstance(self.valid_until, int): - elt.set("valid_until", self.valid_until.toXMLtime()) - return elt + self.gctx.sql.sweep() + assert child_cert and child_cert.sql_in_db + publisher.call_pubd(done, errback) -class list_published_objects_elt(rpki.xml_utils.text_elt, left_right_namespace): - """ - <list_published_objects/> element. - """ + req = q_msg[0] + assert req.tag == rpki.up_down.tag_request - element_name = "list_published_objects" - attributes = ("self_handle", "tag", "uri", "child_handle") - text_attribute = "obj" + # Subsetting not yet implemented, this is the one place where we + # have to handle it, by reporting that we're lame. - obj = None - child_handle = None + if any(req.get(a) for a in ("req_resource_set_as", "req_resource_set_ipv4", "req_resource_set_ipv6")): + raise rpki.exceptions.NotImplementedYet("req_* attributes not implemented yet, sorry") - def __repr__(self): - return rpki.log.log_repr(self, self.self_handle, self.child_handle, self.uri) + class_name = req.get("class_name") + pkcs10 = rpki.x509.PKCS10(Base64 = req.text) + pkcs10.check_valid_request_ca() + ca = self.ca_from_class_name(class_name) + ca_detail = ca.active_ca_detail + if ca_detail is None: + raise rpki.exceptions.NoActiveCA("No active CA for class %r" % class_name) - def serve_dispatch(self, r_msg, cb, eb): - """ - Handle a <list_published_objects/> query. The method name is a - misnomer here, there's no action attribute and no dispatch, we - just dump every published object for the specified <self/> and return. - """ - for parent in self_elt.serve_fetch_handle(self.gctx, None, self.self_handle).parents: - for ca in parent.cas: - ca_detail = ca.active_ca_detail - if ca_detail is not None: - r_msg.append(self.make_reply(ca_detail.crl_uri, ca_detail.latest_crl)) - r_msg.append(self.make_reply(ca_detail.manifest_uri, ca_detail.latest_manifest)) - r_msg.extend(self.make_reply(c.uri, c.cert, c.child.child_handle) - for c in ca_detail.child_certs) - r_msg.extend(self.make_reply(r.uri, r.roa) - for r in ca_detail.roas if r.roa is not None) - r_msg.extend(self.make_reply(g.uri, g.ghostbuster) - for g in ca_detail.ghostbusters) - r_msg.extend(self.make_reply(c.uri, c.cert) - for c in ca_detail.ee_certificates) - cb() - - def make_reply(self, uri, obj, child_handle = None): - """ - Generate one reply PDU. - """ - r_pdu = self.make_pdu(tag = self.tag, self_handle = self.self_handle, - uri = uri, child_handle = child_handle) - r_pdu.obj = obj.get_Base64() - return r_pdu + self.gctx.irdb_query_child_resources(self.self.self_handle, self.child_handle, got_resources, errback) -class list_received_resources_elt(rpki.xml_utils.base_elt, left_right_namespace): - """ - <list_received_resources/> element. - """ - element_name = "list_received_resources" - attributes = ("self_handle", "tag", "parent_handle", - "notBefore", "notAfter", "uri", "sia_uri", "aia_uri", "asn", "ipv4", "ipv6") - - def __repr__(self): - return rpki.log.log_repr(self, self.self_handle, self.parent_handle, self.uri, self.notAfter) + def up_down_handle_revoke(self, q_msg, r_msg, callback, errback): - def serve_dispatch(self, r_msg, cb, eb): - """ - Handle a <list_received_resources/> query. The method name is a - misnomer here, there's no action attribute and no dispatch, we - just dump a bunch of data about every certificate issued to us by - one of our parents, then return. - """ - for parent in self_elt.serve_fetch_handle(self.gctx, None, self.self_handle).parents: - for ca in parent.cas: - ca_detail = ca.active_ca_detail - if ca_detail is not None and ca_detail.latest_ca_cert is not None: - r_msg.append(self.make_reply(parent.parent_handle, ca_detail.ca_cert_uri, ca_detail.latest_ca_cert)) - cb() - - def make_reply(self, parent_handle, uri, cert): - """ - Generate one reply PDU. - """ - resources = cert.get_3779resources() - return self.make_pdu( - tag = self.tag, - self_handle = self.self_handle, - parent_handle = parent_handle, - notBefore = str(cert.getNotBefore()), - notAfter = str(cert.getNotAfter()), - uri = uri, - sia_uri = cert.get_sia_directory_uri(), - aia_uri = cert.get_aia_uri(), - asn = resources.asn, - ipv4 = resources.v4, - ipv6 = resources.v6) - -class report_error_elt(rpki.xml_utils.text_elt, left_right_namespace): - """ - <report_error/> element. - """ + def done(): + SubElement(r_msg, key.tag, class_name = class_name, ski = key.get("ski")) + callback() - element_name = "report_error" - attributes = ("tag", "self_handle", "error_code") - text_attribute = "error_text" + key = q_msg[0] + assert key.tag == rpki.up_down.tag_key + class_name = key.get("class_name") + ski = base64.urlsafe_b64decode(key.get("ski") + "=") - error_text = None + publisher = rpki.rpkid.publication_queue() - def __repr__(self): - return rpki.log.log_repr(self, self.self_handle, self.error_code) + ca = self.ca_from_class_name(class_name) + for ca_detail in ca.ca_details: + for child_cert in self.fetch_child_certs(ca_detail = ca_detail, ski = ski): + child_cert.revoke(publisher = publisher) - @classmethod - def from_exception(cls, e, self_handle = None, tag = None): - """ - Generate a <report_error/> element from an exception. - """ - self = cls() - self.self_handle = self_handle - self.tag = tag - self.error_code = e.__class__.__name__ - self.error_text = str(e) - return self + self.gctx.sql.sweep() + publisher.call_pubd(done, errback) -class msg(rpki.xml_utils.msg, left_right_namespace): - """ - Left-right PDU. - """ - ## @var version - # Protocol version - version = int(rpki.relaxng.left_right.version) - - ## @var pdus - # Dispatch table of PDUs for this protocol. - pdus = dict((x.element_name, x) - for x in (self_elt, child_elt, parent_elt, bsc_elt, - repository_elt, list_resources_elt, - list_roa_requests_elt, list_ghostbuster_requests_elt, - list_ee_certificate_requests_elt, - list_published_objects_elt, - list_received_resources_elt, report_error_elt)) - - def serve_top_level(self, gctx, cb): + def serve_up_down(self, q_der, callback): """ - Serve one msg PDU. + Outer layer of server handling for one up-down PDU from this child. """ - r_msg = self.__class__.reply() - - def loop(iterator, q_pdu): - - def fail(e): - if not isinstance(e, rpki.exceptions.NotFound): - logger.exception("Unhandled exception serving left-right PDU %r", q_pdu) - r_msg.append(report_error_elt.from_exception( - e, self_handle = q_pdu.self_handle, tag = q_pdu.tag)) - cb(r_msg) + def done(): + callback(rpki.up_down.cms_msg().wrap(r_msg, bsc.private_key_id, + bsc.signing_cert, bsc.signing_cert_crl)) - try: - q_pdu.gctx = gctx - q_pdu.serve_dispatch(r_msg, iterator, fail) - except (rpki.async.ExitNow, SystemExit): - raise - except Exception, e: - fail(e) + def lose(e): + logger.exception("Unhandled exception serving child %r", self) + rpki.up_down.generate_error_response_from_exception(r_msg, e, q_type) + done() - def done(): - cb(r_msg) + bsc = self.bsc + if bsc is None: + raise rpki.exceptions.BSCNotFound("Could not find BSC %s" % self.bsc_id) + q_cms = rpki.up_down.cms_msg(DER = q_der) + q_msg = q_cms.unwrap((self.gctx.bpki_ta, + self.self.bpki_cert, + self.self.bpki_glue, + self.bpki_cert, + self.bpki_glue)) + q_cms.check_replay_sql(self, "child", self.child_handle) + q_type = q_msg.get("type") + logger.info("Serving %s query from child %s [sender %s, recipient %s]", + q_type, self.child_handle, q_msg.get("sender"), q_msg.get("recipient")) + if rpki.up_down.enforce_strict_up_down_xml_sender and q_msg.get("sender") != self.child_handle: + raise rpki.exceptions.BadSender("Unexpected XML sender %s" % q_msg.get("sender")) + self.gctx.sql.sweep() - rpki.async.iterator(self, loop, done) + r_msg = Element(rpki.up_down.tag_message, nsmap = rpki.up_down.nsmap, version = rpki.up_down.version, + sender = q_msg.get("recipient"), recipient = q_msg.get("sender"), type = q_type + "_response") -class sax_handler(rpki.xml_utils.sax_handler): - """ - SAX handler for Left-Right protocol. - """ + try: + getattr(self, "up_down_handle_" + q_type)(q_msg, r_msg, done, lose) + except (rpki.async.ExitNow, SystemExit): + raise + except Exception, e: + lose(e) - pdu = msg - name = "msg" - version = rpki.relaxng.left_right.version class cms_msg(rpki.x509.XML_CMS_object): """ - Class to hold a CMS-signed left-right PDU. + CMS-signed left-right PDU. """ encoding = "us-ascii" schema = rpki.relaxng.left_right - saxify = sax_handler.saxify diff --git a/rpki/log.py b/rpki/log.py index 2abb3b2c..7bad6dc2 100644 --- a/rpki/log.py +++ b/rpki/log.py @@ -32,7 +32,7 @@ import traceback as tb try: have_setproctitle = False if os.getenv("DISABLE_SETPROCTITLE") is None: - import setproctitle + import setproctitle # pylint: disable=F0401 have_setproctitle = True except ImportError: pass @@ -48,7 +48,7 @@ show_python_ids = False # Whether tracebacks are enabled globally. Individual classes and # modules may choose to override this. -enable_tracebacks = False +enable_tracebacks = True ## @var use_setproctitle # Whether to use setproctitle (if available) to change name shown for @@ -96,7 +96,10 @@ class Formatter(object): yield time.strftime("%Y-%m-%d %H:%M:%S ", time.gmtime(record.created)) yield "%s[%d]: " % (self.ident, record.process) try: - yield repr(record.context) + " " + if isinstance(record.context, (str, unicode)): + yield record.context + " " + else: + yield repr(record.context) + " " except AttributeError: pass yield record.getMessage() @@ -262,3 +265,17 @@ def log_repr(obj, *tokens): words.append(" at %#x" % id(obj)) return "<" + " ".join(words) + ">" + + +def show_stack(stack_logger = None): + """ + Log a stack trace. + """ + + if stack_logger is None: + stack_logger = logger + + for frame in tb.format_stack(): + for line in frame.split("\n"): + if line: + stack_logger.debug("%s", line.rstrip()) diff --git a/rpki/oids.py b/rpki/oids.py index 9fa30a04..afb95020 100644 --- a/rpki/oids.py +++ b/rpki/oids.py @@ -57,6 +57,7 @@ id_ad_caRepository = "1.3.6.1.5.5.7.48.5" id_ad_signedObjectRepository = "1.3.6.1.5.5.7.48.9" id_ad_rpkiManifest = "1.3.6.1.5.5.7.48.10" id_ad_signedObject = "1.3.6.1.5.5.7.48.11" +id_ad_rpkiNotify = "1.3.6.1.5.5.7.48.13" commonName = "2.5.4.3" serialNumber = "2.5.4.5" countryName = "2.5.4.6" diff --git a/rpki/old_irdbd.py b/rpki/old_irdbd.py index 6c026a31..9294ee84 100644 --- a/rpki/old_irdbd.py +++ b/rpki/old_irdbd.py @@ -30,7 +30,7 @@ import time import logging import argparse import urlparse -import rpki.http +import rpki.http_simple import rpki.config import rpki.resource_set import rpki.relaxng @@ -226,7 +226,7 @@ class main(object): rpki.left_right.list_ghostbuster_requests_elt : handle_list_ghostbuster_requests, rpki.left_right.list_ee_certificate_requests_elt : handle_list_ee_certificate_requests } - def handler(self, query, path, cb): + def handler(self, request, q_der): try: self.db.ping(True) @@ -235,7 +235,7 @@ class main(object): try: - q_msg = rpki.left_right.cms_msg(DER = query).unwrap((self.bpki_ta, self.rpkid_cert)) + q_msg = rpki.left_right.cms_msg_saxify(DER = q_der).unwrap((self.bpki_ta, self.rpkid_cert)) if not isinstance(q_msg, rpki.left_right.msg) or not q_msg.is_query(): raise rpki.exceptions.BadQuery("Unexpected %r PDU" % q_msg) @@ -251,28 +251,19 @@ class main(object): else: h(self, q_pdu, r_msg) - except (rpki.async.ExitNow, SystemExit): - raise - except Exception, e: logger.exception("Exception serving PDU %r", q_pdu) r_msg.append(rpki.left_right.report_error_elt.from_exception(e, q_pdu.self_handle, q_pdu.tag)) - except (rpki.async.ExitNow, SystemExit): - raise - except Exception, e: logger.exception("Exception decoding query") r_msg.append(rpki.left_right.report_error_elt.from_exception(e)) - cb(200, body = rpki.left_right.cms_msg().wrap(r_msg, self.irdbd_key, self.irdbd_cert)) - - except (rpki.async.ExitNow, SystemExit): - raise + request.send_cms_response(rpki.left_right.cms_msg_saxify().wrap(r_msg, self.irdbd_key, self.irdbd_cert)) except Exception, e: logger.exception("Unhandled exception, returning HTTP failure") - cb(500, reason = "Unhandled exception %s: %s" % (e.__class__.__name__, e)) + request.send_error(500, "Unhandled exception %s: %s" % (e.__class__.__name__, e)) def __init__(self): @@ -290,7 +281,7 @@ class main(object): rpki.log.init("irdbd", args) - self.cfg = rpki.config.parser(args.config, "irdbd") + self.cfg = rpki.config.parser(set_filename = args.config, section = "irdbd") startup_msg = self.cfg.get("startup-message", "") if startup_msg: @@ -319,6 +310,6 @@ class main(object): u.query == "" and \ u.fragment == "" - rpki.http.server(host = u.hostname or "localhost", - port = u.port or 443, - handlers = ((u.path, self.handler),)) + rpki.http_simple.server(host = u.hostname or "localhost", + port = u.port or 443, + handlers = ((u.path, self.handler),)) diff --git a/rpki/pubd.py b/rpki/pubd.py index 79315a78..b5d36199 100644 --- a/rpki/pubd.py +++ b/rpki/pubd.py @@ -23,23 +23,27 @@ RPKI publication engine. import os import re +import uuid import time +import socket import logging import argparse + import rpki.resource_set -import rpki.up_down import rpki.x509 -import rpki.sql -import rpki.http import rpki.config import rpki.exceptions -import rpki.relaxng import rpki.log import rpki.publication +import rpki.publication_control import rpki.daemonize +import rpki.http_simple + +from lxml.etree import Element, SubElement logger = logging.getLogger(__name__) + class main(object): """ Main program for pubd. @@ -47,7 +51,8 @@ class main(object): def __init__(self): - os.environ["TZ"] = "UTC" + os.environ.update(TZ = "UTC", + DJANGO_SETTINGS_MODULE = "rpki.django_settings.pubd") time.tzset() self.irbe_cms_timestamp = None @@ -68,7 +73,7 @@ class main(object): rpki.log.init("pubd", args) - self.cfg = rpki.config.parser(args.config, "pubd") + self.cfg = rpki.config.parser(set_filename = args.config, section = "pubd") self.cfg.set_global_flags() if not args.foreground: @@ -90,84 +95,202 @@ class main(object): if self.profile: logger.info("Running in profile mode with output to %s", self.profile) - self.sql = rpki.sql.session(self.cfg) + import django + django.setup() + + global rpki # pylint: disable=W0602 + import rpki.pubdb # pylint: disable=W0621 self.bpki_ta = rpki.x509.X509(Auto_update = self.cfg.get("bpki-ta")) self.irbe_cert = rpki.x509.X509(Auto_update = self.cfg.get("irbe-cert")) self.pubd_cert = rpki.x509.X509(Auto_update = self.cfg.get("pubd-cert")) self.pubd_key = rpki.x509.RSA( Auto_update = self.cfg.get("pubd-key")) + self.pubd_crl = rpki.x509.CRL( Auto_update = self.cfg.get("pubd-crl")) self.http_server_host = self.cfg.get("server-host", "") self.http_server_port = self.cfg.getint("server-port") self.publication_base = self.cfg.get("publication-base", "publication/") - self.publication_multimodule = self.cfg.getboolean("publication-multimodule", False) + self.rrdp_uri_base = self.cfg.get("rrdp-uri-base", + "http://%s/rrdp/" % socket.getfqdn()) + self.rrdp_expiration_interval = rpki.sundial.timedelta.parse(self.cfg.get("rrdp-expiration-interval", "6h")) + self.rrdp_publication_base = self.cfg.get("rrdp-publication-base", + "rrdp-publication/") - rpki.http.server( + try: + self.session = rpki.pubdb.models.Session.objects.get() + except rpki.pubdb.models.Session.DoesNotExist: + self.session = rpki.pubdb.models.Session.objects.create(uuid = str(uuid.uuid4()), serial = 0) + + rpki.http_simple.server( host = self.http_server_host, port = self.http_server_port, handlers = (("/control", self.control_handler), ("/client/", self.client_handler))) - def handler_common(self, query, client, cb, certs, crl = None): + + def control_handler(self, request, q_der): """ - Common PDU handler code. + Process one PDU from the IRBE. """ - def done(r_msg): - reply = rpki.publication.cms_msg().wrap(r_msg, self.pubd_key, self.pubd_cert, crl) - self.sql.sweep() - cb(reply) + from django.db import transaction, connection - q_cms = rpki.publication.cms_msg(DER = query) - q_msg = q_cms.unwrap(certs) - if client is None: + try: + connection.cursor() # Reconnect to mysqld if necessary + q_cms = rpki.publication_control.cms_msg(DER = q_der) + q_msg = q_cms.unwrap((self.bpki_ta, self.irbe_cert)) self.irbe_cms_timestamp = q_cms.check_replay(self.irbe_cms_timestamp, "control") - else: - q_cms.check_replay_sql(client, client.client_handle) - q_msg.serve_top_level(self, client, done) + if q_msg.get("type") != "query": + raise rpki.exceptions.BadQuery("Message type is %s, expected query" % q_msg.get("type")) + r_msg = Element(rpki.publication_control.tag_msg, nsmap = rpki.publication_control.nsmap, + type = "reply", version = rpki.publication_control.version) - def control_handler(self, query, path, cb): - """ - Process one PDU from the IRBE. - """ + try: + q_pdu = None + with transaction.atomic(): - def done(body): - cb(200, body = body) + for q_pdu in q_msg: + if q_pdu.tag != rpki.publication_control.tag_client: + raise rpki.exceptions.BadQuery("PDU is %s, expected client" % q_pdu.tag) + client_handle = q_pdu.get("client_handle") + action = q_pdu.get("action") + if client_handle is None: + logger.info("Control %s request", action) + else: + logger.info("Control %s request for %s", action, client_handle) + + if action in ("get", "list"): + if action == "get": + clients = rpki.pubdb.models.Client.objects.get(client_handle = client_handle), + else: + clients = rpki.pubdb.models.Client.objects.all() + for client in clients: + r_pdu = SubElement(r_msg, q_pdu.tag, action = action, + client_handle = client.client_handle, base_uri = client.base_uri) + if q_pdu.get("tag"): + r_pdu.set("tag", q_pdu.get("tag")) + SubElement(r_pdu, rpki.publication_control.tag_bpki_cert).text = client.bpki_cert.get_Base64() + if client.bpki_glue is not None: + SubElement(r_pdu, rpki.publication_control.tag_bpki_glue).text = client.bpki_glue.get_Base64() + + if action in ("create", "set"): + if action == "create": + client = rpki.pubdb.models.Client(client_handle = client_handle) + else: + client = rpki.pubdb.models.Client.objects.get(client_handle = client_handle) + if q_pdu.get("base_uri"): + client.base_uri = q_pdu.get("base_uri") + bpki_cert = q_pdu.find(rpki.publication_control.tag_bpki_cert) + if bpki_cert is not None: + client.bpki_cert = bpki_cert.text.decode("base64") + bpki_glue = q_pdu.find(rpki.publication_control.tag_bpki_glue) + if bpki_glue is not None: + client.bpki_glue = bpki_glue.text.decode("base64") + if q_pdu.get("clear_replay_protection") == "yes": + client.last_cms_timestamp = None + client.save() + logger.debug("Stored client_handle %s, base_uri %s, bpki_cert %r, bpki_glue %r, last_cms_timestamp %s", + client.client_handle, client.base_uri, client.bpki_cert, client.bpki_glue, + client.last_cms_timestamp) + r_pdu = SubElement(r_msg, q_pdu.tag, action = action, client_handle = client_handle) + if q_pdu.get("tag"): + r_pdu.set("tag", q_pdu.get("tag")) + + if action == "destroy": + rpki.pubdb.models.Client.objects.filter(client_handle = client_handle).delete() + r_pdu = SubElement(r_msg, q_pdu.tag, action = action, client_handle = client_handle) + if q_pdu.get("tag"): + r_pdu.set("tag", q_pdu.get("tag")) + + except Exception, e: + logger.exception("Exception processing PDU %r", q_pdu) + r_pdu = SubElement(r_msg, rpki.publication_control.tag_report_error, error_code = e.__class__.__name__) + r_pdu.text = str(e) + if q_pdu.get("tag") is not None: + r_pdu.set("tag", q_pdu.get("tag")) + + request.send_cms_response(rpki.publication_control.cms_msg().wrap(r_msg, self.pubd_key, self.pubd_cert)) - try: - self.handler_common(query, None, done, (self.bpki_ta, self.irbe_cert)) - except (rpki.async.ExitNow, SystemExit): - raise except Exception, e: - logger.exception("Unhandled exception processing control query, path %r", path) - cb(500, reason = "Unhandled exception %s: %s" % (e.__class__.__name__, e)) + logger.exception("Unhandled exception processing control query, path %r", request.path) + request.send_error(500, "Unhandled exception %s: %s" % (e.__class__.__name__, e)) + client_url_regexp = re.compile("/client/([-A-Z0-9_/]+)$", re.I) - def client_handler(self, query, path, cb): + def client_handler(self, request, q_der): """ Process one PDU from a client. """ - def done(body): - cb(200, body = body) + from django.db import transaction, connection try: - match = self.client_url_regexp.search(path) + connection.cursor() # Reconnect to mysqld if necessary + match = self.client_url_regexp.search(request.path) if match is None: - raise rpki.exceptions.BadContactURL("Bad path: %s" % path) - client_handle = match.group(1) - client = rpki.publication.client_elt.sql_fetch_where1(self, "client_handle = %s", (client_handle,)) - if client is None: - raise rpki.exceptions.ClientNotFound("Could not find client %s" % client_handle) - config = rpki.publication.config_elt.fetch(self) - if config is None or config.bpki_crl is None: - raise rpki.exceptions.CMSCRLNotSet - self.handler_common(query, client, done, (self.bpki_ta, client.bpki_cert, client.bpki_glue), config.bpki_crl) - except (rpki.async.ExitNow, SystemExit): - raise + raise rpki.exceptions.BadContactURL("Bad path: %s" % request.path) + client = rpki.pubdb.models.Client.objects.get(client_handle = match.group(1)) + q_cms = rpki.publication.cms_msg(DER = q_der) + q_msg = q_cms.unwrap((self.bpki_ta, client.bpki_cert, client.bpki_glue)) + client.last_cms_timestamp = q_cms.check_replay(client.last_cms_timestamp, client.client_handle) + client.save() + if q_msg.get("type") != "query": + raise rpki.exceptions.BadQuery("Message type is %s, expected query" % q_msg.get("type")) + r_msg = Element(rpki.publication.tag_msg, nsmap = rpki.publication.nsmap, + type = "reply", version = rpki.publication.version) + delta = None + try: + with transaction.atomic(): + for q_pdu in q_msg: + if q_pdu.get("uri"): + logger.info("Client %s request for %s", q_pdu.tag, q_pdu.get("uri")) + else: + logger.info("Client %s request", q_pdu.tag) + + if q_pdu.tag == rpki.publication.tag_list: + for obj in client.publishedobject_set.all(): + r_pdu = SubElement(r_msg, q_pdu.tag, uri = obj.uri, hash = obj.hash) + if q_pdu.get("tag") is not None: + r_pdu.set("tag", q_pdu.get("tag")) + + else: + assert q_pdu.tag in (rpki.publication.tag_publish, rpki.publication.tag_withdraw) + if delta is None: + delta = self.session.new_delta(rpki.sundial.now() + self.rrdp_expiration_interval) + client.check_allowed_uri(q_pdu.get("uri")) + if q_pdu.tag == rpki.publication.tag_publish: + der = q_pdu.text.decode("base64") + logger.info("Publishing %s", rpki.x509.uri_dispatch(q_pdu.get("uri"))(DER = der).tracking_data(q_pdu.get("uri"))) + delta.publish(client, der, q_pdu.get("uri"), q_pdu.get("hash")) + else: + logger.info("Withdrawing %s", q_pdu.get("uri")) + delta.withdraw(client, q_pdu.get("uri"), q_pdu.get("hash")) + r_pdu = SubElement(r_msg, q_pdu.tag, uri = q_pdu.get("uri")) + if q_pdu.get("tag") is not None: + r_pdu.set("tag", q_pdu.get("tag")) + + if delta is not None: + delta.activate() + self.session.generate_snapshot() + self.session.expire_deltas() + + except Exception, e: + logger.exception("Exception processing PDU %r", q_pdu) + r_pdu = SubElement(r_msg, rpki.publication.tag_report_error, error_code = e.__class__.__name__) + r_pdu.text = str(e) + if q_pdu.get("tag") is not None: + r_pdu.set("tag", q_pdu.get("tag")) + + else: + if delta is not None: + self.session.synchronize_rrdp_files(self.rrdp_publication_base, self.rrdp_uri_base) + delta.update_rsync_files(self.publication_base) + + request.send_cms_response(rpki.publication.cms_msg().wrap(r_msg, self.pubd_key, self.pubd_cert, self.pubd_crl)) + except Exception, e: - logger.exception("Unhandled exception processing client query, path %r", path) - cb(500, reason = "Could not process PDU: %s" % e) + logger.exception("Unhandled exception processing client query, path %r", request.path) + request.send_error(500, "Could not process PDU: %s" % e) diff --git a/rpki/pubdb/__init__.py b/rpki/pubdb/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/rpki/pubdb/__init__.py diff --git a/rpki/pubdb/migrations/0001_initial.py b/rpki/pubdb/migrations/0001_initial.py new file mode 100644 index 00000000..4f312844 --- /dev/null +++ b/rpki/pubdb/migrations/0001_initial.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import rpki.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Client', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('client_handle', models.CharField(unique=True, max_length=255)), + ('base_uri', models.TextField()), + ('bpki_cert', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('bpki_glue', rpki.fields.CertificateField(default=None, serialize=False, null=True, blank=True)), + ('last_cms_timestamp', rpki.fields.SundialField(null=True, blank=True)), + ], + ), + migrations.CreateModel( + name='Delta', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('serial', models.BigIntegerField()), + ('xml', models.TextField()), + ('hash', models.CharField(max_length=64)), + ('expires', rpki.fields.SundialField()), + ], + ), + migrations.CreateModel( + name='PublishedObject', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('uri', models.CharField(max_length=255)), + ('der', rpki.fields.BlobField(default=None, serialize=False, blank=True)), + ('hash', models.CharField(max_length=64)), + ('client', models.ForeignKey(to='pubdb.Client')), + ], + ), + migrations.CreateModel( + name='Session', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('uuid', models.CharField(unique=True, max_length=36)), + ('serial', models.BigIntegerField()), + ('snapshot', models.TextField(blank=True)), + ('hash', models.CharField(max_length=64, blank=True)), + ], + ), + migrations.AddField( + model_name='publishedobject', + name='session', + field=models.ForeignKey(to='pubdb.Session'), + ), + migrations.AddField( + model_name='delta', + name='session', + field=models.ForeignKey(to='pubdb.Session'), + ), + migrations.AlterUniqueTogether( + name='publishedobject', + unique_together=set([('session', 'hash'), ('session', 'uri')]), + ), + ] diff --git a/rpki/pubdb/migrations/__init__.py b/rpki/pubdb/migrations/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/rpki/pubdb/migrations/__init__.py diff --git a/rpki/pubdb/models.py b/rpki/pubdb/models.py new file mode 100644 index 00000000..446867b8 --- /dev/null +++ b/rpki/pubdb/models.py @@ -0,0 +1,313 @@ +# $Id$ +# +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +Django ORM models for pubd. +""" + +from __future__ import unicode_literals +from django.db import models +from rpki.fields import BlobField, CertificateField, SundialField +from lxml.etree import Element, SubElement, tostring as ElementToString + +import os +import logging +import rpki.exceptions +import rpki.relaxng + +logger = logging.getLogger(__name__) + + +# Some of this probably ought to move into a rpki.rrdp module. + +rrdp_xmlns = rpki.relaxng.rrdp.xmlns +rrdp_nsmap = rpki.relaxng.rrdp.nsmap +rrdp_version = "1" + +rrdp_tag_delta = rrdp_xmlns + "delta" +rrdp_tag_notification = rrdp_xmlns + "notification" +rrdp_tag_publish = rrdp_xmlns + "publish" +rrdp_tag_snapshot = rrdp_xmlns + "snapshot" +rrdp_tag_withdraw = rrdp_xmlns + "withdraw" + + +# This would probably be useful to more than just this module, not +# sure quite where to put it at the moment. + +def DERSubElement(elt, name, der, attrib = None, **kwargs): + """ + Convenience wrapper around SubElement for use with Base64 text. + """ + + se = SubElement(elt, name, attrib, **kwargs) + se.text = rpki.x509.base64_with_linebreaks(der) + se.tail = "\n" + return se + + + +class Client(models.Model): + client_handle = models.CharField(unique = True, max_length = 255) + base_uri = models.TextField() + bpki_cert = CertificateField() + bpki_glue = CertificateField(null = True) + last_cms_timestamp = SundialField(blank = True, null = True) + + + def check_allowed_uri(self, uri): + """ + Make sure that a target URI is within this client's allowed URI space. + """ + + if not uri.startswith(self.base_uri): + raise rpki.exceptions.ForbiddenURI + + +class Session(models.Model): + uuid = models.CharField(unique = True, max_length=36) + serial = models.BigIntegerField() + snapshot = models.TextField(blank = True) + hash = models.CharField(max_length = 64, blank = True) + + ## @var keep_all_rrdp_files + # Debugging flag to prevent expiration of old RRDP files. + # This simplifies debugging delta code. Need for this + # may go away once RRDP is fully integrated into rcynic. + keep_all_rrdp_files = False + + def new_delta(self, expires): + """ + Construct a new delta associated with this session. + """ + + delta = Delta(session = self, + serial = self.serial + 1, + expires = expires) + delta.elt = Element(rrdp_tag_delta, + nsmap = rrdp_nsmap, + version = rrdp_version, + session_id = self.uuid, + serial = str(delta.serial)) + return delta + + + def expire_deltas(self): + """ + Delete deltas whose expiration date has passed. + """ + + self.delta_set.filter(expires__lt = rpki.sundial.now()).delete() + + + def generate_snapshot(self): + """ + Generate an XML snapshot of this session. + """ + + xml = Element(rrdp_tag_snapshot, nsmap = rrdp_nsmap, + version = rrdp_version, + session_id = self.uuid, + serial = str(self.serial)) + xml.text = "\n" + for obj in self.publishedobject_set.all(): + DERSubElement(xml, rrdp_tag_publish, + der = obj.der, + uri = obj.uri) + rpki.relaxng.rrdp.assertValid(xml) + self.snapshot = ElementToString(xml, pretty_print = True) + self.hash = rpki.x509.sha256(self.snapshot).encode("hex") + self.save() + + + @property + def snapshot_fn(self): + return "%s/snapshot/%s.xml" % (self.uuid, self.serial) + + + @property + def notification_fn(self): + return "notify.xml" + + + @staticmethod + def _write_rrdp_file(fn, text, rrdp_publication_base, overwrite = False): + if overwrite or not os.path.exists(os.path.join(rrdp_publication_base, fn)): + tn = os.path.join(rrdp_publication_base, fn + ".%s.tmp" % os.getpid()) + if not os.path.isdir(os.path.dirname(tn)): + os.makedirs(os.path.dirname(tn)) + with open(tn, "w") as f: + f.write(text) + os.rename(tn, os.path.join(rrdp_publication_base, fn)) + + + @staticmethod + def _rrdp_filename_to_uri(fn, rrdp_uri_base): + return "%s/%s" % (rrdp_uri_base.rstrip("/"), fn) + + + def _generate_update_xml(self, rrdp_uri_base): + xml = Element(rrdp_tag_notification, nsmap = rrdp_nsmap, + version = rrdp_version, + session_id = self.uuid, + serial = str(self.serial)) + SubElement(xml, rrdp_tag_snapshot, + uri = self._rrdp_filename_to_uri(self.snapshot_fn, rrdp_uri_base), + hash = self.hash) + for delta in self.delta_set.all(): + SubElement(xml, rrdp_tag_delta, + uri = self._rrdp_filename_to_uri(delta.fn, rrdp_uri_base), + hash = delta.hash, + serial = str(delta.serial)) + rpki.relaxng.rrdp.assertValid(xml) + return ElementToString(xml, pretty_print = True) + + + def synchronize_rrdp_files(self, rrdp_publication_base, rrdp_uri_base): + """ + Write current RRDP files to disk, clean up old files and directories. + """ + + current_filenames = set() + + for delta in self.delta_set.all(): + self._write_rrdp_file(delta.fn, delta.xml, rrdp_publication_base) + current_filenames.add(delta.fn) + + self._write_rrdp_file(self.snapshot_fn, self.snapshot, rrdp_publication_base) + current_filenames.add(self.snapshot_fn) + + self._write_rrdp_file(self.notification_fn, self._generate_update_xml(rrdp_uri_base), + rrdp_publication_base, overwrite = True) + current_filenames.add(self.notification_fn) + + if not self.keep_all_rrdp_files: + for root, dirs, files in os.walk(rrdp_publication_base, topdown = False): + for fn in files: + fn = os.path.join(root, fn) + if fn[len(rrdp_publication_base):].lstrip("/") not in current_filenames: + os.remove(fn) + for dn in dirs: + try: + os.rmdir(os.path.join(root, dn)) + except OSError: + pass + + +class Delta(models.Model): + serial = models.BigIntegerField() + xml = models.TextField() + hash = models.CharField(max_length = 64) + expires = SundialField() + session = models.ForeignKey(Session) + + + @staticmethod + def _uri_to_filename(uri, publication_base): + if not uri.startswith("rsync://"): + raise rpki.exceptions.BadURISyntax(uri) + path = uri.split("/")[4:] + path.insert(0, publication_base.rstrip("/")) + filename = "/".join(path) + if "/../" in filename or filename.endswith("/.."): + raise rpki.exceptions.BadURISyntax(filename) + return filename + + + @property + def fn(self): + return "%s/deltas/%s.xml" % (self.session.uuid, self.serial) + + + def activate(self): + rpki.relaxng.rrdp.assertValid(self.elt) + self.xml = ElementToString(self.elt, pretty_print = True) + self.hash = rpki.x509.sha256(self.xml).encode("hex") + self.save() + self.session.serial += 1 + self.session.save() + + + def publish(self, client, der, uri, obj_hash): + try: + obj = client.publishedobject_set.get(session = self.session, uri = uri) + if obj.hash == obj_hash: + obj.delete() + elif obj_hash is None: + raise rpki.exceptions.ExistingObjectAtURI("Object already published at %s" % uri) + else: + raise rpki.exceptions.DifferentObjectAtURI("Found different object at %s (old %s, new %s)" % (uri, obj.hash, obj_hash)) + except rpki.pubdb.models.PublishedObject.DoesNotExist: + pass + logger.debug("Publishing %s", uri) + PublishedObject.objects.create(session = self.session, client = client, der = der, uri = uri, + hash = rpki.x509.sha256(der).encode("hex")) + se = DERSubElement(self.elt, rrdp_tag_publish, der = der, uri = uri) + if obj_hash is not None: + se.set("hash", obj_hash) + rpki.relaxng.rrdp.assertValid(self.elt) + + + def withdraw(self, client, uri, obj_hash): + obj = client.publishedobject_set.get(session = self.session, uri = uri) + if obj.hash != obj_hash: + raise rpki.exceptions.DifferentObjectAtURI("Found different object at %s (old %s, new %s)" % (uri, obj.hash, obj_hash)) + logger.debug("Withdrawing %s", uri) + obj.delete() + SubElement(self.elt, rrdp_tag_withdraw, uri = uri, hash = obj_hash).tail = "\n" + rpki.relaxng.rrdp.assertValid(self.elt) + + + def update_rsync_files(self, publication_base): + from errno import ENOENT + min_path_len = len(publication_base.rstrip("/")) + for pdu in self.elt: + assert pdu.tag in (rrdp_tag_publish, rrdp_tag_withdraw) + fn = self._uri_to_filename(pdu.get("uri"), publication_base) + if pdu.tag == rrdp_tag_publish: + tn = fn + ".tmp" + dn = os.path.dirname(fn) + if not os.path.isdir(dn): + os.makedirs(dn) + with open(tn, "wb") as f: + f.write(pdu.text.decode("base64")) + os.rename(tn, fn) + else: + try: + os.remove(fn) + except OSError, e: + if e.errno != ENOENT: + raise + dn = os.path.dirname(fn) + while len(dn) > min_path_len: + try: + os.rmdir(dn) + except OSError: + break + else: + dn = os.path.dirname(dn) + del self.elt + + +class PublishedObject(models.Model): + uri = models.CharField(max_length = 255) + der = BlobField() + hash = models.CharField(max_length = 64) + client = models.ForeignKey(Client) + session = models.ForeignKey(Session) + + class Meta: # pylint: disable=C1001,W0232 + unique_together = (("session", "hash"), + ("session", "uri")) diff --git a/rpki/publication.py b/rpki/publication.py index 5fc7f3dd..7939b9de 100644 --- a/rpki/publication.py +++ b/rpki/publication.py @@ -1,470 +1,73 @@ # $Id$ # -# Copyright (C) 2009--2012 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. -# +# Copyright (C) 2013--2014 Dragon Research Labs ("DRL") +# Portions copyright (C) 2009--2012 Internet Systems Consortium ("ISC") # Portions copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. +# copyright notices and this permission notice appear in all copies. # -# THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL, ISC, AND ARIN DISCLAIM ALL +# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL, +# ISC, OR ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION +# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -RPKI "publication" protocol. +RPKI publication protocol. """ -import os -import errno import logging -import rpki.resource_set + import rpki.x509 -import rpki.sql import rpki.exceptions -import rpki.xml_utils -import rpki.http -import rpki.up_down import rpki.relaxng -import rpki.sundial -import rpki.log logger = logging.getLogger(__name__) -class publication_namespace(object): - """ - XML namespace parameters for publication protocol. - """ - - xmlns = rpki.relaxng.publication.xmlns - nsmap = rpki.relaxng.publication.nsmap +nsmap = rpki.relaxng.publication.nsmap +version = rpki.relaxng.publication.version -class control_elt(rpki.xml_utils.data_elt, rpki.sql.sql_persistent, publication_namespace): - """ - Virtual class for control channel objects. - """ +tag_msg = rpki.relaxng.publication.xmlns + "msg" +tag_list = rpki.relaxng.publication.xmlns + "list" +tag_publish = rpki.relaxng.publication.xmlns + "publish" +tag_withdraw = rpki.relaxng.publication.xmlns + "withdraw" +tag_report_error = rpki.relaxng.publication.xmlns + "report_error" - def serve_dispatch(self, r_msg, cb, eb): - """ - Action dispatch handler. This needs special handling because we - need to make sure that this PDU arrived via the control channel. - """ - if self.client is not None: - raise rpki.exceptions.BadQuery("Control query received on client channel") - rpki.xml_utils.data_elt.serve_dispatch(self, r_msg, cb, eb) -class config_elt(control_elt): +def raise_if_error(pdu): """ - <config/> element. This is a little weird because there should - never be more than one row in the SQL config table, but we have to - put the BPKI CRL somewhere and SQL is the least bad place available. + Raise an appropriate error if this is a <report_error/> PDU. - So we reuse a lot of the SQL machinery, but we nail config_id at 1, - we don't expose it in the XML protocol, and we only support the get - and set actions. + As a convenience, this will also accept a <msg/> PDU and raise an + appropriate error if it contains any <report_error/> PDUs or if + the <msg/> is not a reply. """ - attributes = ("action", "tag") - element_name = "config" - elements = ("bpki_crl",) - - sql_template = rpki.sql.template( - "config", - "config_id", - ("bpki_crl", rpki.x509.CRL)) - - wired_in_config_id = 1 - - def startElement(self, stack, name, attrs): - """ - StartElement() handler for config object. This requires special - handling because of the weird way we treat config_id. - """ - control_elt.startElement(self, stack, name, attrs) - self.config_id = self.wired_in_config_id - - @classmethod - def fetch(cls, gctx): - """ - Fetch the config object from SQL. This requires special handling - because of the weird way we treat config_id. - """ - return cls.sql_fetch(gctx, cls.wired_in_config_id) - - def serve_set(self, r_msg, cb, eb): - """ - Handle a set action. This requires special handling because - config doesn't support the create method. - """ - if self.sql_fetch(self.gctx, self.config_id) is None: - control_elt.serve_create(self, r_msg, cb, eb) + if pdu.tag == tag_report_error: + code = pdu.get("error_code") + logger.debug("<report_error/> code %r", code) + e = getattr(rpki.exceptions, code, None) + if e is not None and issubclass(e, rpki.exceptions.RPKI_Exception): + raise e(pdu.text) else: - control_elt.serve_set(self, r_msg, cb, eb) - - def serve_fetch_one_maybe(self): - """ - Find the config object on which a get or set method should - operate. - """ - return self.sql_fetch(self.gctx, self.config_id) - -class client_elt(control_elt): - """ - <client/> element. - """ - - element_name = "client" - attributes = ("action", "tag", "client_handle", "base_uri") - elements = ("bpki_cert", "bpki_glue") - booleans = ("clear_replay_protection",) - - sql_template = rpki.sql.template( - "client", - "client_id", - "client_handle", - "base_uri", - ("bpki_cert", rpki.x509.X509), - ("bpki_glue", rpki.x509.X509), - ("last_cms_timestamp", rpki.sundial.datetime)) - - base_uri = None - bpki_cert = None - bpki_glue = None - last_cms_timestamp = None - - def serve_post_save_hook(self, q_pdu, r_pdu, cb, eb): - """ - Extra server actions for client_elt. - """ - actions = [] - if q_pdu.clear_replay_protection: - actions.append(self.serve_clear_replay_protection) - def loop(iterator, action): - action(iterator, eb) - rpki.async.iterator(actions, loop, cb) - - def serve_clear_replay_protection(self, cb, eb): - """ - Handle a clear_replay_protection action for this client. - """ - self.last_cms_timestamp = None - self.sql_mark_dirty() - cb() - - def serve_fetch_one_maybe(self): - """ - Find the client object on which a get, set, or destroy method - should operate, or which would conflict with a create method. - """ - return self.sql_fetch_where1(self.gctx, "client_handle = %s", (self.client_handle,)) - - def serve_fetch_all(self): - """ - Find client objects on which a list method should operate. - """ - return self.sql_fetch_all(self.gctx) - - def check_allowed_uri(self, uri): - """ - Make sure that a target URI is within this client's allowed URI space. - """ - if not uri.startswith(self.base_uri): - raise rpki.exceptions.ForbiddenURI - -class publication_object_elt(rpki.xml_utils.base_elt, publication_namespace): - """ - Virtual class for publishable objects. These have very similar - syntax, differences lie in underlying datatype and methods. XML - methods are a little different from the pattern used for objects - that support the create/set/get/list/destroy actions, but - publishable objects don't go in SQL either so these classes would be - different in any case. - """ - - attributes = ("action", "tag", "client_handle", "uri") - payload_type = None - payload = None - - def endElement(self, stack, name, text): - """ - Handle a publishable element element. - """ - assert name == self.element_name, "Unexpected name %s, stack %s" % (name, stack) - if text: - self.payload = self.payload_type(Base64 = text) # pylint: disable=E1102 - stack.pop() - - def toXML(self): - """ - Generate XML element for publishable object. - """ - elt = self.make_elt() - if self.payload: - elt.text = self.payload.get_Base64() - return elt - - def serve_dispatch(self, r_msg, cb, eb): - """ - Action dispatch handler. - """ - # pylint: disable=E0203 - try: - if self.client is None: - raise rpki.exceptions.BadQuery("Client query received on control channel") - dispatch = { "publish" : self.serve_publish, - "withdraw" : self.serve_withdraw } - if self.action not in dispatch: - raise rpki.exceptions.BadQuery("Unexpected query: action %s" % self.action) - self.client.check_allowed_uri(self.uri) - dispatch[self.action]() - r_pdu = self.__class__() - r_pdu.action = self.action - r_pdu.tag = self.tag - r_pdu.uri = self.uri - r_msg.append(r_pdu) - cb() - except rpki.exceptions.NoObjectAtURI, e: - # This can happen when we're cleaning up from a prior mess, so - # we generate a <report_error/> PDU then carry on. - r_msg.append(report_error_elt.from_exception(e, self.tag)) - cb() - - def serve_publish(self): - """ - Publish an object. - """ - logger.info("Publishing %s", self.payload.tracking_data(self.uri)) - filename = self.uri_to_filename() - filename_tmp = filename + ".tmp" - dirname = os.path.dirname(filename) - if not os.path.isdir(dirname): - os.makedirs(dirname) - f = open(filename_tmp, "wb") - f.write(self.payload.get_DER()) - f.close() - os.rename(filename_tmp, filename) - - def serve_withdraw(self): - """ - Withdraw an object, then recursively delete empty directories. - """ - logger.info("Withdrawing %s", self.uri) - filename = self.uri_to_filename() - try: - os.remove(filename) - except OSError, e: - if e.errno == errno.ENOENT: - raise rpki.exceptions.NoObjectAtURI("No object published at %s" % self.uri) - else: - raise - min_path_len = len(self.gctx.publication_base.rstrip("/")) - dirname = os.path.dirname(filename) - while len(dirname) > min_path_len: - try: - os.rmdir(dirname) - except OSError: - break - else: - dirname = os.path.dirname(dirname) - - def uri_to_filename(self): - """ - Convert a URI to a local filename. - """ - if not self.uri.startswith("rsync://"): - raise rpki.exceptions.BadURISyntax(self.uri) - path = self.uri.split("/")[3:] - if not self.gctx.publication_multimodule: - del path[0] - path.insert(0, self.gctx.publication_base.rstrip("/")) - filename = "/".join(path) - if "/../" in filename or filename.endswith("/.."): - raise rpki.exceptions.BadURISyntax(filename) - return filename - - @classmethod - def make_publish(cls, uri, obj, tag = None): - """ - Construct a publication PDU. - """ - assert cls.payload_type is not None and type(obj) is cls.payload_type - return cls.make_pdu(action = "publish", uri = uri, payload = obj, tag = tag) - - @classmethod - def make_withdraw(cls, uri, obj, tag = None): - """ - Construct a withdrawal PDU. - """ - assert cls.payload_type is not None and type(obj) is cls.payload_type - return cls.make_pdu(action = "withdraw", uri = uri, tag = tag) - - def raise_if_error(self): - """ - No-op, since this is not a <report_error/> PDU. - """ - pass - -class certificate_elt(publication_object_elt): - """ - <certificate/> element. - """ - - element_name = "certificate" - payload_type = rpki.x509.X509 - -class crl_elt(publication_object_elt): - """ - <crl/> element. - """ - - element_name = "crl" - payload_type = rpki.x509.CRL - -class manifest_elt(publication_object_elt): - """ - <manifest/> element. - """ - - element_name = "manifest" - payload_type = rpki.x509.SignedManifest - -class roa_elt(publication_object_elt): - """ - <roa/> element. - """ - - element_name = "roa" - payload_type = rpki.x509.ROA - -class ghostbuster_elt(publication_object_elt): - """ - <ghostbuster/> element. - """ - - element_name = "ghostbuster" - payload_type = rpki.x509.Ghostbuster - -publication_object_elt.obj2elt = dict( - (e.payload_type, e) for e in - (certificate_elt, crl_elt, manifest_elt, roa_elt, ghostbuster_elt)) - -class report_error_elt(rpki.xml_utils.text_elt, publication_namespace): - """ - <report_error/> element. - """ - - element_name = "report_error" - attributes = ("tag", "error_code") - text_attribute = "error_text" - - error_text = None - - @classmethod - def from_exception(cls, e, tag = None): - """ - Generate a <report_error/> element from an exception. - """ - self = cls() - self.tag = tag - self.error_code = e.__class__.__name__ - self.error_text = str(e) - return self - - def __str__(self): - s = "" - if getattr(self, "tag", None) is not None: - s += "[%s] " % self.tag - s += self.error_code - if getattr(self, "error_text", None) is not None: - s += ": " + self.error_text - return s - - def raise_if_error(self): - """ - Raise exception associated with this <report_error/> PDU. - """ - t = rpki.exceptions.__dict__.get(self.error_code) - if isinstance(t, type) and issubclass(t, rpki.exceptions.RPKI_Exception): - raise t(getattr(self, "text", None)) - else: - raise rpki.exceptions.BadPublicationReply("Unexpected response from pubd: %s" % self) - -class msg(rpki.xml_utils.msg, publication_namespace): - """ - Publication PDU. - """ - - ## @var version - # Protocol version - version = int(rpki.relaxng.publication.version) - - ## @var pdus - # Dispatch table of PDUs for this protocol. - pdus = dict((x.element_name, x) for x in - (config_elt, client_elt, certificate_elt, crl_elt, manifest_elt, roa_elt, ghostbuster_elt, report_error_elt)) - - def serve_top_level(self, gctx, client, cb): - """ - Serve one msg PDU. - """ - if not self.is_query(): - raise rpki.exceptions.BadQuery("Message type is not query") - r_msg = self.__class__.reply() - - def loop(iterator, q_pdu): - - def fail(e): - if not isinstance(e, rpki.exceptions.NotFound): - logger.exception("Exception processing PDU %r", q_pdu) - r_msg.append(report_error_elt.from_exception(e, q_pdu.tag)) - cb(r_msg) - - try: - q_pdu.gctx = gctx - q_pdu.client = client - q_pdu.serve_dispatch(r_msg, iterator, fail) - except (rpki.async.ExitNow, SystemExit): - raise - except Exception, e: - fail(e) - - def done(): - cb(r_msg) - - rpki.async.iterator(self, loop, done) - -class sax_handler(rpki.xml_utils.sax_handler): - """ - SAX handler for publication protocol. - """ + raise rpki.exceptions.BadPublicationReply("Unexpected response from pubd: %r, %r" % (code, pdu)) - pdu = msg - name = "msg" - version = rpki.relaxng.publication.version + if pdu.tag == tag_msg: + if pdu.get("type") != "reply": + raise rpki.exceptions.BadPublicationReply("Unexpected response from pubd: expected reply, got %r" % pdu.get("type")) + for p in pdu: + raise_if_error(p) class cms_msg(rpki.x509.XML_CMS_object): """ - Class to hold a CMS-signed publication PDU. + CMS-signed publication PDU. """ encoding = "us-ascii" schema = rpki.relaxng.publication - saxify = sax_handler.saxify diff --git a/rpki/publication_control.py b/rpki/publication_control.py new file mode 100644 index 00000000..ddb9d417 --- /dev/null +++ b/rpki/publication_control.py @@ -0,0 +1,74 @@ +# $Id$ +# +# Copyright (C) 2013--2014 Dragon Research Labs ("DRL") +# Portions copyright (C) 2009--2012 Internet Systems Consortium ("ISC") +# Portions copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notices and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL, ISC, AND ARIN DISCLAIM ALL +# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL, +# ISC, OR ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION +# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +""" +RPKI publication control protocol. + +Per IETF SIDR WG discussion, this is now separate from the publication +protocol itself. +""" + +import logging + +import rpki.x509 +import rpki.exceptions +import rpki.relaxng + +logger = logging.getLogger(__name__) + + +nsmap = rpki.relaxng.publication_control.nsmap +version = rpki.relaxng.publication_control.version + +tag_msg = rpki.relaxng.publication_control.xmlns + "msg" +tag_client = rpki.relaxng.publication_control.xmlns + "client" +tag_bpki_cert = rpki.relaxng.publication_control.xmlns + "bpki_cert" +tag_bpki_glue = rpki.relaxng.publication_control.xmlns + "bpki_glue" +tag_report_error = rpki.relaxng.publication_control.xmlns + "report_error" + + +def raise_if_error(pdu): + """ + Raise an appropriate error if this is a <report_error/> PDU. + + As a convience, this will also accept a <msg/> PDU and raise an + appropriate error if it contains any <report_error/> PDUs. + """ + + if pdu.tag == tag_report_error: + code = pdu.get("error_code") + logger.debug("<report_error/> code %r", code) + e = getattr(rpki.exceptions, code, None) + if e is not None and issubclass(e, rpki.exceptions.RPKI_Exception): + raise e(pdu.text) + else: + raise rpki.exceptions.BadPublicationReply("Unexpected response from pubd: %r, %r" % (code, pdu)) + + if pdu.tag == tag_msg: + for p in pdu: + raise_if_error(p) + + +class cms_msg(rpki.x509.XML_CMS_object): + """ + CMS-signed publication control PDU. + """ + + encoding = "us-ascii" + schema = rpki.relaxng.publication_control diff --git a/rpki/rcynic.py b/rpki/rcynic.py index 10ad7516..a36e4a4e 100644 --- a/rpki/rcynic.py +++ b/rpki/rcynic.py @@ -53,6 +53,7 @@ class rcynic_object(object): Print a bunch of object attributes, quietly ignoring any that might be missing. """ + for a in attrs: try: print "%s: %s" % (a.capitalize(), getattr(self, a)) @@ -63,6 +64,7 @@ class rcynic_object(object): """ Print common object attributes. """ + self.show_attrs("filename", "uri", "status", "timestamp") class rcynic_certificate(rcynic_object): @@ -91,6 +93,7 @@ class rcynic_certificate(rcynic_object): """ Print certificate attributes. """ + rcynic_object.show(self) self.show_attrs("notBefore", "notAfter", "aia_uri", "sia_directory_uri", "resources") @@ -128,6 +131,7 @@ class rcynic_roa(rcynic_object): """ Print ROA attributes. """ + rcynic_object.show(self) self.show_attrs("notBefore", "notAfter", "aia_uri", "resources", "asID") if self.prefix_sets: diff --git a/rpki/relaxng.py b/rpki/relaxng.py index e43384e7..1b16073b 100644 --- a/rpki/relaxng.py +++ b/rpki/relaxng.py @@ -6,7 +6,7 @@ from rpki.relaxng_parser import RelaxNGParser ## Parsed RelaxNG left_right schema left_right = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> <!-- - $Id: left-right-schema.rnc 5902 2014-07-18 16:37:04Z sra $ + $Id: left-right.rnc 5981 2014-10-02 04:54:51Z sra $ RelaxNG schema for RPKI left-right protocol. @@ -552,12 +552,12 @@ left_right = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> </attribute> </optional> <optional> - <element name="bpki_cms_cert"> + <element name="bpki_cert"> <ref name="base64"/> </element> </optional> <optional> - <element name="bpki_cms_glue"> + <element name="bpki_glue"> <ref name="base64"/> </element> </optional> @@ -768,6 +768,11 @@ left_right = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> <ref name="bsc_handle"/> </optional> <optional> + <attribute name="rrdp_notification_uri"> + <ref name="uri"/> + </attribute> + </optional> + <optional> <element name="bpki_cert"> <ref name="base64"/> </element> @@ -967,14 +972,12 @@ left_right = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> <ref name="ipv6_list"/> </attribute> </optional> - <optional> - <attribute name="cn"> - <data type="string"> - <param name="maxLength">64</param> - <param name="pattern">[\-0-9A-Za-z_ ]+</param> - </data> - </attribute> - </optional> + <attribute name="cn"> + <data type="string"> + <param name="maxLength">64</param> + <param name="pattern">[\-0-9A-Za-z_ ]+</param> + </data> + </attribute> <optional> <attribute name="sn"> <data type="string"> @@ -1102,7 +1105,7 @@ left_right = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> ## Parsed RelaxNG myrpki schema myrpki = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> <!-- - $Id: myrpki.rnc 5757 2014-04-05 22:42:12Z sra $ + $Id: myrpki.rnc 5876 2014-06-26 19:00:12Z sra $ RelaxNG schema for MyRPKI XML messages. @@ -1481,11 +1484,183 @@ myrpki = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> --> ''') -## @var publication -## Parsed RelaxNG publication schema -publication = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> +## @var oob_setup +## Parsed RelaxNG oob_setup schema +oob_setup = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> +<!-- $Id: rpki-setup.rnc 3429 2015-10-14 23:46:50Z sra $ --> +<grammar ns="http://www.hactrn.net/uris/rpki/rpki-setup/" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes"> + <define name="version"> + <value>1</value> + </define> + <define name="base64"> + <data type="base64Binary"> + <param name="maxLength">512000</param> + </data> + </define> + <define name="handle"> + <data type="string"> + <param name="maxLength">255</param> + <param name="pattern">[\-_A-Za-z0-9/]*</param> + </data> + </define> + <define name="uri"> + <data type="anyURI"> + <param name="maxLength">4096</param> + </data> + </define> + <define name="any"> + <element> + <anyName/> + <zeroOrMore> + <attribute> + <anyName/> + </attribute> + </zeroOrMore> + <zeroOrMore> + <choice> + <ref name="any"/> + <text/> + </choice> + </zeroOrMore> + </element> + </define> + <define name="authorization_token"> + <ref name="base64"/> + </define> + <define name="bpki_ta"> + <ref name="base64"/> + </define> + <start combine="choice"> + <element name="child_request"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="child_handle"> + <ref name="handle"/> + </attribute> + <element name="child_bpki_ta"> + <ref name="bpki_ta"/> + </element> + </element> + </start> + <start combine="choice"> + <element name="parent_response"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="service_uri"> + <ref name="uri"/> + </attribute> + <attribute name="child_handle"> + <ref name="handle"/> + </attribute> + <attribute name="parent_handle"> + <ref name="handle"/> + </attribute> + <element name="parent_bpki_ta"> + <ref name="bpki_ta"/> + </element> + <optional> + <element name="offer"> + <empty/> + </element> + </optional> + <zeroOrMore> + <element name="referral"> + <attribute name="referrer"> + <ref name="handle"/> + </attribute> + <optional> + <attribute name="contact_uri"> + <ref name="uri"/> + </attribute> + </optional> + <ref name="authorization_token"/> + </element> + </zeroOrMore> + </element> + </start> + <start combine="choice"> + <element name="publisher_request"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="publisher_handle"> + <ref name="handle"/> + </attribute> + <element name="publisher_bpki_ta"> + <ref name="bpki_ta"/> + </element> + <zeroOrMore> + <element name="referral"> + <attribute name="referrer"> + <ref name="handle"/> + </attribute> + <ref name="authorization_token"/> + </element> + </zeroOrMore> + </element> + </start> + <start combine="choice"> + <element name="repository_response"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="service_uri"> + <ref name="uri"/> + </attribute> + <attribute name="publisher_handle"> + <ref name="handle"/> + </attribute> + <attribute name="sia_base"> + <ref name="uri"/> + </attribute> + <optional> + <attribute name="rrdp_notification_uri"> + <ref name="uri"/> + </attribute> + </optional> + <element name="repository_bpki_ta"> + <ref name="bpki_ta"/> + </element> + </element> + </start> + <start combine="choice"> + <element name="authorization"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="authorized_sia_base"> + <ref name="uri"/> + </attribute> + <ref name="bpki_ta"/> + </element> + </start> + <start combine="choice"> + <element name="error"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="reason"> + <choice> + <value>syntax-error</value> + <value>authentication-failure</value> + <value>refused</value> + </choice> + </attribute> + <optional> + <ref name="any"/> + </optional> + </element> + </start> +</grammar> +''') + +## @var publication_control +## Parsed RelaxNG publication_control schema +publication_control = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> <!-- - $Id: publication-schema.rnc 5902 2014-07-18 16:37:04Z sra $ + $Id: publication-control.rnc 5903 2014-07-18 17:08:13Z sra $ RelaxNG schema for RPKI publication protocol. @@ -1506,7 +1681,7 @@ publication = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. --> -<grammar ns="http://www.hactrn.net/uris/rpki/publication-spec/" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes"> +<grammar ns="http://www.hactrn.net/uris/rpki/publication-control/" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes"> <define name="version"> <value>1</value> </define> @@ -1540,26 +1715,12 @@ publication = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> </start> <!-- PDUs allowed in a query --> <define name="query_elt"> - <choice> - <ref name="config_query"/> - <ref name="client_query"/> - <ref name="certificate_query"/> - <ref name="crl_query"/> - <ref name="manifest_query"/> - <ref name="roa_query"/> - <ref name="ghostbuster_query"/> - </choice> + <ref name="client_query"/> </define> <!-- PDUs allowed in a reply --> <define name="reply_elt"> <choice> - <ref name="config_reply"/> <ref name="client_reply"/> - <ref name="certificate_reply"/> - <ref name="crl_reply"/> - <ref name="manifest_reply"/> - <ref name="roa_reply"/> - <ref name="ghostbuster_reply"/> <ref name="report_error_reply"/> </choice> </define> @@ -1603,60 +1764,7 @@ publication = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> <param name="pattern">[\-_A-Za-z0-9/]+</param> </data> </define> - <!-- - <config/> element (use restricted to repository operator) - config_handle attribute, create, list, and destroy commands omitted deliberately, see code for details - --> - <define name="config_payload"> - <optional> - <element name="bpki_crl"> - <ref name="base64"/> - </element> - </optional> - </define> - <define name="config_query" combine="choice"> - <element name="config"> - <attribute name="action"> - <value>set</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="config_payload"/> - </element> - </define> - <define name="config_reply" combine="choice"> - <element name="config"> - <attribute name="action"> - <value>set</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - </element> - </define> - <define name="config_query" combine="choice"> - <element name="config"> - <attribute name="action"> - <value>get</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - </element> - </define> - <define name="config_reply" combine="choice"> - <element name="config"> - <attribute name="action"> - <value>get</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="config_payload"/> - </element> - </define> - <!-- <client/> element (use restricted to repository operator) --> + <!-- <client/> element --> <define name="client_handle"> <attribute name="client_handle"> <ref name="object_handle"/> @@ -1801,242 +1909,217 @@ publication = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> <ref name="client_handle"/> </element> </define> - <!-- <certificate/> element --> - <define name="certificate_query" combine="choice"> - <element name="certificate"> - <attribute name="action"> - <value>publish</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - <ref name="base64"/> - </element> + <!-- <report_error/> element --> + <define name="error"> + <data type="token"> + <param name="maxLength">1024</param> + </data> </define> - <define name="certificate_reply" combine="choice"> - <element name="certificate"> - <attribute name="action"> - <value>publish</value> - </attribute> + <define name="report_error_reply"> + <element name="report_error"> <optional> <ref name="tag"/> </optional> - <ref name="uri"/> - </element> - </define> - <define name="certificate_query" combine="choice"> - <element name="certificate"> - <attribute name="action"> - <value>withdraw</value> + <attribute name="error_code"> + <ref name="error"/> </attribute> <optional> - <ref name="tag"/> + <data type="string"> + <param name="maxLength">512000</param> + </data> </optional> - <ref name="uri"/> </element> </define> - <define name="certificate_reply" combine="choice"> - <element name="certificate"> - <attribute name="action"> - <value>withdraw</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> +</grammar> +<!-- + Local Variables: + indent-tabs-mode: nil + comment-start: "# " + comment-start-skip: "#[ \t]*" + End: +--> +''') + +## @var publication +## Parsed RelaxNG publication schema +publication = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> +<!-- + $Id: publication.rnc 5896 2014-07-15 19:34:32Z sra $ + + RelaxNG schema for RPKI publication protocol, from current I-D. + + Copyright (c) 2014 IETF Trust and the persons identified as authors + of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Internet Society, IETF or IETF Trust, nor the + names of specific contributors, may be used to endorse or promote + products derived from this software without specific prior written + permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +--> +<grammar ns="http://www.hactrn.net/uris/rpki/publication-spec/" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes"> + <!-- This is version 3 of the protocol. --> + <define name="version"> + <value>3</value> </define> - <!-- <crl/> element --> - <define name="crl_query" combine="choice"> - <element name="crl"> - <attribute name="action"> - <value>publish</value> + <!-- Top level PDU is either a query or a reply. --> + <start combine="choice"> + <element name="msg"> + <attribute name="version"> + <ref name="version"/> </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - <ref name="base64"/> - </element> - </define> - <define name="crl_reply" combine="choice"> - <element name="crl"> - <attribute name="action"> - <value>publish</value> + <attribute name="type"> + <value>query</value> </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> + <zeroOrMore> + <ref name="query_elt"/> + </zeroOrMore> </element> - </define> - <define name="crl_query" combine="choice"> - <element name="crl"> - <attribute name="action"> - <value>withdraw</value> + </start> + <start combine="choice"> + <element name="msg"> + <attribute name="version"> + <ref name="version"/> </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> - </define> - <define name="crl_reply" combine="choice"> - <element name="crl"> - <attribute name="action"> - <value>withdraw</value> + <attribute name="type"> + <value>reply</value> </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> + <zeroOrMore> + <ref name="reply_elt"/> + </zeroOrMore> </element> + </start> + <!-- PDUs allowed in queries and replies. --> + <define name="query_elt"> + <choice> + <ref name="publish_query"/> + <ref name="withdraw_query"/> + <ref name="list_query"/> + </choice> </define> - <!-- <manifest/> element --> - <define name="manifest_query" combine="choice"> - <element name="manifest"> - <attribute name="action"> - <value>publish</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - <ref name="base64"/> - </element> + <define name="reply_elt"> + <choice> + <ref name="publish_reply"/> + <ref name="withdraw_reply"/> + <ref name="list_reply"/> + <ref name="report_error_reply"/> + </choice> </define> - <define name="manifest_reply" combine="choice"> - <element name="manifest"> - <attribute name="action"> - <value>publish</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> + <!-- Tag attributes for bulk operations. --> + <define name="tag"> + <attribute name="tag"> + <data type="token"> + <param name="maxLength">1024</param> + </data> + </attribute> </define> - <define name="manifest_query" combine="choice"> - <element name="manifest"> - <attribute name="action"> - <value>withdraw</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> + <!-- Base64 encoded DER stuff. --> + <define name="base64"> + <data type="base64Binary"/> </define> - <define name="manifest_reply" combine="choice"> - <element name="manifest"> - <attribute name="action"> - <value>withdraw</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> + <!-- Publication URIs. --> + <define name="uri"> + <attribute name="uri"> + <data type="anyURI"> + <param name="maxLength">4096</param> + </data> + </attribute> </define> - <!-- <roa/> element --> - <define name="roa_query" combine="choice"> - <element name="roa"> - <attribute name="action"> - <value>publish</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - <ref name="base64"/> - </element> + <!-- Digest of objects being withdrawn --> + <define name="hash"> + <attribute name="hash"> + <data type="string"> + <param name="pattern">[0-9a-fA-F]+</param> + </data> + </attribute> </define> - <define name="roa_reply" combine="choice"> - <element name="roa"> - <attribute name="action"> - <value>publish</value> - </attribute> + <!-- Error codes. --> + <define name="error"> + <data type="token"> + <param name="maxLength">1024</param> + </data> + </define> + <!-- <publish/> element --> + <define name="publish_query"> + <element name="publish"> <optional> <ref name="tag"/> </optional> <ref name="uri"/> - </element> - </define> - <define name="roa_query" combine="choice"> - <element name="roa"> - <attribute name="action"> - <value>withdraw</value> - </attribute> <optional> - <ref name="tag"/> + <ref name="hash"/> </optional> - <ref name="uri"/> + <ref name="base64"/> </element> </define> - <define name="roa_reply" combine="choice"> - <element name="roa"> - <attribute name="action"> - <value>withdraw</value> - </attribute> + <define name="publish_reply"> + <element name="publish"> <optional> <ref name="tag"/> </optional> <ref name="uri"/> </element> </define> - <!-- <ghostbuster/> element --> - <define name="ghostbuster_query" combine="choice"> - <element name="ghostbuster"> - <attribute name="action"> - <value>publish</value> - </attribute> + <!-- <withdraw/> element --> + <define name="withdraw_query"> + <element name="withdraw"> <optional> <ref name="tag"/> </optional> <ref name="uri"/> - <ref name="base64"/> + <ref name="hash"/> </element> </define> - <define name="ghostbuster_reply" combine="choice"> - <element name="ghostbuster"> - <attribute name="action"> - <value>publish</value> - </attribute> + <define name="withdraw_reply"> + <element name="withdraw"> <optional> <ref name="tag"/> </optional> <ref name="uri"/> </element> </define> - <define name="ghostbuster_query" combine="choice"> - <element name="ghostbuster"> - <attribute name="action"> - <value>withdraw</value> - </attribute> + <!-- <list/> element --> + <define name="list_query"> + <element name="list"> <optional> <ref name="tag"/> </optional> - <ref name="uri"/> </element> </define> - <define name="ghostbuster_reply" combine="choice"> - <element name="ghostbuster"> - <attribute name="action"> - <value>withdraw</value> - </attribute> + <define name="list_reply"> + <element name="list"> <optional> <ref name="tag"/> </optional> <ref name="uri"/> + <ref name="hash"/> </element> </define> <!-- <report_error/> element --> - <define name="error"> - <data type="token"> - <param name="maxLength">1024</param> - </data> - </define> <define name="report_error_reply"> <element name="report_error"> <optional> @@ -2066,7 +2149,7 @@ publication = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> ## Parsed RelaxNG router_certificate schema router_certificate = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> <!-- - $Id: router-certificate-schema.rnc 5757 2014-04-05 22:42:12Z sra $ + $Id: router-certificate.rnc 5881 2014-07-03 16:55:02Z sra $ RelaxNG schema for BGPSEC router certificate interchange format. @@ -2164,11 +2247,165 @@ router_certificate = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> --> ''') +## @var rrdp +## Parsed RelaxNG rrdp schema +rrdp = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> +<!-- + $Id: rrdp.rnc 6010 2014-11-08 18:01:58Z sra $ + + RelaxNG schema for RPKI Repository Delta Protocol (RRDP). + + Copyright (C) 2014 Dragon Research Labs ("DRL") + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH + REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, + INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + PERFORMANCE OF THIS SOFTWARE. +--> +<grammar ns="http://www.ripe.net/rpki/rrdp" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes"> + <define name="version"> + <data type="positiveInteger"> + <param name="maxInclusive">1</param> + </data> + </define> + <define name="serial"> + <data type="nonNegativeInteger"/> + </define> + <define name="uri"> + <data type="anyURI"/> + </define> + <define name="uuid"> + <data type="string"> + <param name="pattern">[\-0-9a-fA-F]+</param> + </data> + </define> + <define name="hash"> + <data type="string"> + <param name="pattern">[0-9a-fA-F]+</param> + </data> + </define> + <define name="base64"> + <data type="base64Binary"/> + </define> + <!-- Notification file: lists current snapshots and deltas --> + <start combine="choice"> + <element name="notification"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="session_id"> + <ref name="uuid"/> + </attribute> + <attribute name="serial"> + <ref name="serial"/> + </attribute> + <element name="snapshot"> + <attribute name="uri"> + <ref name="uri"/> + </attribute> + <attribute name="hash"> + <ref name="hash"/> + </attribute> + </element> + <zeroOrMore> + <element name="delta"> + <attribute name="serial"> + <ref name="serial"/> + </attribute> + <attribute name="uri"> + <ref name="uri"/> + </attribute> + <attribute name="hash"> + <ref name="hash"/> + </attribute> + </element> + </zeroOrMore> + </element> + </start> + <!-- Snapshot segment: think DNS AXFR. --> + <start combine="choice"> + <element name="snapshot"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="session_id"> + <ref name="uuid"/> + </attribute> + <attribute name="serial"> + <ref name="serial"/> + </attribute> + <zeroOrMore> + <element name="publish"> + <attribute name="uri"> + <ref name="uri"/> + </attribute> + <ref name="base64"/> + </element> + </zeroOrMore> + </element> + </start> + <!-- Delta segment: think DNS IXFR. --> + <start combine="choice"> + <element name="delta"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="session_id"> + <ref name="uuid"/> + </attribute> + <attribute name="serial"> + <ref name="serial"/> + </attribute> + <oneOrMore> + <ref name="delta_element"/> + </oneOrMore> + </element> + </start> + <define name="delta_element" combine="choice"> + <element name="publish"> + <attribute name="uri"> + <ref name="uri"/> + </attribute> + <optional> + <attribute name="hash"> + <ref name="hash"/> + </attribute> + </optional> + <ref name="base64"/> + </element> + </define> + <define name="delta_element" combine="choice"> + <element name="withdraw"> + <attribute name="uri"> + <ref name="uri"/> + </attribute> + <attribute name="hash"> + <ref name="hash"/> + </attribute> + </element> + </define> +</grammar> +<!-- + Local Variables: + indent-tabs-mode: nil + comment-start: "# " + comment-start-skip: "#[ \t]*" + End: +--> +''') + ## @var up_down ## Parsed RelaxNG up_down schema up_down = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?> <!-- - $Id: up-down-schema.rnc 5757 2014-04-05 22:42:12Z sra $ + $Id: up-down.rnc 5881 2014-07-03 16:55:02Z sra $ RelaxNG schema for the up-down protocol, extracted from RFC 6492. diff --git a/rpki/resource_set.py b/rpki/resource_set.py index fea6ad2d..130bf4e7 100644 --- a/rpki/resource_set.py +++ b/rpki/resource_set.py @@ -86,6 +86,7 @@ class resource_range_as(resource_range): """ Convert a resource_range_as to string format. """ + if self.min == self.max: return str(self.min) else: @@ -96,6 +97,7 @@ class resource_range_as(resource_range): """ Parse ASN resource range from text (eg, XML attributes). """ + r = re_asn_range.match(x) if r: return cls(long(r.group(1)), long(r.group(2))) @@ -107,6 +109,7 @@ class resource_range_as(resource_range): """ Construct ASN range from strings. """ + if b is None: b = a return cls(long(a), long(b)) @@ -133,6 +136,7 @@ class resource_range_ip(resource_range): prefix. Returns prefix length if it can, otherwise raises MustBePrefix exception. """ + mask = self.min ^ self.max if self.min & mask != 0: raise rpki.exceptions.MustBePrefix @@ -154,6 +158,7 @@ class resource_range_ip(resource_range): the logic in one place. This property is useful primarily in context where catching an exception isn't practical. """ + try: self.prefixlen() return True @@ -164,6 +169,7 @@ class resource_range_ip(resource_range): """ Convert a resource_range_ip to string format. """ + try: return str(self.min) + "/" + str(self.prefixlen()) except rpki.exceptions.MustBePrefix: @@ -174,6 +180,7 @@ class resource_range_ip(resource_range): """ Parse IP address range or prefix from text (eg, XML attributes). """ + r = re_address_range.match(x) if r: return cls.from_strings(r.group(1), r.group(2)) @@ -192,6 +199,7 @@ class resource_range_ip(resource_range): """ Construct a resource range corresponding to a prefix. """ + assert isinstance(prefix, rpki.POW.IPAddress) and isinstance(prefixlen, (int, long)) assert prefixlen >= 0 and prefixlen <= prefix.bits, "Nonsensical prefix length: %s" % prefixlen mask = (1 << (prefix.bits - prefixlen)) - 1 @@ -203,6 +211,7 @@ class resource_range_ip(resource_range): Chop up a resource_range_ip into ranges that can be represented as prefixes. """ + try: self.prefixlen() result.append(self) @@ -226,6 +235,7 @@ class resource_range_ip(resource_range): """ Construct IP address range from strings. """ + if b is None: b = a a = rpki.POW.IPAddress(a) @@ -300,6 +310,7 @@ class resource_set(list): """ Initialize a resource_set. """ + list.__init__(self) if isinstance(ini, (int, long)): ini = str(ini) @@ -317,6 +328,7 @@ class resource_set(list): """ Whack this resource_set into canonical form. """ + assert not self.inherit or len(self) == 0 if not self.canonical: self.sort() @@ -339,6 +351,7 @@ class resource_set(list): """ Wrapper around list.append() (q.v.) to reset canonical flag. """ + list.append(self, item) self.canonical = False @@ -346,6 +359,7 @@ class resource_set(list): """ Wrapper around list.extend() (q.v.) to reset canonical flag. """ + list.extend(self, item) self.canonical = False @@ -353,6 +367,7 @@ class resource_set(list): """ Convert a resource_set to string format. """ + if self.inherit: return inherit_token else: @@ -428,6 +443,7 @@ class resource_set(list): """ Set intersection for resource sets. """ + return self._comm(other)[2] __and__ = intersection @@ -436,6 +452,7 @@ class resource_set(list): """ Set difference for resource sets. """ + return self._comm(other)[0] __sub__ = difference @@ -444,6 +461,7 @@ class resource_set(list): """ Set symmetric difference (XOR) for resource sets. """ + com = self._comm(other) return com[0] | com[1] @@ -453,6 +471,7 @@ class resource_set(list): """ Set membership test for resource sets. """ + assert not self.inherit self.canonize() if not self: @@ -479,6 +498,7 @@ class resource_set(list): """ Test whether self is a subset (possibly improper) of other. """ + for i in self: if not other.contains(i): return False @@ -490,6 +510,7 @@ class resource_set(list): """ Test whether self is a superset (possibly improper) of other. """ + return other.issubset(self) __ge__ = issuperset @@ -506,6 +527,7 @@ class resource_set(list): we can't know the answer here. This is also consistent with __nonzero__ which returns True for inherit sets, and False for empty sets. """ + return self.inherit or other.inherit or list.__ne__(self, other) def __eq__(self, other): @@ -516,6 +538,7 @@ class resource_set(list): Tests whether or not this set is empty. Note that sets with the inherit bit set are considered non-empty, despite having zero length. """ + return self.inherit or len(self) @classmethod @@ -553,6 +576,7 @@ class resource_set(list): a backwards compatability wrapper, real functionality is now part of the range classes. """ + return cls.range_type.parse_str(s) class resource_set_as(resource_set): @@ -577,6 +601,7 @@ class resource_set_ip(resource_set): """ Convert from a resource set to a ROA prefix set. """ + prefix_ranges = [] for r in self: r.chop_into_prefixes(prefix_ranges) @@ -632,6 +657,7 @@ class resource_bag(object): """ True iff self is oversized with respect to other. """ + return not self.asn.issubset(other.asn) or \ not self.v4.issubset(other.v4) or \ not self.v6.issubset(other.v6) @@ -640,6 +666,7 @@ class resource_bag(object): """ True iff self is undersized with respect to other. """ + return not other.asn.issubset(self.asn) or \ not other.v4.issubset(self.v4) or \ not other.v6.issubset(self.v6) @@ -650,6 +677,7 @@ class resource_bag(object): Build a resource bag that just inherits everything from its parent. """ + self = cls() self.asn = resource_set_as() self.v4 = resource_set_ipv4() @@ -665,6 +693,7 @@ class resource_bag(object): Parse a comma-separated text string into a resource_bag. Not particularly efficient, fix that if and when it becomes an issue. """ + asns = [] v4s = [] v6s = [] @@ -689,6 +718,7 @@ class resource_bag(object): temporary: in the long run, we should be using rpki.POW.IPAddress rather than long here. """ + asn = inherit_token if resources[0] == "inherit" else [resource_range_as( r[0], r[1]) for r in resources[0] or ()] v4 = inherit_token if resources[1] == "inherit" else [resource_range_ipv4(r[0], r[1]) for r in resources[1] or ()] v6 = inherit_token if resources[2] == "inherit" else [resource_range_ipv6(r[0], r[1]) for r in resources[2] or ()] @@ -700,6 +730,7 @@ class resource_bag(object): """ True iff all resource sets in this bag are empty. """ + return not self.asn and not self.v4 and not self.v6 def __nonzero__(self): @@ -719,6 +750,7 @@ class resource_bag(object): Compute intersection with another resource_bag. valid_until attribute (if any) inherits from self. """ + return self.__class__(self.asn & other.asn, self.v4 & other.v4, self.v6 & other.v6, @@ -731,6 +763,7 @@ class resource_bag(object): Compute union with another resource_bag. valid_until attribute (if any) inherits from self. """ + return self.__class__(self.asn | other.asn, self.v4 | other.v4, self.v6 | other.v6, @@ -743,6 +776,7 @@ class resource_bag(object): Compute difference against another resource_bag. valid_until attribute (if any) inherits from self """ + return self.__class__(self.asn - other.asn, self.v4 - other.v4, self.v6 - other.v6, @@ -755,6 +789,7 @@ class resource_bag(object): Compute symmetric difference against another resource_bag. valid_until attribute (if any) inherits from self """ + return self.__class__(self.asn ^ other.asn, self.v4 ^ other.v4, self.v6 ^ other.v6, @@ -816,6 +851,7 @@ class roa_prefix(object): Initialize a ROA prefix. max_prefixlen is optional and defaults to prefixlen. max_prefixlen must not be smaller than prefixlen. """ + if max_prefixlen is None: max_prefixlen = prefixlen assert max_prefixlen >= prefixlen, "Bad max_prefixlen: %d must not be shorter than %d" % (max_prefixlen, prefixlen) @@ -828,6 +864,7 @@ class roa_prefix(object): Compare two ROA prefix objects. Comparision is based on prefix, prefixlen, and max_prefixlen, in that order. """ + assert self.__class__ is other.__class__ return (cmp(self.prefix, other.prefix) or cmp(self.prefixlen, other.prefixlen) or @@ -837,6 +874,7 @@ class roa_prefix(object): """ Convert a ROA prefix to string format. """ + if self.prefixlen == self.max_prefixlen: return str(self.prefix) + "/" + str(self.prefixlen) else: @@ -848,24 +886,28 @@ class roa_prefix(object): object. This is an irreversable transformation because it loses the max_prefixlen attribute, nothing we can do about that. """ + return self.range_type.make_prefix(self.prefix, self.prefixlen) def min(self): """ Return lowest address covered by prefix. """ + return self.prefix def max(self): """ Return highest address covered by prefix. """ + return self.prefix | ((1 << (self.prefix.bits - self.prefixlen)) - 1) def to_POW_roa_tuple(self): """ Convert a resource_range_ip to rpki.POW.ROA.setPrefixes() format. """ + return self.prefix, self.prefixlen, self.max_prefixlen @classmethod @@ -873,6 +915,7 @@ class roa_prefix(object): """ Parse ROA prefix from text (eg, an XML attribute). """ + r = re_prefix_with_maxlen.match(x) if r: return cls(rpki.POW.IPAddress(r.group(1)), int(r.group(2)), int(r.group(3))) @@ -910,6 +953,7 @@ class roa_prefix_set(list): """ Initialize a ROA prefix set. """ + list.__init__(self) if isinstance(ini, str) and len(ini): self.extend(self.parse_str(s) for s in ini.split(",")) @@ -923,6 +967,7 @@ class roa_prefix_set(list): """ Convert a ROA prefix set to string format. """ + return ",".join(str(x) for x in self) @classmethod @@ -931,6 +976,7 @@ class roa_prefix_set(list): Parse ROA prefix from text (eg, an XML attribute). This method is a backwards compatability shim. """ + return cls.prefix_type.parse_str(s) def to_resource_set(self): @@ -942,6 +988,7 @@ class roa_prefix_set(list): a more efficient way to do this, but start by getting the output right before worrying about making it fast or pretty. """ + r = self.resource_set_type() s = self.resource_set_type() s.append(None) @@ -982,6 +1029,7 @@ class roa_prefix_set(list): """ Convert ROA prefix set to form used by rpki.POW.ROA.setPrefixes(). """ + if self: return tuple(a.to_POW_roa_tuple() for a in self) else: diff --git a/rpki/rootd.py b/rpki/rootd.py index 78a71bba..1d4f5659 100644 --- a/rpki/rootd.py +++ b/rpki/rootd.py @@ -18,127 +18,67 @@ # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -Trivial RPKI up-down protocol root server. Not recommended for -production use. Overrides a bunch of method definitions from the -rpki.* classes in order to reuse as much code as possible. +Trivial RPKI up-down protocol root server. """ import os import time import logging import argparse + import rpki.resource_set import rpki.up_down -import rpki.left_right import rpki.x509 -import rpki.http +import rpki.http_simple import rpki.config import rpki.exceptions import rpki.relaxng import rpki.sundial import rpki.log import rpki.daemonize +import rpki.publication + +from lxml.etree import Element, SubElement logger = logging.getLogger(__name__) -rootd = None - -class list_pdu(rpki.up_down.list_pdu): - def serve_pdu(self, q_msg, r_msg, ignored, callback, errback): - r_msg.payload = rpki.up_down.list_response_pdu() - rootd.compose_response(r_msg) - callback() - -class issue_pdu(rpki.up_down.issue_pdu): - def serve_pdu(self, q_msg, r_msg, ignored, callback, errback): - self.pkcs10.check_valid_request_ca() - r_msg.payload = rpki.up_down.issue_response_pdu() - rootd.compose_response(r_msg, self.pkcs10) - callback() - -class revoke_pdu(rpki.up_down.revoke_pdu): - def serve_pdu(self, q_msg, r_msg, ignored, callback, errback): - logger.debug("Revocation requested for SKI %s", self.ski) - subject_cert = rootd.get_subject_cert() - if subject_cert is None: - logger.debug("No subject certificate, nothing to revoke") - raise rpki.exceptions.NotInDatabase - if subject_cert.gSKI() != self.ski: - logger.debug("Subject certificate has different SKI %s, not revoking", subject_cert.gSKI()) - raise rpki.exceptions.NotInDatabase - logger.debug("Revoking certificate %s", self.ski) - now = rpki.sundial.now() - rootd.revoke_subject_cert(now) - rootd.del_subject_cert() - rootd.del_subject_pkcs10() - rootd.generate_crl_and_manifest(now) - r_msg.payload = rpki.up_down.revoke_response_pdu() - r_msg.payload.class_name = self.class_name - r_msg.payload.ski = self.ski - callback() - -class error_response_pdu(rpki.up_down.error_response_pdu): - exceptions = rpki.up_down.error_response_pdu.exceptions.copy() - exceptions[rpki.exceptions.ClassNameUnknown, revoke_pdu] = 1301 - exceptions[rpki.exceptions.NotInDatabase, revoke_pdu] = 1302 - -class message_pdu(rpki.up_down.message_pdu): - - name2type = { - "list" : list_pdu, - "list_response" : rpki.up_down.list_response_pdu, - "issue" : issue_pdu, - "issue_response" : rpki.up_down.issue_response_pdu, - "revoke" : revoke_pdu, - "revoke_response" : rpki.up_down.revoke_response_pdu, - "error_response" : error_response_pdu } - - type2name = dict((v, k) for k, v in name2type.items()) - - error_pdu_type = error_response_pdu - - def log_query(self, child): - """ - Log query we're handling. - """ - logger.info("Serving %s query", self.type) - -class sax_handler(rpki.up_down.sax_handler): - pdu = message_pdu - -class cms_msg(rpki.up_down.cms_msg): - saxify = sax_handler.saxify + +class ReplayTracker(object): + """ + Stash for replay protection timestamps. + """ + + def __init__(self): + self.cms_timestamp = None + + class main(object): - def get_root_cert(self): - logger.debug("Read root cert %s", self.rpki_root_cert_file) - self.rpki_root_cert = rpki.x509.X509(Auto_file = self.rpki_root_cert_file) def root_newer_than_subject(self): - return os.stat(self.rpki_root_cert_file).st_mtime > \ - os.stat(os.path.join(self.rpki_root_dir, self.rpki_subject_cert)).st_mtime + return self.rpki_root_cert.mtime > os.stat(self.rpki_subject_cert_file).st_mtime + def get_subject_cert(self): - filename = os.path.join(self.rpki_root_dir, self.rpki_subject_cert) try: - x = rpki.x509.X509(Auto_file = filename) - logger.debug("Read subject cert %s", filename) + x = rpki.x509.X509(Auto_file = self.rpki_subject_cert_file) + logger.debug("Read subject cert %s", self.rpki_subject_cert_file) return x except IOError: return None + def set_subject_cert(self, cert): - filename = os.path.join(self.rpki_root_dir, self.rpki_subject_cert) - logger.debug("Writing subject cert %s, SKI %s", filename, cert.hSKI()) - f = open(filename, "wb") - f.write(cert.get_DER()) - f.close() + logger.debug("Writing subject cert %s, SKI %s", self.rpki_subject_cert_file, cert.hSKI()) + with open(self.rpki_subject_cert_file, "wb") as f: + f.write(cert.get_DER()) + def del_subject_cert(self): - filename = os.path.join(self.rpki_root_dir, self.rpki_subject_cert) - logger.debug("Deleting subject cert %s", filename) - os.remove(filename) + logger.debug("Deleting subject cert %s", self.rpki_subject_cert_file) + os.remove(self.rpki_subject_cert_file) + def get_subject_pkcs10(self): try: @@ -148,11 +88,12 @@ class main(object): except IOError: return None + def set_subject_pkcs10(self, pkcs10): logger.debug("Writing subject PKCS #10 %s", self.rpki_subject_pkcs10) - f = open(self.rpki_subject_pkcs10, "wb") - f.write(pkcs10.get_DER()) - f.close() + with open(self.rpki_subject_pkcs10, "wb") as f: + f.write(pkcs10.get_DER()) + def del_subject_pkcs10(self): logger.debug("Deleting subject PKCS #10 %s", self.rpki_subject_pkcs10) @@ -161,9 +102,14 @@ class main(object): except OSError: pass + def issue_subject_cert_maybe(self, new_pkcs10): now = rpki.sundial.now() subject_cert = self.get_subject_cert() + if subject_cert is None: + subject_cert_hash = None + else: + subject_cert_hash = rpki.x509.sha256(subject_cert.get_DER()).encode("hex") old_pkcs10 = self.get_subject_pkcs10() if new_pkcs10 is not None and new_pkcs10 != old_pkcs10: self.set_subject_pkcs10(new_pkcs10) @@ -179,17 +125,16 @@ class main(object): logger.debug("Root certificate has changed, regenerating subject") self.revoke_subject_cert(now) subject_cert = None - self.get_root_cert() if subject_cert is not None: - return subject_cert + return subject_cert, None pkcs10 = old_pkcs10 if new_pkcs10 is None else new_pkcs10 if pkcs10 is None: logger.debug("No PKCS #10 request, can't generate subject certificate yet") - return None + return None, None resources = self.rpki_root_cert.get_3779resources() notAfter = now + self.rpki_subject_lifetime logger.info("Generating subject cert %s with resources %s, expires %s", - self.rpki_base_uri + self.rpki_subject_cert, resources, notAfter) + self.rpki_subject_cert_uri, resources, notAfter) req_key = pkcs10.getPublicKey() req_sia = pkcs10.get_SIA() self.next_serial_number() @@ -199,15 +144,22 @@ class main(object): serial = self.serial_number, sia = req_sia, aia = self.rpki_root_cert_uri, - crldp = self.rpki_base_uri + self.rpki_root_crl, + crldp = self.rpki_root_crl_uri, resources = resources, notBefore = now, notAfter = notAfter) self.set_subject_cert(subject_cert) - self.generate_crl_and_manifest(now) - return subject_cert + pubd_msg = Element(rpki.publication.tag_msg, nsmap = rpki.publication.nsmap, + type = "query", version = rpki.publication.version) + pdu = SubElement(pubd_msg, rpki.publication.tag_publish, uri = self.rpki_subject_cert_uri) + pdu.text = subject_cert.get_Base64() + if subject_cert_hash is not None: + pdu.set("hash", subject_cert_hash) + self.generate_crl_and_manifest(now, pubd_msg) + return subject_cert, pubd_msg + - def generate_crl_and_manifest(self, now): + def generate_crl_and_manifest(self, now, pubd_msg): subject_cert = self.get_subject_cert() self.next_serial_number() self.next_crl_number() @@ -220,23 +172,26 @@ class main(object): thisUpdate = now, nextUpdate = now + self.rpki_subject_regen, revokedCertificates = self.revoked) - fn = os.path.join(self.rpki_root_dir, self.rpki_root_crl) - logger.debug("Writing CRL %s", fn) - f = open(fn, "wb") - f.write(crl.get_DER()) - f.close() - manifest_content = [(self.rpki_root_crl, crl)] + crl_hash = self.read_hash_maybe(self.rpki_root_crl_file) + logger.debug("Writing CRL %s", self.rpki_root_crl_file) + with open(self.rpki_root_crl_file, "wb") as f: + f.write(crl.get_DER()) + pdu = SubElement(pubd_msg, rpki.publication.tag_publish, uri = self.rpki_root_crl_uri) + pdu.text = crl.get_Base64() + if crl_hash is not None: + pdu.set("hash", crl_hash) + manifest_content = [(os.path.basename(self.rpki_root_crl_uri), crl)] if subject_cert is not None: - manifest_content.append((self.rpki_subject_cert, subject_cert)) + manifest_content.append((os.path.basename(self.rpki_subject_cert_uri), subject_cert)) manifest_resources = rpki.resource_set.resource_bag.from_inheritance() manifest_keypair = rpki.x509.RSA.generate() manifest_cert = self.rpki_root_cert.issue( keypair = self.rpki_root_key, subject_key = manifest_keypair.get_public(), serial = self.serial_number, - sia = (None, None, self.rpki_base_uri + self.rpki_root_manifest), + sia = (None, None, self.rpki_root_manifest_uri, self.rrdp_notification_uri), aia = self.rpki_root_cert_uri, - crldp = self.rpki_base_uri + self.rpki_root_crl, + crldp = self.rpki_root_crl_uri, resources = manifest_resources, notBefore = now, notAfter = now + self.rpki_subject_lifetime, @@ -248,63 +203,171 @@ class main(object): names_and_objs = manifest_content, keypair = manifest_keypair, certs = manifest_cert) - fn = os.path.join(self.rpki_root_dir, self.rpki_root_manifest) - logger.debug("Writing manifest %s", fn) - f = open(fn, "wb") - f.write(manifest.get_DER()) - f.close() + mft_hash = self.read_hash_maybe(self.rpki_root_manifest_file) + logger.debug("Writing manifest %s", self.rpki_root_manifest_file) + with open(self.rpki_root_manifest_file, "wb") as f: + f.write(manifest.get_DER()) + pdu = SubElement(pubd_msg, rpki.publication.tag_publish, uri = self.rpki_root_manifest_uri) + pdu.text = manifest.get_Base64() + if mft_hash is not None: + pdu.set("hash", mft_hash) + cer_hash = rpki.x509.sha256(self.rpki_root_cert.get_DER()).encode("hex") + if cer_hash != self.rpki_root_cert_hash: + pdu = SubElement(pubd_msg, rpki.publication.tag_publish, uri = self.rpki_root_cert_uri) + pdu.text = self.rpki_root_cert.get_Base64() + if self.rpki_root_cert_hash is not None: + pdu.set("hash", self.rpki_root_cert_hash) + self.rpki_root_cert_hash = cer_hash + + + @staticmethod + def read_hash_maybe(fn): + try: + with open(fn, "rb") as f: + return rpki.x509.sha256(f.read()).encode("hex") + except IOError: + return None + def revoke_subject_cert(self, now): self.revoked.append((self.get_subject_cert().getSerial(), now)) + + def publish(self, q_msg): + if q_msg is None: + return + assert len(q_msg) > 0 + + if not all(q_pdu.get("hash") is not None for q_pdu in q_msg): + logger.debug("Some publication PDUs are missing hashes, checking published data...") + q = Element(rpki.publication.tag_msg, nsmap = rpki.publication.nsmap, + type = "query", version = rpki.publication.version) + SubElement(q, rpki.publication.tag_list) + published_hash = dict((r.get("uri"), r.get("hash")) for r in self.call_pubd(q)) + for q_pdu in q_msg: + q_uri = q_pdu.get("uri") + if q_pdu.get("hash") is None and published_hash.get(q_uri) is not None: + logger.debug("Updating hash of %s to %s from previously published data", q_uri, published_hash[q_uri]) + q_pdu.set("hash", published_hash[q_uri]) + + r_msg = self.call_pubd(q_msg) + if len(q_msg) != len(r_msg): + raise rpki.exceptions.BadPublicationReply("Wrong number of response PDUs from pubd: sent %s, got %s" % (len(q_msg), len(r_msg))) + + + def call_pubd(self, q_msg): + for q_pdu in q_msg: + logger.info("Sending %s to pubd", q_pdu.get("uri")) + r_msg = rpki.http_simple.client( + proto_cms_msg = rpki.publication.cms_msg, + client_key = self.rootd_bpki_key, + client_cert = self.rootd_bpki_cert, + client_crl = self.rootd_bpki_crl, + server_ta = self.bpki_ta, + server_cert = self.pubd_bpki_cert, + url = self.pubd_url, + q_msg = q_msg, + replay_track = self.pubd_replay_tracker) + rpki.publication.raise_if_error(r_msg) + return r_msg + + def compose_response(self, r_msg, pkcs10 = None): - subject_cert = self.issue_subject_cert_maybe(pkcs10) - rc = rpki.up_down.class_elt() - rc.class_name = self.rpki_class_name - rc.cert_url = rpki.up_down.multi_uri(self.rpki_root_cert_uri) - rc.from_resource_bag(self.rpki_root_cert.get_3779resources()) - rc.issuer = self.rpki_root_cert - r_msg.payload.classes.append(rc) + subject_cert, pubd_msg = self.issue_subject_cert_maybe(pkcs10) + bag = self.rpki_root_cert.get_3779resources() + rc = SubElement(r_msg, rpki.up_down.tag_class, + class_name = self.rpki_class_name, + cert_url = str(rpki.up_down.multi_uri(self.rpki_root_cert_uri)), + resource_set_as = str(bag.asn), + resource_set_ipv4 = str(bag.v4), + resource_set_ipv6 = str(bag.v6), + resource_set_notafter = str(bag.valid_until)) if subject_cert is not None: - rc.certs.append(rpki.up_down.certificate_elt()) - rc.certs[0].cert_url = rpki.up_down.multi_uri(self.rpki_base_uri + self.rpki_subject_cert) - rc.certs[0].cert = subject_cert + c = SubElement(rc, rpki.up_down.tag_certificate, + cert_url = str(rpki.up_down.multi_uri(self.rpki_subject_cert_uri))) + c.text = subject_cert.get_Base64() + SubElement(rc, rpki.up_down.tag_issuer).text = self.rpki_root_cert.get_Base64() + self.publish(pubd_msg) - def up_down_handler(self, query, path, cb): - try: - q_cms = cms_msg(DER = query) - q_msg = q_cms.unwrap((self.bpki_ta, self.child_bpki_cert)) - self.cms_timestamp = q_cms.check_replay(self.cms_timestamp, path) - except (rpki.async.ExitNow, SystemExit): - raise - except Exception, e: - logger.exception("Problem decoding PDU") - return cb(400, reason = "Could not decode PDU: %s" % e) - def done(r_msg): - cb(200, body = cms_msg().wrap( - r_msg, self.rootd_bpki_key, self.rootd_bpki_cert, - self.rootd_bpki_crl if self.include_bpki_crl else None)) + def handle_list(self, q_msg, r_msg): + self.compose_response(r_msg) + + def handle_issue(self, q_msg, r_msg): + # This is where we'd check q_msg[0].get("class_name") if this weren't rootd. + self.compose_response(r_msg, rpki.x509.PKCS10(Base64 = q_msg[0].text)) + + + def handle_revoke(self, q_msg, r_msg): + class_name = q_msg[0].get("class_name") + ski = q_msg[0].get("ski") + logger.debug("Revocation requested for class %s SKI %s", class_name, ski) + subject_cert = self.get_subject_cert() + if subject_cert is None: + logger.debug("No subject certificate, nothing to revoke") + raise rpki.exceptions.NotInDatabase + if subject_cert.gSKI() != ski: + logger.debug("Subject certificate has different SKI %s, not revoking", subject_cert.gSKI()) + raise rpki.exceptions.NotInDatabase + logger.debug("Revoking certificate %s", ski) + now = rpki.sundial.now() + pubd_msg = Element(rpki.publication.tag_msg, nsmap = rpki.publication.nsmap, + type = "query", version = rpki.publication.version) + self.revoke_subject_cert(now) + self.del_subject_cert() + self.del_subject_pkcs10() + SubElement(r_msg, q_msg[0].tag, class_name = class_name, ski = ski) + self.generate_crl_and_manifest(now, pubd_msg) + self.publish(pubd_msg) + + + # Need to do something about mapping exceptions to up-down error + # codes, right now everything shows up as "internal error". + # + #exceptions = { + # rpki.exceptions.ClassNameUnknown : 1201, + # rpki.exceptions.NoActiveCA : 1202, + # (rpki.exceptions.ClassNameUnknown, revoke_pdu) : 1301, + # (rpki.exceptions.NotInDatabase, revoke_pdu) : 1302 } + # + # Might be that what we want here is a subclass of + # rpki.exceptions.RPKI_Exception which carries an extra data field + # for the up-down error code, so that we can add the correct code + # when we instantiate it. + # + # There are also a few that are also schema violations, which means + # we'd have to catch them before validating or pick them out of a + # message that failed validation or otherwise break current + # modularity. Maybe an optional pre-validation check method hook in + # rpki.x509.XML_CMS_object which we can use to intercept such things? + + + def handler(self, request, q_der): try: - q_msg.serve_top_level(None, done) - except (rpki.async.ExitNow, SystemExit): - raise - except Exception, e: + q_cms = rpki.up_down.cms_msg(DER = q_der) + q_msg = q_cms.unwrap((self.bpki_ta, self.child_bpki_cert)) + q_type = q_msg.get("type") + logger.info("Serving %s query", q_type) + r_msg = Element(rpki.up_down.tag_message, nsmap = rpki.up_down.nsmap, version = rpki.up_down.version, + sender = q_msg.get("recipient"), recipient = q_msg.get("sender"), type = q_type + "_response") try: - logger.exception("Exception serving up-down request %r", q_msg) - done(q_msg.serve_error(e)) - except (rpki.async.ExitNow, SystemExit): - raise + self.rpkid_cms_timestamp = q_cms.check_replay(self.rpkid_cms_timestamp, request.path) + getattr(self, "handle_" + q_type)(q_msg, r_msg) except Exception, e: - logger.exception("Exception while generating error report") - cb(500, reason = "Could not process PDU: %s" % e) + logger.exception("Exception processing up-down %s message", q_type) + rpki.up_down.generate_error_response_from_exception(r_msg, e, q_type) + request.send_cms_response(rpki.up_down.cms_msg().wrap(r_msg, self.rootd_bpki_key, self.rootd_bpki_cert, + self.rootd_bpki_crl if self.include_bpki_crl else None)) + except Exception, e: + logger.exception("Unhandled exception processing up-down message") + request.send_error(500, "Unhandled exception %s: %s" % (e.__class__.__name__, e)) def next_crl_number(self): if self.crl_number is None: try: - crl = rpki.x509.CRL(DER_file = os.path.join(self.rpki_root_dir, self.rpki_root_crl)) + crl = rpki.x509.CRL(DER_file = self.rpki_root_crl_file) self.crl_number = crl.getCRLNumber() except: # pylint: disable=W0702 self.crl_number = 0 @@ -324,15 +387,11 @@ class main(object): def __init__(self): - - global rootd - rootd = self # Gross, but simpler than what we'd have to do otherwise - - self.rpki_root_cert = None self.serial_number = None self.crl_number = None self.revoked = [] - self.cms_timestamp = None + self.rpkid_cms_timestamp = None + self.pubd_replay_tracker = ReplayTracker() os.environ["TZ"] = "UTC" time.tzset() @@ -349,7 +408,7 @@ class main(object): rpki.log.init("rootd", args) - self.cfg = rpki.config.parser(args.config, "rootd") + self.cfg = rpki.config.parser(set_filename = args.config, section = "rootd") self.cfg.set_global_flags() if not args.foreground: @@ -361,28 +420,40 @@ class main(object): self.rootd_bpki_crl = rpki.x509.CRL( Auto_update = self.cfg.get("rootd-bpki-crl")) self.child_bpki_cert = rpki.x509.X509(Auto_update = self.cfg.get("child-bpki-cert")) + if self.cfg.has_option("pubd-bpki-cert"): + self.pubd_bpki_cert = rpki.x509.X509(Auto_update = self.cfg.get("pubd-bpki-cert")) + else: + self.pubd_bpki_cert = None + self.http_server_host = self.cfg.get("server-host", "") self.http_server_port = self.cfg.getint("server-port") - self.rpki_class_name = self.cfg.get("rpki-class-name", "wombat") + self.rpki_class_name = self.cfg.get("rpki-class-name") - self.rpki_root_dir = self.cfg.get("rpki-root-dir") - self.rpki_base_uri = self.cfg.get("rpki-base-uri", "rsync://" + self.rpki_class_name + ".invalid/") + self.rpki_root_key = rpki.x509.RSA( Auto_update = self.cfg.get("rpki-root-key-file")) + self.rpki_root_cert = rpki.x509.X509(Auto_update = self.cfg.get("rpki-root-cert-file")) + self.rpki_root_cert_uri = self.cfg.get("rpki-root-cert-uri") + self.rpki_root_cert_hash = None - self.rpki_root_key = rpki.x509.RSA(Auto_update = self.cfg.get("rpki-root-key")) - self.rpki_root_cert_file = self.cfg.get("rpki-root-cert") - self.rpki_root_cert_uri = self.cfg.get("rpki-root-cert-uri", self.rpki_base_uri + "root.cer") + self.rpki_root_manifest_file = self.cfg.get("rpki-root-manifest-file") + self.rpki_root_manifest_uri = self.cfg.get("rpki-root-manifest-uri") - self.rpki_root_manifest = self.cfg.get("rpki-root-manifest", "root.mft") - self.rpki_root_crl = self.cfg.get("rpki-root-crl", "root.crl") - self.rpki_subject_cert = self.cfg.get("rpki-subject-cert", "child.cer") - self.rpki_subject_pkcs10 = self.cfg.get("rpki-subject-pkcs10", "child.pkcs10") + self.rpki_root_crl_file = self.cfg.get("rpki-root-crl-file") + self.rpki_root_crl_uri = self.cfg.get("rpki-root-crl-uri") + self.rpki_subject_cert_file = self.cfg.get("rpki-subject-cert-file") + self.rpki_subject_cert_uri = self.cfg.get("rpki-subject-cert-uri") + self.rpki_subject_pkcs10 = self.cfg.get("rpki-subject-pkcs10-file") self.rpki_subject_lifetime = rpki.sundial.timedelta.parse(self.cfg.get("rpki-subject-lifetime", "8w")) - self.rpki_subject_regen = rpki.sundial.timedelta.parse(self.cfg.get("rpki-subject-regen", self.rpki_subject_lifetime.convert_to_seconds() / 2)) + self.rpki_subject_regen = rpki.sundial.timedelta.parse(self.cfg.get("rpki-subject-regen", + self.rpki_subject_lifetime.convert_to_seconds() / 2)) self.include_bpki_crl = self.cfg.getboolean("include-bpki-crl", False) - rpki.http.server(host = self.http_server_host, - port = self.http_server_port, - handlers = (("/", self.up_down_handler, rpki.up_down.allowed_content_types),)) + self.pubd_url = self.cfg.get("pubd-contact-uri") + + self.rrdp_notification_uri = self.cfg.get("rrdp-notification-uri") + + rpki.http_simple.server(host = self.http_server_host, + port = self.http_server_port, + handlers = (("/", self.handler, rpki.up_down.allowed_content_types),)) diff --git a/rpki/rpkic.py b/rpki/rpkic.py index 8ce28b59..ad695976 100644 --- a/rpki/rpkic.py +++ b/rpki/rpkic.py @@ -38,7 +38,6 @@ import time import rpki.config import rpki.sundial import rpki.log -import rpki.http import rpki.resource_set import rpki.relaxng import rpki.exceptions @@ -47,13 +46,9 @@ import rpki.x509 import rpki.async import rpki.version -from rpki.cli import Cmd, parsecmd, cmdarg +from lxml.etree import SubElement -class BadPrefixSyntax(Exception): "Bad prefix syntax." -class CouldntTalkToDaemon(Exception): "Couldn't talk to daemon." -class BadXMLMessage(Exception): "Bad XML message." -class PastExpiration(Exception): "Expiration date has already passed." -class CantRunRootd(Exception): "Can't run rootd." +from rpki.cli import Cmd, parsecmd, cmdarg module_doc = __doc__ @@ -124,7 +119,7 @@ class main(Cmd): global rpki # pylint: disable=W0602 try: - cfg = rpki.config.parser(self.cfg_file, "myrpki") + cfg = rpki.config.parser(set_filename = self.cfg_file, section = "myrpki") cfg.set_global_flags() except IOError, e: sys.exit("%s: %s" % (e.strerror, e.filename)) @@ -132,26 +127,10 @@ class main(Cmd): self.histfile = cfg.get("history_file", os.path.expanduser("~/.rpkic_history")) self.autosync = cfg.getboolean("autosync", True, section = "rpkic") - import django + os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings.irdb") - from django.conf import settings - - settings.configure( - DATABASES = { "default" : { - "ENGINE" : "django.db.backends.mysql", - "NAME" : cfg.get("sql-database", section = "irdbd"), - "USER" : cfg.get("sql-username", section = "irdbd"), - "PASSWORD" : cfg.get("sql-password", section = "irdbd"), - "HOST" : "", - "PORT" : "", - "OPTIONS" : { "init_command": "SET storage_engine=INNODB" }}}, - INSTALLED_APPS = ("rpki.irdb",), - MIDDLEWARE_CLASSES = (), # API change, feh - ) - - if django.VERSION >= (1, 7): # API change, feh - from django.apps import apps - apps.populate(settings.INSTALLED_APPS) + import django + django.setup() import rpki.irdb # pylint: disable=W0621 @@ -173,9 +152,6 @@ class main(Cmd): except rpki.config.ConfigParser.Error: pass - import django.core.management - django.core.management.call_command("syncdb", verbosity = 0, load_initial_data = False) - self.zoo = rpki.irdb.Zookeeper(cfg = cfg, handle = self.handle, logstream = sys.stdout) @@ -229,7 +205,7 @@ class main(Cmd): self.zoo.reset_identity(args.handle) def complete_select_identity(self, *args): - return self.irdb_handle_complete(rpki.irdb.ResourceHolderCA.objects, *args) + return self.irdb_handle_complete(rpki.irdb.models.ResourceHolderCA.objects, *args) @parsecmd(argsubparsers) @@ -340,9 +316,9 @@ class main(Cmd): try: self.zoo.delete_child(args.child_handle) self.zoo.synchronize_ca() - except rpki.irdb.ResourceHolderCA.DoesNotExist: + except rpki.irdb.models.ResourceHolderCA.DoesNotExist: print "No such resource holder \"%s\"" % self.zoo.handle - except rpki.irdb.Child.DoesNotExist: + except rpki.irdb.models.Child.DoesNotExist: print "No such child \"%s\"" % args.child_handle def complete_delete_child(self, *args): @@ -385,9 +361,9 @@ class main(Cmd): try: self.zoo.delete_parent(args.parent_handle) self.zoo.synchronize_ca() - except rpki.irdb.ResourceHolderCA.DoesNotExist: + except rpki.irdb.models.ResourceHolderCA.DoesNotExist: print "No such resource holder \"%s\"" % self.zoo.handle - except rpki.irdb.Parent.DoesNotExist: + except rpki.irdb.models.Parent.DoesNotExist: print "No such parent \"%s\"" % args.parent_handle def complete_delete_parent(self, *args): @@ -421,9 +397,9 @@ class main(Cmd): try: self.zoo.delete_rootd() self.zoo.synchronize_ca() - except rpki.irdb.ResourceHolderCA.DoesNotExist: + except rpki.irdb.models.ResourceHolderCA.DoesNotExist: print "No such resource holder \"%s\"" % self.zoo.handle - except rpki.irdb.Rootd.DoesNotExist: + except rpki.irdb.models.Rootd.DoesNotExist: print "No associated rootd" @@ -444,7 +420,7 @@ class main(Cmd): r.save("%s.repository-response.xml" % client_handle.replace("/", "."), sys.stdout) try: self.zoo.synchronize_pubd() - except rpki.irdb.Repository.DoesNotExist: + except rpki.irdb.models.Repository.DoesNotExist: pass @@ -458,9 +434,9 @@ class main(Cmd): try: self.zoo.delete_publication_client(args.client_handle) self.zoo.synchronize_pubd() - except rpki.irdb.ResourceHolderCA.DoesNotExist: + except rpki.irdb.models.ResourceHolderCA.DoesNotExist: print "No such resource holder \"%s\"" % self.zoo.handle - except rpki.irdb.Client.DoesNotExist: + except rpki.irdb.models.Client.DoesNotExist: print "No such client \"%s\"" % args.client_handle def complete_delete_publication_client(self, *args): @@ -494,9 +470,9 @@ class main(Cmd): try: self.zoo.delete_repository(args.repository_handle) self.zoo.synchronize_ca() - except rpki.irdb.ResourceHolderCA.DoesNotExist: + except rpki.irdb.models.ResourceHolderCA.DoesNotExist: print "No such resource holder \"%s\"" % self.zoo.handle - except rpki.irdb.Repository.DoesNotExist: + except rpki.irdb.models.Repository.DoesNotExist: print "No such repository \"%s\"" % args.repository_handle def complete_delete_repository(self, *args): @@ -512,7 +488,7 @@ class main(Cmd): try: self.zoo.delete_self() self.zoo.synchronize_deleted_ca() - except rpki.irdb.ResourceHolderCA.DoesNotExist: + except rpki.irdb.models.ResourceHolderCA.DoesNotExist: print "No such resource holder \"%s\"" % self.zoo.handle @@ -607,18 +583,20 @@ class main(Cmd): Show resources received by this entity from its parent(s). """ - for pdu in self.zoo.call_rpkid( - rpki.left_right.list_received_resources_elt.make_pdu(self_handle = self.zoo.handle)): + q_msg = self.zoo._compose_left_right_query() + SubElement(q_msg, rpki.left_right.tag_list_received_resources, self_handle = self.zoo.handle) + + for r_pdu in self.zoo.call_rpkid(q_msg): - print "Parent: ", pdu.parent_handle - print " notBefore:", pdu.notBefore - print " notAfter: ", pdu.notAfter - print " URI: ", pdu.uri - print " SIA URI: ", pdu.sia_uri - print " AIA URI: ", pdu.aia_uri - print " ASN: ", pdu.asn - print " IPv4: ", pdu.ipv4 - print " IPv6: ", pdu.ipv6 + print "Parent: ", r_pdu.get("parent_handle") + print " notBefore:", r_pdu.get("notBefore") + print " notAfter: ", r_pdu.get("notAfter") + print " URI: ", r_pdu.get("uri") + print " SIA URI: ", r_pdu.get("sia_uri") + print " AIA URI: ", r_pdu.get("aia_uri") + print " ASN: ", r_pdu.get("asn") + print " IPv4: ", r_pdu.get("ipv4") + print " IPv6: ", r_pdu.get("ipv6") @parsecmd(argsubparsers) @@ -627,16 +605,18 @@ class main(Cmd): Show published objects. """ - for pdu in self.zoo.call_rpkid( - rpki.left_right.list_published_objects_elt.make_pdu(self_handle = self.zoo.handle)): + q_msg = self.zoo._compose_left_right_query() + SubElement(q_msg, rpki.left_right.tag_list_published_objects, self_handle = self.zoo.handle) - track = rpki.x509.uri_dispatch(pdu.uri)(Base64 = pdu.obj).tracking_data(pdu.uri) - child = pdu.child_handle + for r_pdu in self.zoo.call_rpkid(q_msg): + uri = r_pdu.get("uri") + track = rpki.x509.uri_dispatch(uri)(Base64 = r_pdu.text).tracking_data(uri) + child_handle = r_pdu.get("child_handle") - if child is None: + if child_handle is None: print track else: - print track, child + print track, child_handle @parsecmd(argsubparsers) @@ -737,9 +717,9 @@ class main(Cmd): self.zoo.delete_router_certificate_request(args.gski) if self.autosync: self.zoo.run_rpkid_now() - except rpki.irdb.ResourceHolderCA.DoesNotExist: + except rpki.irdb.models.ResourceHolderCA.DoesNotExist: print "No such resource holder \"%s\"" % self.zoo.handle - except rpki.irdb.EECertificateRequest.DoesNotExist: + except rpki.irdb.models.EECertificateRequest.DoesNotExist: print "No certificate request matching g(SKI) \"%s\"" % args.gski def complete_delete_router_certificate_request(self, text, line, begidx, endidx): @@ -881,6 +861,6 @@ class main(Cmd): List all <self/> handles in this rpkid instance. """ - for ca in rpki.irdb.ResourceHolderCA.objects.all(): + for ca in rpki.irdb.models.ResourceHolderCA.objects.all(): print ca.handle diff --git a/rpki/rpkid.py b/rpki/rpkid.py index 628209af..bc13cd9a 100644 --- a/rpki/rpkid.py +++ b/rpki/rpkid.py @@ -28,6 +28,7 @@ import random import base64 import logging import argparse + import rpki.resource_set import rpki.up_down import rpki.left_right @@ -42,8 +43,11 @@ import rpki.async import rpki.daemonize import rpki.rpkid_tasks +from lxml.etree import Element, SubElement, tostring as ElementToString + logger = logging.getLogger(__name__) + class main(object): """ Main program for rpkid. @@ -75,7 +79,7 @@ class main(object): rpki.log.init("rpkid", args) - self.cfg = rpki.config.parser(args.config, "rpkid") + self.cfg = rpki.config.parser(set_filename = args.config, section = "rpkid") self.cfg.set_global_flags() if not args.foreground: @@ -101,6 +105,12 @@ class main(object): if self.profile: logger.info("Running in profile mode with output to %s", self.profile) + import django + django.setup() + + global rpki # pylint: disable=W0602 + import rpki.rpkidb # pylint: disable=W0621 + self.sql = rpki.sql.session(self.cfg) self.bpki_ta = rpki.x509.X509(Auto_update = self.cfg.get("bpki-ta")) @@ -116,17 +126,6 @@ class main(object): self.publication_kludge_base = self.cfg.get("publication-kludge-base", "publication/") - # Icky hack to let Iain do some testing quickly, should go away - # once we sort out whether we can make this change permanent. - # - # OK, the stuff to add router certificate support makes enough - # other changes that we're going to need a migration program in - # any case, so might as well throw the switch here too, or at - # least find out if it (still) works as expected. - - self.merge_publication_directories = self.cfg.getboolean("merge_publication_directories", - True) - self.use_internal_cron = self.cfg.getboolean("use-internal-cron", True) self.initial_delay = random.randint(self.cfg.getint("initial-delay-min", 10), @@ -161,19 +160,23 @@ class main(object): else: logger.debug("Not using internal clock, start_cron() call ignored") - def irdb_query(self, callback, errback, *q_pdus, **kwargs): + @staticmethod + def _compose_left_right_query(): + """ + Compose top level element of a left-right query to irdbd. + """ + + return Element(rpki.left_right.tag_msg, nsmap = rpki.left_right.nsmap, + type = "query", version = rpki.left_right.version) + + def irdb_query(self, q_msg, callback, errback): """ Perform an IRDB callback query. """ try: - q_types = tuple(type(q_pdu) for q_pdu in q_pdus) + q_tags = set(q_pdu.tag for q_pdu in q_msg) - expected_pdu_count = kwargs.pop("expected_pdu_count", None) - assert len(kwargs) == 0 - - q_msg = rpki.left_right.msg.query() - q_msg.extend(q_pdus) q_der = rpki.left_right.cms_msg().wrap(q_msg, self.rpkid_key, self.rpkid_cert) def unwrap(r_der): @@ -181,15 +184,10 @@ class main(object): r_cms = rpki.left_right.cms_msg(DER = r_der) r_msg = r_cms.unwrap((self.bpki_ta, self.irdb_cert)) self.irdbd_cms_timestamp = r_cms.check_replay(self.irdbd_cms_timestamp, self.irdb_url) - if not r_msg.is_reply() or not all(type(r_pdu) in q_types for r_pdu in r_msg): + #rpki.left_right.check_response(r_msg) + if r_msg.get("type") != "reply" or not all(r_pdu.tag in q_tags for r_pdu in r_msg): raise rpki.exceptions.BadIRDBReply( "Unexpected response to IRDB query: %s" % r_cms.pretty_print_content()) - if expected_pdu_count is not None and len(r_msg) != expected_pdu_count: - assert isinstance(expected_pdu_count, (int, long)) - raise rpki.exceptions.BadIRDBReply( - "Expected exactly %d PDU%s from IRDB: %s" % ( - expected_pdu_count, "" if expected_pdu_count == 1 else "s", - r_cms.pretty_print_content())) callback(r_msg) except Exception, e: errback(e) @@ -209,100 +207,335 @@ class main(object): Ask IRDB about a child's resources. """ - q_pdu = rpki.left_right.list_resources_elt() - q_pdu.self_handle = self_handle - q_pdu.child_handle = child_handle + q_msg = self._compose_left_right_query() + SubElement(q_msg, rpki.left_right.tag_list_resources, + self_handle = self_handle, child_handle = child_handle) def done(r_msg): + if len(r_msg) != 1: + raise rpki.exceptions.BadIRDBReply( + "Expected exactly one PDU from IRDB: %s" % r_msg.pretty_print_content()) callback(rpki.resource_set.resource_bag( - asn = r_msg[0].asn, - v4 = r_msg[0].ipv4, - v6 = r_msg[0].ipv6, - valid_until = r_msg[0].valid_until)) + asn = rpki.resource_set.resource_set_as(r_msg[0].get("asn")), + v4 = rpki.resource_set.resource_set_ipv4(r_msg[0].get("ipv4")), + v6 = rpki.resource_set.resource_set_ipv6(r_msg[0].get("ipv6")), + valid_until = rpki.sundial.datetime.fromXMLtime(r_msg[0].get("valid_until")))) - self.irdb_query(done, errback, q_pdu, expected_pdu_count = 1) + self.irdb_query(q_msg, done, errback) def irdb_query_roa_requests(self, self_handle, callback, errback): """ Ask IRDB about self's ROA requests. """ - q_pdu = rpki.left_right.list_roa_requests_elt() - q_pdu.self_handle = self_handle - - self.irdb_query(callback, errback, q_pdu) + q_msg = self._compose_left_right_query() + SubElement(q_msg, rpki.left_right.tag_list_roa_requests, self_handle = self_handle) + self.irdb_query(q_msg, callback, errback) def irdb_query_ghostbuster_requests(self, self_handle, parent_handles, callback, errback): """ Ask IRDB about self's ghostbuster record requests. """ - q_pdus = [] - + q_msg = self._compose_left_right_query() for parent_handle in parent_handles: - q_pdu = rpki.left_right.list_ghostbuster_requests_elt() - q_pdu.self_handle = self_handle - q_pdu.parent_handle = parent_handle - q_pdus.append(q_pdu) - - self.irdb_query(callback, errback, *q_pdus) + SubElement(q_msg, rpki.left_right.tag_list_ghostbuster_requests, + self_handle = self_handle, parent_handle = parent_handle) + self.irdb_query(q_msg, callback, errback) def irdb_query_ee_certificate_requests(self, self_handle, callback, errback): """ Ask IRDB about self's EE certificate requests. """ - q_pdu = rpki.left_right.list_ee_certificate_requests_elt() - q_pdu.self_handle = self_handle + q_msg = self._compose_left_right_query() + SubElement(q_msg, rpki.left_right.tag_list_ee_certificate_requests, self_handle = self_handle) + self.irdb_query(q_msg, callback, errback) + + @property + def left_right_models(self): + """ + Map element tag to rpkidb model. + """ + + try: + return self._left_right_models + except AttributeError: + import rpki.rpkidb.models # pylint: disable=W0621 + self._left_right_models = { + rpki.left_right.tag_self : rpki.rpkidb.models.Self, + rpki.left_right.tag_bsc : rpki.rpkidb.models.BSC, + rpki.left_right.tag_parent : rpki.rpkidb.models.Parent, + rpki.left_right.tag_child : rpki.rpkidb.models.Child, + rpki.left_right.tag_repository : rpki.rpkidb.models.Repository } + return self._left_right_models + + @property + def left_right_trivial_handlers(self): + """ + Map element tag to bound handler methods for trivial PDU types. + """ + + try: + return self._left_right_trivial_handlers + except AttributeError: + self._left_right_trivial_handlers = { + rpki.left_right.tag_list_published_objects : self.handle_list_published_objects, + rpki.left_right.tag_list_received_resources : self.handle_list_received_resources } + return self._left_right_trivial_handlers + + def handle_list_published_objects(self, q_pdu, r_msg): + """ + <list_published_objects/> server. + + This is written for the old SQL API, will need rewriting once we + switch rpkid to Django ORM. + """ + + logger.debug(".handle_list_published_objects() %s", ElementToString(q_pdu)) + + self_handle = q_pdu.get("self_handle") + msg_tag = q_pdu.get("tag") + + kw = dict(self_handle = self_handle) + if msg_tag is not None: + kw.update(tag = msg_tag) + + for parent in rpki.left_right.self_elt.serve_fetch_handle(self, None, self_handle).parents: + for ca in parent.cas: + ca_detail = ca.active_ca_detail + if ca_detail is not None: + + SubElement(r_msg, rpki.left_right.tag_list_published_objects, + uri = ca_detail.crl_uri, **kw).text = ca_detail.latest_crl.get_Base64() + + SubElement(r_msg, rpki.left_right.tag_list_published_objects, + uri = ca_detail.manifest_uri, **kw).text = ca_detail.latest_manifest.get_Base64() + + for c in ca_detail.child_certs: + SubElement(r_msg, rpki.left_right.tag_list_published_objects, + uri = c.uri, child_handle = c.child.child_handle, **kw).text = c.cert.get_Base64() + + for r in ca_detail.roas: + if r.roa is not None: + SubElement(r_msg, rpki.left_right.tag_list_published_objects, + uri = r.uri, **kw).text = r.roa.get_Base64() + + for g in ca_detail.ghostbusters: + SubElement(r_msg, rpki.left_right.tag_list_published_objects, + uri = g.uri, **kw).text = g.ghostbuster.get_Base64() + + for c in ca_detail.ee_certificates: + SubElement(r_msg, rpki.left_right.tag_list_published_objects, + uri = c.uri, **kw).text = c.cert.get_Base64() + + def handle_list_received_resources(self, q_pdu, r_msg): + """ + <list_received_resources/> server. + + This is written for the old SQL API, will need rewriting once we + switch rpkid to Django ORM. + """ + + logger.debug(".handle_list_received_resources() %s", ElementToString(q_pdu)) + + self_handle = q_pdu.get("self_handle") + msg_tag = q_pdu.get("tag") + + for parent in rpki.left_right.self_elt.serve_fetch_handle(self, None, self_handle).parents: + for ca in parent.cas: + ca_detail = ca.active_ca_detail + if ca_detail is not None and ca_detail.latest_ca_cert is not None: + + cert = ca_detail.latest_ca_cert + resources = cert.get_3779resources() + + r_pdu = SubElement(r_msg, rpki.left_right.tag_list_received_resources, + self_handle = self_handle, + parent_handle = parent.parent_handle, + uri = ca_detail.ca_cert_uri, + notBefore = str(cert.getNotBefore()), + notAfter = str(cert.getNotAfter()), + sia_uri = cert.get_sia_directory_uri(), + aia_uri = cert.get_aia_uri(), + asn = str(resources.asn), + ipv4 = str(resources.v4), + ipv6 = str(resources.v6)) + + if msg_tag is not None: + r_pdu.set("tag", msg_tag) - self.irdb_query(callback, errback, q_pdu) def left_right_handler(self, query, path, cb): """ Process one left-right PDU. """ - def done(r_msg): - reply = rpki.left_right.cms_msg().wrap(r_msg, self.rpkid_key, self.rpkid_cert) - self.sql.sweep() - cb(200, body = reply) + # This handles five persistent classes (self, bsc, parent, child, + # repository) and two simple queries (list_published_objects and + # list_received_resources). The former probably need to dispatch + # via methods to the corresponding model classes; the latter + # probably just become calls to ordinary methods of this + # (rpki.rpkid.main) class. + # + # Need to clone logic from rpki.pubd.main.control_handler(). try: q_cms = rpki.left_right.cms_msg(DER = query) q_msg = q_cms.unwrap((self.bpki_ta, self.irbe_cert)) + r_msg = Element(rpki.left_right.tag_msg, nsmap = rpki.left_right.nsmap, + type = "reply", version = rpki.left_right.version) self.irbe_cms_timestamp = q_cms.check_replay(self.irbe_cms_timestamp, path) - if not q_msg.is_query(): + + assert q_msg.tag.startswith(rpki.left_right.xmlns) + assert all(q_pdu.tag.startswith(rpki.left_right.xmlns) for q_pdu in q_msg) + + if q_msg.get("version") != rpki.left_right.version: + raise rpki.exceptions.BadQuery("Unrecognized protocol version") + + if q_msg.get("type") != "query": raise rpki.exceptions.BadQuery("Message type is not query") - q_msg.serve_top_level(self, done) + + def done(): + self.sql.sweep() + cb(200, body = rpki.left_right.cms_msg().wrap(r_msg, self.rpkid_key, self.rpkid_cert)) + + def loop(iterator, q_pdu): + + def fail(e): + if not isinstance(e, rpki.exceptions.NotFound): + logger.exception("Unhandled exception serving left-right PDU %r", q_pdu) + # Compatability kludge + if isinstance(q_pdu, rpki.left_right.base_elt): + error_self_handle = q_pdu.self_handle + error_tag = q_pdu.tag + else: + error_self_handle = q_pdu.get("self_handle") + error_tag = q_pdu.get("tag") + r_pdu = SubElement(r_msg, rpki.left_right.tag_report_error, error_code = e.__class__.__name__) + r_pdu.text = str(e) + if error_tag is not None: + r_pdu.set("tag", error_tag) + if error_self_handle is not None: + r_pdu.set("self_handle", error_self_handle) + self.sql.sweep() + cb(200, body = rpki.left_right.cms_msg().wrap(r_msg, self.rpkid_key, self.rpkid_cert)) + + try: + if q_pdu.tag in self.left_right_trivial_handlers: + self.left_right_trivial_handlers[q_pdu.tag](q_pdu, r_msg) + iterator() + + elif True: # Old-style handlers + + q_map = { rpki.left_right.tag_self : rpki.left_right.self_elt, + rpki.left_right.tag_bsc : rpki.left_right.bsc_elt, + rpki.left_right.tag_parent : rpki.left_right.parent_elt, + rpki.left_right.tag_child : rpki.left_right.child_elt, + rpki.left_right.tag_repository : rpki.left_right.repository_elt } + q_pdu = q_map[q_pdu.tag].fromXML(q_pdu) + q_pdu.gctx = self + q_pdu.serve_dispatch(r_msg, iterator, fail) + + else: # New-style handlers + + # Notes on hooks in old code + # + # .serve_pre_save_hook(): used by all classes to do some + # kind of handle fixup which I think is now OBE. Also + # used by BSC for key generation, because schema (and + # corresponding new model) don't allow NULL for private + # key or PKCS10 request, so either we have to relax the + # schema constraint or generate key before saving. + # (bsc) + # + # .serve_destroy_hook(): used by several objects to + # trigger revocation of related objects. Will probably + # need to preserve this behavior. + # (self, parent, child) + # + # .serve_post_save_hook(): used to trigger various actions + # based on boolean attributes in XML. + # (self, repository, parent, child) + + action = q_pdu.get("action") + model = self.left_right_models[q_pdu.tag] + + if action in ("get", "list"): + for obj in model.objects.xml_list(q_pdu): + obj.xml_template.encode(obj, r_msg) + + elif action == "destroy": + obj = model.objects.xml_get_for_delete(q_pdu) + try: + hook = obj.xml_pre_delete_hook + except AttributeError: + pass + else: + hook() + obj.delete() + obj.xml_template.acknowledge(obj, q_pdu, r_msg) + + elif action in ("create", "set"): + obj = model.objects.xml_get_or_create(q_pdu) + obj.xml_template.decode(obj, q_pdu) + try: + hook = obj.xml_pre_save_hook + except AttributeError: + pass + else: + hook(q_pdu) + obj.save() + try: + hook = obj.xml_post_save_hook + except AttributeError: + pass + else: + hook(q_pdu) + obj.xml_template.acknowledge(obj, q_pdu, r_msg) + + else: + raise rpki.exceptions.BadQuery + + except (rpki.async.ExitNow, SystemExit): + raise + except Exception, e: + fail(e) + + rpki.async.iterator(q_msg, loop, done) + except (rpki.async.ExitNow, SystemExit): raise + except Exception, e: logger.exception("Unhandled exception serving left-right request") cb(500, reason = "Unhandled exception %s: %s" % (e.__class__.__name__, e)) up_down_url_regexp = re.compile("/up-down/([-A-Z0-9_]+)/([-A-Z0-9_]+)$", re.I) - def up_down_handler(self, query, path, cb): + def up_down_handler(self, q_der, path, cb): """ Process one up-down PDU. """ - def done(reply): + def done(r_der): self.sql.sweep() - cb(200, body = reply) + cb(200, body = r_der) try: match = self.up_down_url_regexp.search(path) if match is None: raise rpki.exceptions.BadContactURL("Bad URL path received in up_down_handler(): %s" % path) self_handle, child_handle = match.groups() - child = rpki.left_right.child_elt.sql_fetch_where1(self, - "self.self_handle = %s AND child.child_handle = %s AND child.self_id = self.self_id", - (self_handle, child_handle), - "self") + child = rpki.left_right.child_elt.sql_fetch_where1( + gctx = self, + where = "self.self_handle = %s AND child.child_handle = %s AND child.self_id = self.self_id", + args = (self_handle, child_handle), + also_from = "self") if child is None: - raise rpki.exceptions.ChildNotFound("Could not find child %s of self %s in up_down_handler()" % (child_handle, self_handle)) - child.serve_up_down(query, done) + raise rpki.exceptions.ChildNotFound("Could not find child %s of self %s in up_down_handler()" % ( + child_handle, self_handle)) + child.serve_up_down(q_der, done) except (rpki.async.ExitNow, SystemExit): raise except (rpki.exceptions.ChildNotFound, rpki.exceptions.BadContactURL), e: @@ -317,6 +550,7 @@ class main(object): Record that we were still alive when we got here, by resetting keepalive timer. """ + if force or self.cron_timeout is not None: self.cron_timeout = rpki.sundial.now() + self.cron_keepalive @@ -324,6 +558,7 @@ class main(object): """ Add a task to the scheduler task queue, unless it's already queued. """ + if task not in self.task_queue: logger.debug("Adding %r to task queue", task) self.task_queue.append(task) @@ -338,6 +573,7 @@ class main(object): queue (we don't want to run it directly, as that could eventually blow out our call stack). """ + try: self.task_current = self.task_queue.pop(0) except IndexError: @@ -349,6 +585,7 @@ class main(object): """ Run first task on the task queue, unless one is running already. """ + if self.task_current is None: self.task_next() @@ -445,6 +682,7 @@ class ca_obj(rpki.sql.sql_persistent): """ Fetch parent object to which this CA object links. """ + return rpki.left_right.parent_elt.sql_fetch(self.gctx, self.parent_id) @property @@ -452,6 +690,7 @@ class ca_obj(rpki.sql.sql_persistent): """ Fetch all ca_detail objects that link to this CA object. """ + return ca_detail_obj.sql_fetch_where(self.gctx, "ca_id = %s", (self.ca_id,)) @property @@ -459,6 +698,7 @@ class ca_obj(rpki.sql.sql_persistent): """ Fetch the pending ca_details for this CA, if any. """ + return ca_detail_obj.sql_fetch_where(self.gctx, "ca_id = %s AND state = 'pending'", (self.ca_id,)) @property @@ -466,6 +706,7 @@ class ca_obj(rpki.sql.sql_persistent): """ Fetch the active ca_detail for this CA, if any. """ + return ca_detail_obj.sql_fetch_where1(self.gctx, "ca_id = %s AND state = 'active'", (self.ca_id,)) @property @@ -473,6 +714,7 @@ class ca_obj(rpki.sql.sql_persistent): """ Fetch deprecated ca_details for this CA, if any. """ + return ca_detail_obj.sql_fetch_where(self.gctx, "ca_id = %s AND state = 'deprecated'", (self.ca_id,)) @property @@ -480,6 +722,7 @@ class ca_obj(rpki.sql.sql_persistent): """ Fetch active and deprecated ca_details for this CA, if any. """ + return ca_detail_obj.sql_fetch_where(self.gctx, "ca_id = %s AND (state = 'active' OR state = 'deprecated')", (self.ca_id,)) @property @@ -487,6 +730,7 @@ class ca_obj(rpki.sql.sql_persistent): """ Fetch revoked ca_details for this CA, if any. """ + return ca_detail_obj.sql_fetch_where(self.gctx, "ca_id = %s AND state = 'revoked'", (self.ca_id,)) @property @@ -495,7 +739,7 @@ class ca_obj(rpki.sql.sql_persistent): Fetch ca_details which are candidates for consideration when processing an up-down issue_response PDU. """ - #return ca_detail_obj.sql_fetch_where(self.gctx, "ca_id = %s AND latest_ca_cert IS NOT NULL AND state != 'revoked'", (self.ca_id,)) + return ca_detail_obj.sql_fetch_where(self.gctx, "ca_id = %s AND state != 'revoked'", (self.ca_id,)) def construct_sia_uri(self, parent, rc): @@ -504,16 +748,12 @@ class ca_obj(rpki.sql.sql_persistent): information and the parent's up-down protocol list_response PDU. """ - sia_uri = rc.suggested_sia_head and rc.suggested_sia_head.rsync() - if not sia_uri or not sia_uri.startswith(parent.sia_base): + sia_uri = rc.get("suggested_sia_head", "") + if not sia_uri.startswith("rsync://") or not sia_uri.startswith(parent.sia_base): sia_uri = parent.sia_base if not sia_uri.endswith("/"): raise rpki.exceptions.BadURISyntax("SIA URI must end with a slash: %s" % sia_uri) - # With luck this can go away sometime soon. - if self.gctx.merge_publication_directories: - return sia_uri - else: - return sia_uri + str(self.ca_id) + "/" + return sia_uri def check_for_updates(self, parent, rc, cb, eb): """ @@ -530,28 +770,40 @@ class ca_obj(rpki.sql.sql_persistent): self.sia_uri = sia_uri self.sql_mark_dirty() - rc_resources = rc.to_resource_bag() - cert_map = dict((c.cert.get_SKI(), c) for c in rc.certs) + class_name = rc.get("class_name") + + rc_resources = rpki.resource_set.resource_bag( + rc.get("resource_set_as"), + rc.get("resource_set_ipv4"), + rc.get("resource_set_ipv6"), + rc.get("resource_set_notafter")) + + cert_map = {} + for c in rc.getiterator(rpki.up_down.tag_certificate): + x = rpki.x509.X509(Base64 = c.text) + u = rpki.up_down.multi_uri(c.get("cert_url")).rsync() + cert_map[x.gSKI()] = (x, u) def loop(iterator, ca_detail): self.gctx.checkpoint() - rc_cert = cert_map.pop(ca_detail.public_key.get_SKI(), None) + rc_cert, rc_cert_uri = cert_map.pop(ca_detail.public_key.gSKI(), (None, None)) if rc_cert is None: - logger.warning("SKI %s in resource class %s is in database but missing from list_response to %s from %s, maybe parent certificate went away?", - ca_detail.public_key.gSKI(), rc.class_name, parent.self.self_handle, parent.parent_handle) + logger.warning("SKI %s in resource class %s is in database but missing from list_response to %s from %s, " + "maybe parent certificate went away?", + ca_detail.public_key.gSKI(), class_name, parent.self.self_handle, parent.parent_handle) publisher = publication_queue() - ca_detail.delete(ca = ca_detail.ca, publisher = publisher) + ca_detail.destroy(ca = ca_detail.ca, publisher = publisher) return publisher.call_pubd(iterator, eb) else: - if ca_detail.state == "active" and ca_detail.ca_cert_uri != rc_cert.cert_url.rsync(): - logger.debug("AIA changed: was %s now %s", ca_detail.ca_cert_uri, rc_cert.cert_url.rsync()) - ca_detail.ca_cert_uri = rc_cert.cert_url.rsync() + if ca_detail.state == "active" and ca_detail.ca_cert_uri != rc_cert_uri: + logger.debug("AIA changed: was %s now %s", ca_detail.ca_cert_uri, rc_cert_uri) + ca_detail.ca_cert_uri = rc_cert_uri ca_detail.sql_mark_dirty() if ca_detail.state in ("pending", "active"): @@ -563,7 +815,7 @@ class ca_obj(rpki.sql.sql_persistent): if (ca_detail.state == "pending" or sia_uri_changed or - ca_detail.latest_ca_cert != rc_cert.cert or + ca_detail.latest_ca_cert != rc_cert or ca_detail.latest_ca_cert.getNotAfter() != rc_resources.valid_until or current_resources.undersized(rc_resources) or current_resources.oversized(rc_resources)): @@ -581,9 +833,7 @@ class ca_obj(rpki.sql.sql_persistent): def done(): if cert_map: logger.warning("Unknown certificate SKI%s %s in resource class %s in list_response to %s from %s, maybe you want to \"revoke_forgotten\"?", - "" if len(cert_map) == 1 else "s", - ", ".join(c.cert.gSKI() for c in cert_map.values()), - rc.class_name, parent.self.self_handle, parent.parent_handle) + "" if len(cert_map) == 1 else "s", ", ".join(cert_map), class_name, parent.self.self_handle, parent.parent_handle) self.gctx.sql.sweep() self.gctx.checkpoint() cb() @@ -591,29 +841,30 @@ class ca_obj(rpki.sql.sql_persistent): ca_details = self.issue_response_candidate_ca_details if True: - skis_parent = set(x.cert.gSKI() - for x in cert_map.itervalues()) + skis_parent = set(cert_map) skis_me = set(x.latest_ca_cert.gSKI() for x in ca_details if x.latest_ca_cert is not None) for ski in skis_parent & skis_me: logger.debug("Parent %s agrees that %s has SKI %s in resource class %s", - parent.parent_handle, parent.self.self_handle, ski, rc.class_name) + parent.parent_handle, parent.self.self_handle, ski, class_name) for ski in skis_parent - skis_me: logger.debug("Parent %s thinks %s has SKI %s in resource class %s but I don't think so", - parent.parent_handle, parent.self.self_handle, ski, rc.class_name) + parent.parent_handle, parent.self.self_handle, ski, class_name) for ski in skis_me - skis_parent: logger.debug("I think %s has SKI %s in resource class %s but parent %s doesn't think so", - parent.self.self_handle, ski, rc.class_name, parent.parent_handle) + parent.self.self_handle, ski, class_name, parent.parent_handle) if ca_details: rpki.async.iterator(ca_details, loop, done) else: logger.warning("Existing resource class %s to %s from %s with no certificates, rekeying", - rc.class_name, parent.self.self_handle, parent.parent_handle) + class_name, parent.self.self_handle, parent.parent_handle) self.gctx.checkpoint() self.rekey(cb, eb) + # Called from exactly one place, in rpki.rpkid_tasks.PollParentTask.class_loop(). + # Probably want to refactor. @classmethod def create(cls, parent, rc, cb, eb): """ @@ -624,7 +875,7 @@ class ca_obj(rpki.sql.sql_persistent): self = cls() self.gctx = parent.gctx self.parent_id = parent.parent_id - self.parent_resource_class = rc.class_name + self.parent_resource_class = rc.get("class_name") self.sql_store() try: self.sia_uri = self.construct_sia_uri(parent, rc) @@ -633,20 +884,20 @@ class ca_obj(rpki.sql.sql_persistent): raise ca_detail = ca_detail_obj.create(self) - def done(issue_response): - c = issue_response.payload.classes[0].certs[0] - logger.debug("CA %r received certificate %s", self, c.cert_url) + def done(r_msg): + c = r_msg[0][0] + logger.debug("CA %r received certificate %s", self, c.get("cert_url")) ca_detail.activate( ca = self, - cert = c.cert, - uri = c.cert_url, + cert = rpki.x509.X509(Base64 = c.text), + uri = c.get("cert_url"), callback = cb, errback = eb) logger.debug("Sending issue request to %r from %r", parent, self.create) - rpki.up_down.issue_pdu.query(parent, self, ca_detail, done, eb) + parent.up_down_issue_query(self, ca_detail, done, eb) - def delete(self, parent, callback): + def destroy(self, parent, callback): """ The list of current resource classes received from parent does not include the class corresponding to this CA, so we need to delete @@ -669,13 +920,14 @@ class ca_obj(rpki.sql.sql_persistent): publisher = publication_queue() for ca_detail in self.ca_details: - ca_detail.delete(ca = self, publisher = publisher, allow_failure = True) + ca_detail.destroy(ca = self, publisher = publisher, allow_failure = True) publisher.call_pubd(done, lose) def next_serial_number(self): """ Allocate a certificate serial number. """ + self.last_issued_sn += 1 self.sql_mark_dirty() return self.last_issued_sn @@ -684,6 +936,7 @@ class ca_obj(rpki.sql.sql_persistent): """ Allocate a manifest serial number. """ + self.last_manifest_sn += 1 self.sql_mark_dirty() return self.last_manifest_sn @@ -692,6 +945,7 @@ class ca_obj(rpki.sql.sql_persistent): """ Allocate a CRL serial number. """ + self.last_crl_sn += 1 self.sql_mark_dirty() return self.last_crl_sn @@ -708,19 +962,19 @@ class ca_obj(rpki.sql.sql_persistent): old_detail = self.active_ca_detail new_detail = ca_detail_obj.create(self) - def done(issue_response): - c = issue_response.payload.classes[0].certs[0] - logger.debug("CA %r received certificate %s", self, c.cert_url) + def done(r_msg): + c = r_msg[0][0] + logger.debug("CA %r received certificate %s", self, c.get("cert_url")) new_detail.activate( ca = self, - cert = c.cert, - uri = c.cert_url, + cert = rpki.x509.X509(Base64 = c.text), + uri = c.get("cert_url"), predecessor = old_detail, callback = cb, errback = eb) logger.debug("Sending issue request to %r from %r", parent, self.rekey) - rpki.up_down.issue_pdu.query(parent, self, new_detail, done, eb) + parent.up_down_issue_query(self, new_detail, done, eb) def revoke(self, cb, eb, revoke_all = False): """ @@ -782,6 +1036,7 @@ class ca_detail_obj(rpki.sql.sql_persistent): """ Extra assertions for SQL decode of a ca_detail_obj. """ + rpki.sql.sql_persistent.sql_decode(self, vals) assert self.public_key is None or self.private_key_id is None or self.public_key.get_DER() == self.private_key_id.get_public_DER() assert self.manifest_public_key is None or self.manifest_private_key_id is None or self.manifest_public_key.get_DER() == self.manifest_private_key_id.get_public_DER() @@ -792,12 +1047,14 @@ class ca_detail_obj(rpki.sql.sql_persistent): """ Fetch CA object to which this ca_detail links. """ + return ca_obj.sql_fetch(self.gctx, self.ca_id) def fetch_child_certs(self, child = None, ski = None, unique = False, unpublished = None): """ Fetch all child_cert objects that link to this ca_detail. """ + return rpki.rpkid.child_cert_obj.fetch(self.gctx, child, self, ski, unique, unpublished) @property @@ -805,6 +1062,7 @@ class ca_detail_obj(rpki.sql.sql_persistent): """ Fetch all child_cert objects that link to this ca_detail. """ + return self.fetch_child_certs() def unpublished_child_certs(self, when): @@ -812,6 +1070,7 @@ class ca_detail_obj(rpki.sql.sql_persistent): Fetch all unpublished child_cert objects linked to this ca_detail with attempted publication dates older than when. """ + return self.fetch_child_certs(unpublished = when) @property @@ -819,6 +1078,7 @@ class ca_detail_obj(rpki.sql.sql_persistent): """ Fetch all revoked_cert objects that link to this ca_detail. """ + return revoked_cert_obj.sql_fetch_where(self.gctx, "ca_detail_id = %s", (self.ca_detail_id,)) @property @@ -826,6 +1086,7 @@ class ca_detail_obj(rpki.sql.sql_persistent): """ Fetch all ROA objects that link to this ca_detail. """ + return rpki.rpkid.roa_obj.sql_fetch_where(self.gctx, "ca_detail_id = %s", (self.ca_detail_id,)) def unpublished_roas(self, when): @@ -833,34 +1094,52 @@ class ca_detail_obj(rpki.sql.sql_persistent): Fetch all unpublished ROA objects linked to this ca_detail with attempted publication dates older than when. """ - return rpki.rpkid.roa_obj.sql_fetch_where(self.gctx, "ca_detail_id = %s AND published IS NOT NULL and published < %s", (self.ca_detail_id, when)) + + return rpki.rpkid.roa_obj.sql_fetch_where(self.gctx, "ca_detail_id = %s AND published IS NOT NULL and published < %s", + (self.ca_detail_id, when)) @property def ghostbusters(self): """ Fetch all Ghostbuster objects that link to this ca_detail. """ + return rpki.rpkid.ghostbuster_obj.sql_fetch_where(self.gctx, "ca_detail_id = %s", (self.ca_detail_id,)) + def unpublished_ghostbusters(self, when): + """ + Fetch all unpublished Ghostbusters objects linked to this + ca_detail with attempted publication dates older than when. + """ + + return rpki.rpkid.ghostbuster_obj.sql_fetch_where(self.gctx, + "ca_detail_id = %s AND published IS NOT NULL and published < %s", + (self.ca_detail_id, when)) + @property def ee_certificates(self): """ Fetch all EE certificate objects that link to this ca_detail. """ + return rpki.rpkid.ee_cert_obj.sql_fetch_where(self.gctx, "ca_detail_id = %s", (self.ca_detail_id,)) - def unpublished_ghostbusters(self, when): + def unpublished_ee_certificates(self, when): """ - Fetch all unpublished Ghostbusters objects linked to this + Fetch all unpublished EE certificate objects linked to this ca_detail with attempted publication dates older than when. """ - return rpki.rpkid.ghostbuster_obj.sql_fetch_where(self.gctx, "ca_detail_id = %s AND published IS NOT NULL and published < %s", (self.ca_detail_id, when)) + + return rpki.rpkid.ee_cert_obj.sql_fetch_where(self.gctx, + "ca_detail_id = %s AND published IS NOT NULL and published < %s", + (self.ca_detail_id, when)) @property def crl_uri(self): """ Return publication URI for this ca_detail's CRL. """ + return self.ca.sia_uri + self.crl_uri_tail @property @@ -868,6 +1147,7 @@ class ca_detail_obj(rpki.sql.sql_persistent): """ Return tail (filename portion) of publication URI for this ca_detail's CRL. """ + return self.public_key.gSKI() + ".crl" @property @@ -875,17 +1155,19 @@ class ca_detail_obj(rpki.sql.sql_persistent): """ Return publication URI for this ca_detail's manifest. """ + return self.ca.sia_uri + self.public_key.gSKI() + ".mft" def has_expired(self): """ Return whether this ca_detail's certificate has expired. """ + return self.latest_ca_cert.getNotAfter() <= rpki.sundial.now() def covers(self, target): """ - Test whether this ca-detail covers a given set of resources. + Test whether this ca_detail covers a given set of resources. """ assert not target.asn.inherit and not target.v4.inherit and not target.v6.inherit @@ -900,7 +1182,7 @@ class ca_detail_obj(rpki.sql.sql_persistent): publisher = publication_queue() self.latest_ca_cert = cert - self.ca_cert_uri = uri.rsync() + self.ca_cert_uri = uri self.generate_manifest_cert() self.state = "active" self.generate_crl(publisher = publisher) @@ -921,7 +1203,7 @@ class ca_detail_obj(rpki.sql.sql_persistent): publisher.call_pubd(callback, errback) - def delete(self, ca, publisher, allow_failure = False): + def destroy(self, ca, publisher, allow_failure = False): """ Delete this ca_detail and all of the certs it issued. @@ -932,11 +1214,10 @@ class ca_detail_obj(rpki.sql.sql_persistent): repository = ca.parent.repository handler = False if allow_failure else None for child_cert in self.child_certs: - publisher.withdraw(cls = rpki.publication.certificate_elt, - uri = child_cert.uri, - obj = child_cert.cert, - repository = repository, - handler = handler) + publisher.queue(uri = child_cert.uri, + old_obj = child_cert.cert, + repository = repository, + handler = handler) child_cert.sql_mark_deleted() for roa in self.roas: roa.revoke(publisher = publisher, allow_failure = allow_failure, fast = True) @@ -947,21 +1228,19 @@ class ca_detail_obj(rpki.sql.sql_persistent): except AttributeError: latest_manifest = None if latest_manifest is not None: - publisher.withdraw(cls = rpki.publication.manifest_elt, - uri = self.manifest_uri, - obj = self.latest_manifest, - repository = repository, - handler = handler) + publisher.queue(uri = self.manifest_uri, + old_obj = self.latest_manifest, + repository = repository, + handler = handler) try: latest_crl = self.latest_crl except AttributeError: latest_crl = None if latest_crl is not None: - publisher.withdraw(cls = rpki.publication.crl_elt, - uri = self.crl_uri, - obj = self.latest_crl, - repository = repository, - handler = handler) + publisher.queue(uri = self.crl_uri, + old_obj = self.latest_crl, + repository = repository, + handler = handler) self.gctx.sql.sweep() for cert in self.revoked_certs: # + self.child_certs logger.debug("Deleting %r", cert) @@ -994,13 +1273,18 @@ class ca_detail_obj(rpki.sql.sql_persistent): ca = self.ca parent = ca.parent + class_name = ca.parent_resource_class + gski = self.latest_ca_cert.gSKI() def parent_revoked(r_msg): - if r_msg.payload.ski != self.latest_ca_cert.gSKI(): + if r_msg[0].get("class_name") != class_name: + raise rpki.exceptions.ResourceClassMismatch + + if r_msg[0].get("ski") != gski: raise rpki.exceptions.SKIMismatch - logger.debug("Parent revoked %s, starting cleanup", self.latest_ca_cert.gSKI()) + logger.debug("Parent revoked %s, starting cleanup", gski) crl_interval = rpki.sundial.timedelta(seconds = parent.self.crl_interval) @@ -1038,8 +1322,9 @@ class ca_detail_obj(rpki.sql.sql_persistent): self.sql_mark_dirty() publisher.call_pubd(cb, eb) - logger.debug("Asking parent to revoke CA certificate %s", self.latest_ca_cert.gSKI()) - rpki.up_down.revoke_pdu.query(ca, self.latest_ca_cert.gSKI(), parent_revoked, eb) + logger.debug("Asking parent to revoke CA certificate %s", gski) + parent.up_down_revoke_query(class_name, gski, parent_revoked, eb) + def update(self, parent, ca, rc, sia_uri_changed, old_resources, callback, errback): """ @@ -1047,24 +1332,27 @@ class ca_detail_obj(rpki.sql.sql_persistent): children of this ca_detail. """ - def issued(issue_response): - c = issue_response.payload.classes[0].certs[0] - logger.debug("CA %r received certificate %s", self, c.cert_url) + def issued(r_msg): + c = r_msg[0][0] + cert = rpki.x509.X509(Base64 = c.text) + cert_url = c.get("cert_url") + + logger.debug("CA %r received certificate %s", self, cert_url) if self.state == "pending": return self.activate( ca = ca, - cert = c.cert, - uri = c.cert_url, + cert = cert, + uri = cert_url, callback = callback, errback = errback) - validity_changed = self.latest_ca_cert is None or self.latest_ca_cert.getNotAfter() != c.cert.getNotAfter() + validity_changed = self.latest_ca_cert is None or self.latest_ca_cert.getNotAfter() != cert.getNotAfter() publisher = publication_queue() - if self.latest_ca_cert != c.cert: - self.latest_ca_cert = c.cert + if self.latest_ca_cert != cert: + self.latest_ca_cert = cert self.sql_mark_dirty() self.generate_manifest_cert() self.generate_crl(publisher = publisher) @@ -1092,7 +1380,8 @@ class ca_detail_obj(rpki.sql.sql_persistent): publisher.call_pubd(callback, errback) logger.debug("Sending issue request to %r from %r", parent, self.update) - rpki.up_down.issue_pdu.query(parent, ca, self, issued, errback) + parent.up_down_issue_query(ca, self, issued, errback) + @classmethod def create(cls, ca): @@ -1146,7 +1435,7 @@ class ca_detail_obj(rpki.sql.sql_persistent): ca = self.ca, resources = resources, subject_key = self.manifest_public_key, - sia = (None, None, self.manifest_uri)) + sia = (None, None, self.manifest_uri, self.ca.parent.repository.rrdp_notification_uri)) def issue(self, ca, child, subject_key, sia, resources, publisher, child_cert = None): """ @@ -1171,6 +1460,7 @@ class ca_detail_obj(rpki.sql.sql_persistent): notAfter = resources.valid_until) if child_cert is None: + old_cert = None child_cert = rpki.rpkid.child_cert_obj( gctx = child.gctx, child_id = child.child_id, @@ -1178,6 +1468,7 @@ class ca_detail_obj(rpki.sql.sql_persistent): cert = cert) logger.debug("Created new child_cert %r", child_cert) else: + old_cert = child_cert.cert child_cert.cert = cert del child_cert.ca_detail child_cert.ca_detail_id = self.ca_detail_id @@ -1186,10 +1477,10 @@ class ca_detail_obj(rpki.sql.sql_persistent): child_cert.ski = cert.get_SKI() child_cert.published = rpki.sundial.now() child_cert.sql_store() - publisher.publish( - cls = rpki.publication.certificate_elt, + publisher.queue( uri = child_cert.uri, - obj = child_cert.cert, + old_obj = old_cert, + new_obj = child_cert.cert, repository = ca.parent.repository, handler = child_cert.published_callback) self.generate_manifest(publisher = publisher) @@ -1220,6 +1511,8 @@ class ca_detail_obj(rpki.sql.sql_persistent): certlist.append((revoked_cert.serial, revoked_cert.revoked)) certlist.sort() + old_crl = self.latest_crl + self.latest_crl = rpki.x509.CRL.generate( keypair = self.private_key_id, issuer = self.latest_ca_cert, @@ -1230,10 +1523,10 @@ class ca_detail_obj(rpki.sql.sql_persistent): self.crl_published = rpki.sundial.now() self.sql_mark_dirty() - publisher.publish( - cls = rpki.publication.crl_elt, + publisher.queue( uri = self.crl_uri, - obj = self.latest_crl, + old_obj = old_crl, + new_obj = self.latest_crl, repository = parent.repository, handler = self.crl_published_callback) @@ -1241,7 +1534,8 @@ class ca_detail_obj(rpki.sql.sql_persistent): """ Check result of CRL publication. """ - pdu.raise_if_error() + + rpki.publication.raise_if_error(pdu) self.crl_published = None self.sql_mark_dirty() @@ -1277,6 +1571,7 @@ class ca_detail_obj(rpki.sql.sql_persistent): objs.extend((e.uri_tail, e.cert) for e in self.ee_certificates) logger.debug("Building manifest object %s", uri) + old_manifest = self.latest_manifest self.latest_manifest = rpki.x509.SignedManifest.build( serial = ca.next_manifest_number(), thisUpdate = now, @@ -1289,17 +1584,18 @@ class ca_detail_obj(rpki.sql.sql_persistent): self.manifest_published = rpki.sundial.now() self.sql_mark_dirty() - publisher.publish(cls = rpki.publication.manifest_elt, - uri = uri, - obj = self.latest_manifest, - repository = parent.repository, - handler = self.manifest_published_callback) + publisher.queue(uri = uri, + old_obj = old_manifest, + new_obj = self.latest_manifest, + repository = parent.repository, + handler = self.manifest_published_callback) def manifest_published_callback(self, pdu): """ Check result of manifest publication. """ - pdu.raise_if_error() + + rpki.publication.raise_if_error(pdu) self.manifest_published = None self.sql_mark_dirty() @@ -1360,21 +1656,19 @@ class ca_detail_obj(rpki.sql.sql_persistent): self.crl_published is not None and \ self.crl_published < stale: logger.debug("Retrying publication for %s", self.crl_uri) - publisher.publish(cls = rpki.publication.crl_elt, - uri = self.crl_uri, - obj = self.latest_crl, - repository = repository, - handler = self.crl_published_callback) + publisher.queue(uri = self.crl_uri, + new_obj = self.latest_crl, + repository = repository, + handler = self.crl_published_callback) if self.latest_manifest is not None and \ self.manifest_published is not None and \ self.manifest_published < stale: logger.debug("Retrying publication for %s", self.manifest_uri) - publisher.publish(cls = rpki.publication.manifest_elt, - uri = self.manifest_uri, - obj = self.latest_manifest, - repository = repository, - handler = self.manifest_published_callback) + publisher.queue(uri = self.manifest_uri, + new_obj = self.latest_manifest, + repository = repository, + handler = self.manifest_published_callback) if not check_all: return @@ -1384,31 +1678,37 @@ class ca_detail_obj(rpki.sql.sql_persistent): for child_cert in self.unpublished_child_certs(stale): logger.debug("Retrying publication for %s", child_cert) - publisher.publish( - cls = rpki.publication.certificate_elt, + publisher.queue( uri = child_cert.uri, - obj = child_cert.cert, + new_obj = child_cert.cert, repository = repository, handler = child_cert.published_callback) for roa in self.unpublished_roas(stale): logger.debug("Retrying publication for %s", roa) - publisher.publish( - cls = rpki.publication.roa_elt, + publisher.queue( uri = roa.uri, - obj = roa.roa, + new_obj = roa.roa, repository = repository, handler = roa.published_callback) for ghostbuster in self.unpublished_ghostbusters(stale): logger.debug("Retrying publication for %s", ghostbuster) - publisher.publish( - cls = rpki.publication.ghostbuster_elt, + publisher.queue( uri = ghostbuster.uri, - obj = ghostbuster.ghostbuster, + new_obj = ghostbuster.ghostbuster, repository = repository, handler = ghostbuster.published_callback) + for ee_cert in self.unpublished_ee_certificates(stale): + logger.debug("Retrying publication for %s", ee_cert) + publisher.queue( + uri = ee_cert.uri, + new_obj = ee_cert.cert, + repository = repository, + handler = ee_cert.published_callback) + + class child_cert_obj(rpki.sql.sql_persistent): """ Certificate that has been issued to a child. @@ -1435,6 +1735,7 @@ class child_cert_obj(rpki.sql.sql_persistent): """ Initialize a child_cert_obj. """ + rpki.sql.sql_persistent.__init__(self) self.gctx = gctx self.child_id = child_id @@ -1450,6 +1751,7 @@ class child_cert_obj(rpki.sql.sql_persistent): """ Fetch child object to which this child_cert object links. """ + return rpki.left_right.child_elt.sql_fetch(self.gctx, self.child_id) @property @@ -1458,6 +1760,7 @@ class child_cert_obj(rpki.sql.sql_persistent): """ Fetch ca_detail object to which this child_cert object links. """ + return ca_detail_obj.sql_fetch(self.gctx, self.ca_detail_id) @ca_detail.deleter @@ -1472,6 +1775,7 @@ class child_cert_obj(rpki.sql.sql_persistent): """ Return the tail (filename) portion of the URI for this child_cert. """ + return self.cert.gSKI() + ".cer" @property @@ -1479,6 +1783,7 @@ class child_cert_obj(rpki.sql.sql_persistent): """ Return the publication URI for this child_cert. """ + return self.ca_detail.ca.sia_uri + self.uri_tail def revoke(self, publisher, generate_crl_and_manifest = True): @@ -1490,10 +1795,9 @@ class child_cert_obj(rpki.sql.sql_persistent): ca = ca_detail.ca logger.debug("Revoking %r %r", self, self.uri) revoked_cert_obj.revoke(cert = self.cert, ca_detail = ca_detail) - publisher.withdraw( - cls = rpki.publication.certificate_elt, - uri = self.uri, - obj = self.cert, + publisher.queue( + uri = self.uri, + old_obj = self.cert, repository = ca.parent.repository) self.gctx.sql.sweep() self.sql_delete() @@ -1624,7 +1928,8 @@ class child_cert_obj(rpki.sql.sql_persistent): """ Publication callback: check result and mark published. """ - pdu.raise_if_error() + + rpki.publication.raise_if_error(pdu) self.published = None self.sql_mark_dirty() @@ -1648,6 +1953,7 @@ class revoked_cert_obj(rpki.sql.sql_persistent): """ Initialize a revoked_cert_obj. """ + rpki.sql.sql_persistent.__init__(self) self.gctx = gctx self.serial = serial @@ -1663,6 +1969,7 @@ class revoked_cert_obj(rpki.sql.sql_persistent): """ Fetch ca_detail object to which this revoked_cert_obj links. """ + return ca_detail_obj.sql_fetch(self.gctx, self.ca_detail_id) @classmethod @@ -1670,6 +1977,7 @@ class revoked_cert_obj(rpki.sql.sql_persistent): """ Revoke a certificate. """ + return cls( serial = cert.getSerial(), expires = cert.getNotAfter(), @@ -1711,6 +2019,7 @@ class roa_obj(rpki.sql.sql_persistent): """ Fetch ca_detail object to which this roa_obj links. """ + return rpki.rpkid.ca_detail_obj.sql_fetch(self.gctx, self.ca_detail_id) @ca_detail.deleter @@ -1724,6 +2033,7 @@ class roa_obj(rpki.sql.sql_persistent): """ Extra SQL fetch actions for roa_obj -- handle prefix lists. """ + for version, datatype, attribute in ((4, rpki.resource_set.roa_prefix_set_ipv4, "ipv4"), (6, rpki.resource_set.roa_prefix_set_ipv6, "ipv6")): setattr(self, attribute, datatype.from_sql( @@ -1738,6 +2048,7 @@ class roa_obj(rpki.sql.sql_persistent): """ Extra SQL insert actions for roa_obj -- handle prefix lists. """ + for version, prefix_set in ((4, self.ipv4), (6, self.ipv6)): if prefix_set: self.gctx.sql.executemany( @@ -1752,6 +2063,7 @@ class roa_obj(rpki.sql.sql_persistent): """ Extra SQL delete actions for roa_obj -- handle prefix lists. """ + self.gctx.sql.execute("DELETE FROM roa_prefix WHERE roa_id = %s", (self.roa_id,)) def __repr__(self): @@ -1887,16 +2199,15 @@ class roa_obj(rpki.sql.sql_persistent): ca = ca, resources = resources, subject_key = keypair.get_public(), - sia = (None, None, self.uri_from_key(keypair))) + sia = (None, None, self.uri_from_key(keypair), ca.parent.repository.rrdp_notification_uri)) self.roa = rpki.x509.ROA.build(self.asn, self.ipv4, self.ipv6, keypair, (self.cert,)) self.published = rpki.sundial.now() self.sql_store() logger.debug("Generating %r URI %s", self, self.uri) - publisher.publish( - cls = rpki.publication.roa_elt, + publisher.queue( uri = self.uri, - obj = self.roa, + new_obj = self.roa, repository = ca.parent.repository, handler = self.published_callback) if not fast: @@ -1907,7 +2218,8 @@ class roa_obj(rpki.sql.sql_persistent): """ Check publication result. """ - pdu.raise_if_error() + + rpki.publication.raise_if_error(pdu) self.published = None self.sql_mark_dirty() @@ -1941,9 +2253,10 @@ class roa_obj(rpki.sql.sql_persistent): logger.debug("Withdrawing %r %s and revoking its EE cert", self, uri) rpki.rpkid.revoked_cert_obj.revoke(cert = cert, ca_detail = ca_detail) - publisher.withdraw(cls = rpki.publication.roa_elt, uri = uri, obj = roa, - repository = ca_detail.ca.parent.repository, - handler = False if allow_failure else None) + publisher.queue(uri = uri, + old_obj = roa, + repository = ca_detail.ca.parent.repository, + handler = False if allow_failure else None) if not regenerate: self.sql_mark_deleted() @@ -1957,6 +2270,7 @@ class roa_obj(rpki.sql.sql_persistent): """ Reissue ROA associated with this roa_obj. """ + if self.ca_detail is None: self.generate(publisher = publisher, fast = fast) else: @@ -1966,6 +2280,7 @@ class roa_obj(rpki.sql.sql_persistent): """ Return publication URI for a public key. """ + return self.ca_detail.ca.sia_uri + key.gSKI() + ".roa" @property @@ -1973,6 +2288,7 @@ class roa_obj(rpki.sql.sql_persistent): """ Return the publication URI for this roa_obj's ROA. """ + return self.ca_detail.ca.sia_uri + self.uri_tail @property @@ -1981,6 +2297,7 @@ class roa_obj(rpki.sql.sql_persistent): Return the tail (filename portion) of the publication URI for this roa_obj's ROA. """ + return self.cert.gSKI() + ".roa" @@ -2023,6 +2340,7 @@ class ghostbuster_obj(rpki.sql.sql_persistent): """ Fetch self object to which this ghostbuster_obj links. """ + return rpki.left_right.self_elt.sql_fetch(self.gctx, self.self_id) @property @@ -2031,6 +2349,7 @@ class ghostbuster_obj(rpki.sql.sql_persistent): """ Fetch ca_detail object to which this ghostbuster_obj links. """ + return rpki.rpkid.ca_detail_obj.sql_fetch(self.gctx, self.ca_detail_id) def __init__(self, gctx = None, self_id = None, ca_detail_id = None, vcard = None): @@ -2090,16 +2409,15 @@ class ghostbuster_obj(rpki.sql.sql_persistent): ca = ca, resources = resources, subject_key = keypair.get_public(), - sia = (None, None, self.uri_from_key(keypair))) + sia = (None, None, self.uri_from_key(keypair), ca.parent.repository.rrdp_notification_uri)) self.ghostbuster = rpki.x509.Ghostbuster.build(self.vcard, keypair, (self.cert,)) self.published = rpki.sundial.now() self.sql_store() logger.debug("Generating Ghostbuster record %r", self.uri) - publisher.publish( - cls = rpki.publication.ghostbuster_elt, + publisher.queue( uri = self.uri, - obj = self.ghostbuster, + new_obj = self.ghostbuster, repository = ca.parent.repository, handler = self.published_callback) if not fast: @@ -2109,7 +2427,8 @@ class ghostbuster_obj(rpki.sql.sql_persistent): """ Check publication result. """ - pdu.raise_if_error() + + rpki.publication.raise_if_error(pdu) self.published = None self.sql_mark_dirty() @@ -2143,9 +2462,10 @@ class ghostbuster_obj(rpki.sql.sql_persistent): logger.debug("Withdrawing %r %s and revoking its EE cert", self, uri) rpki.rpkid.revoked_cert_obj.revoke(cert = cert, ca_detail = ca_detail) - publisher.withdraw(cls = rpki.publication.ghostbuster_elt, uri = uri, obj = ghostbuster, - repository = ca_detail.ca.parent.repository, - handler = False if allow_failure else None) + publisher.queue(uri = uri, + old_obj = ghostbuster, + repository = ca_detail.ca.parent.repository, + handler = False if allow_failure else None) if not regenerate: self.sql_mark_deleted() @@ -2159,6 +2479,7 @@ class ghostbuster_obj(rpki.sql.sql_persistent): """ Reissue Ghostbuster associated with this ghostbuster_obj. """ + if self.ghostbuster is None: self.generate(publisher = publisher, fast = fast) else: @@ -2168,6 +2489,7 @@ class ghostbuster_obj(rpki.sql.sql_persistent): """ Return publication URI for a public key. """ + return self.ca_detail.ca.sia_uri + key.gSKI() + ".gbr" @property @@ -2175,6 +2497,7 @@ class ghostbuster_obj(rpki.sql.sql_persistent): """ Return the publication URI for this ghostbuster_obj's ghostbuster. """ + return self.ca_detail.ca.sia_uri + self.uri_tail @property @@ -2183,6 +2506,7 @@ class ghostbuster_obj(rpki.sql.sql_persistent): Return the tail (filename portion) of the publication URI for this ghostbuster_obj's ghostbuster. """ + return self.cert.gSKI() + ".gbr" @@ -2220,6 +2544,7 @@ class ee_cert_obj(rpki.sql.sql_persistent): """ Fetch self object to which this ee_cert_obj links. """ + return rpki.left_right.self_elt.sql_fetch(self.gctx, self.self_id) @property @@ -2228,6 +2553,7 @@ class ee_cert_obj(rpki.sql.sql_persistent): """ Fetch ca_detail object to which this ee_cert_obj links. """ + return rpki.rpkid.ca_detail_obj.sql_fetch(self.gctx, self.ca_detail_id) @ca_detail.deleter @@ -2245,6 +2571,7 @@ class ee_cert_obj(rpki.sql.sql_persistent): Although, really, one has to ask why we don't just store g(SKI) in rpkid.sql instead of ski.... """ + return base64.urlsafe_b64encode(self.ski).rstrip("=") @gski.setter @@ -2256,6 +2583,7 @@ class ee_cert_obj(rpki.sql.sql_persistent): """ Return the publication URI for this ee_cert_obj. """ + return self.ca_detail.ca.sia_uri + self.uri_tail @property @@ -2264,6 +2592,7 @@ class ee_cert_obj(rpki.sql.sql_persistent): Return the tail (filename portion) of the publication URI for this ee_cert_obj. """ + return self.cert.gSKI() + ".cer" @classmethod @@ -2275,10 +2604,12 @@ class ee_cert_obj(rpki.sql.sql_persistent): cn, sn = subject_name.extract_cn_and_sn() ca = ca_detail.ca + sia = (None, None, ca_detail.ca.sia_uri + subject_key.gSKI() + ".cer", ca.parent.repository.rrdp_notification_uri) + cert = ca_detail.issue_ee( ca = ca, subject_key = subject_key, - sia = None, + sia = sia, resources = resources, notAfter = resources.valid_until, cn = cn, @@ -2291,12 +2622,11 @@ class ee_cert_obj(rpki.sql.sql_persistent): ca_detail_id = ca_detail.ca_detail_id, cert = cert) - publisher.publish( - cls = rpki.publication.certificate_elt, - uri = self.uri, - obj = self.cert, + publisher.queue( + uri = self.uri, + new_obj = self.cert, repository = ca.parent.repository, - handler = self.published_callback) + handler = self.published_callback) self.sql_store() @@ -2315,10 +2645,9 @@ class ee_cert_obj(rpki.sql.sql_persistent): ca = ca_detail.ca logger.debug("Revoking %r %r", self, self.uri) revoked_cert_obj.revoke(cert = self.cert, ca_detail = ca_detail) - publisher.withdraw(cls = rpki.publication.certificate_elt, - uri = self.uri, - obj = self.cert, - repository = ca.parent.repository) + publisher.queue(uri = self.uri, + old_obj = self.cert, + repository = ca.parent.repository) self.gctx.sql.sweep() self.sql_delete() if generate_crl_and_manifest: @@ -2392,7 +2721,7 @@ class ee_cert_obj(rpki.sql.sql_persistent): ca = ca_detail.ca, subject_key = self.cert.getPublicKey(), eku = self.cert.get_EKU(), - sia = None, + sia = (None, None, self.uri, ca_detail.ca.parent.repository.rrdp_notification_uri), resources = resources, notAfter = resources.valid_until, cn = cn, @@ -2400,12 +2729,12 @@ class ee_cert_obj(rpki.sql.sql_persistent): self.sql_mark_dirty() - publisher.publish( - cls = rpki.publication.certificate_elt, - uri = self.uri, - obj = self.cert, + publisher.queue( + uri = self.uri, + old_obj = old_cert, + new_obj = self.cert, repository = ca_detail.ca.parent.repository, - handler = self.published_callback) + handler = self.published_callback) if must_revoke: revoked_cert_obj.revoke(cert = old_cert.cert, ca_detail = old_ca_detail) @@ -2422,7 +2751,8 @@ class ee_cert_obj(rpki.sql.sql_persistent): """ Publication callback: check result and mark published. """ - pdu.raise_if_error() + + rpki.publication.raise_if_error(pdu) self.published = None self.sql_mark_dirty() @@ -2450,29 +2780,54 @@ class publication_queue(object): if self.replace: self.uris = {} - def _add(self, uri, obj, repository, handler, make_pdu): + def queue(self, uri, repository, handler = None, + old_obj = None, new_obj = None, old_hash = None): + + assert old_obj is not None or new_obj is not None or old_hash is not None + assert old_obj is None or old_hash is None + assert old_obj is None or isinstance(old_obj, rpki.x509.uri_dispatch(uri)) + assert new_obj is None or isinstance(new_obj, rpki.x509.uri_dispatch(uri)) + + logger.debug("Queuing publication action: uri %s, old %r, new %r, hash %s", + uri, old_obj, new_obj, old_hash) + + # id(repository) may need to change to repository.peer_contact_uri + # once we convert from our custom SQL cache to Django ORM. + rid = id(repository) if rid not in self.repositories: self.repositories[rid] = repository - self.msgs[rid] = rpki.publication.msg.query() + self.msgs[rid] = Element(rpki.publication.tag_msg, nsmap = rpki.publication.nsmap, + type = "query", version = rpki.publication.version) + if self.replace and uri in self.uris: - logger.debug("Removing publication duplicate <%s %r %r>", - self.uris[uri].action, self.uris[uri].uri, self.uris[uri].payload) - self.msgs[rid].remove(self.uris.pop(uri)) - pdu = make_pdu(uri = uri, obj = obj) + logger.debug("Removing publication duplicate %r", self.uris[uri]) + old_pdu = self.uris.pop(uri) + self.msgs[rid].remove(old_pdu) + pdu_hash = old_pdu.get("hash") + elif old_hash is not None: + pdu_hash = old_hash + elif old_obj is None: + pdu_hash = None + else: + pdu_hash = rpki.x509.sha256(old_obj.get_DER()).encode("hex") + + if new_obj is None: + pdu = SubElement(self.msgs[rid], rpki.publication.tag_withdraw, uri = uri, hash = pdu_hash) + else: + pdu = SubElement(self.msgs[rid], rpki.publication.tag_publish, uri = uri) + pdu.text = new_obj.get_Base64() + if pdu_hash is not None: + pdu.set("hash", pdu_hash) + if handler is not None: - self.handlers[id(pdu)] = handler - pdu.tag = id(pdu) - self.msgs[rid].append(pdu) + tag = str(id(pdu)) + self.handlers[tag] = handler + pdu.set("tag", tag) + if self.replace: self.uris[uri] = pdu - def publish(self, cls, uri, obj, repository, handler = None): - return self._add( uri, obj, repository, handler, cls.make_publish) - - def withdraw(self, cls, uri, obj, repository, handler = None): - return self._add( uri, obj, repository, handler, cls.make_withdraw) - def call_pubd(self, cb, eb): def loop(iterator, rid): logger.debug("Calling pubd[%r]", self.repositories[rid]) @@ -2487,5 +2842,5 @@ class publication_queue(object): return sum(len(self.msgs[rid]) for rid in self.repositories) def empty(self): - assert (not self.msgs) == (self.size == 0) + assert (not self.msgs) == (self.size == 0), "Assertion failure: not self.msgs: %r, self.size %r" % (not self.msgs, self.size) return not self.msgs diff --git a/rpki/rpkid_tasks.py b/rpki/rpkid_tasks.py index 58b4bcfe..c44b2220 100644 --- a/rpki/rpkid_tasks.py +++ b/rpki/rpkid_tasks.py @@ -176,12 +176,12 @@ class PollParentTask(AbstractTask): def parent_loop(self, parent_iterator, parent): self.parent_iterator = parent_iterator self.parent = parent - rpki.up_down.list_pdu.query(parent, self.got_list, self.list_failed) + parent.up_down_list_query(self.got_list, self.list_failed) def got_list(self, r_msg): self.ca_map = dict((ca.parent_resource_class, ca) for ca in self.parent.cas) self.gctx.checkpoint() - rpki.async.iterator(r_msg.payload.classes, self.class_loop, self.class_done) + rpki.async.iterator(r_msg.getiterator(rpki.up_down.tag_class), self.class_loop, self.class_done) def list_failed(self, e): logger.exception("Couldn't get resource class list from parent %r, skipping", self.parent) @@ -191,7 +191,7 @@ class PollParentTask(AbstractTask): self.gctx.checkpoint() self.class_iterator = class_iterator try: - ca = self.ca_map.pop(rc.class_name) + ca = self.ca_map.pop(rc.get("class_name")) except KeyError: rpki.rpkid.ca_obj.create(self.parent, rc, class_iterator, self.class_create_failed) else: @@ -210,7 +210,7 @@ class PollParentTask(AbstractTask): def ca_loop(self, iterator, ca): self.gctx.checkpoint() - ca.delete(self.parent, iterator) + ca.destroy(self.parent, iterator) def ca_done(self): self.gctx.checkpoint() @@ -310,10 +310,9 @@ class UpdateChildrenTask(AbstractTask): self.child.child_handle, child_cert.cert.gSKI(), old_resources.valid_until, irdb_resources.valid_until) child_cert.sql_delete() - self.publisher.withdraw( - cls = rpki.publication.certificate_elt, + self.publisher.queue( uri = child_cert.uri, - obj = child_cert.cert, + old_obj = child_cert.cert, repository = ca.parent.repository) ca_detail.generate_manifest(publisher = self.publisher) @@ -359,7 +358,7 @@ class UpdateROAsTask(AbstractTask): logger.debug("Issuing query for ROA requests") self.gctx.irdb_query_roa_requests(self.self_handle, self.got_roa_requests, self.roa_requests_failed) - def got_roa_requests(self, roa_requests): + def got_roa_requests(self, r_msg): self.gctx.checkpoint() logger.debug("Received response to query for ROA requests") @@ -385,15 +384,17 @@ class UpdateROAsTask(AbstractTask): else: self.orphans.append(roa) - for roa_request in roa_requests: - k = (roa_request.asn, str(roa_request.ipv4), str(roa_request.ipv6)) + for r_pdu in r_msg: + k = (r_pdu.get("asn"), r_pdu.get("ipv4"), r_pdu.get("ipv6")) if k in seen: - logger.warning("Skipping duplicate ROA request %r", roa_request) + logger.warning("Skipping duplicate ROA request %r", r_pdu) else: seen.add(k) roa = roas.pop(k, None) if roa is None: - roa = rpki.rpkid.roa_obj(self.gctx, self.self_id, roa_request.asn, roa_request.ipv4, roa_request.ipv6) + roa = rpki.rpkid.roa_obj(self.gctx, self.self_id, long(r_pdu.get("asn")), + rpki.resource_set.roa_prefix_set_ipv4(r_pdu.get("ipv4")), + rpki.resource_set.roa_prefix_set_ipv6(r_pdu.get("ipv6"))) logger.debug("Created new %r", roa) else: logger.debug("Found existing %r", roa) @@ -485,7 +486,7 @@ class UpdateGhostbustersTask(AbstractTask): self.got_ghostbuster_requests, self.ghostbuster_requests_failed) - def got_ghostbuster_requests(self, ghostbuster_requests): + def got_ghostbuster_requests(self, r_msg): try: self.gctx.checkpoint() @@ -508,24 +509,24 @@ class UpdateGhostbustersTask(AbstractTask): else: ghostbusters[k] = ghostbuster - for ghostbuster_request in ghostbuster_requests: - if ghostbuster_request.parent_handle not in parents: - logger.warning("Unknown parent_handle %r in Ghostbuster request, skipping", ghostbuster_request.parent_handle) + for r_pdu in r_msg: + if r_pdu.get("parent_handle") not in parents: + logger.warning("Unknown parent_handle %r in Ghostbuster request, skipping", r_pdu.get("parent_handle")) continue - k = (ghostbuster_request.parent_handle, ghostbuster_request.vcard) + k = (r_pdu.get("parent_handle"), r_pdu.text) if k in seen: - logger.warning("Skipping duplicate Ghostbuster request %r", ghostbuster_request) + logger.warning("Skipping duplicate Ghostbuster request %r", r_pdu) continue seen.add(k) - for ca in parents[ghostbuster_request.parent_handle].cas: + for ca in parents[r_pdu.get("parent_handle")].cas: ca_detail = ca.active_ca_detail if ca_detail is not None: - ghostbuster = ghostbusters.pop((ca_detail.ca_detail_id, ghostbuster_request.vcard), None) + ghostbuster = ghostbusters.pop((ca_detail.ca_detail_id, r_pdu.text), None) if ghostbuster is None: - ghostbuster = rpki.rpkid.ghostbuster_obj(self.gctx, self.self_id, ca_detail.ca_detail_id, ghostbuster_request.vcard) - logger.debug("Created new %r for %r", ghostbuster, ghostbuster_request.parent_handle) + ghostbuster = rpki.rpkid.ghostbuster_obj(self.gctx, self.self_id, ca_detail.ca_detail_id, r_pdu.text) + logger.debug("Created new %r for %r", ghostbuster, r_pdu.get("parent_handle")) else: - logger.debug("Found existing %r for %s", ghostbuster, ghostbuster_request.parent_handle) + logger.debug("Found existing %r for %s", ghostbuster, r_pdu.get("parent_handle")) ghostbuster.update(publisher = publisher, fast = True) ca_details.add(ca_detail) @@ -576,7 +577,7 @@ class UpdateEECertificatesTask(AbstractTask): self.got_requests, self.get_requests_failed) - def got_requests(self, requests): + def got_requests(self, r_msg): try: self.gctx.checkpoint() @@ -595,39 +596,43 @@ class UpdateEECertificatesTask(AbstractTask): ca_details = set() - for req in requests: - ees = existing.pop(req.gski, ()) + for r_pdu in r_msg: + gski = r_pdu.get("gski") + ees = existing.pop(gski, ()) resources = rpki.resource_set.resource_bag( - asn = req.asn, - v4 = req.ipv4, - v6 = req.ipv6, - valid_until = req.valid_until) + asn = rpki.resource_set.resource_set_as(r_pdu.get("asn")), + v4 = rpki.resource_set.resource_set_ipv4(r_pdu.get("ipv4")), + v6 = rpki.resource_set.resource_set_ipv6(r_pdu.get("ipv6")), + valid_until = rpki.sundial.datetime.fromXMLtime(r_pdu.get("valid_until"))) covering = self.find_covering_ca_details(resources) ca_details.update(covering) for ee in ees: if ee.ca_detail in covering: logger.debug("Updating existing EE certificate for %s %s", - req.gski, resources) + gski, resources) ee.reissue( resources = resources, publisher = publisher) covering.remove(ee.ca_detail) else: logger.debug("Existing EE certificate for %s %s is no longer covered", - req.gski, resources) + gski, resources) ee.revoke(publisher = publisher) + subject_name = rpki.x509.X501DN.from_cn(r_pdu.get("cn"), r_pdu.get("sn")) + subject_key = rpki.x509.PKCS10(Base64 = r_pdu[0].text).getPublicKey() + for ca_detail in covering: logger.debug("No existing EE certificate for %s %s", - req.gski, resources) + gski, resources) rpki.rpkid.ee_cert_obj.create( ca_detail = ca_detail, - subject_name = rpki.x509.X501DN.from_cn(req.cn, req.sn), - subject_key = req.pkcs10.getPublicKey(), + subject_name = subject_name, + subject_key = subject_key, resources = resources, publisher = publisher, - eku = req.eku or None) + eku = r_pdu.get("eku", "").split(",") or None) # Anything left is an orphan for ees in existing.values(): @@ -690,7 +695,7 @@ class RegenerateCRLsAndManifestsTask(AbstractTask): try: for ca_detail in ca.revoked_ca_details: if now > ca_detail.latest_crl.getNextUpdate(): - ca_detail.delete(ca = ca, publisher = publisher) + ca_detail.destroy(ca = ca, publisher = publisher) for ca_detail in ca.active_or_deprecated_ca_details: if now + regen_margin > ca_detail.latest_crl.getNextUpdate(): ca_detail.generate_crl(publisher = publisher) diff --git a/rpki/rpkidb/__init__.py b/rpki/rpkidb/__init__.py new file mode 100644 index 00000000..7764913c --- /dev/null +++ b/rpki/rpkidb/__init__.py @@ -0,0 +1,3 @@ +# $Id$ +# +# Placeholder for rpkidb Django models not yet written. diff --git a/rpki/rpkidb/migrations/0001_initial.py b/rpki/rpkidb/migrations/0001_initial.py new file mode 100644 index 00000000..f88f0cdd --- /dev/null +++ b/rpki/rpkidb/migrations/0001_initial.py @@ -0,0 +1,224 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import rpki.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='BSC', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('bsc_handle', models.SlugField(max_length=255)), + ('private_key_id', rpki.fields.KeyField(default=None, serialize=False, blank=True)), + ('pkcs10_request', rpki.fields.PKCS10Field(default=None, serialize=False, blank=True)), + ('hash_alg', rpki.fields.EnumField(choices=[(1, 'sha256')])), + ('signing_cert', rpki.fields.CertificateField(default=None, serialize=False, null=True, blank=True)), + ('signing_cert_crl', rpki.fields.CRLField(default=None, serialize=False, null=True, blank=True)), + ], + ), + migrations.CreateModel( + name='CA', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('last_crl_sn', models.BigIntegerField(default=1)), + ('last_manifest_sn', models.BigIntegerField(default=1)), + ('next_manifest_update', rpki.fields.SundialField(null=True)), + ('next_crl_update', rpki.fields.SundialField(null=True)), + ('last_issued_sn', models.BigIntegerField(default=1)), + ('sia_uri', models.TextField(null=True)), + ('parent_resource_class', models.TextField(null=True)), + ], + ), + migrations.CreateModel( + name='CADetail', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('public_key', rpki.fields.KeyField(default=None, serialize=False, null=True, blank=True)), + ('private_key_id', rpki.fields.KeyField(default=None, serialize=False, null=True, blank=True)), + ('latest_crl', rpki.fields.CRLField(default=None, serialize=False, null=True, blank=True)), + ('crl_published', rpki.fields.SundialField(null=True)), + ('latest_ca_cert', rpki.fields.CertificateField(default=None, serialize=False, null=True, blank=True)), + ('manifest_private_key_id', rpki.fields.KeyField(default=None, serialize=False, null=True, blank=True)), + ('manifest_public_key', rpki.fields.KeyField(default=None, serialize=False, null=True, blank=True)), + ('latest_manifest_cert', rpki.fields.CertificateField(default=None, serialize=False, null=True, blank=True)), + ('latest_manifest', rpki.fields.ManifestField(default=None, serialize=False, null=True, blank=True)), + ('manifest_published', rpki.fields.SundialField(null=True)), + ('state', rpki.fields.EnumField(choices=[(1, 'pending'), (2, 'active'), (3, 'deprecated'), (4, 'revoked')])), + ('ca_cert_uri', models.TextField(null=True)), + ('ca', models.ForeignKey(related_name='ca_details', to='rpkidb.CA')), + ], + ), + migrations.CreateModel( + name='Child', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('child_handle', models.SlugField(max_length=255)), + ('bpki_cert', rpki.fields.CertificateField(default=None, serialize=False, null=True, blank=True)), + ('bpki_glue', rpki.fields.CertificateField(default=None, serialize=False, null=True, blank=True)), + ('last_cms_timestamp', rpki.fields.SundialField(null=True)), + ('bsc', models.ForeignKey(related_name='children', to='rpkidb.BSC')), + ], + ), + migrations.CreateModel( + name='ChildCert', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('cert', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('published', rpki.fields.SundialField(null=True)), + ('ski', rpki.fields.BlobField(default=None, serialize=False, blank=True)), + ('ca_detail', models.ForeignKey(related_name='child_certs', to='rpkidb.CADetail')), + ('child', models.ForeignKey(related_name='child_certs', to='rpkidb.Child')), + ], + ), + migrations.CreateModel( + name='EECert', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('ski', rpki.fields.BlobField(default=None, serialize=False, blank=True)), + ('cert', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('published', rpki.fields.SundialField(null=True)), + ('ca_detail', models.ForeignKey(related_name='ee_certs', to='rpkidb.CADetail')), + ], + ), + migrations.CreateModel( + name='Ghostbuster', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('vcard', models.TextField()), + ('cert', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('ghostbuster', rpki.fields.GhostbusterField(default=None, serialize=False, blank=True)), + ('published', rpki.fields.SundialField(null=True)), + ('ca_detail', models.ForeignKey(related_name='ghostbusters', to='rpkidb.CADetail')), + ], + ), + migrations.CreateModel( + name='Parent', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('parent_handle', models.SlugField(max_length=255)), + ('bpki_cms_cert', rpki.fields.CertificateField(default=None, serialize=False, null=True, blank=True)), + ('bpki_cms_glue', rpki.fields.CertificateField(default=None, serialize=False, null=True, blank=True)), + ('peer_contact_uri', models.TextField(null=True)), + ('sia_base', models.TextField(null=True)), + ('sender_name', models.TextField(null=True)), + ('recipient_name', models.TextField(null=True)), + ('last_cms_timestamp', rpki.fields.SundialField(null=True)), + ('bsc', models.ForeignKey(related_name='parents', to='rpkidb.BSC')), + ], + ), + migrations.CreateModel( + name='Repository', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('repository_handle', models.SlugField(max_length=255)), + ('peer_contact_uri', models.TextField(null=True)), + ('bpki_cert', rpki.fields.CertificateField(default=None, serialize=False, null=True, blank=True)), + ('bpki_glue', rpki.fields.CertificateField(default=None, serialize=False, null=True, blank=True)), + ('last_cms_timestamp', rpki.fields.SundialField(null=True)), + ('bsc', models.ForeignKey(related_name='repositories', to='rpkidb.BSC')), + ], + ), + migrations.CreateModel( + name='RevokedCert', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('serial', models.BigIntegerField()), + ('revoked', rpki.fields.SundialField()), + ('expires', rpki.fields.SundialField()), + ('ca_detail', models.ForeignKey(related_name='revoked_certs', to='rpkidb.CADetail')), + ], + ), + migrations.CreateModel( + name='ROA', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('asn', models.BigIntegerField()), + ('ipv4', models.TextField(null=True)), + ('ipv6', models.TextField(null=True)), + ('cert', rpki.fields.CertificateField(default=None, serialize=False, blank=True)), + ('roa', rpki.fields.ROAField(default=None, serialize=False, blank=True)), + ('published', rpki.fields.SundialField(null=True)), + ('ca_detail', models.ForeignKey(related_name='roas', to='rpkidb.CADetail')), + ], + ), + migrations.CreateModel( + name='Self', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('self_handle', models.SlugField(max_length=255)), + ('use_hsm', models.BooleanField(default=False)), + ('crl_interval', models.BigIntegerField(null=True)), + ('regen_margin', models.BigIntegerField(null=True)), + ('bpki_cert', rpki.fields.CertificateField(default=None, serialize=False, null=True, blank=True)), + ('bpki_glue', rpki.fields.CertificateField(default=None, serialize=False, null=True, blank=True)), + ], + ), + migrations.AddField( + model_name='roa', + name='self', + field=models.ForeignKey(related_name='roas', to='rpkidb.Self'), + ), + migrations.AddField( + model_name='repository', + name='self', + field=models.ForeignKey(related_name='repositories', to='rpkidb.Self'), + ), + migrations.AddField( + model_name='parent', + name='repository', + field=models.ForeignKey(related_name='parents', to='rpkidb.Repository'), + ), + migrations.AddField( + model_name='parent', + name='self', + field=models.ForeignKey(related_name='parents', to='rpkidb.Self'), + ), + migrations.AddField( + model_name='ghostbuster', + name='self', + field=models.ForeignKey(related_name='ghostbusters', to='rpkidb.Self'), + ), + migrations.AddField( + model_name='eecert', + name='self', + field=models.ForeignKey(related_name='ee_certs', to='rpkidb.Self'), + ), + migrations.AddField( + model_name='child', + name='self', + field=models.ForeignKey(related_name='children', to='rpkidb.Self'), + ), + migrations.AddField( + model_name='ca', + name='parent', + field=models.ForeignKey(related_name='cas', to='rpkidb.Parent'), + ), + migrations.AddField( + model_name='bsc', + name='self', + field=models.ForeignKey(related_name='bscs', to='rpkidb.Self'), + ), + migrations.AlterUniqueTogether( + name='repository', + unique_together=set([('self', 'repository_handle')]), + ), + migrations.AlterUniqueTogether( + name='parent', + unique_together=set([('self', 'parent_handle')]), + ), + migrations.AlterUniqueTogether( + name='child', + unique_together=set([('self', 'child_handle')]), + ), + migrations.AlterUniqueTogether( + name='bsc', + unique_together=set([('self', 'bsc_handle')]), + ), + ] diff --git a/rpki/rpkidb/migrations/0002_auto_20151015_2213.py b/rpki/rpkidb/migrations/0002_auto_20151015_2213.py new file mode 100644 index 00000000..f602b42b --- /dev/null +++ b/rpki/rpkidb/migrations/0002_auto_20151015_2213.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('rpkidb', '0001_initial'), + ] + + operations = [ + migrations.RenameField( + model_name='parent', + old_name='bpki_cms_cert', + new_name='bpki_cert', + ), + migrations.RenameField( + model_name='parent', + old_name='bpki_cms_glue', + new_name='bpki_glue', + ), + migrations.AddField( + model_name='repository', + name='rrdp_notification_uri', + field=models.TextField(null=True), + ), + ] diff --git a/rpki/rpkidb/migrations/__init__.py b/rpki/rpkidb/migrations/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/rpki/rpkidb/migrations/__init__.py diff --git a/rpki/rpkidb/models.py b/rpki/rpkidb/models.py new file mode 100644 index 00000000..0a4346e0 --- /dev/null +++ b/rpki/rpkidb/models.py @@ -0,0 +1,2250 @@ +""" +Django ORM models for rpkid. +""" + +from __future__ import unicode_literals + +import logging +import base64 + +from django.db import models + +import rpki.left_right + +from rpki.fields import (EnumField, SundialField, BlobField, + CertificateField, KeyField, CRLField, PKCS10Field, + ManifestField, ROAField, GhostbusterField) + +from lxml.etree import Element, SubElement + +logger = logging.getLogger(__name__) + +# The objects available via the left-right protocol allow NULL values +# in places we wouldn't otherwise (eg, bpki_cert fields), to support +# existing protocol which allows back-end to build up objects +# gradually. We may want to rethink this eventually, but that yak can +# wait for its shave, particularly since disallowing null should be a +# very simple change given migrations. + +# The <self/> element was really badly named, but we weren't using +# Python when we named it. Perhaps <tenant/> would be a better name? +# Would want to rename it in left-right too. +# +# To make things worse, <self/> elements are handled slightly +# differently in many places, so there are a number of occurances of +# "self" or "self_handle" as special case magic. Feh. +# +# Cope for now, just be careful. + +class XMLTemplate(object): + """ + Encapsulate all the voodoo for transcoding between lxml and ORM. + """ + + # Type map to simplify declaration of Base64 sub-elements. + + element_type = dict(bpki_cert = rpki.x509.X509, + bpki_glue = rpki.x509.X509, + pkcs10_request = rpki.x509.PKCS10, + signing_cert = rpki.x509.X509, + signing_cert_crl = rpki.x509.CRL) + + + def __init__(self, name, attributes = (), booleans = (), elements = (), readonly = (), handles = ()): + self.name = name + self.handles = handles + self.attributes = attributes + self.booleans = booleans + self.elements = elements + self.readonly = readonly + + + def encode(self, obj, r_msg): + """ + Encode an ORM object as XML. + """ + + r_pdu = SubElement(r_msg, rpki.left_right.xmlns + self.name, nsmap = rpki.left_right.nsmap) + r_pdu.set(self.name + "_handle", getattr(obj, self.name + "_handle")) + if self.name != "self": + r_pdu.set("self_handle", getattr(obj, "self_handle")) + for h in self.handles: + k = h.xml_template.name + v = getattr(obj, k) + if v is not None: + r_pdu.set(k + "_handle", getattr(v, k + "_handle")) + for k in self.attributes: + v = getattr(obj, k) + if v is not None: + r_pdu.set(k, str(v)) + for k in self.booleans: + if getattr(obj, k): + r_pdu.set(k, "yes") + for k in self.elements + self.readonly: + v = getattr(obj, k) + if v is not None and not v.empty(): + SubElement(r_pdu, rpki.left_right.xmlns + k).text = v.get_Base64() + + + def acknowledge(self, obj, q_pdu, r_msg): + """ + Add an acknowledgement PDU in response to a create, set, or + destroy action. + + This includes a bit of special-case code for BSC objects which has + to go somewhere; we could handle it via some kind method of + call-out to the BSC model, but it's not worth building a general + mechanism for one case, so we do it inline and have done. + """ + + assert q_pdu.tag == rpki.left_right.xmlns + self.name + r_pdu = SubElement(r_msg, rpki.left_right.xmlns + self.name, nsmap = rpki.left_right.nsmap) + r_pdu.set(self.name + "_handle", getattr(obj, self.name + "_handle")) + if self.name != "self": + r_pdu.set("self_handle", getattr(obj, "self_handle")) + if self.name == "bsc" and q_pdu.get("action") != "destroy" and obj.pkcs11_request is not None: + assert not obj.pkcs11_request.empty() + SubElement(r_pdu, rpki.left_right.xmlns + "pkcs11_request").text = obj.pkcs11_request.get_Base64() + + + def decode(self, obj, q_pdu): + """ + Decode XML into an ORM object. + """ + + assert q_pdu.tag == rpki.left_right.xmlns + self.name + for h in self.handles: + k = h.xml_template.name + v = q_pdu.get(k + "_handle") + if v is not None: + setattr(obj, k, h.objects.get(**{k + "_handle" : v, "self" : obj.self})) + for k in self.attributes: + v = q_pdu.get(k) + if v is not None: + v.encode("ascii") + if v.isdigit(): + v = long(v) + setattr(obj, k, v) + for k in self.booleans: + v = q_pdu.get(k) + if v is not None: + setattr(obj, k, v == "yes") + for k in self.elements: + v = q_pdu.findtext(rpki.left_right.xmlns + k) + if v and v.strip(): + setattr(obj, k, self.element_type[k](Base64 = v)) + + +class XMLManager(models.Manager): # pylint: disable=W0232 + """ + Add a few methods which locate or create an object or objects + corresponding to the handles in an XML element, as appropriate. + + This assumes that models which use it have an "xml" class attribute + holding an XMLTemplate object (above). + """ + + + def xml_get_or_create(self, xml): + name = self.model.xml_template.name + action = xml.get("action") + assert xml.tag == rpki.left_right.xmlns + name and action in ("create", "set") + d = { name + "_handle" : xml.get(name + "_handle") } + if name != "self" and action == "create": + d["self"] = Self.objects.get(self_handle = xml.get("self_handle")) + elif name != "self": + d["self__self_handle"] = xml.get("self_handle") + return self.model(**d) if action == "create" else self.get(**d) + + + def xml_list(self, xml): + name = self.model.xml_template.name + action = xml.get("action") + assert xml.tag == rpki.left_right.xmlns + name and action in ("get", "list") + d = {} + if action == "get": + d[name + "_handle"] = xml.get(name + "_handle") + if name != "self": + d["self__self_handle"] = xml.get("self_handle") + return self.filter(**d) if d else self.all() + + + def xml_get_for_delete(self, xml): + name = self.model.xml_template.name + action = xml.get("action") + assert xml.tag == rpki.left_right.xmlns + name and action == "destroy" + d = { name + "_handle" : xml.get(name + "_handle") } + if name != "self": + d["self__self_handle"] = xml.get("self_handle") + return self.get(**d) + +# Models + +class Self(models.Model): + self_handle = models.SlugField(max_length = 255) + use_hsm = models.BooleanField(default = False) + crl_interval = models.BigIntegerField(null = True) + regen_margin = models.BigIntegerField(null = True) + bpki_cert = CertificateField(null = True) + bpki_glue = CertificateField(null = True) + objects = XMLManager() + + xml_template = XMLTemplate( + name = "self", + attributes = ("crl_interval", "regen_margin"), + booleans = ("use_hsm",), + elements = ("bpki_cert", "bpki_glue")) + + + def xml_pre_delete_hook(self): + raise NotImplementedError + + + def xml_post_save_hook(self, q_pdu, cb, eb): + if q_pdu.get("clear_replay_protection"): + for parent in self.parents.all(): + parent.clear_replay_protection() + for child in self.children.all(): + child.clear_replay_protection() + for repository in self.repositories.all(): + repository.clear_replay_protection() + actions = [] + rekey = q_pdu.get("rekey") + revoke = q_pdu.get("revoke") + reissue = q_pdu.get("reissue") + revoke_forgotten = q_pdu.get("revoke_forgotten") + if rekey or revoke or reissue or revoke_forgotten: + for parent in self.parents.all(): + if rekey: + actions.append(parent.serve_rekey) + if revoke: + actions.append(parent.serve_revoke) + if reissue: + actions.append(parent.serve_reissue) + if revoke_forgotten: + actions.append(parent.serve_revoke_forgotten) + if q_pdu.get("publish_world_now"): + actions.append(self.serve_publish_world_now) + if q_pdu.get("run_now"): + actions.append(self.serve_run_now) + def loop(iterator, action): + action(iterator, eb) + rpki.async.iterator(actions, loop, cb) + + + def serve_publish_world_now(self, cb, eb): + publisher = rpki.rpkid.publication_queue() + repositories = set() + objects = dict() + + def loop(iterator, parent): + repository = parent.repository + if repository.peer_contact_uri in repositories: + return iterator() + repositories.add(repository.peer_contact_uri) + q_msg = Element(rpki.publication.tag_msg, nsmap = rpki.publication.nsmap, + type = "query", version = rpki.publication.version) + SubElement(q_msg, rpki.publication.tag_list, tag = "list") + + def list_handler(r_pdu): + rpki.publication.raise_if_error(r_pdu) + assert r_pdu.tag == rpki.publication.tag_list + assert r_pdu.get("uri") not in objects + objects[r_pdu.get("uri")] = (r_pdu.get("hash"), repository) + + repository.call_pubd(iterator, eb, q_msg, length_check = False, handlers = dict(list = list_handler)) + + def reconcile(uri, obj, repository): + h, r = objects.pop(uri, (None, None)) + if h is not None: + assert r == repository + publisher.queue(uri = uri, new_obj = obj, old_hash = h, repository = repository) + + def done(): + for ca_detail in CADetail.objects.filter(ca__parent__self = self, state = "active"): + repository = ca_detail.ca.parent.repository + reconcile(uri = ca_detail.crl_uri, obj = ca_detail.latest_crl, repository = repository) + reconcile(uri = ca_detail.manifest_uri, obj = ca_detail.latest_manifest, repository = repository) + for c in ca_detail.child_certs.all(): + reconcile(uri = c.uri, obj = c.cert, repository = repository) + for r in ca_detail.roas.filter(roa__isnull = False): + reconcile(uri = r.uri, obj = r.roa, repository = repository) + for g in ca_detail.ghostbusters.all(): + reconcile(uri = g.uri, obj = g.ghostbuster, repository = repository) + for c in ca_detail.ee_certificates.all(): + reconcile(uri = c.uri, obj = c.cert, repository = repository) + for u in objects: + h, r = objects[u] + publisher.queue(uri = u, old_hash = h, repository = r) + publisher.call_pubd(cb, eb) + + rpki.async.iterator(self.parents.all(), loop, done) + + + def serve_run_now(self, cb, eb): + logger.debug("Forced immediate run of periodic actions for self %s[%d]", self.self_handle, self.self_id) + completion = rpki.rpkid_tasks.CompletionHandler(cb) + self.schedule_cron_tasks(completion) + assert completion.count > 0 + self.gctx.task_run() + + + def schedule_cron_tasks(self, completion): + try: + tasks = self.cron_tasks + except AttributeError: + tasks = self.cron_tasks = tuple(task(self) for task in rpki.rpkid_tasks.task_classes) + for task in tasks: + self.gctx.task_add(task) + completion.register(task) + + + def find_covering_ca_details(self, resources): + """ + Return all active CADetails for this <self/> which cover a + particular set of resources. + + If we expected there to be a large number of CADetails, we + could add index tables and write fancy SQL query to do this, but + for the expected common case where there are only one or two + active CADetails per <self/>, it's probably not worth it. In + any case, this is an optimization we can leave for later. + """ + + return set(ca_detail + for ca_detail in CADetail.objects.filter(ca__parent__self = self, state = "active") + if ca_detail.covers(resources)) + + +class BSC(models.Model): + bsc_handle = models.SlugField(max_length = 255) + private_key_id = KeyField() + pkcs10_request = PKCS10Field() + hash_alg = EnumField(choices = ("sha256",)) + signing_cert = CertificateField(null = True) + signing_cert_crl = CRLField(null = True) + self = models.ForeignKey(Self, related_name = "bscs") + objects = XMLManager() + + class Meta: # pylint: disable=C1001,W0232 + unique_together = ("self", "bsc_handle") + + xml_template = XMLTemplate( + name = "bsc", + elements = ("signing_cert", "signing_cert_crl"), + readonly = ("pkcs10_request",)) + + + def xml_pre_save_hook(self, q_pdu): + # Handle key generation, only supports RSA with SHA-256 for now. + if q_pdu.get("generate_keypair"): + assert q_pdu.get("key_type") in (None, "rsa") and q_pdu.get("hash_alg") in (None, "sha256") + self.private_key_id = rpki.x509.RSA.generate(keylength = int(q_pdu.get("key_length", 2048))) + self.pkcs10_request = rpki.x509.PKCS10.create(keypair = self.private_key_id) + + +class Repository(models.Model): + repository_handle = models.SlugField(max_length = 255) + peer_contact_uri = models.TextField(null = True) + rrdp_notification_uri = models.TextField(null = True) + bpki_cert = CertificateField(null = True) + bpki_glue = CertificateField(null = True) + last_cms_timestamp = SundialField(null = True) + bsc = models.ForeignKey(BSC, related_name = "repositories") + self = models.ForeignKey(Self, related_name = "repositories") + objects = XMLManager() + + class Meta: # pylint: disable=C1001,W0232 + unique_together = ("self", "repository_handle") + + xml_template = XMLTemplate( + name = "repository", + handles = (BSC,), + attributes = ("peer_contact_uri", "rrdp_notification_uri"), + elements = ("bpki_cert", "bpki_glue")) + + + def xml_post_save_hook(self, q_pdu, cb, eb): + if q_pdu.get("clear_replay_protection"): + self.clear_replay_protection() + cb() + + + def clear_replay_protection(self): + self.last_cms_timestamp = None + self.save() + + + def call_pubd(self, callback, errback, q_msg, handlers = {}, length_check = True): # pylint: disable=W0102 + """ + Send a message to publication daemon and return the response. + + As a convenience, attempting to send an empty message returns + immediate success without sending anything. + + handlers is a dict of handler functions to process the response + PDUs. If the tag value in the response PDU appears in the dict, + the associated handler is called to process the PDU. If no tag + matches, a default handler is called to check for errors; a + handler value of False suppresses calling of the default handler. + """ + + try: + if len(q_msg) == 0: + return callback() + + for q_pdu in q_msg: + logger.info("Sending %r to pubd", q_pdu) + + bsc = self.bsc + q_der = rpki.publication.cms_msg().wrap(q_msg, bsc.private_key_id, bsc.signing_cert, bsc.signing_cert_crl) + bpki_ta_path = (self.gctx.bpki_ta, self.self.bpki_cert, self.self.bpki_glue, self.bpki_cert, self.bpki_glue) + + def done(r_der): + try: + logger.debug("Received response from pubd") + r_cms = rpki.publication.cms_msg(DER = r_der) + r_msg = r_cms.unwrap(bpki_ta_path) + r_cms.check_replay_sql(self, self.peer_contact_uri) + for r_pdu in r_msg: + handler = handlers.get(r_pdu.get("tag"), rpki.publication.raise_if_error) + if handler: + logger.debug("Calling pubd handler %r", handler) + handler(r_pdu) + if length_check and len(q_msg) != len(r_msg): + raise rpki.exceptions.BadPublicationReply("Wrong number of response PDUs from pubd: sent %r, got %r" % (q_msg, r_msg)) + callback() + except (rpki.async.ExitNow, SystemExit): + raise + except Exception, e: + errback(e) + + logger.debug("Sending request to pubd") + rpki.http.client( + url = self.peer_contact_uri, + msg = q_der, + callback = done, + errback = errback) + + except (rpki.async.ExitNow, SystemExit): + raise + except Exception, e: + errback(e) + + +class Parent(models.Model): + parent_handle = models.SlugField(max_length = 255) + bpki_cert = CertificateField(null = True) + bpki_glue = CertificateField(null = True) + peer_contact_uri = models.TextField(null = True) + sia_base = models.TextField(null = True) + sender_name = models.TextField(null = True) + recipient_name = models.TextField(null = True) + last_cms_timestamp = SundialField(null = True) + self = models.ForeignKey(Self, related_name = "parents") + bsc = models.ForeignKey(BSC, related_name = "parents") + repository = models.ForeignKey(Repository, related_name = "parents") + objects = XMLManager() + + class Meta: # pylint: disable=C1001,W0232 + unique_together = ("self", "parent_handle") + + xml_template = XMLTemplate( + name = "parent", + handles = (BSC, Repository), + attributes = ("peer_contact_uri", "sia_base", "sender_name", "recipient_name"), + elements = ("bpki_cert", "bpki_glue")) + + + def xml_pre_delete_hook(self, cb, eb): + self.destroy(cb, delete_parent = False) + + + def xml_post_save_hook(self, q_pdu, cb, eb): + if q_pdu.get("clear_replay_protection"): + self.clear_replay_protection() + actions = [] + if q_pdu.get("rekey"): + actions.append(self.serve_rekey) + if q_pdu.get("revoke"): + actions.append(self.serve_revoke) + if q_pdu.get("reissue"): + actions.append(self.serve_reissue) + if q_pdu.get("revoke_forgotten"): + actions.append(self.serve_revoke_forgotten) + def loop(iterator, action): + action(iterator, eb) + rpki.async.iterator(actions, loop, cb) + + + def serve_rekey(self, cb, eb): + def loop(iterator, ca): + ca.rekey(iterator, eb) + rpki.async.iterator(self.cas.all(), loop, cb) + + + def serve_revoke(self, cb, eb): + def loop(iterator, ca): + ca.revoke(cb = iterator, eb = eb) + rpki.async.iterator(self.cas.all(), loop, cb) + + + def serve_reissue(self, cb, eb): + def loop(iterator, ca): + ca.reissue(cb = iterator, eb = eb) + rpki.async.iterator(self.cas.all(), loop, cb) + + + def clear_replay_protection(self): + self.last_cms_timestamp = None + self.save() + + + def get_skis(self, cb, eb): + """ + Fetch SKIs that this parent thinks we have. In theory this should + agree with our own database, but in practice stuff can happen, so + sometimes we need to know what our parent thinks. + + Result is a dictionary with the resource class name as key and a + set of SKIs as value. + """ + + def done(r_msg): + cb(dict((rc.get("class_name"), + set(rpki.x509.X509(Base64 = c.text).gSKI() + for c in rc.getiterator(rpki.up_down.tag_certificate))) + for rc in r_msg.getiterator(rpki.up_down.tag_class))) + self.up_down_list_query(done, eb) + + + def revoke_skis(self, rc_name, skis_to_revoke, cb, eb): + """ + Revoke a set of SKIs within a particular resource class. + """ + + def loop(iterator, ski): + def revoked(r_pdu): + iterator() + logger.debug("Asking parent %r to revoke class %r, SKI %s", self, rc_name, ski) + self.up_down_revoke_query(rc_name, ski, revoked, eb) + rpki.async.iterator(skis_to_revoke, loop, cb) + + + def serve_revoke_forgotten(self, cb, eb): + """ + Handle a left-right revoke_forgotten action for this parent. + + This is a bit fiddly: we have to compare the result of an up-down + list query with what we have locally and identify the SKIs of any + certificates that have gone missing. This should never happen in + ordinary operation, but can arise if we have somehow lost a + private key, in which case there is nothing more we can do with + the issued cert, so we have to clear it. As this really is not + supposed to happen, we don't clear it automatically, instead we + require an explicit trigger. + """ + + def got_skis(skis_from_parent): + def loop(iterator, item): + rc_name, skis_to_revoke = item + if rc_name in ca_map: + for ca_detail in ca_map[rc_name].issue_response_candidate_ca_details: + skis_to_revoke.discard(ca_detail.latest_ca_cert.gSKI()) + self.revoke_skis(rc_name, skis_to_revoke, iterator, eb) + ca_map = dict((ca.parent_resource_class, ca) for ca in self.cas.all()) + rpki.async.iterator(skis_from_parent.items(), loop, cb) + self.get_skis(got_skis, eb) + + + def destroy(self, cb, delete_parent = True): + """ + Delete all the CA stuff under this parent, and perhaps the parent + itself. + """ + + def loop(iterator, ca): + ca.destroy(self, iterator) + def revoke(): + self.serve_revoke_forgotten(done, fail) + def fail(e): + logger.warning("Trouble getting parent to revoke certificates, blundering onwards: %s", e) + done() + def done(): + if delete_parent: + self.delete() + cb() + rpki.async.iterator(self.cas, loop, revoke) + + + def _compose_up_down_query(self, query_type): + return Element(rpki.up_down.tag_message, nsmap = rpki.up_down.nsmap, version = rpki.up_down.version, + sender = self.sender_name, recipient = self.recipient_name, type = query_type) + + + def up_down_list_query(self, cb, eb): + q_msg = self._compose_up_down_query("list") + self.query_up_down(q_msg, cb, eb) + + + def up_down_issue_query(self, ca, ca_detail, cb, eb): + pkcs10 = rpki.x509.PKCS10.create( + keypair = ca_detail.private_key_id, + is_ca = True, + caRepository = ca.sia_uri, + rpkiManifest = ca_detail.manifest_uri, + rpkiNotify = ca.parent.repository.rrdp_notification_uri) + q_msg = self._compose_up_down_query("issue") + q_pdu = SubElement(q_msg, rpki.up_down.tag_request, class_name = ca.parent_resource_class) + q_pdu.text = pkcs10.get_Base64() + self.query_up_down(q_msg, cb, eb) + + + def up_down_revoke_query(self, class_name, ski, cb, eb): + q_msg = self._compose_up_down_query("revoke") + SubElement(q_msg, rpki.up_down.tag_key, class_name = class_name, ski = ski) + self.query_up_down(q_msg, cb, eb) + + + def query_up_down(self, q_msg, cb, eb): + + if self.bsc is None: + raise rpki.exceptions.BSCNotFound("Could not find BSC") + + if self.bsc.signing_cert is None: + raise rpki.exceptions.BSCNotReady("BSC %r is not yet usable" % self.bsc.bsc_handle) + + q_der = rpki.up_down.cms_msg().wrap(q_msg, + self.bsc.private_key_id, + self.bsc.signing_cert, + self.bsc.signing_cert_crl) + + def unwrap(r_der): + try: + r_cms = rpki.up_down.cms_msg(DER = r_der) + r_msg = r_cms.unwrap((self.gctx.bpki_ta, + self.self.bpki_cert, + self.self.bpki_glue, + self.bpki_cert, + self.bpki_glue)) + r_cms.check_replay_sql(self, self.peer_contact_uri) + rpki.up_down.check_response(r_msg, q_msg.get("type")) + + except (SystemExit, rpki.async.ExitNow): + raise + except Exception, e: + eb(e) + else: + cb(r_msg) + + rpki.http.client( + msg = q_der, + url = self.peer_contact_uri, + callback = unwrap, + errback = eb, + content_type = rpki.up_down.content_type) + + + def construct_sia_uri(self, rc): + """ + Construct the sia_uri value for a CA under this parent given + configured information and the parent's up-down protocol + list_response PDU. + """ + + sia_uri = rc.get("suggested_sia_head", "") + if not sia_uri.startswith("rsync://") or not sia_uri.startswith(self.sia_base): + sia_uri = self.sia_base + if not sia_uri.endswith("/"): + raise rpki.exceptions.BadURISyntax("SIA URI must end with a slash: %s" % sia_uri) + return sia_uri + + +class CA(models.Model): + last_crl_sn = models.BigIntegerField(default = 1) + last_manifest_sn = models.BigIntegerField(default = 1) + next_manifest_update = SundialField(null = True) + next_crl_update = SundialField(null = True) + last_issued_sn = models.BigIntegerField(default = 1) + sia_uri = models.TextField(null = True) + parent_resource_class = models.TextField(null = True) # Not sure this should allow NULL + parent = models.ForeignKey(Parent, related_name = "cas") + + # So it turns out that there's always a 1:1 mapping between the + # class_name we receive from our parent and the class_name we issue + # to our children: in spite of the obfuscated way that we used to + # handle class names, we never actually added a way for the back-end + # to create new classes. Not clear we want to encourage this, but + # if we wanted to support it, simple approach would probably be an + # optional class_name attribute in the left-right <list_resources/> + # response; if not present, we'd use parent's class_name as now, + # otherwise we'd use the supplied class_name. + + # ca_obj has a zillion properties encoding various specialized + # ca_detail queries. ORM query syntax probably renders this OBE, + # but need to translate in existing code. + # + #def pending_ca_details(self): return self.ca_details.filter(state = "pending") + #def active_ca_detail(self): return self.ca_details.get(state = "active") + #def deprecated_ca_details(self): return self.ca_details.filter(state = "deprecated") + #def active_or_deprecated_ca_details(self): return self.ca_details.filter(state__in = ("active", "deprecated")) + #def revoked_ca_details(self): return self.ca_details.filter(state = "revoked") + #def issue_response_candidate_ca_details(self): return self.ca_details.exclude(state = "revoked") + + + def check_for_updates(self, parent, rc, cb, eb): + """ + Parent has signaled continued existance of a resource class we + already knew about, so we need to check for an updated + certificate, changes in resource coverage, revocation and reissue + with the same key, etc. + """ + + sia_uri = parent.construct_sia_uri(rc) + sia_uri_changed = self.sia_uri != sia_uri + if sia_uri_changed: + logger.debug("SIA changed: was %s now %s", self.sia_uri, sia_uri) + self.sia_uri = sia_uri + self.sql_mark_dirty() + class_name = rc.get("class_name") + rc_resources = rpki.resource_set.resource_bag( + rc.get("resource_set_as"), + rc.get("resource_set_ipv4"), + rc.get("resource_set_ipv6"), + rc.get("resource_set_notafter")) + cert_map = {} + for c in rc.getiterator(rpki.up_down.tag_certificate): + x = rpki.x509.X509(Base64 = c.text) + u = rpki.up_down.multi_uri(c.get("cert_url")).rsync() + cert_map[x.gSKI()] = (x, u) + def loop(iterator, ca_detail): + rc_cert, rc_cert_uri = cert_map.pop(ca_detail.public_key.gSKI(), (None, None)) + if rc_cert is None: + logger.warning("SKI %s in resource class %s is in database but missing from list_response to %s from %s, " + "maybe parent certificate went away?", + ca_detail.public_key.gSKI(), class_name, parent.self.self_handle, parent.parent_handle) + publisher = rpki.rpkid.publication_queue() + ca_detail.destroy(ca = ca_detail.ca, publisher = publisher) + return publisher.call_pubd(iterator, eb) + if ca_detail.state == "active" and ca_detail.ca_cert_uri != rc_cert_uri: + logger.debug("AIA changed: was %s now %s", ca_detail.ca_cert_uri, rc_cert_uri) + ca_detail.ca_cert_uri = rc_cert_uri + ca_detail.save() + if ca_detail.state not in ("pending", "active"): + return iterator() + if ca_detail.state == "pending": + current_resources = rpki.resource_set.resource_bag() + else: + current_resources = ca_detail.latest_ca_cert.get_3779resources() + if (ca_detail.state == "pending" or + sia_uri_changed or + ca_detail.latest_ca_cert != rc_cert or + ca_detail.latest_ca_cert.getNotAfter() != rc_resources.valid_until or + current_resources.undersized(rc_resources) or + current_resources.oversized(rc_resources)): + return ca_detail.update( + parent = parent, + ca = self, + rc = rc, + sia_uri_changed = sia_uri_changed, + old_resources = current_resources, + callback = iterator, + errback = eb) + iterator() + def done(): + if cert_map: + logger.warning("Unknown certificate SKI%s %s in resource class %s in list_response to %s from %s, maybe you want to \"revoke_forgotten\"?", + "" if len(cert_map) == 1 else "s", ", ".join(cert_map), class_name, parent.self.self_handle, parent.parent_handle) + cb() + ca_details = self.ca_details.exclude(state = "revoked") + if ca_details: + rpki.async.iterator(ca_details, loop, done) + else: + logger.warning("Existing resource class %s to %s from %s with no certificates, rekeying", + class_name, parent.self.self_handle, parent.parent_handle) + self.rekey(cb, eb) + + + # Called from exactly one place, in rpki.rpkid_tasks.PollParentTask.class_loop(). + # Might want to refactor. + + @classmethod + def create(cls, parent, rc, cb, eb): + """ + Parent has signaled existance of a new resource class, so we need + to create and set up a corresponding CA object. + """ + + self = cls.objects.create(parent = parent, + parent_resource_class = rc.get("class_name"), + sia_uri = parent.construct_sia_uri(rc)) + ca_detail = CADetail.create(self) + def done(r_msg): + c = r_msg[0][0] + logger.debug("CA %r received certificate %s", self, c.get("cert_url")) + ca_detail.activate( + ca = self, + cert = rpki.x509.X509(Base64 = c.text), + uri = c.get("cert_url"), + callback = cb, + errback = eb) + logger.debug("Sending issue request to %r from %r", parent, self.create) + parent.up_down_issue_query(self, ca_detail, done, eb) + + + def destroy(self, parent, callback): + """ + The list of current resource classes received from parent does not + include the class corresponding to this CA, so we need to delete + it (and its little dog too...). + + All certs published by this CA are now invalid, so need to + withdraw them, the CRL, and the manifest from the repository, + delete all child_cert and ca_detail records associated with this + CA, then finally delete this CA itself. + """ + + def lose(e): + logger.exception("Could not delete CA %r, skipping", self) + callback() + def done(): + logger.debug("Deleting %r", self) + self.delete() + callback() + publisher = rpki.rpkid.publication_queue() + for ca_detail in self.ca_details.all(): + ca_detail.destroy(ca = self, publisher = publisher, allow_failure = True) + publisher.call_pubd(done, lose) + + + def next_serial_number(self): + """ + Allocate a certificate serial number. + """ + + self.last_issued_sn += 1 + self.save() + return self.last_issued_sn + + + def next_manifest_number(self): + """ + Allocate a manifest serial number. + """ + + self.last_manifest_sn += 1 + self.save() + return self.last_manifest_sn + + + def next_crl_number(self): + """ + Allocate a CRL serial number. + """ + + self.last_crl_sn += 1 + self.save() + return self.last_crl_sn + + + def rekey(self, cb, eb): + """ + Initiate a rekey operation for this CA. Generate a new keypair. + Request cert from parent using new keypair. Mark result as our + active ca_detail. Reissue all child certs issued by this CA using + the new ca_detail. + """ + + old_detail = self.ca_details.get(state = "active") + new_detail = CADetail.create(self) + def done(r_msg): + c = r_msg[0][0] + logger.debug("CA %r received certificate %s", self, c.get("cert_url")) + new_detail.activate( + ca = self, + cert = rpki.x509.X509(Base64 = c.text), + uri = c.get("cert_url"), + predecessor = old_detail, + callback = cb, + errback = eb) + logger.debug("Sending issue request to %r from %r", self.parent, self.rekey) + self.parent.up_down_issue_query(self, new_detail, done, eb) + + + def revoke(self, cb, eb, revoke_all = False): + """ + Revoke deprecated ca_detail objects associated with this CA, or + all ca_details associated with this CA if revoke_all is set. + """ + + def loop(iterator, ca_detail): + ca_detail.revoke(cb = iterator, eb = eb) + rpki.async.iterator(self.ca_details.all() if revoke_all else self.ca_details.filter(state = "deprecated"), + loop, cb) + + + def reissue(self, cb, eb): + """ + Reissue all current certificates issued by this CA. + """ + + ca_detail = self.ca_details.get(state = "active") + if ca_detail: + ca_detail.reissue(cb, eb) + else: + cb() + + +class CADetail(models.Model): + public_key = KeyField(null = True) + private_key_id = KeyField(null = True) + latest_crl = CRLField(null = True) + crl_published = SundialField(null = True) + latest_ca_cert = CertificateField(null = True) + manifest_private_key_id = KeyField(null = True) + manifest_public_key = KeyField(null = True) + latest_manifest_cert = CertificateField(null = True) + latest_manifest = ManifestField(null = True) + manifest_published = SundialField(null = True) + state = EnumField(choices = ("pending", "active", "deprecated", "revoked")) + ca_cert_uri = models.TextField(null = True) + ca = models.ForeignKey(CA, related_name = "ca_details") + + + # Like the old ca_obj class, the old ca_detail_obj class had ten + # zillion properties and methods encapsulating SQL queries. + # Translate as we go. + + + @property + def crl_uri(self): + """ + Return publication URI for this ca_detail's CRL. + """ + + return self.ca.sia_uri + self.crl_uri_tail + + + @property + def crl_uri_tail(self): + """ + Return tail (filename portion) of publication URI for this ca_detail's CRL. + """ + + return self.public_key.gSKI() + ".crl" + + + @property + def manifest_uri(self): + """ + Return publication URI for this ca_detail's manifest. + """ + + return self.ca.sia_uri + self.public_key.gSKI() + ".mft" + + + def has_expired(self): + """ + Return whether this ca_detail's certificate has expired. + """ + + return self.latest_ca_cert.getNotAfter() <= rpki.sundial.now() + + + def covers(self, target): + """ + Test whether this ca-detail covers a given set of resources. + """ + + assert not target.asn.inherit and not target.v4.inherit and not target.v6.inherit + me = self.latest_ca_cert.get_3779resources() + return target.asn <= me.asn and target.v4 <= me.v4 and target.v6 <= me.v6 + + + def activate(self, ca, cert, uri, callback, errback, predecessor = None): + """ + Activate this ca_detail. + """ + + publisher = rpki.rpkid.publication_queue() + self.latest_ca_cert = cert + self.ca_cert_uri = uri + self.generate_manifest_cert() + self.state = "active" + self.generate_crl(publisher = publisher) + self.generate_manifest(publisher = publisher) + self.save() + if predecessor is not None: + predecessor.state = "deprecated" + predecessor.save() + for child_cert in predecessor.child_certs.all(): + child_cert.reissue(ca_detail = self, publisher = publisher) + for roa in predecessor.roas.all(): + roa.regenerate(publisher = publisher) + for ghostbuster in predecessor.ghostbusters.all(): + ghostbuster.regenerate(publisher = publisher) + predecessor.generate_crl(publisher = publisher) + predecessor.generate_manifest(publisher = publisher) + publisher.call_pubd(callback, errback) + + + def destroy(self, ca, publisher, allow_failure = False): + """ + Delete this ca_detail and all of the certs it issued. + + If allow_failure is true, we clean up as much as we can but don't + raise an exception. + """ + + repository = ca.parent.repository + handler = False if allow_failure else None + for child_cert in self.child_certs.all(): + publisher.queue(uri = child_cert.uri, old_obj = child_cert.cert, repository = repository, handler = handler) + child_cert.delete() + for roa in self.roas.all(): + roa.revoke(publisher = publisher, allow_failure = allow_failure, fast = True) + for ghostbuster in self.ghostbusters.all(): + ghostbuster.revoke(publisher = publisher, allow_failure = allow_failure, fast = True) + if self.latest_manifest is not None: + publisher.queue(uri = self.manifest_uri, old_obj = self.latest_manifest, repository = repository, handler = handler) + if self.latest_crl is not None: + publisher.queue(uri = self.crl_uri, old_obj = self.latest_crl, repository = repository, handler = handler) + for cert in self.revoked_certs.all(): # + self.child_certs.all() + logger.debug("Deleting %r", cert) + cert.delete() + logger.debug("Deleting %r", self) + self.delete() + + def revoke(self, cb, eb): + """ + Request revocation of all certificates whose SKI matches the key + for this ca_detail. + + Tasks: + + - Request revocation of old keypair by parent. + + - Revoke all child certs issued by the old keypair. + + - Generate a final CRL, signed with the old keypair, listing all + the revoked certs, with a next CRL time after the last cert or + CRL signed by the old keypair will have expired. + + - Generate a corresponding final manifest. + + - Destroy old keypairs. + + - Leave final CRL and manifest in place until their nextupdate + time has passed. + """ + + ca = self.ca + parent = ca.parent + class_name = ca.parent_resource_class + gski = self.latest_ca_cert.gSKI() + + def parent_revoked(r_msg): + if r_msg[0].get("class_name") != class_name: + raise rpki.exceptions.ResourceClassMismatch + if r_msg[0].get("ski") != gski: + raise rpki.exceptions.SKIMismatch + logger.debug("Parent revoked %s, starting cleanup", gski) + crl_interval = rpki.sundial.timedelta(seconds = parent.self.crl_interval) + nextUpdate = rpki.sundial.now() + if self.latest_manifest is not None: + self.latest_manifest.extract_if_needed() + nextUpdate = nextUpdate.later(self.latest_manifest.getNextUpdate()) + if self.latest_crl is not None: + nextUpdate = nextUpdate.later(self.latest_crl.getNextUpdate()) + publisher = rpki.rpkid.publication_queue() + for child_cert in self.child_certs.all(): + nextUpdate = nextUpdate.later(child_cert.cert.getNotAfter()) + child_cert.revoke(publisher = publisher) + for roa in self.roas.all(): + nextUpdate = nextUpdate.later(roa.cert.getNotAfter()) + roa.revoke(publisher = publisher) + for ghostbuster in self.ghostbusters.all(): + nextUpdate = nextUpdate.later(ghostbuster.cert.getNotAfter()) + ghostbuster.revoke(publisher = publisher) + nextUpdate += crl_interval + self.generate_crl(publisher = publisher, nextUpdate = nextUpdate) + self.generate_manifest(publisher = publisher, nextUpdate = nextUpdate) + self.private_key_id = None + self.manifest_private_key_id = None + self.manifest_public_key = None + self.latest_manifest_cert = None + self.state = "revoked" + self.save() + publisher.call_pubd(cb, eb) + logger.debug("Asking parent to revoke CA certificate %s", gski) + parent.up_down_revoke_query(class_name, gski, parent_revoked, eb) + + + def update(self, parent, ca, rc, sia_uri_changed, old_resources, callback, errback): + """ + Need to get a new certificate for this ca_detail and perhaps frob + children of this ca_detail. + """ + + def issued(r_msg): + c = r_msg[0][0] + cert = rpki.x509.X509(Base64 = c.text) + cert_url = c.get("cert_url") + logger.debug("CA %r received certificate %s", self, cert_url) + if self.state == "pending": + return self.activate(ca = ca, cert = cert, uri = cert_url, callback = callback, errback = errback) + validity_changed = self.latest_ca_cert is None or self.latest_ca_cert.getNotAfter() != cert.getNotAfter() + publisher = rpki.rpkid.publication_queue() + if self.latest_ca_cert != cert: + self.latest_ca_cert = cert + self.save() + self.generate_manifest_cert() + self.generate_crl(publisher = publisher) + self.generate_manifest(publisher = publisher) + new_resources = self.latest_ca_cert.get_3779resources() + if sia_uri_changed or old_resources.oversized(new_resources): + for child_cert in self.child_certs.all(): + child_resources = child_cert.cert.get_3779resources() + if sia_uri_changed or child_resources.oversized(new_resources): + child_cert.reissue(ca_detail = self, resources = child_resources & new_resources, publisher = publisher) + if sia_uri_changed or validity_changed or old_resources.oversized(new_resources): + for roa in self.roas.all(): + roa.update(publisher = publisher, fast = True) + if sia_uri_changed or validity_changed: + for ghostbuster in self.ghostbusters.all(): + ghostbuster.update(publisher = publisher, fast = True) + publisher.call_pubd(callback, errback) + logger.debug("Sending issue request to %r from %r", parent, self.update) + parent.up_down_issue_query(ca, self, issued, errback) + + + @classmethod + def create(cls, ca): + """ + Create a new ca_detail object for a specified CA. + """ + + cer_keypair = rpki.x509.RSA.generate() + mft_keypair = rpki.x509.RSA.generate() + return cls.objects.create(ca = ca, state = "pending", + private_key_id = cer_keypair, public_key = cer_keypair.get_public(), + manifest_private_key_id = mft_keypair, manifest_public_key = mft_keypair.get_public()) + + + def issue_ee(self, ca, resources, subject_key, sia, + cn = None, sn = None, notAfter = None, eku = None): + """ + Issue a new EE certificate. + """ + + if notAfter is None: + notAfter = self.latest_ca_cert.getNotAfter() + return self.latest_ca_cert.issue( + keypair = self.private_key_id, + subject_key = subject_key, + serial = ca.next_serial_number(), + sia = sia, + aia = self.ca_cert_uri, + crldp = self.crl_uri, + resources = resources, + notAfter = notAfter, + is_ca = False, + cn = cn, + sn = sn, + eku = eku) + + + def generate_manifest_cert(self): + """ + Generate a new manifest certificate for this ca_detail. + """ + + resources = rpki.resource_set.resource_bag.from_inheritance() + self.latest_manifest_cert = self.issue_ee( + ca = self.ca, + resources = resources, + subject_key = self.manifest_public_key, + sia = (None, None, self.manifest_uri, self.ca.parent.repository.rrdp_notification_uri)) + + + def issue(self, ca, child, subject_key, sia, resources, publisher, child_cert = None): + """ + Issue a new certificate to a child. Optional child_cert argument + specifies an existing child_cert object to update in place; if not + specified, we create a new one. Returns the child_cert object + containing the newly issued cert. + """ + + self.check_failed_publication(publisher) + cert = self.latest_ca_cert.issue( + keypair = self.private_key_id, + subject_key = subject_key, + serial = ca.next_serial_number(), + aia = self.ca_cert_uri, + crldp = self.crl_uri, + sia = sia, + resources = resources, + notAfter = resources.valid_until) + if child_cert is None: + old_cert = None + child_cert = ChildCert(child = child, ca_detail = self, cert = cert) + logger.debug("Created new child_cert %r", child_cert) + else: + old_cert = child_cert.cert + child_cert.cert = cert + child_cert.ca_detail = self + logger.debug("Reusing existing child_cert %r", child_cert) + child_cert.ski = cert.get_SKI() + child_cert.published = rpki.sundial.now() + child_cert.save() + publisher.queue( + uri = child_cert.uri, + old_obj = old_cert, + new_obj = child_cert.cert, + repository = ca.parent.repository, + handler = child_cert.published_callback) + self.generate_manifest(publisher = publisher) + return child_cert + + + def generate_crl(self, publisher, nextUpdate = None): + """ + Generate a new CRL for this ca_detail. At the moment this is + unconditional, that is, it is up to the caller to decide whether a + new CRL is needed. + """ + + self.check_failed_publication(publisher) + crl_interval = rpki.sundial.timedelta(seconds = self.ca.parent.self.crl_interval) + now = rpki.sundial.now() + if nextUpdate is None: + nextUpdate = now + crl_interval + certlist = [] + for revoked_cert in self.revoked_certs.all(): + if now > revoked_cert.expires + crl_interval: + revoked_cert.delete() + else: + certlist.append((revoked_cert.serial, revoked_cert.revoked)) + certlist.sort() + old_crl = self.latest_crl + self.latest_crl = rpki.x509.CRL.generate( + keypair = self.private_key_id, + issuer = self.latest_ca_cert, + serial = self.ca.next_crl_number(), + thisUpdate = now, + nextUpdate = nextUpdate, + revokedCertificates = certlist) + self.crl_published = now + self.save() + publisher.queue( + uri = self.crl_uri, + old_obj = old_crl, + new_obj = self.latest_crl, + repository = self.ca.parent.repository, + handler = self.crl_published_callback) + + + def crl_published_callback(self, pdu): + """ + Check result of CRL publication. + """ + + rpki.publication.raise_if_error(pdu) + self.crl_published = None + self.save() + + + def generate_manifest(self, publisher, nextUpdate = None): + """ + Generate a new manifest for this ca_detail. + """ + + self.check_failed_publication(publisher) + + crl_interval = rpki.sundial.timedelta(seconds = self.ca.parent.self.crl_interval) + now = rpki.sundial.now() + uri = self.manifest_uri + if nextUpdate is None: + nextUpdate = now + crl_interval + if (self.latest_manifest_cert is None or + (self.latest_manifest_cert.getNotAfter() < nextUpdate and + self.latest_manifest_cert.getNotAfter() < self.latest_ca_cert.getNotAfter())): + logger.debug("Generating EE certificate for %s", uri) + self.generate_manifest_cert() + logger.debug("Latest CA cert notAfter %s, new %s EE notAfter %s", + self.latest_ca_cert.getNotAfter(), uri, self.latest_manifest_cert.getNotAfter()) + logger.debug("Constructing manifest object list for %s", uri) + objs = [(self.crl_uri_tail, self.latest_crl)] + objs.extend((c.uri_tail, c.cert) for c in self.child_certs.all()) + objs.extend((r.uri_tail, r.roa) for r in self.roas.filter(roa__isnull = False)) + objs.extend((g.uri_tail, g.ghostbuster) for g in self.ghostbusters.all()) + objs.extend((e.uri_tail, e.cert) for e in self.ee_certificates.all()) + logger.debug("Building manifest object %s", uri) + old_manifest = self.latest_manifest + self.latest_manifest = rpki.x509.SignedManifest.build( + serial = self.ca.next_manifest_number(), + thisUpdate = now, + nextUpdate = nextUpdate, + names_and_objs = objs, + keypair = self.manifest_private_key_id, + certs = self.latest_manifest_cert) + logger.debug("Manifest generation took %s", rpki.sundial.now() - now) + self.manifest_published = now + self.save() + publisher.queue(uri = uri, + old_obj = old_manifest, + new_obj = self.latest_manifest, + repository = self.ca.parent.repository, + handler = self.manifest_published_callback) + + + def manifest_published_callback(self, pdu): + """ + Check result of manifest publication. + """ + + rpki.publication.raise_if_error(pdu) + self.manifest_published = None + self.save() + + + def reissue(self, cb, eb): + """ + Reissue all current certificates issued by this ca_detail. + """ + + publisher = rpki.rpkid.publication_queue() + self.check_failed_publication(publisher) + for roa in self.roas.all(): + roa.regenerate(publisher, fast = True) + for ghostbuster in self.ghostbusters.all(): + ghostbuster.regenerate(publisher, fast = True) + for ee_certificate in self.ee_certificates.all(): + ee_certificate.reissue(publisher, force = True) + for child_cert in self.child_certs.all(): + child_cert.reissue(self, publisher, force = True) + self.generate_manifest_cert() + self.save() + self.generate_crl(publisher = publisher) + self.generate_manifest(publisher = publisher) + self.save() + publisher.call_pubd(cb, eb) + + + def check_failed_publication(self, publisher, check_all = True): + """ + Check for failed publication of objects issued by this ca_detail. + + All publishable objects have timestamp fields recording time of + last attempted publication, and callback methods which clear these + timestamps once publication has succeeded. Our task here is to + look for objects issued by this ca_detail which have timestamps + set (indicating that they have not been published) and for which + the timestamps are not very recent (for some definition of very + recent -- intent is to allow a bit of slack in case pubd is just + being slow). In such cases, we want to retry publication. + + As an optimization, we can probably skip checking other products + if manifest and CRL have been published, thus saving ourselves + several complex SQL queries. Not sure yet whether this + optimization is worthwhile. + + For the moment we check everything without optimization, because + it simplifies testing. + + For the moment our definition of staleness is hardwired; this + should become configurable. + """ + + logger.debug("Checking for failed publication for %r", self) + + stale = rpki.sundial.now() - rpki.sundial.timedelta(seconds = 60) + repository = self.ca.parent.repository + if self.latest_crl is not None and self.crl_published is not None and self.crl_published < stale: + logger.debug("Retrying publication for %s", self.crl_uri) + publisher.queue(uri = self.crl_uri, + new_obj = self.latest_crl, + repository = repository, + handler = self.crl_published_callback) + if self.latest_manifest is not None and self.manifest_published is not None and self.manifest_published < stale: + logger.debug("Retrying publication for %s", self.manifest_uri) + publisher.queue(uri = self.manifest_uri, + new_obj = self.latest_manifest, + repository = repository, + handler = self.manifest_published_callback) + if not check_all: + return + for child_cert in self.child_certs.filter(published__isnull = False, published__lt = stale): + logger.debug("Retrying publication for %s", child_cert) + publisher.queue( + uri = child_cert.uri, + new_obj = child_cert.cert, + repository = repository, + handler = child_cert.published_callback) + for roa in self.roas.filter(published__isnull = False, published__lt = stale): + logger.debug("Retrying publication for %s", roa) + publisher.queue( + uri = roa.uri, + new_obj = roa.roa, + repository = repository, + handler = roa.published_callback) + for ghostbuster in self.ghostbusters.filter(published__isnull = False, published__lt = stale): + logger.debug("Retrying publication for %s", ghostbuster) + publisher.queue( + uri = ghostbuster.uri, + new_obj = ghostbuster.ghostbuster, + repository = repository, + handler = ghostbuster.published_callback) + for ee_cert in self.ee_certs.filter(published__isnull = False, published__lt = stale): + logger.debug("Retrying publication for %s", ee_cert) + publisher.queue( + uri = ee_cert.uri, + new_obj = ee_cert.cert, + repository = repository, + handler = ee_cert.published_callback) + + +class Child(models.Model): + child_handle = models.SlugField(max_length = 255) + bpki_cert = CertificateField(null = True) + bpki_glue = CertificateField(null = True) + last_cms_timestamp = SundialField(null = True) + self = models.ForeignKey(Self, related_name = "children") + bsc = models.ForeignKey(BSC, related_name = "children") + objects = XMLManager() + + class Meta: # pylint: disable=C1001,W0232 + unique_together = ("self", "child_handle") + + xml_template = XMLTemplate( + name = "child", + handles = (BSC,), + elements = ("bpki_cert", "bpki_glue")) + + + def xml_pre_delete_hook(self, cb, eb): + publisher = rpki.rpkid.publication_queue() + for child_cert in self.child_certs.all(): + child_cert.revoke(publisher = publisher, generate_crl_and_manifest = True) + publisher.call_pubd(cb, eb) + + + def xml_post_save_hook(self, q_pdu, cb, eb): + if q_pdu.get("clear_replay_protection"): + self.clear_replay_protection() + if q_pdu.get("reissue"): + self.serve_reissue(cb, eb) + else: + cb() + + + def serve_reissue(self, cb, eb): + publisher = rpki.rpkid.publication_queue() + for child_cert in self.child_certs.all(): + child_cert.reissue(child_cert.ca_detail, publisher, force = True) + publisher.call_pubd(cb, eb) + + + def clear_replay_protection(self): + self.last_cms_timestamp = None + self.save() + + + def up_down_handle_list(self, q_msg, r_msg, callback, errback): + def got_resources(irdb_resources): + if irdb_resources.valid_until < rpki.sundial.now(): + logger.debug("Child %s's resources expired %s", self.child_handle, irdb_resources.valid_until) + else: + for ca_detail in CADetail.objects.filter(ca__parent__self = self.self, state = "active"): + resources = ca_detail.latest_ca_cert.get_3779resources() & irdb_resources + if resources.empty(): + logger.debug("No overlap between received resources and what child %s should get ([%s], [%s])", + self.child_handle, ca_detail.latest_ca_cert.get_3779resources(), irdb_resources) + continue + rc = SubElement(r_msg, rpki.up_down.tag_class, + class_name = ca_detail.ca.parent_resource_class, + cert_url = ca_detail.ca_cert_uri, + resource_set_as = str(resources.asn), + resource_set_ipv4 = str(resources.v4), + resource_set_ipv6 = str(resources.v6), + resource_set_notafter = str(resources.valid_until)) + for child_cert in self.child_certs.filter(ca_detail = ca_detail): + c = SubElement(rc, rpki.up_down.tag_certificate, cert_url = child_cert.uri) + c.text = child_cert.cert.get_Base64() + SubElement(rc, rpki.up_down.tag_issuer).text = ca_detail.latest_ca_cert.get_Base64() + callback() + self.gctx.irdb_query_child_resources(self.self.self_handle, self.child_handle, got_resources, errback) + + + def up_down_handle_issue(self, q_msg, r_msg, callback, errback): + + def got_resources(irdb_resources): + + def done(): + rc = SubElement(r_msg, rpki.up_down.tag_class, + class_name = class_name, + cert_url = ca_detail.ca_cert_uri, + resource_set_as = str(resources.asn), + resource_set_ipv4 = str(resources.v4), + resource_set_ipv6 = str(resources.v6), + resource_set_notafter = str(resources.valid_until)) + c = SubElement(rc, rpki.up_down.tag_certificate, cert_url = child_cert.uri) + c.text = child_cert.cert.get_Base64() + SubElement(rc, rpki.up_down.tag_issuer).text = ca_detail.latest_ca_cert.get_Base64() + callback() + + if irdb_resources.valid_until < rpki.sundial.now(): + raise rpki.exceptions.IRDBExpired("IRDB entry for child %s expired %s" % ( + self.child_handle, irdb_resources.valid_until)) + + resources = irdb_resources & ca_detail.latest_ca_cert.get_3779resources() + resources.valid_until = irdb_resources.valid_until + req_key = pkcs10.getPublicKey() + req_sia = pkcs10.get_SIA() + + # Generate new cert or regenerate old one if necessary + + publisher = rpki.rpkid.publication_queue() + + try: + child_cert = self.child_certs.get(ca_detail = ca_detail, ski = req_key.get_SKI()) + + except ChildCert.NotFound: + child_cert = ca_detail.issue( + ca = ca_detail.ca, + child = self, + subject_key = req_key, + sia = req_sia, + resources = resources, + publisher = publisher) + + else: + child_cert = child_cert.reissue( + ca_detail = ca_detail, + sia = req_sia, + resources = resources, + publisher = publisher) + + publisher.call_pubd(done, errback) + + req = q_msg[0] + assert req.tag == rpki.up_down.tag_request + + # Subsetting not yet implemented, this is the one place where we have to handle it, by reporting that we're lame. + + if any(req.get(a) for a in ("req_resource_set_as", "req_resource_set_ipv4", "req_resource_set_ipv6")): + raise rpki.exceptions.NotImplementedYet("req_* attributes not implemented yet, sorry") + + class_name = req.get("class_name") + pkcs10 = rpki.x509.PKCS10(Base64 = req.text) + pkcs10.check_valid_request_ca() + ca_detail = CADetail.objects.get(ca__parent__self = self.self, + ca__parent_class_name = class_name, + state = "active") + self.gctx.irdb_query_child_resources(self.self.self_handle, self.child_handle, got_resources, errback) + + + def up_down_handle_revoke(self, q_msg, r_msg, callback, errback): + def done(): + SubElement(r_msg, key.tag, class_name = class_name, ski = key.get("ski")) + callback() + key = q_msg[0] + assert key.tag == rpki.up_down.tag_key + class_name = key.get("class_name") + ski = base64.urlsafe_b64decode(key.get("ski") + "=") + publisher = rpki.rpkid.publication_queue() + for child_cert in ChildCert.objects.filter(ca_detail__ca__parent__self = self.self, + ca_detail__ca__parent_class_name = class_name, + ski = ski): + child_cert.revoke(publisher = publisher) + publisher.call_pubd(done, errback) + + + def serve_up_down(self, q_der, callback): + """ + Outer layer of server handling for one up-down PDU from this child. + """ + + def done(): + callback(rpki.up_down.cms_msg().wrap(r_msg, + self.bsc.private_key_id, + self.bsc.signing_cert, + self.bsc.signing_cert_crl)) + + def lose(e): + logger.exception("Unhandled exception serving child %r", self) + rpki.up_down.generate_error_response_from_exception(r_msg, e, q_type) + done() + + if self.bsc is None: + raise rpki.exceptions.BSCNotFound("Could not find BSC") + q_cms = rpki.up_down.cms_msg(DER = q_der) + q_msg = q_cms.unwrap((self.gctx.bpki_ta, + self.self.bpki_cert, + self.self.bpki_glue, + self.bpki_cert, + self.bpki_glue)) + q_cms.check_replay_sql(self, "child", self.child_handle) + q_type = q_msg.get("type") + logger.info("Serving %s query from child %s [sender %s, recipient %s]", + q_type, self.child_handle, q_msg.get("sender"), q_msg.get("recipient")) + if rpki.up_down.enforce_strict_up_down_xml_sender and q_msg.get("sender") != self.child_handle: + raise rpki.exceptions.BadSender("Unexpected XML sender %s" % q_msg.get("sender")) + + r_msg = Element(rpki.up_down.tag_message, nsmap = rpki.up_down.nsmap, version = rpki.up_down.version, + sender = q_msg.get("recipient"), recipient = q_msg.get("sender"), type = q_type + "_response") + + try: + getattr(self, "up_down_handle_" + q_type)(q_msg, r_msg, done, lose) + except (rpki.async.ExitNow, SystemExit): + raise + except Exception, e: + lose(e) + + +class ChildCert(models.Model): + cert = CertificateField() + published = SundialField(null = True) + ski = BlobField() + child = models.ForeignKey(Child, related_name = "child_certs") + ca_detail = models.ForeignKey(CADetail, related_name = "child_certs") + + + @property + def uri_tail(self): + """ + Return the tail (filename) portion of the URI for this child_cert. + """ + + return self.cert.gSKI() + ".cer" + + + @property + def uri(self): + """ + Return the publication URI for this child_cert. + """ + + return self.ca_detail.ca.sia_uri + self.uri_tail + + + def revoke(self, publisher, generate_crl_and_manifest = True): + """ + Revoke a child cert. + """ + + ca_detail = self.ca_detail + logger.debug("Revoking %r %r", self, self.uri) + RevokedCert.revoke(cert = self.cert, ca_detail = ca_detail) + publisher.queue(uri = self.uri, old_obj = self.cert, repository = ca_detail.ca.parent.repository) + self.delete() + if generate_crl_and_manifest: + ca_detail.generate_crl(publisher = publisher) + ca_detail.generate_manifest(publisher = publisher) + + + def reissue(self, ca_detail, publisher, resources = None, sia = None, force = False): + """ + Reissue an existing child cert, reusing the public key. If the + child cert we would generate is identical to the one we already + have, we just return the one we already have. If we have to + revoke the old child cert when generating the new one, we have to + generate a new child_cert_obj, so calling code that needs the + updated child_cert_obj must use the return value from this method. + """ + + ca = ca_detail.ca + child = self.child + old_resources = self.cert.get_3779resources() + old_sia = self.cert.get_SIA() + old_aia = self.cert.get_AIA()[0] + old_ca_detail = self.ca_detail + needed = False + if resources is None: + resources = old_resources + if sia is None: + sia = old_sia + assert resources.valid_until is not None and old_resources.valid_until is not None + if resources.asn != old_resources.asn or resources.v4 != old_resources.v4 or resources.v6 != old_resources.v6: + logger.debug("Resources changed for %r: old %s new %s", self, old_resources, resources) + needed = True + if resources.valid_until != old_resources.valid_until: + logger.debug("Validity changed for %r: old %s new %s", + self, old_resources.valid_until, resources.valid_until) + needed = True + if sia != old_sia: + logger.debug("SIA changed for %r: old %r new %r", self, old_sia, sia) + needed = True + if ca_detail != old_ca_detail: + logger.debug("Issuer changed for %r: old %r new %r", self, old_ca_detail, ca_detail) + needed = True + if ca_detail.ca_cert_uri != old_aia: + logger.debug("AIA changed for %r: old %r new %r", self, old_aia, ca_detail.ca_cert_uri) + needed = True + must_revoke = old_resources.oversized(resources) or old_resources.valid_until > resources.valid_until + if must_revoke: + logger.debug("Must revoke any existing cert(s) for %r", self) + needed = True + if not needed and force: + logger.debug("No change needed for %r, forcing reissuance anyway", self) + needed = True + if not needed: + logger.debug("No change to %r", self) + return self + if must_revoke: + for x in child.child_certs.filter(ca_detail = ca_detail, ski = self.ski): + logger.debug("Revoking child_cert %r", x) + x.revoke(publisher = publisher) + ca_detail.generate_crl(publisher = publisher) + ca_detail.generate_manifest(publisher = publisher) + child_cert = ca_detail.issue( + ca = ca, + child = child, + subject_key = self.cert.getPublicKey(), + sia = sia, + resources = resources, + child_cert = None if must_revoke else self, + publisher = publisher) + logger.debug("New child_cert %r uri %s", child_cert, child_cert.uri) + return child_cert + + + def published_callback(self, pdu): + """ + Publication callback: check result and mark published. + """ + + rpki.publication.raise_if_error(pdu) + self.published = None + self.save() + + +class EECert(models.Model): + ski = BlobField() + cert = CertificateField() + published = SundialField(null = True) + self = models.ForeignKey(Self, related_name = "ee_certs") + ca_detail = models.ForeignKey(CADetail, related_name = "ee_certs") + + + @property + def gski(self): + """ + Calculate g(SKI), for ease of comparison with XML. + + Although, really, one has to ask why we don't just store g(SKI) + in rpkid.sql instead of ski.... + """ + + return base64.urlsafe_b64encode(self.ski).rstrip("=") + + @gski.setter + def gski(self, val): + self.ski = base64.urlsafe_b64decode(val + ("=" * ((4 - len(val)) % 4))) + + + @property + def uri(self): + """ + Return the publication URI for this ee_cert_obj. + """ + + return self.ca_detail.ca.sia_uri + self.uri_tail + + + @property + def uri_tail(self): + """ + Return the tail (filename portion) of the publication URI for this + ee_cert_obj. + """ + + return self.cert.gSKI() + ".cer" + + + @classmethod + def create(cls, ca_detail, subject_name, subject_key, resources, publisher, eku = None): + """ + Generate a new EE certificate. + """ + + cn, sn = subject_name.extract_cn_and_sn() + sia = (None, None, ca_detail.ca.sia_uri + subject_key.gSKI() + ".cer", ca_detail.ca.parent.repository.rrdp_notification_uri) + cert = ca_detail.issue_ee( + ca = ca_detail.ca, + subject_key = subject_key, + sia = sia, + resources = resources, + notAfter = resources.valid_until, + cn = cn, + sn = sn, + eku = eku) + self = cls(self = ca_detail.ca.parent.self, ca_detail_id = ca_detail.ca_detail_id, cert = cert) + publisher.queue( + uri = self.uri, + new_obj = self.cert, + repository = ca_detail.ca.parent.repository, + handler = self.published_callback) + self.save() + ca_detail.generate_manifest(publisher = publisher) + logger.debug("New ee_cert %r", self) + return self + + + def revoke(self, publisher, generate_crl_and_manifest = True): + """ + Revoke and withdraw an EE certificate. + """ + + ca_detail = self.ca_detail + logger.debug("Revoking %r %r", self, self.uri) + RevokedCert.revoke(cert = self.cert, ca_detail = ca_detail) + publisher.queue(uri = self.uri, old_obj = self.cert, repository = ca_detail.ca.parent.repository) + self.delete() + if generate_crl_and_manifest: + ca_detail.generate_crl(publisher = publisher) + ca_detail.generate_manifest(publisher = publisher) + + + def reissue(self, publisher, ca_detail = None, resources = None, force = False): + """ + Reissue an existing EE cert, reusing the public key. If the EE + cert we would generate is identical to the one we already have, we + just return; if we need to reissue, we reuse this ee_cert_obj and + just update its contents, as the publication URI will not have + changed. + """ + + needed = False + old_cert = self.cert + old_ca_detail = self.ca_detail + if ca_detail is None: + ca_detail = old_ca_detail + assert ca_detail.ca is old_ca_detail.ca + old_resources = old_cert.get_3779resources() + if resources is None: + resources = old_resources + assert resources.valid_until is not None and old_resources.valid_until is not None + assert ca_detail.covers(resources) + if ca_detail != self.ca_detail: + logger.debug("ca_detail changed for %r: old %r new %r", self, self.ca_detail, ca_detail) + needed = True + if ca_detail.ca_cert_uri != old_cert.get_AIA()[0]: + logger.debug("AIA changed for %r: old %s new %s", self, old_cert.get_AIA()[0], ca_detail.ca_cert_uri) + needed = True + if resources.valid_until != old_resources.valid_until: + logger.debug("Validity changed for %r: old %s new %s", self, old_resources.valid_until, resources.valid_until) + needed = True + if resources.asn != old_resources.asn or resources.v4 != old_resources.v4 or resources.v6 != old_resources.v6: + logger.debug("Resources changed for %r: old %s new %s", self, old_resources, resources) + needed = True + must_revoke = old_resources.oversized(resources) or old_resources.valid_until > resources.valid_until + if must_revoke: + logger.debug("Must revoke existing cert(s) for %r", self) + needed = True + if not needed and force: + logger.debug("No change needed for %r, forcing reissuance anyway", self) + needed = True + if not needed: + logger.debug("No change to %r", self) + return + cn, sn = self.cert.getSubject().extract_cn_and_sn() + self.cert = ca_detail.issue_ee( + ca = ca_detail.ca, + subject_key = self.cert.getPublicKey(), + eku = self.cert.get_EKU(), + sia = (None, None, self.uri, ca_detail.ca.parent.repository.rrdp_notification_uri), + resources = resources, + notAfter = resources.valid_until, + cn = cn, + sn = sn) + self.save() + publisher.queue( + uri = self.uri, + old_obj = old_cert, + new_obj = self.cert, + repository = ca_detail.ca.parent.repository, + handler = self.published_callback) + if must_revoke: + RevokedCert.revoke(cert = old_cert.cert, ca_detail = old_ca_detail) + ca_detail.generate_crl(publisher = publisher) + ca_detail.generate_manifest(publisher = publisher) + + + def published_callback(self, pdu): + """ + Publication callback: check result and mark published. + """ + + rpki.publication.raise_if_error(pdu) + self.published = None + self.save() + + + +class Ghostbuster(models.Model): + vcard = models.TextField() + cert = CertificateField() + ghostbuster = GhostbusterField() + published = SundialField(null = True) + self = models.ForeignKey(Self, related_name = "ghostbusters") + ca_detail = models.ForeignKey(CADetail, related_name = "ghostbusters") + + + def update(self, publisher, fast = False): + """ + Bring this ghostbuster_obj up to date if necesssary. + """ + + if self.ghostbuster is None: + logger.debug("Ghostbuster record doesn't exist, generating") + return self.generate(publisher = publisher, fast = fast) + + now = rpki.sundial.now() + regen_time = self.cert.getNotAfter() - rpki.sundial.timedelta(seconds = self.self.regen_margin) + + if now > regen_time and self.cert.getNotAfter() < self.ca_detail.latest_ca_cert.getNotAfter(): + logger.debug("%r past threshold %s, regenerating", self, regen_time) + return self.regenerate(publisher = publisher, fast = fast) + + if now > regen_time: + logger.warning("%r is past threshold %s but so is issuer %r, can't regenerate", self, regen_time, self.ca_detail) + + if self.cert.get_AIA()[0] != self.ca_detail.ca_cert_uri: + logger.debug("%r AIA changed, regenerating", self) + return self.regenerate(publisher = publisher, fast = fast) + + + def generate(self, publisher, fast = False): + """ + Generate a Ghostbuster record + + Once we have the right covering certificate, we generate the + ghostbuster payload, generate a new EE certificate, use the EE + certificate to sign the ghostbuster payload, publish the result, + then throw away the private key for the EE cert. This is modeled + after the way we handle ROAs. + + If fast is set, we leave generating the new manifest for our + caller to handle, presumably at the end of a bulk operation. + """ + + resources = rpki.resource_set.resource_bag.from_inheritance() + keypair = rpki.x509.RSA.generate() + self.cert = self.ca_detail.issue_ee( + ca = self.ca_detail.ca, + resources = resources, + subject_key = keypair.get_public(), + sia = (None, None, self.uri_from_key(keypair), self.ca_detail.ca.parent.repository.rrdp_notification_uri)) + self.ghostbuster = rpki.x509.Ghostbuster.build(self.vcard, keypair, (self.cert,)) + self.published = rpki.sundial.now() + self.save() + logger.debug("Generating Ghostbuster record %r", self.uri) + publisher.queue( + uri = self.uri, + new_obj = self.ghostbuster, + repository = self.ca_detail.ca.parent.repository, + handler = self.published_callback) + if not fast: + self.ca_detail.generate_manifest(publisher = publisher) + + + def published_callback(self, pdu): + """ + Check publication result. + """ + + rpki.publication.raise_if_error(pdu) + self.published = None + self.save() + + + def revoke(self, publisher, regenerate = False, allow_failure = False, fast = False): + """ + Withdraw Ghostbuster associated with this ghostbuster_obj. + + In order to preserve make-before-break properties without + duplicating code, this method also handles generating a + replacement ghostbuster when requested. + + If allow_failure is set, failing to withdraw the ghostbuster will not be + considered an error. + + If fast is set, SQL actions will be deferred, on the assumption + that our caller will handle regenerating CRL and manifest and + flushing the SQL cache. + """ + + ca_detail = self.ca_detail + logger.debug("%s %r, ca_detail %r state is %s", + "Regenerating" if regenerate else "Not regenerating", + self, ca_detail, ca_detail.state) + if regenerate: + self.generate(publisher = publisher, fast = fast) + logger.debug("Withdrawing %r %s and revoking its EE cert", self, self.uri) + RevokedCert.revoke(cert = self.cert, ca_detail = ca_detail) + publisher.queue(uri = self.uri, + old_obj = self.ghostbuster, + repository = ca_detail.ca.parent.repository, + handler = False if allow_failure else None) + if not regenerate: + self.delete() + if not fast: + ca_detail.generate_crl(publisher = publisher) + ca_detail.generate_manifest(publisher = publisher) + + + def regenerate(self, publisher, fast = False): + """ + Reissue Ghostbuster associated with this ghostbuster_obj. + """ + + if self.ghostbuster is None: + self.generate(publisher = publisher, fast = fast) + else: + self.revoke(publisher = publisher, regenerate = True, fast = fast) + + + def uri_from_key(self, key): + """ + Return publication URI for a public key. + """ + + return self.ca_detail.ca.sia_uri + key.gSKI() + ".gbr" + + + @property + def uri(self): + """ + Return the publication URI for this ghostbuster_obj's ghostbuster. + """ + + return self.ca_detail.ca.sia_uri + self.uri_tail + + + @property + def uri_tail(self): + """ + Return the tail (filename portion) of the publication URI for this + ghostbuster_obj's ghostbuster. + """ + + return self.cert.gSKI() + ".gbr" + + +class RevokedCert(models.Model): + serial = models.BigIntegerField() + revoked = SundialField() + expires = SundialField() + ca_detail = models.ForeignKey(CADetail, related_name = "revoked_certs") + + @classmethod + def revoke(cls, cert, ca_detail): + """ + Revoke a certificate. + """ + + return cls.objects.create( + serial = cert.getSerial(), + expires = cert.getNotAfter(), + revoked = rpki.sundial.now(), + ca_detail = ca_detail) + + +class ROA(models.Model): + asn = models.BigIntegerField() + ipv4 = models.TextField(null = True) + ipv6 = models.TextField(null = True) + cert = CertificateField() + roa = ROAField() + published = SundialField(null = True) + self = models.ForeignKey(Self, related_name = "roas") + ca_detail = models.ForeignKey(CADetail, related_name = "roas") + + + def update(self, publisher, fast = False): + """ + Bring ROA up to date if necesssary. + """ + + if self.roa is None: + logger.debug("%r doesn't exist, generating", self) + return self.generate(publisher = publisher, fast = fast) + + if self.ca_detail is None: + logger.debug("%r has no associated ca_detail, generating", self) + return self.generate(publisher = publisher, fast = fast) + + if self.ca_detail.state != "active": + logger.debug("ca_detail associated with %r not active (state %s), regenerating", self, self.ca_detail.state) + return self.regenerate(publisher = publisher, fast = fast) + + now = rpki.sundial.now() + regen_time = self.cert.getNotAfter() - rpki.sundial.timedelta(seconds = self.self.regen_margin) + + if now > regen_time and self.cert.getNotAfter() < self.ca_detail.latest_ca_cert.getNotAfter(): + logger.debug("%r past threshold %s, regenerating", self, regen_time) + return self.regenerate(publisher = publisher, fast = fast) + + if now > regen_time: + logger.warning("%r is past threshold %s but so is issuer %r, can't regenerate", self, regen_time, self.ca_detail) + + ca_resources = self.ca_detail.latest_ca_cert.get_3779resources() + ee_resources = self.cert.get_3779resources() + + if ee_resources.oversized(ca_resources): + logger.debug("%r oversized with respect to CA, regenerating", self) + return self.regenerate(publisher = publisher, fast = fast) + + v4 = rpki.resource_set.resource_set_ipv4(self.ipv4) + v6 = rpki.resource_set.resource_set_ipv6(self.ipv6) + + if ee_resources.v4 != v4 or ee_resources.v6 != v6: + logger.debug("%r resources do not match EE, regenerating", self) + return self.regenerate(publisher = publisher, fast = fast) + + if self.cert.get_AIA()[0] != self.ca_detail.ca_cert_uri: + logger.debug("%r AIA changed, regenerating", self) + return self.regenerate(publisher = publisher, fast = fast) + + + def generate(self, publisher, fast = False): + """ + Generate a ROA. + + At present we have no way of performing a direct lookup from a + desired set of resources to a covering certificate, so we have to + search. This could be quite slow if we have a lot of active + ca_detail objects. Punt on the issue for now, revisit if + profiling shows this as a hotspot. + + Once we have the right covering certificate, we generate the ROA + payload, generate a new EE certificate, use the EE certificate to + sign the ROA payload, publish the result, then throw away the + private key for the EE cert, all per the ROA specification. This + implies that generating a lot of ROAs will tend to thrash + /dev/random, but there is not much we can do about that. + + If fast is set, we leave generating the new manifest for our + caller to handle, presumably at the end of a bulk operation. + """ + + if self.ipv4 is None and self.ipv6 is None: + raise rpki.exceptions.EmptyROAPrefixList + + v4 = rpki.resource_set.resource_set_ipv4(self.ipv4) + v6 = rpki.resource_set.resource_set_ipv6(self.ipv6) + + if self.ca_detail is not None and self.ca_detail.state == "active" and not self.ca_detail.has_expired(): + logger.debug("Keeping old ca_detail %r for ROA %r", self.ca_detail, self) + else: + logger.debug("Searching for new ca_detail for ROA %r", self) + for ca_detail in CADetail.objects.filter(ca__parent__self = self.self, state = "active"): + resources = ca_detail.latest_ca_cert.get_3779resources() + if not ca_detail.has_expired() and v4.issubset(resources.v4) and v6.issubset(resources.v6): + logger.debug("Using new ca_detail %r for ROA %r", ca_detail, self) + self.ca_detail = ca_detail + break + else: + raise rpki.exceptions.NoCoveringCertForROA("Could not find a certificate covering %r" % self) + + resources = rpki.resource_set.resource_bag(v4 = v4, v6 = v6) + keypair = rpki.x509.RSA.generate() + + self.cert = self.ca_detail.issue_ee( + ca = self.ca_detail.ca, + resources = resources, + subject_key = keypair.get_public(), + sia = (None, None, self.uri_from_key(keypair), self.ca_detail.ca.parent.repository.rrdp_notification_uri)) + self.roa = rpki.x509.ROA.build(self.asn, self.ipv4, self.ipv6, keypair, (self.cert,)) + self.published = rpki.sundial.now() + self.save() + + logger.debug("Generating %r URI %s", self, self.uri) + publisher.queue(uri = self.uri, new_obj = self.roa, + repository = self.ca_detail.ca.parent.repository, + handler = self.published_callback) + if not fast: + self.ca_detail.generate_manifest(publisher = publisher) + + + def published_callback(self, pdu): + """ + Check publication result. + """ + + rpki.publication.raise_if_error(pdu) + self.published = None + self.save() + + + def revoke(self, publisher, regenerate = False, allow_failure = False, fast = False): + """ + Withdraw ROA associated with this roa_obj. + + In order to preserve make-before-break properties without + duplicating code, this method also handles generating a + replacement ROA when requested. + + If allow_failure is set, failing to withdraw the ROA will not be + considered an error. + + If fast is set, SQL actions will be deferred, on the assumption + that our caller will handle regenerating CRL and manifest and + flushing the SQL cache. + """ + + ca_detail = self.ca_detail + logger.debug("%s %r, ca_detail %r state is %s", + "Regenerating" if regenerate else "Not regenerating", + self, ca_detail, ca_detail.state) + if regenerate: + self.generate(publisher = publisher, fast = fast) + logger.debug("Withdrawing %r %s and revoking its EE cert", self, self.uri) + RevokedCert.revoke(cert = self.cert, ca_detail = ca_detail) + publisher.queue(uri = self.uri, old_obj = self.roa, + repository = ca_detail.ca.parent.repository, + handler = False if allow_failure else None) + if not regenerate: + self.delete() + if not fast: + ca_detail.generate_crl(publisher = publisher) + ca_detail.generate_manifest(publisher = publisher) + + + def regenerate(self, publisher, fast = False): + """ + Reissue ROA associated with this roa_obj. + """ + + if self.ca_detail is None: + self.generate(publisher = publisher, fast = fast) + else: + self.revoke(publisher = publisher, regenerate = True, fast = fast) + + + def uri_from_key(self, key): + """ + Return publication URI for a public key. + """ + + return self.ca_detail.ca.sia_uri + key.gSKI() + ".roa" + + + @property + def uri(self): + """ + Return the publication URI for this roa_obj's ROA. + """ + + return self.ca_detail.ca.sia_uri + self.uri_tail + + + @property + def uri_tail(self): + """ + Return the tail (filename portion) of the publication URI for this + roa_obj's ROA. + """ + + return self.cert.gSKI() + ".roa" diff --git a/rpki/rtr/bgpdump.py b/rpki/rtr/bgpdump.py index fc3ae9df..5ffabc4d 100755 --- a/rpki/rtr/bgpdump.py +++ b/rpki/rtr/bgpdump.py @@ -295,7 +295,7 @@ def bgpdump_server_main(args): rpki.rtr.server.read_current = clock.read_current try: - server = rpki.rtr.server.ServerChannel(logger = logger) + server = rpki.rtr.server.ServerChannel(logger = logger, refresh = args.refresh, retry = args.retry, expire = args.expire) old_serial = server.get_serial() logger.debug("[Starting at serial %d (%s)]", old_serial, old_serial) while clock: diff --git a/rpki/sql.py b/rpki/sql.py index 31ed40ee..55e6f7cb 100644 --- a/rpki/sql.py +++ b/rpki/sql.py @@ -56,11 +56,12 @@ class session(object): ping_threshold = rpki.sundial.timedelta(seconds = 60) - def __init__(self, cfg): + def __init__(self, cfg, autocommit = True): self.username = cfg.get("sql-username") self.database = cfg.get("sql-database") self.password = cfg.get("sql-password") + self.autocommit = autocommit self.conv = MySQLdb.converters.conversions.copy() self.conv.update({ @@ -78,7 +79,7 @@ class session(object): passwd = self.password, conv = self.conv) self.cur = self.db.cursor() - self.db.autocommit(True) + self.db.autocommit(self.autocommit) self.timestamp = rpki.sundial.now() def close(self): @@ -113,11 +114,37 @@ class session(object): def lastrowid(self): return self.cur.lastrowid + def commit(self): + """ + Sweep cache, then commit SQL. + """ + + self.sweep() + logger.debug("Executing SQL COMMIT") + self.db.commit() + + def rollback(self): + """ + SQL rollback, then clear cache and dirty cache. + + NB: We have no way of clearing other references to cached objects, + so if you call this method you MUST forget any state that might + cause you to retain such references. This is probably tricky, and + is itself a good argument for switching to something like the + Django ORM's @commit_on_success semantics, but we do what we can. + """ + + logger.debug("Executing SQL ROLLBACK, discarding SQL cache and dirty set") + self.db.rollback() + self.dirty.clear() + self.cache.clear() + def cache_clear(self): """ Clear the SQL object cache. Shouldn't be necessary now that the cache uses weak references, but should be harmless. """ + logger.debug("Clearing SQL cache") self.assert_pristine() self.cache.clear() @@ -126,14 +153,15 @@ class session(object): """ Assert that there are no dirty objects in the cache. """ + assert not self.dirty, "Dirty objects in SQL cache: %s" % self.dirty def sweep(self): """ Write any dirty objects out to SQL. """ + for s in self.dirty.copy(): - #if s.sql_cache_debug: logger.debug("Sweeping (%s) %r", "deleting" if s.sql_deleted else "storing", s) if s.sql_deleted: s.sql_delete() @@ -150,6 +178,7 @@ class template(object): """ Build a SQL template. """ + type_map = dict((x[0], x[1]) for x in data_columns if isinstance(x, tuple)) data_columns = tuple(isinstance(x, tuple) and x[0] or x for x in data_columns) columns = (index_column,) + data_columns @@ -220,6 +249,7 @@ class sql_persistent(object): """ Fetch one object from SQL, based on an arbitrary SQL WHERE expression. """ + results = cls.sql_fetch_where(gctx, where, args, also_from) if len(results) == 0: return None @@ -235,6 +265,7 @@ class sql_persistent(object): """ Fetch all objects of this type from SQL. """ + return cls.sql_fetch_where(gctx, None) @classmethod @@ -242,6 +273,7 @@ class sql_persistent(object): """ Fetch objects of this type matching an arbitrary SQL WHERE expression. """ + if where is None: assert args is None and also_from is None if cls.sql_debug: @@ -269,6 +301,7 @@ class sql_persistent(object): """ Initialize one Python object from the result of a SQL query. """ + self = cls() self.gctx = gctx self.sql_decode(dict(zip(cls.sql_template.columns, row))) @@ -281,6 +314,7 @@ class sql_persistent(object): """ Mark this object as needing to be written back to SQL. """ + if self.sql_cache_debug and not self.sql_is_dirty: logger.debug("Marking %r SQL dirty", self) self.gctx.sql.dirty.add(self) @@ -289,6 +323,7 @@ class sql_persistent(object): """ Mark this object as not needing to be written back to SQL. """ + if self.sql_cache_debug and self.sql_is_dirty: logger.debug("Marking %r SQL clean", self) self.gctx.sql.dirty.discard(self) @@ -298,12 +333,14 @@ class sql_persistent(object): """ Query whether this object needs to be written back to SQL. """ + return self in self.gctx.sql.dirty def sql_mark_deleted(self): """ Mark this object as needing to be deleted in SQL. """ + self.sql_deleted = True self.sql_mark_dirty() @@ -311,6 +348,7 @@ class sql_persistent(object): """ Store this object to SQL. """ + args = self.sql_encode() if not self.sql_in_db: if self.sql_debug: @@ -333,6 +371,7 @@ class sql_persistent(object): """ Delete this object from SQL. """ + if self.sql_in_db: id = getattr(self, self.sql_template.index) # pylint: disable=W0622 if self.sql_debug: @@ -352,6 +391,7 @@ class sql_persistent(object): mapping between column names in SQL and attribute names in Python. If you need something fancier, override this. """ + d = dict((a, getattr(self, a, None)) for a in self.sql_template.columns) for i in self.sql_template.map: if d.get(i) is not None: @@ -365,6 +405,7 @@ class sql_persistent(object): between column names in SQL and attribute names in Python. If you need something fancier, override this. """ + for a in self.sql_template.columns: if vals.get(a) is not None and a in self.sql_template.map: setattr(self, a, self.sql_template.map[a].from_sql(vals[a])) @@ -375,18 +416,21 @@ class sql_persistent(object): """ Customization hook. """ + pass def sql_insert_hook(self): """ Customization hook. """ + pass def sql_update_hook(self): """ Customization hook. """ + self.sql_delete_hook() self.sql_insert_hook() @@ -394,6 +438,7 @@ class sql_persistent(object): """ Customization hook. """ + pass diff --git a/rpki/sql_schemas.py b/rpki/sql_schemas.py index 07037970..a3c039af 100644 --- a/rpki/sql_schemas.py +++ b/rpki/sql_schemas.py @@ -2,35 +2,24 @@ ## @var rpkid ## SQL schema rpkid -rpkid = '''-- $Id: rpkid.sql 5845 2014-05-29 22:31:15Z sra $ +rpkid = '''-- $Id: rpkid.sql 5881 2014-07-03 16:55:02Z sra $ --- Copyright (C) 2009--2011 Internet Systems Consortium ("ISC") +-- Copyright (C) 2012--2014 Dragon Research Labs ("DRL") +-- Portions copyright (C) 2009--2011 Internet Systems Consortium ("ISC") +-- Portions copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") -- -- Permission to use, copy, modify, and distribute this software for any -- purpose with or without fee is hereby granted, provided that the above --- copyright notice and this permission notice appear in all copies. +-- copyright notices and this permission notice appear in all copies. -- --- THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH --- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY --- AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, --- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM --- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE --- OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR --- PERFORMANCE OF THIS SOFTWARE. - --- Copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") --- --- Permission to use, copy, modify, and distribute this software for any --- purpose with or without fee is hereby granted, provided that the above --- copyright notice and this permission notice appear in all copies. --- --- THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH --- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY --- AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, --- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM --- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE --- OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR --- PERFORMANCE OF THIS SOFTWARE. +-- THE SOFTWARE IS PROVIDED "AS IS" AND DRL, ISC, AND ARIN DISCLAIM ALL +-- WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +-- WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL, +-- ISC, OR ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +-- CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +-- OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +-- NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION +-- WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -- SQL objects needed by the RPKI engine (rpkid.py). @@ -82,6 +71,7 @@ CREATE TABLE repository ( repository_id SERIAL NOT NULL, repository_handle VARCHAR(255) NOT NULL, peer_contact_uri TEXT, + rrdp_notification_uri TEXT, bpki_cert LONGBLOB, bpki_glue LONGBLOB, last_cms_timestamp DATETIME, @@ -98,8 +88,8 @@ CREATE TABLE repository ( CREATE TABLE parent ( parent_id SERIAL NOT NULL, parent_handle VARCHAR(255) NOT NULL, - bpki_cms_cert LONGBLOB, - bpki_cms_glue LONGBLOB, + bpki_cert LONGBLOB, + bpki_glue LONGBLOB, peer_contact_uri TEXT, sia_base TEXT, sender_name TEXT, @@ -256,50 +246,39 @@ CREATE TABLE ee_cert ( ## @var pubd ## SQL schema pubd -pubd = '''-- $Id: pubd.sql 5757 2014-04-05 22:42:12Z sra $ +pubd = '''-- $Id: pubd.sql 5914 2014-08-06 22:52:28Z sra $ --- Copyright (C) 2009--2010 Internet Systems Consortium ("ISC") +-- Copyright (C) 2012--2014 Dragon Research Labs ("DRL") +-- Portions copyright (C) 2009--2010 Internet Systems Consortium ("ISC") +-- Portions copyright (C) 2008 American Registry for Internet Numbers ("ARIN") -- -- Permission to use, copy, modify, and distribute this software for any -- purpose with or without fee is hereby granted, provided that the above --- copyright notice and this permission notice appear in all copies. --- --- THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH --- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY --- AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, --- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM --- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE --- OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR --- PERFORMANCE OF THIS SOFTWARE. - --- Copyright (C) 2008 American Registry for Internet Numbers ("ARIN") +-- copyright notices and this permission notice appear in all copies. -- --- Permission to use, copy, modify, and distribute this software for any --- purpose with or without fee is hereby granted, provided that the above --- copyright notice and this permission notice appear in all copies. --- --- THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH --- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY --- AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, --- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM --- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE --- OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR --- PERFORMANCE OF THIS SOFTWARE. +-- THE SOFTWARE IS PROVIDED "AS IS" AND DRL, ISC, AND ARIN DISCLAIM ALL +-- WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +-- WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL, +-- ISC, OR ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +-- CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +-- OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +-- NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION +-- WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -- SQL objects needed by pubd.py. --- The config table is weird because we're really only using it --- to store one BPKI CRL, but putting this here lets us use a lot of --- existing machinery and the alternatives are whacky in other ways. +-- Old tables that should just be flushed if present at all. -DROP TABLE IF EXISTS client; DROP TABLE IF EXISTS config; +DROP TABLE IF EXISTS snapshot; -CREATE TABLE config ( - config_id SERIAL NOT NULL, - bpki_crl LONGBLOB, - PRIMARY KEY (config_id) -) ENGINE=InnoDB; +-- DROP TABLE commands must be in correct (reverse dependency) order +-- to satisfy FOREIGN KEY constraints. + +DROP TABLE IF EXISTS object; +DROP TABLE IF EXISTS delta; +DROP TABLE IF EXISTS session; +DROP TABLE IF EXISTS client; CREATE TABLE client ( client_id SERIAL NOT NULL, @@ -312,6 +291,43 @@ CREATE TABLE client ( UNIQUE (client_handle) ) ENGINE=InnoDB; +CREATE TABLE session ( + session_id SERIAL NOT NULL, + uuid VARCHAR(36) NOT NULL, + serial BIGINT UNSIGNED NOT NULL, + snapshot LONGTEXT, + hash CHAR(64), + PRIMARY KEY (session_id), + UNIQUE (uuid) +) ENGINE=InnoDB; + +CREATE TABLE delta ( + delta_id SERIAL NOT NULL, + serial BIGINT UNSIGNED NOT NULL, + xml LONGTEXT NOT NULL, + hash CHAR(64) NOT NULL, + expires DATETIME NOT NULL, + session_id BIGINT UNSIGNED NOT NULL, + PRIMARY KEY (delta_id), + CONSTRAINT delta_session_id + FOREIGN KEY (session_id) REFERENCES session (session_id) ON DELETE CASCADE +) ENGINE=InnoDB; + +CREATE TABLE object ( + object_id SERIAL NOT NULL, + uri VARCHAR(255) NOT NULL, + der LONGBLOB NOT NULL, + hash CHAR(64) NOT NULL, + client_id BIGINT UNSIGNED NOT NULL, + session_id BIGINT UNSIGNED NOT NULL, + PRIMARY KEY (object_id), + CONSTRAINT object_client_id + FOREIGN KEY (client_id) REFERENCES client (client_id) ON DELETE CASCADE, + CONSTRAINT object_session_id + FOREIGN KEY (session_id) REFERENCES session (session_id) ON DELETE CASCADE, + UNIQUE (session_id, hash) +) ENGINE=InnoDB; + -- Local Variables: -- indent-tabs-mode: nil -- End: diff --git a/rpki/sundial.py b/rpki/sundial.py index 7be122c8..60037277 100644 --- a/rpki/sundial.py +++ b/rpki/sundial.py @@ -51,6 +51,7 @@ def now(): """ Get current timestamp. """ + return datetime.utcnow() class ParseFailure(Exception): @@ -69,6 +70,7 @@ class datetime(pydatetime.datetime): Convert to seconds from epoch (like time.time()). Conversion method is a bit silly, but avoids time module timezone whackiness. """ + return int(self.strftime("%s")) @classmethod @@ -76,6 +78,7 @@ class datetime(pydatetime.datetime): """ Convert from XML time representation. """ + if x is None: return None else: @@ -85,6 +88,7 @@ class datetime(pydatetime.datetime): """ Convert to XML time representation. """ + return self.strftime("%Y-%m-%dT%H:%M:%SZ") def __str__(self): @@ -96,6 +100,7 @@ class datetime(pydatetime.datetime): Convert a datetime.datetime object into this subclass. This is whacky due to the weird constructors for datetime. """ + return cls.combine(x.date(), x.time()) def to_datetime(self): @@ -104,6 +109,7 @@ class datetime(pydatetime.datetime): shouldn't be necessary, but convincing SQL interfaces to use subclasses of datetime can be hard. """ + return pydatetime.datetime(year = self.year, month = self.month, day = self.day, hour = self.hour, minute = self.minute, second = self.second, microsecond = 0, tzinfo = None) @@ -115,6 +121,7 @@ class datetime(pydatetime.datetime): Convert from the format OpenSSL's command line tool uses into this subclass. May require rewriting if we run into locale problems. """ + if x.startswith("notBefore=") or x.startswith("notAfter="): x = x.partition("=")[2] return cls.strptime(x, "%b %d %H:%M:%S %Y GMT") @@ -124,24 +131,28 @@ class datetime(pydatetime.datetime): """ Convert from SQL storage format. """ + return cls.from_datetime(x) def to_sql(self): """ Convert to SQL storage format. """ + return self.to_datetime() def later(self, other): """ Return the later of two timestamps. """ + return other if other > self else self def earlier(self, other): """ Return the earlier of two timestamps. """ + return other if other < self else self def __add__(self, y): return _cast(pydatetime.datetime.__add__(self, y)) @@ -216,6 +227,7 @@ class timedelta(pydatetime.timedelta): """ Parse text into a timedelta object. """ + if not isinstance(arg, str): return cls(seconds = arg) elif arg.isdigit(): @@ -237,6 +249,7 @@ class timedelta(pydatetime.timedelta): """ Convert a timedelta interval to seconds. """ + return self.days * 24 * 60 * 60 + self.seconds @classmethod @@ -244,6 +257,7 @@ class timedelta(pydatetime.timedelta): """ Convert a datetime.timedelta object into this subclass. """ + return cls(days = x.days, seconds = x.seconds, microseconds = x.microseconds) def __abs__(self): return _cast(pydatetime.timedelta.__abs__(self)) @@ -264,6 +278,7 @@ def _cast(x): """ Cast result of arithmetic operations back into correct subtype. """ + if isinstance(x, pydatetime.datetime): return datetime.from_datetime(x) if isinstance(x, pydatetime.timedelta): diff --git a/rpki/up_down.py b/rpki/up_down.py index 5339e9a7..1303647e 100644 --- a/rpki/up_down.py +++ b/rpki/up_down.py @@ -21,20 +21,20 @@ RPKI "up-down" protocol. """ -import base64 import logging -import lxml.etree import rpki.resource_set import rpki.x509 import rpki.exceptions import rpki.log -import rpki.xml_utils import rpki.relaxng +from lxml.etree import SubElement, tostring as ElementToString + logger = logging.getLogger(__name__) -xmlns = rpki.relaxng.up_down.xmlns -nsmap = rpki.relaxng.up_down.nsmap +xmlns = rpki.relaxng.up_down.xmlns +nsmap = rpki.relaxng.up_down.nsmap +version = "1" ## @var content_type # MIME content type to use when sending up-down queries. @@ -46,69 +46,26 @@ content_type = "application/x-rpki" # queries. allowed_content_types = ("application/rpki-updown", "application/x-rpki") -class base_elt(object): - """ - Generic PDU object. - - Virtual class, just provides some default methods. - """ - - def startElement(self, stack, name, attrs): - """ - Ignore startElement() if there's no specific handler. +## @var enforce_strict_up_down_xml_sender +# Enforce strict checking of XML "sender" field in up-down protocol - Some elements have no attributes and we only care about their - text content. - """ - pass +enforce_strict_up_down_xml_sender = False - def endElement(self, stack, name, text): - """ - Ignore endElement() if there's no specific handler. - - If we don't need to do anything else, just pop the stack. - """ - stack.pop() +tag_certificate = xmlns + "certificate" +tag_class = xmlns + "class" +tag_description = xmlns + "description" +tag_issuer = xmlns + "issuer" +tag_message = xmlns + "message" +tag_request = xmlns + "request" +tag_status = xmlns + "status" - def make_elt(self, name, *attrs): - """ - Construct a element, copying over a set of attributes. - """ - elt = lxml.etree.Element(xmlns + name, nsmap = nsmap) - for key in attrs: - val = getattr(self, key, None) - if val is not None: - elt.set(key, str(val)) - return elt - - def make_b64elt(self, elt, name, value): - """ - Construct a sub-element with Base64 text content. - """ - if value is not None and not value.empty(): - lxml.etree.SubElement(elt, xmlns + name, nsmap = nsmap).text = value.get_Base64() - - def serve_pdu(self, q_msg, r_msg, child, callback, errback): - """ - Default PDU handler to catch unexpected types. - """ - raise rpki.exceptions.BadQuery("Unexpected query type %s" % q_msg.type) - - def check_response(self): - """ - Placeholder for response checking. - """ - pass class multi_uri(list): """ - Container for a set of URIs. + Container for a set of URIs. This probably could be simplified. """ def __init__(self, ini): - """ - Initialize a set of URIs, which includes basic some syntax checking. - """ list.__init__(self) if isinstance(ini, (list, tuple)): self[:] = ini @@ -121,624 +78,95 @@ class multi_uri(list): raise TypeError def __str__(self): - """ - Convert a multi_uri back to a string representation. - """ return ",".join(self) def rsync(self): """ Find first rsync://... URI in self. """ + for s in self: if s.startswith("rsync://"): return s return None -class certificate_elt(base_elt): - """ - Up-Down protocol representation of an issued certificate. - """ - - def startElement(self, stack, name, attrs): - """ - Handle attributes of <certificate/> element. - """ - assert name == "certificate", "Unexpected name %s, stack %s" % (name, stack) - self.cert_url = multi_uri(attrs["cert_url"]) - self.req_resource_set_as = rpki.resource_set.resource_set_as(attrs.get("req_resource_set_as")) - self.req_resource_set_ipv4 = rpki.resource_set.resource_set_ipv4(attrs.get("req_resource_set_ipv4")) - self.req_resource_set_ipv6 = rpki.resource_set.resource_set_ipv6(attrs.get("req_resource_set_ipv6")) - - def endElement(self, stack, name, text): - """ - Handle text content of a <certificate/> element. - """ - assert name == "certificate", "Unexpected name %s, stack %s" % (name, stack) - self.cert = rpki.x509.X509(Base64 = text) - stack.pop() - - def toXML(self): - """ - Generate a <certificate/> element. - """ - elt = self.make_elt("certificate", "cert_url", - "req_resource_set_as", "req_resource_set_ipv4", "req_resource_set_ipv6") - elt.text = self.cert.get_Base64() - return elt - -class class_elt(base_elt): - """ - Up-Down protocol representation of a resource class. - """ - - issuer = None - - def __init__(self): - """ - Initialize class_elt. - """ - base_elt.__init__(self) - self.certs = [] - - def startElement(self, stack, name, attrs): - """ - Handle <class/> elements and their children. - """ - if name == "certificate": - cert = certificate_elt() - self.certs.append(cert) - stack.append(cert) - cert.startElement(stack, name, attrs) - elif name != "issuer": - assert name == "class", "Unexpected name %s, stack %s" % (name, stack) - self.class_name = attrs["class_name"] - self.cert_url = multi_uri(attrs["cert_url"]) - self.suggested_sia_head = attrs.get("suggested_sia_head") - self.resource_set_as = rpki.resource_set.resource_set_as(attrs["resource_set_as"]) - self.resource_set_ipv4 = rpki.resource_set.resource_set_ipv4(attrs["resource_set_ipv4"]) - self.resource_set_ipv6 = rpki.resource_set.resource_set_ipv6(attrs["resource_set_ipv6"]) - self.resource_set_notafter = rpki.sundial.datetime.fromXMLtime(attrs.get("resource_set_notafter")) - - def endElement(self, stack, name, text): - """ - Handle <class/> elements and their children. - """ - if name == "issuer": - self.issuer = rpki.x509.X509(Base64 = text) - else: - assert name == "class", "Unexpected name %s, stack %s" % (name, stack) - stack.pop() - - def toXML(self): - """ - Generate a <class/> element. - """ - elt = self.make_elt("class", "class_name", "cert_url", "resource_set_as", - "resource_set_ipv4", "resource_set_ipv6", - "resource_set_notafter", "suggested_sia_head") - elt.extend([i.toXML() for i in self.certs]) - self.make_b64elt(elt, "issuer", self.issuer) - return elt - - def to_resource_bag(self): - """ - Build a resource_bag from from this <class/> element. - """ - return rpki.resource_set.resource_bag(self.resource_set_as, - self.resource_set_ipv4, - self.resource_set_ipv6, - self.resource_set_notafter) - - def from_resource_bag(self, bag): - """ - Set resources of this class element from a resource_bag. - """ - self.resource_set_as = bag.asn - self.resource_set_ipv4 = bag.v4 - self.resource_set_ipv6 = bag.v6 - self.resource_set_notafter = bag.valid_until - -class list_pdu(base_elt): - """ - Up-Down protocol "list" PDU. - """ - - def toXML(self): - """Generate (empty) payload of "list" PDU.""" - return [] - - def serve_pdu(self, q_msg, r_msg, child, callback, errback): - """ - Serve one "list" PDU. - """ - - def handle(irdb_resources): - - r_msg.payload = list_response_pdu() - - if irdb_resources.valid_until < rpki.sundial.now(): - logger.debug("Child %s's resources expired %s", child.child_handle, irdb_resources.valid_until) - else: - for parent in child.parents: - for ca in parent.cas: - ca_detail = ca.active_ca_detail - if not ca_detail: - logger.debug("No active ca_detail, can't issue to %s", child.child_handle) - continue - resources = ca_detail.latest_ca_cert.get_3779resources() & irdb_resources - if resources.empty(): - logger.debug("No overlap between received resources and what child %s should get ([%s], [%s])", - child.child_handle, ca_detail.latest_ca_cert.get_3779resources(), irdb_resources) - continue - rc = class_elt() - rc.class_name = str(ca.ca_id) - rc.cert_url = multi_uri(ca_detail.ca_cert_uri) - rc.from_resource_bag(resources) - for child_cert in child.fetch_child_certs(ca_detail = ca_detail): - c = certificate_elt() - c.cert_url = multi_uri(child_cert.uri) - c.cert = child_cert.cert - rc.certs.append(c) - rc.issuer = ca_detail.latest_ca_cert - r_msg.payload.classes.append(rc) - - callback() - - self.gctx.irdb_query_child_resources(child.self.self_handle, child.child_handle, handle, errback) - - @classmethod - def query(cls, parent, cb, eb): - """ - Send a "list" query to parent. - """ - try: - logger.info('Sending "list" request to parent %s', parent.parent_handle) - parent.query_up_down(cls(), cb, eb) - except (rpki.async.ExitNow, SystemExit): - raise - except Exception, e: - eb(e) - -class class_response_syntax(base_elt): - """ - Syntax for Up-Down protocol "list_response" and "issue_response" PDUs. - """ - - def __init__(self): - """ - Initialize class_response_syntax. - """ - base_elt.__init__(self) - self.classes = [] - - def startElement(self, stack, name, attrs): - """ - Handle "list_response" and "issue_response" PDUs. - """ - assert name == "class", "Unexpected name %s, stack %s" % (name, stack) - c = class_elt() - self.classes.append(c) - stack.append(c) - c.startElement(stack, name, attrs) - - def toXML(self): - """Generate payload of "list_response" and "issue_response" PDUs.""" - return [c.toXML() for c in self.classes] -class list_response_pdu(class_response_syntax): - """ - Up-Down protocol "list_response" PDU. - """ - pass - -class issue_pdu(base_elt): - """ - Up-Down protocol "issue" PDU. - """ +error_response_codes = { + 1101 : "Already processing request", + 1102 : "Version number error", + 1103 : "Unrecognised request type", + 1201 : "Request - no such resource class", + 1202 : "Request - no resources allocated in resource class", + 1203 : "Request - badly formed certificate request", + 1301 : "Revoke - no such resource class", + 1302 : "Revoke - no such key", + 2001 : "Internal Server Error - Request not performed" } - def startElement(self, stack, name, attrs): - """ - Handle "issue" PDU. - """ - assert name == "request", "Unexpected name %s, stack %s" % (name, stack) - self.class_name = attrs["class_name"] - self.req_resource_set_as = rpki.resource_set.resource_set_as(attrs.get("req_resource_set_as")) - self.req_resource_set_ipv4 = rpki.resource_set.resource_set_ipv4(attrs.get("req_resource_set_ipv4")) - self.req_resource_set_ipv6 = rpki.resource_set.resource_set_ipv6(attrs.get("req_resource_set_ipv6")) - def endElement(self, stack, name, text): - """ - Handle "issue" PDU. - """ - assert name == "request", "Unexpected name %s, stack %s" % (name, stack) - self.pkcs10 = rpki.x509.PKCS10(Base64 = text) - stack.pop() - - def toXML(self): - """ - Generate payload of "issue" PDU. - """ - elt = self.make_elt("request", "class_name", "req_resource_set_as", - "req_resource_set_ipv4", "req_resource_set_ipv6") - elt.text = self.pkcs10.get_Base64() - return [elt] - - def serve_pdu(self, q_msg, r_msg, child, callback, errback): - """ - Serve one issue request PDU. - """ - - # Subsetting not yet implemented, this is the one place where we - # have to handle it, by reporting that we're lame. - - if self.req_resource_set_as or \ - self.req_resource_set_ipv4 or \ - self.req_resource_set_ipv6: - raise rpki.exceptions.NotImplementedYet("req_* attributes not implemented yet, sorry") - - # Check the request - self.pkcs10.check_valid_request_ca() - ca = child.ca_from_class_name(self.class_name) - ca_detail = ca.active_ca_detail - if ca_detail is None: - raise rpki.exceptions.NoActiveCA("No active CA for class %r" % self.class_name) - - # Check current cert, if any - - def got_resources(irdb_resources): - - if irdb_resources.valid_until < rpki.sundial.now(): - raise rpki.exceptions.IRDBExpired("IRDB entry for child %s expired %s" % ( - child.child_handle, irdb_resources.valid_until)) - - resources = irdb_resources & ca_detail.latest_ca_cert.get_3779resources() - resources.valid_until = irdb_resources.valid_until - req_key = self.pkcs10.getPublicKey() - req_sia = self.pkcs10.get_SIA() - child_cert = child.fetch_child_certs(ca_detail = ca_detail, ski = req_key.get_SKI(), unique = True) - - # Generate new cert or regenerate old one if necessary - - publisher = rpki.rpkid.publication_queue() - - if child_cert is None: - child_cert = ca_detail.issue( - ca = ca, - child = child, - subject_key = req_key, - sia = req_sia, - resources = resources, - publisher = publisher) - else: - child_cert = child_cert.reissue( - ca_detail = ca_detail, - sia = req_sia, - resources = resources, - publisher = publisher) - - def done(): - c = certificate_elt() - c.cert_url = multi_uri(child_cert.uri) - c.cert = child_cert.cert - rc = class_elt() - rc.class_name = self.class_name - rc.cert_url = multi_uri(ca_detail.ca_cert_uri) - rc.from_resource_bag(resources) - rc.certs.append(c) - rc.issuer = ca_detail.latest_ca_cert - r_msg.payload = issue_response_pdu() - r_msg.payload.classes.append(rc) - callback() - - self.gctx.sql.sweep() - assert child_cert and child_cert.sql_in_db - publisher.call_pubd(done, errback) - - self.gctx.irdb_query_child_resources(child.self.self_handle, child.child_handle, got_resources, errback) - - @classmethod - def query(cls, parent, ca, ca_detail, callback, errback): - """ - Send an "issue" request to parent associated with ca. - """ - assert ca_detail is not None and ca_detail.state in ("pending", "active") - self = cls() - self.class_name = ca.parent_resource_class - self.pkcs10 = rpki.x509.PKCS10.create( - keypair = ca_detail.private_key_id, - is_ca = True, - caRepository = ca.sia_uri, - rpkiManifest = ca_detail.manifest_uri) - logger.info('Sending "issue" request to parent %s', parent.parent_handle) - parent.query_up_down(self, callback, errback) - -class issue_response_pdu(class_response_syntax): - """ - Up-Down protocol "issue_response" PDU. - """ +exception_map = { + rpki.exceptions.NoActiveCA : 1202, + (rpki.exceptions.ClassNameUnknown, "revoke") : 1301, + rpki.exceptions.ClassNameUnknown : 1201, + (rpki.exceptions.NotInDatabase, "revoke") : 1302 } - def check_response(self): - """ - Check whether this looks like a reasonable issue_response PDU. - XML schema should be tighter for this response. - """ - if len(self.classes) != 1 or len(self.classes[0].certs) != 1: - raise rpki.exceptions.BadIssueResponse -class revoke_syntax(base_elt): +def check_response(r_msg, q_type): """ - Syntax for Up-Down protocol "revoke" and "revoke_response" PDUs. + Additional checks beyond the XML schema for whether this looks like + a reasonable up-down response message. """ - def startElement(self, stack, name, attrs): - """Handle "revoke" PDU.""" - self.class_name = attrs["class_name"] - self.ski = attrs["ski"] + r_type = r_msg.get("type") - def toXML(self): - """Generate payload of "revoke" PDU.""" - return [self.make_elt("key", "class_name", "ski")] + if r_type == "error_response": + raise rpki.exceptions.UpstreamError(error_response_codes[int(r_msg.findtext(tag_status))]) -class revoke_pdu(revoke_syntax): - """ - Up-Down protocol "revoke" PDU. - """ + if r_type != q_type + "_response": + raise rpki.exceptions.UnexpectedUpDownResponse - def get_SKI(self): - """ - Convert g(SKI) encoding from PDU back to raw SKI. - """ - return base64.urlsafe_b64decode(self.ski + "=") + if r_type == "issue_response" and (len(r_msg) != 1 or len(r_msg[0]) != 2): + logger.debug("Weird issue_response %r: len(r_msg) %s len(r_msg[0]) %s", + r_msg, len(r_msg), len(r_msg[0]) if len(r_msg) else None) + logger.debug("Offending message\n%s", ElementToString(r_msg)) + raise rpki.exceptions.BadIssueResponse - def serve_pdu(self, q_msg, r_msg, child, cb, eb): - """ - Serve one revoke request PDU. - """ - def done(): - r_msg.payload = revoke_response_pdu() - r_msg.payload.class_name = self.class_name - r_msg.payload.ski = self.ski - cb() - - ca = child.ca_from_class_name(self.class_name) - publisher = rpki.rpkid.publication_queue() - for ca_detail in ca.ca_details: - for child_cert in child.fetch_child_certs(ca_detail = ca_detail, ski = self.get_SKI()): - child_cert.revoke(publisher = publisher) - self.gctx.sql.sweep() - publisher.call_pubd(done, eb) - - @classmethod - def query(cls, ca, gski, cb, eb): - """ - Send a "revoke" request for certificate(s) named by gski to parent associated with ca. - """ - parent = ca.parent - self = cls() - self.class_name = ca.parent_resource_class - self.ski = gski - logger.info('Sending "revoke" request for SKI %s to parent %s', gski, parent.parent_handle) - parent.query_up_down(self, cb, eb) - -class revoke_response_pdu(revoke_syntax): +def generate_error_response(r_msg, status = 2001, description = None): """ - Up-Down protocol "revoke_response" PDU. + Generate an error response. If status is given, it specifies the + numeric code to use, otherwise we default to "internal error". + If description is specified, we use it as the description, otherwise + we just use the default string associated with status. """ - pass + assert status in error_response_codes + del r_msg[:] + r_msg.set("type", "error_response") + SubElement(r_msg, tag_status).text = str(status) + se = SubElement(r_msg, tag_description) + se.set("{http://www.w3.org/XML/1998/namespace}lang", "en-US") + se.text = str(description or error_response_codes[status]) -class error_response_pdu(base_elt): - """ - Up-Down protocol "error_response" PDU. - """ - codes = { - 1101 : "Already processing request", - 1102 : "Version number error", - 1103 : "Unrecognised request type", - 1201 : "Request - no such resource class", - 1202 : "Request - no resources allocated in resource class", - 1203 : "Request - badly formed certificate request", - 1301 : "Revoke - no such resource class", - 1302 : "Revoke - no such key", - 2001 : "Internal Server Error - Request not performed" } - - exceptions = { - rpki.exceptions.NoActiveCA : 1202, - (rpki.exceptions.ClassNameUnknown, revoke_pdu) : 1301, - rpki.exceptions.ClassNameUnknown : 1201, - (rpki.exceptions.NotInDatabase, revoke_pdu) : 1302 } - - def __init__(self, exception = None, request_payload = None): - """ - Initialize an error_response PDU from an exception object. - """ - base_elt.__init__(self) - if exception is not None: - logger.debug("Constructing up-down error response from exception %s", exception) - exception_type = type(exception) - request_type = None if request_payload is None else type(request_payload) - logger.debug("Constructing up-down error response: exception_type %s, request_type %s", - exception_type, request_type) - if False: - self.status = self.exceptions.get((exception_type, request_type), - self.exceptions.get(exception_type, 2001)) - else: - self.status = self.exceptions.get((exception_type, request_type)) - if self.status is None: - logger.debug("No request-type-specific match, trying exception match") - self.status = self.exceptions.get(exception_type) - if self.status is None: - logger.debug("No exception match either, defaulting") - self.status = 2001 - self.description = str(exception) - logger.debug("Chosen status code: %s", self.status) - - def endElement(self, stack, name, text): - """ - Handle "error_response" PDU. - """ - if name == "status": - code = int(text) - if code not in self.codes: - raise rpki.exceptions.BadStatusCode("%s is not a known status code" % code) - self.status = code - elif name == "description": - self.description = text - else: - assert name == "message", "Unexpected name %s, stack %s" % (name, stack) - stack.pop() - stack[-1].endElement(stack, name, text) - - def toXML(self): - """ - Generate payload of "error_response" PDU. - """ - assert self.status in self.codes - elt = self.make_elt("status") - elt.text = str(self.status) - payload = [elt] - if self.description: - elt = self.make_elt("description") - elt.text = str(self.description) - elt.set("{http://www.w3.org/XML/1998/namespace}lang", "en-US") - payload.append(elt) - return payload - - def check_response(self): - """ - Handle an error response. For now, just raise an exception, - perhaps figure out something more clever to do later. - """ - raise rpki.exceptions.UpstreamError(self.codes[self.status]) - -class message_pdu(base_elt): +def generate_error_response_from_exception(r_msg, e, q_type): """ - Up-Down protocol message wrapper PDU. + Construct an error response from an exception. q_type + specifies the kind of query to which this is a response, since the + same exception can generate different codes in response to different + queries. """ - version = 1 - - name2type = { - "list" : list_pdu, - "list_response" : list_response_pdu, - "issue" : issue_pdu, - "issue_response" : issue_response_pdu, - "revoke" : revoke_pdu, - "revoke_response" : revoke_response_pdu, - "error_response" : error_response_pdu } - - type2name = dict((v, k) for k, v in name2type.items()) - - error_pdu_type = error_response_pdu - - def toXML(self): - """ - Generate payload of message PDU. - """ - elt = self.make_elt("message", "version", "sender", "recipient", "type") - elt.extend(self.payload.toXML()) - return elt - - def startElement(self, stack, name, attrs): - """ - Handle message PDU. - - Payload of the <message/> element varies depending on the "type" - attribute, so after some basic checks we have to instantiate the - right class object to handle whatever kind of PDU this is. - """ - assert name == "message", "Unexpected name %s, stack %s" % (name, stack) - assert self.version == int(attrs["version"]) - self.sender = attrs["sender"] - self.recipient = attrs["recipient"] - self.type = attrs["type"] - self.payload = self.name2type[attrs["type"]]() - stack.append(self.payload) - - def __str__(self): - """ - Convert a message PDU to a string. - """ - return lxml.etree.tostring(self.toXML(), pretty_print = True, encoding = "UTF-8") - - def serve_top_level(self, child, callback): - """ - Serve one message request PDU. - """ - - r_msg = message_pdu() - r_msg.sender = self.recipient - r_msg.recipient = self.sender - - def done(): - r_msg.type = self.type2name[type(r_msg.payload)] - callback(r_msg) - - def lose(e): - logger.exception("Unhandled exception serving child %r", child) - callback(self.serve_error(e)) - - try: - self.log_query(child) - self.payload.serve_pdu(self, r_msg, child, done, lose) - except (rpki.async.ExitNow, SystemExit): - raise - except Exception, e: - lose(e) - - def log_query(self, child): - """ - Log query we're handling. Separate method so rootd can override. - """ - logger.info("Serving %s query from child %s [sender %s, recipient %s]", self.type, child.child_handle, self.sender, self.recipient) - - def serve_error(self, exception): - """ - Generate an error_response message PDU. - """ - r_msg = message_pdu() - r_msg.sender = self.recipient - r_msg.recipient = self.sender - r_msg.payload = self.error_pdu_type(exception, self.payload) - r_msg.type = self.type2name[type(r_msg.payload)] - return r_msg - - @classmethod - def make_query(cls, payload, sender, recipient): - """ - Construct one message PDU. - """ - assert not cls.type2name[type(payload)].endswith("_response") - if sender is None: - sender = "tweedledee" - if recipient is None: - recipient = "tweedledum" - self = cls() - self.sender = sender - self.recipient = recipient - self.payload = payload - self.type = self.type2name[type(payload)] - return self - -class sax_handler(rpki.xml_utils.sax_handler): - """ - SAX handler for Up-Down protocol. - """ + t = type(e) + code = (exception_map.get((t, q_type)) or exception_map.get(t) or 2001) + generate_error_response(r_msg, code, e) - pdu = message_pdu - name = "message" - version = "1" class cms_msg(rpki.x509.XML_CMS_object): """ - Class to hold a CMS-signed up-down PDU. + CMS-signed up-down PDU. """ encoding = "UTF-8" schema = rpki.relaxng.up_down - saxify = sax_handler.saxify allow_extra_certs = True allow_extra_crls = True diff --git a/rpki/x509.py b/rpki/x509.py index a7e4d17a..2d50b129 100644 --- a/rpki/x509.py +++ b/rpki/x509.py @@ -57,6 +57,7 @@ def base64_with_linebreaks(der): Encode DER (really, anything) as Base64 text, with linebreaks to keep the result (sort of) readable. """ + b = base64.b64encode(der) n = len(b) return "\n" + "\n".join(b[i : min(i + 64, n)] for i in xrange(0, n, 64)) + "\n" @@ -69,18 +70,63 @@ def looks_like_PEM(text): i = text.find("-----BEGIN ") return i >= 0 and text.find("\n-----END ", i) > i -def first_rsync_uri(xia): +def first_uri_matching_prefix(xia, prefix): """ - Find first rsync URI in a sequence of AIA or SIA URIs. - Returns the URI if found, otherwise None. + Find first URI in a sequence of AIA or SIA URIs which matches a + particular prefix string. Returns the URI if found, otherwise None. """ if xia is not None: for uri in xia: - if uri.startswith("rsync://"): + if uri.startswith(prefix): return uri return None +def first_rsync_uri(xia): + """ + Find first rsync URI in a sequence of AIA or SIA URIs. + Returns the URI if found, otherwise None. + """ + + return first_uri_matching_prefix(xia, "rsync://") + +def first_http_uri(xia): + """ + Find first HTTP URI in a sequence of AIA or SIA URIs. + Returns the URI if found, otherwise None. + """ + + return first_uri_matching_prefix(xia, "http://") + +def first_https_uri(xia): + """ + Find first HTTPS URI in a sequence of AIA or SIA URIs. + Returns the URI if found, otherwise None. + """ + + return first_uri_matching_prefix(xia, "https://") + +def sha1(data): + """ + Calculate SHA-1 digest of some data. + Convenience wrapper around rpki.POW.Digest class. + """ + + d = rpki.POW.Digest(rpki.POW.SHA1_DIGEST) + d.update(data) + return d.digest() + +def sha256(data): + """ + Calculate SHA-256 digest of some data. + Convenience wrapper around rpki.POW.Digest class. + """ + + d = rpki.POW.Digest(rpki.POW.SHA256_DIGEST) + d.update(data) + return d.digest() + + class X501DN(object): """ Class to hold an X.501 Distinguished Name. @@ -207,12 +253,14 @@ class DER_object(object): """ Test whether this object is empty. """ + return all(getattr(self, a, None) is None for a in self.formats) def clear(self): """ Make this object empty. """ + for a in self.formats + self.other_clear: setattr(self, a, None) self.filename = None @@ -223,6 +271,7 @@ class DER_object(object): """ Initialize a DER_object. """ + self.clear() if len(kw): self.set(**kw) @@ -271,6 +320,7 @@ class DER_object(object): """ Check for updates to a DER object that auto-updates from a file. """ + if self.filename is None: return try: @@ -297,10 +347,19 @@ class DER_object(object): else: self.lastfail = None + @property + def mtime(self): + """ + Retrieve os.stat().st_mtime for auto-update files. + """ + + return os.stat(self.filename).st_mtime + def check(self): """ Perform basic checks on a DER object. """ + self.check_auto_update() assert not self.empty() @@ -309,6 +368,7 @@ class DER_object(object): Set the POW value of this object based on a PEM input value. Subclasses may need to override this. """ + assert self.empty() self.POW = self.POW_class.pemRead(pem) @@ -317,6 +377,7 @@ class DER_object(object): Get the DER value of this object. Subclasses may need to override this method. """ + self.check() if self.DER: return self.DER @@ -330,6 +391,7 @@ class DER_object(object): Get the rpki.POW value of this object. Subclasses may need to override this method. """ + self.check() if not self.POW: # pylint: disable=E0203 self.POW = self.POW_class.derRead(self.get_DER()) @@ -339,18 +401,21 @@ class DER_object(object): """ Get the Base64 encoding of the DER value of this object. """ + return base64_with_linebreaks(self.get_DER()) def get_PEM(self): """ Get the PEM representation of this object. """ + return self.get_POW().pemWrite() def __cmp__(self, other): """ Compare two DER-encoded objects. """ + if self is None and other is None: return 0 elif self is None: @@ -367,6 +432,7 @@ class DER_object(object): Return hexadecimal string representation of SKI for this object. Only work for subclasses that implement get_SKI(). """ + ski = self.get_SKI() return ":".join(("%02X" % ord(i) for i in ski)) if ski else "" @@ -375,6 +441,7 @@ class DER_object(object): Calculate g(SKI) for this object. Only work for subclasses that implement get_SKI(). """ + return base64.urlsafe_b64encode(self.get_SKI()).rstrip("=") def hAKI(self): @@ -382,6 +449,7 @@ class DER_object(object): Return hexadecimal string representation of AKI for this object. Only work for subclasses that implement get_AKI(). """ + aki = self.get_AKI() return ":".join(("%02X" % ord(i) for i in aki)) if aki else "" @@ -390,24 +458,28 @@ class DER_object(object): Calculate g(AKI) for this object. Only work for subclasses that implement get_AKI(). """ + return base64.urlsafe_b64encode(self.get_AKI()).rstrip("=") def get_AKI(self): """ Get the AKI extension from this object, if supported. """ + return self.get_POW().getAKI() def get_SKI(self): """ Get the SKI extension from this object, if supported. """ + return self.get_POW().getSKI() def get_EKU(self): """ Get the Extended Key Usage extension from this object, if supported. """ + return self.get_POW().getEKU() def get_SIA(self): @@ -415,6 +487,7 @@ class DER_object(object): Get the SIA extension from this object. Only works for subclasses that support getSIA(). """ + return self.get_POW().getSIA() def get_sia_directory_uri(self): @@ -422,6 +495,7 @@ class DER_object(object): Get SIA directory (id-ad-caRepository) URI from this object. Only works for subclasses that support getSIA(). """ + sia = self.get_POW().getSIA() return None if sia is None else first_rsync_uri(sia[0]) @@ -430,6 +504,7 @@ class DER_object(object): Get SIA manifest (id-ad-rpkiManifest) URI from this object. Only works for subclasses that support getSIA(). """ + sia = self.get_POW().getSIA() return None if sia is None else first_rsync_uri(sia[1]) @@ -438,14 +513,26 @@ class DER_object(object): Get SIA object (id-ad-signedObject) URI from this object. Only works for subclasses that support getSIA(). """ + sia = self.get_POW().getSIA() return None if sia is None else first_rsync_uri(sia[2]) + def get_sia_rrdp_notify(self): + """ + Get SIA RRDP (id-ad-rpkiNotify) URI from this object. + We prefer HTTPS over HTTP if both are present. + Only works for subclasses that support getSIA(). + """ + + sia = self.get_POW().getSIA() + return None if sia is None else first_https_uri(sia[3]) or first_http_uri(sia[3]) + def get_AIA(self): """ Get the SIA extension from this object. Only works for subclasses that support getAIA(). """ + return self.get_POW().getAIA() def get_aia_uri(self): @@ -453,6 +540,7 @@ class DER_object(object): Get AIA (id-ad-caIssuers) URI from this object. Only works for subclasses that support getAIA(). """ + return first_rsync_uri(self.get_POW().getAIA()) def get_basicConstraints(self): @@ -460,6 +548,7 @@ class DER_object(object): Get the basicConstraints extension from this object. Only works for subclasses that support getExtension(). """ + return self.get_POW().getBasicConstraints() def is_CA(self): @@ -467,6 +556,7 @@ class DER_object(object): Return True if and only if object has the basicConstraints extension and its cA value is true. """ + basicConstraints = self.get_basicConstraints() return basicConstraints is not None and basicConstraints[0] @@ -474,6 +564,7 @@ class DER_object(object): """ Get RFC 3779 resources as rpki.resource_set objects. """ + resources = rpki.resource_set.resource_bag.from_POW_rfc3779(self.get_POW().getRFC3779()) try: resources.valid_until = self.getNotAfter() @@ -486,12 +577,14 @@ class DER_object(object): """ Convert from SQL storage format. """ + return cls(DER = x) def to_sql(self): """ Convert to SQL storage format. """ + return self.get_DER() def dumpasn1(self): @@ -522,11 +615,11 @@ class DER_object(object): provide more information, but should make sure to include at least this information at the start of the tracking line. """ + try: - d = rpki.POW.Digest(rpki.POW.SHA1_DIGEST) - d.update(self.get_DER()) - return "%s %s %s" % (uri, self.creation_timestamp, - "".join(("%02X" % ord(b) for b in d.digest()))) + return "%s %s %s" % (uri, + self.creation_timestamp, + "".join(("%02X" % ord(b) for b in sha1(self.get_DER())))) except: # pylint: disable=W0702 return uri @@ -534,12 +627,14 @@ class DER_object(object): """ Pickling protocol -- pickle the DER encoding. """ + return self.get_DER() def __setstate__(self, state): """ Pickling protocol -- unpickle the DER encoding. """ + self.set(DER = state) class X509(DER_object): @@ -559,48 +654,56 @@ class X509(DER_object): """ Get the issuer of this certificate. """ + return X501DN.from_POW(self.get_POW().getIssuer()) def getSubject(self): """ Get the subject of this certificate. """ + return X501DN.from_POW(self.get_POW().getSubject()) def getNotBefore(self): """ Get the inception time of this certificate. """ + return self.get_POW().getNotBefore() def getNotAfter(self): """ Get the expiration time of this certificate. """ + return self.get_POW().getNotAfter() def getSerial(self): """ Get the serial number of this certificate. """ + return self.get_POW().getSerial() def getPublicKey(self): """ Extract the public key from this certificate. """ + return PublicKey(POW = self.get_POW().getPublicKey()) def get_SKI(self): """ Get the SKI extension from this object. """ + return self.get_POW().getSKI() def expired(self): """ Test whether this certificate has expired. """ + return self.getNotAfter() <= rpki.sundial.now() def issue(self, keypair, subject_key, serial, sia, aia, crldp, notAfter, @@ -670,6 +773,10 @@ class X509(DER_object): Common code to issue an RPKI certificate. """ + if not sia or len(sia) != 4 or not sia[3]: + logger.debug("Oops! _issue() sia: %r", sia) + rpki.log.show_stack(logger) + now = rpki.sundial.now() ski = subject_key.get_SKI() @@ -715,11 +822,12 @@ class X509(DER_object): assert sia is not None or not is_ca if sia is not None: - caRepository, rpkiManifest, signedObject = sia + caRepository, rpkiManifest, signedObject, rpkiNotify = sia cert.setSIA( (caRepository,) if isinstance(caRepository, str) else caRepository, (rpkiManifest,) if isinstance(rpkiManifest, str) else rpkiManifest, - (signedObject,) if isinstance(signedObject, str) else signedObject) + (signedObject,) if isinstance(signedObject, str) else signedObject, + (rpkiNotify,) if isinstance(rpkiNotify, str) else rpkiNotify) if resources is not None: cert.setRFC3779( @@ -743,6 +851,7 @@ class X509(DER_object): """ Issue a BPKI certificate with values taking from an existing certificate. """ + return self.bpki_certify( keypair = keypair, subject_name = source_cert.getSubject(), @@ -759,6 +868,7 @@ class X509(DER_object): """ Issue a self-signed BPKI CA certificate. """ + return cls._bpki_certify( keypair = keypair, issuer_name = subject_name, @@ -775,6 +885,7 @@ class X509(DER_object): """ Issue a normal BPKI certificate. """ + assert keypair.get_public() == self.getPublicKey() return self._bpki_certify( keypair = keypair, @@ -833,6 +944,7 @@ class X509(DER_object): allowed cases. So this method allows X509, None, lists, and tuples, and returns a tuple of X509 objects. """ + if isinstance(chain, cls): chain = (chain,) return tuple(x for x in chain if x is not None) @@ -842,6 +954,7 @@ class X509(DER_object): """ Time at which this object was created. """ + return self.getNotBefore() class PKCS10(DER_object): @@ -869,6 +982,7 @@ class PKCS10(DER_object): """ Get the DER value of this certification request. """ + self.check() if self.DER: return self.DER @@ -881,6 +995,7 @@ class PKCS10(DER_object): """ Get the rpki.POW value of this certification request. """ + self.check() if not self.POW: # pylint: disable=E0203 self.POW = rpki.POW.PKCS10.derRead(self.get_DER()) @@ -890,18 +1005,21 @@ class PKCS10(DER_object): """ Extract the subject name from this certification request. """ + return X501DN.from_POW(self.get_POW().getSubject()) def getPublicKey(self): """ Extract the public key from this certification request. """ + return PublicKey(POW = self.get_POW().getPublicKey()) def get_SKI(self): """ Compute SKI for public key from this certification request. """ + return self.getPublicKey().get_SKI() @@ -966,7 +1084,7 @@ class PKCS10(DER_object): if sias is None: raise rpki.exceptions.BadPKCS10("PKCS #10 CA SIA missing") - caRepository, rpkiManifest, signedObject = sias + caRepository, rpkiManifest, signedObject, rpkiNotify = sias if signedObject: raise rpki.exceptions.BadPKCS10("PKCS #10 CA SIA must not have id-ad-signedObject") @@ -989,6 +1107,8 @@ class PKCS10(DER_object): if any(uri.startswith("rsync://") and uri.endswith("/") for uri in rpkiManifest): raise rpki.exceptions.BadPKCS10("PKCS #10 CA SIA id-ad-rpkiManifest ends with slash") + if any(not uri.startswith("http://") and not uri.startswith("https://") for uri in rpkiNotify): + raise rpki.exceptions.BadPKCS10("PKCS #10 CA SIA id-ad-rpkiNotify neither HTTP nor HTTPS") def check_valid_request_ee(self): """ @@ -1016,7 +1136,7 @@ class PKCS10(DER_object): bc = self.get_POW().getBasicConstraints() sia = self.get_POW().getSIA() - caRepository, rpkiManifest, signedObject = sia or (None, None, None) + caRepository, rpkiManifest, signedObject, rpkiNotify = sia or (None, None, None, None) if alg not in (rpki.oids.sha256WithRSAEncryption, rpki.oids.ecdsa_with_SHA256): raise rpki.exceptions.BadPKCS10("PKCS #10 has bad signature algorithm for EE: %s" % alg) @@ -1033,6 +1153,8 @@ class PKCS10(DER_object): if signedObject and not any(uri.startswith("rsync://") for uri in signedObject): raise rpki.exceptions.BadPKCS10("PKCS #10 EE SIA id-ad-signedObject contains no rsync URIs") + if rpkiNotify and any(not uri.startswith("http://") and not uri.startswith("https://") for uri in rpkiNotify): + raise rpki.exceptions.BadPKCS10("PKCS #10 EE SIA id-ad-rpkiNotify neither HTTP nor HTTPS") def check_valid_request_router(self): """ @@ -1070,7 +1192,7 @@ class PKCS10(DER_object): @classmethod def create(cls, keypair, exts = None, is_ca = False, caRepository = None, rpkiManifest = None, signedObject = None, - cn = None, sn = None, eku = None): + cn = None, sn = None, eku = None, rpkiNotify = None): """ Create a new request for a given keypair. """ @@ -1089,6 +1211,9 @@ class PKCS10(DER_object): if isinstance(signedObject, str): signedObject = (signedObject,) + if isinstance(rpkiNotify, str): + rpkiNotify = (rpkiNotify,) + req = rpki.POW.PKCS10() req.setVersion(0) req.setSubject(X501DN.from_cn(cn, sn).get_POW()) @@ -1098,8 +1223,8 @@ class PKCS10(DER_object): req.setBasicConstraints(True, None) req.setKeyUsage(cls.expected_ca_keyUsage) - if caRepository or rpkiManifest or signedObject: - req.setSIA(caRepository, rpkiManifest, signedObject) + if caRepository or rpkiManifest or signedObject or rpkiNotify: + req.setSIA(caRepository, rpkiManifest, signedObject, rpkiNotify) if eku: req.setEKU(eku) @@ -1150,6 +1275,7 @@ class PrivateKey(DER_object): """ Get the DER value of this keypair. """ + self.check() if self.DER: return self.DER @@ -1162,6 +1288,7 @@ class PrivateKey(DER_object): """ Get the rpki.POW value of this keypair. """ + self.check() if not self.POW: # pylint: disable=E0203 self.POW = rpki.POW.Asymmetric.derReadPrivate(self.get_DER()) @@ -1171,12 +1298,14 @@ class PrivateKey(DER_object): """ Get the PEM representation of this keypair. """ + return self.get_POW().pemWritePrivate() def _set_PEM(self, pem): """ Set the POW value of this keypair from a PEM string. """ + assert self.empty() self.POW = self.POW_class.pemReadPrivate(pem) @@ -1184,18 +1313,21 @@ class PrivateKey(DER_object): """ Get the DER encoding of the public key from this keypair. """ + return self.get_POW().derWritePublic() def get_SKI(self): """ Calculate the SKI of this keypair. """ + return self.get_POW().calculateSKI() def get_public(self): """ Convert the public key of this keypair into a PublicKey object. """ + return PublicKey(DER = self.get_public_DER()) class PublicKey(DER_object): @@ -1209,6 +1341,7 @@ class PublicKey(DER_object): """ Get the DER value of this public key. """ + self.check() if self.DER: return self.DER @@ -1221,6 +1354,7 @@ class PublicKey(DER_object): """ Get the rpki.POW value of this public key. """ + self.check() if not self.POW: # pylint: disable=E0203 self.POW = rpki.POW.Asymmetric.derReadPublic(self.get_DER()) @@ -1230,12 +1364,14 @@ class PublicKey(DER_object): """ Get the PEM representation of this public key. """ + return self.get_POW().pemWritePublic() def _set_PEM(self, pem): """ Set the POW value of this public key from a PEM string. """ + assert self.empty() self.POW = self.POW_class.pemReadPublic(pem) @@ -1243,6 +1379,7 @@ class PublicKey(DER_object): """ Calculate the SKI of this public key. """ + return self.get_POW().calculateSKI() class KeyParams(DER_object): @@ -1266,6 +1403,7 @@ class RSA(PrivateKey): """ Generate a new keypair. """ + if not quiet: logger.debug("Generating new %d-bit RSA key", keylength) if generate_insecure_debug_only_rsa_key is not None: @@ -1348,6 +1486,7 @@ class CMS_object(DER_object): """ Get the DER value of this CMS_object. """ + self.check() if self.DER: return self.DER @@ -1360,6 +1499,7 @@ class CMS_object(DER_object): """ Get the rpki.POW value of this CMS_object. """ + self.check() if not self.POW: # pylint: disable=E0203 self.POW = self.POW_class.derRead(self.get_DER()) @@ -1369,6 +1509,7 @@ class CMS_object(DER_object): """ Extract signingTime from CMS signed attributes. """ + return self.get_POW().signingTime() def verify(self, ta): @@ -1540,6 +1681,7 @@ class CMS_object(DER_object): """ Time at which this object was created. """ + return self.get_signingTime() @@ -1561,6 +1703,7 @@ class Wrapped_CMS_object(CMS_object): """ Get the inner content of this Wrapped_CMS_object. """ + if self.content is None: raise rpki.exceptions.CMSContentNotSet("Inner content of CMS object %r is not set" % self) return self.content @@ -1569,6 +1712,7 @@ class Wrapped_CMS_object(CMS_object): """ Set the (inner) content of this Wrapped_CMS_object, clearing the wrapper. """ + self.clear() self.content = content @@ -1651,12 +1795,14 @@ class SignedManifest(DER_CMS_object): """ Get thisUpdate value from this manifest. """ + return self.get_POW().getThisUpdate() def getNextUpdate(self): """ Get nextUpdate value from this manifest. """ + return self.get_POW().getNextUpdate() @classmethod @@ -1667,9 +1813,7 @@ class SignedManifest(DER_CMS_object): filelist = [] for name, obj in names_and_objs: - d = rpki.POW.Digest(rpki.POW.SHA256_DIGEST) - d.update(obj.get_DER()) - filelist.append((name.rpartition("/")[2], d.digest())) + filelist.append((name.rpartition("/")[2], sha256(obj.get_DER()))) filelist.sort(key = lambda x: x[0]) obj = cls.POW_class() @@ -1697,6 +1841,7 @@ class ROA(DER_CMS_object): """ Build a ROA. """ + ipv4 = ipv4.to_POW_roa_tuple() if ipv4 else None ipv6 = ipv6.to_POW_roa_tuple() if ipv6 else None obj = cls.POW_class() @@ -1712,6 +1857,7 @@ class ROA(DER_CMS_object): Return a string containing data we want to log when tracking how objects move through the RPKI system. """ + msg = DER_CMS_object.tracking_data(self, uri) try: self.extract_if_needed() @@ -1788,12 +1934,13 @@ class XML_CMS_object(Wrapped_CMS_object): ## @var check_outbound_schema # If set, perform RelaxNG schema check on outbound messages. - check_outbound_schema = False + check_outbound_schema = True def encode(self): """ Encode inner content for signing. """ + return lxml.etree.tostring(self.get_content(), pretty_print = True, encoding = self.encoding, @@ -1803,12 +1950,14 @@ class XML_CMS_object(Wrapped_CMS_object): """ Decode XML and set inner content. """ + self.content = lxml.etree.fromstring(xml) def pretty_print_content(self): """ Pretty print XML content of this message. """ + return lxml.etree.tostring(self.get_content(), pretty_print = True, encoding = self.encoding, @@ -1818,6 +1967,7 @@ class XML_CMS_object(Wrapped_CMS_object): """ Handle XML RelaxNG schema check. """ + try: self.schema.assertValid(self.get_content()) except lxml.etree.DocumentInvalid: @@ -1830,6 +1980,7 @@ class XML_CMS_object(Wrapped_CMS_object): """ Write DER of current message to disk, for debugging. """ + f = open(prefix + rpki.sundial.now().isoformat() + "Z.cms", "wb") f.write(self.get_DER()) f.close() @@ -1838,10 +1989,8 @@ class XML_CMS_object(Wrapped_CMS_object): """ Wrap an XML PDU in CMS and return its DER encoding. """ - if self.saxify is None: - self.set_content(msg) - else: - self.set_content(msg.toXML()) + + self.set_content(msg) if self.check_outbound_schema: self.schema_check() self.sign(keypair, certs, crls) @@ -1853,15 +2002,13 @@ class XML_CMS_object(Wrapped_CMS_object): """ Unwrap a CMS-wrapped XML PDU and return Python objects. """ + if self.dump_inbound_cms: self.dump_inbound_cms.dump(self) self.verify(ta) if self.check_inbound_schema: self.schema_check() - if self.saxify is None: - return self.get_content() - else: - return self.saxify(self.get_content()) # pylint: disable=E1102 + return self.get_content() def check_replay(self, timestamp, *context): """ @@ -1869,6 +2016,7 @@ class XML_CMS_object(Wrapped_CMS_object): timestamp. Raises an exception if the recorded timestamp is more recent, otherwise returns the new timestamp. """ + new_timestamp = self.get_signingTime() if timestamp is not None and timestamp > new_timestamp: if context: @@ -1884,20 +2032,13 @@ class XML_CMS_object(Wrapped_CMS_object): "last_cms_timestamp" field of an SQL object and stores the new timestamp back in that same field. """ + obj.last_cms_timestamp = self.check_replay(obj.last_cms_timestamp, *context) obj.sql_mark_dirty() - ## @var saxify - # SAX handler hook. Subclasses can set this to a SAX handler, in - # which case .unwrap() will call it and return the result. - # Otherwise, .unwrap() just returns a verified element tree. - - saxify = None - class SignedReferral(XML_CMS_object): encoding = "us-ascii" - schema = rpki.relaxng.myrpki - saxify = None + schema = rpki.relaxng.oob_setup class Ghostbuster(Wrapped_CMS_object): """ @@ -1913,6 +2054,7 @@ class Ghostbuster(Wrapped_CMS_object): Encode inner content for signing. At the moment we're treating the VCard as an opaque byte string, so no encoding needed here. """ + return self.get_content() def decode(self, vcard): @@ -1920,6 +2062,7 @@ class Ghostbuster(Wrapped_CMS_object): Decode XML and set inner content. At the moment we're treating the VCard as an opaque byte string, so no encoding needed here. """ + self.content = vcard @classmethod @@ -1927,6 +2070,7 @@ class Ghostbuster(Wrapped_CMS_object): """ Build a Ghostbuster record. """ + self = cls() self.set_content(vcard) self.sign(keypair, certs) @@ -1944,6 +2088,7 @@ class CRL(DER_object): """ Get the DER value of this CRL. """ + self.check() if self.DER: return self.DER @@ -1956,6 +2101,7 @@ class CRL(DER_object): """ Get the rpki.POW value of this CRL. """ + self.check() if not self.POW: # pylint: disable=E0203 self.POW = rpki.POW.CRL.derRead(self.get_DER()) @@ -1965,24 +2111,28 @@ class CRL(DER_object): """ Get thisUpdate value from this CRL. """ + return self.get_POW().getThisUpdate() def getNextUpdate(self): """ Get nextUpdate value from this CRL. """ + return self.get_POW().getNextUpdate() def getIssuer(self): """ Get issuer value of this CRL. """ + return X501DN.from_POW(self.get_POW().getIssuer()) def getCRLNumber(self): """ Get CRL Number value for this CRL. """ + return self.get_POW().getCRLNumber() @classmethod @@ -1990,6 +2140,7 @@ class CRL(DER_object): """ Generate a new CRL. """ + crl = rpki.POW.CRL() crl.setVersion(version) crl.setIssuer(issuer.getSubject().get_POW()) @@ -2006,6 +2157,7 @@ class CRL(DER_object): """ Time at which this object was created. """ + return self.getThisUpdate() ## @var uri_dispatch_map @@ -2024,4 +2176,5 @@ def uri_dispatch(uri): """ Return the Python class object corresponding to a given URI. """ + return uri_dispatch_map[os.path.splitext(uri)[1]] diff --git a/rpki/xml_utils.py b/rpki/xml_utils.py deleted file mode 100644 index c276ce98..00000000 --- a/rpki/xml_utils.py +++ /dev/null @@ -1,494 +0,0 @@ -# $Id$ -# -# Copyright (C) 2009-2012 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. -# -# Portions copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -XML utilities. -""" - -import xml.sax -import lxml.sax -import lxml.etree -import rpki.exceptions - -class sax_handler(xml.sax.handler.ContentHandler): - """ - SAX handler for RPKI protocols. - - This class provides some basic amenities for parsing protocol XML of - the kind we use in the RPKI protocols, including whacking all the - protocol element text into US-ASCII, simplifying accumulation of - text fields, and hiding some of the fun relating to XML namespaces. - - General assumption: by the time this parsing code gets invoked, the - XML has already passed RelaxNG validation, so we only have to check - for errors that the schema can't catch, and we don't have to play as - many XML namespace games. - """ - - def __init__(self): - """ - Initialize SAX handler. - """ - xml.sax.handler.ContentHandler.__init__(self) - self.text = "" - self.stack = [] - - def startElementNS(self, name, qname, attrs): - """ - Redirect startElementNS() events to startElement(). - """ - return self.startElement(name[1], attrs) - - def endElementNS(self, name, qname): - """ - Redirect endElementNS() events to endElement(). - """ - return self.endElement(name[1]) - - def characters(self, content): - """ - Accumulate a chuck of element content (text). - """ - self.text += content - - def startElement(self, name, attrs): - """ - Handle startElement() events. - - We maintain a stack of nested elements under construction so that - we can feed events directly to the current element rather than - having to pass them through all the nesting elements. - - If the stack is empty, this event is for the outermost element, so - we call a virtual method to create the corresponding object and - that's the object we'll be returning as our final result. - """ - - a = dict() - for k, v in attrs.items(): - if isinstance(k, tuple): - if k == ("http://www.w3.org/XML/1998/namespace", "lang"): - k = "xml:lang" - else: - assert k[0] is None - k = k[1] - a[k.encode("ascii")] = v.encode("ascii") - if len(self.stack) == 0: - assert not hasattr(self, "result") - self.result = self.create_top_level(name, a) - self.stack.append(self.result) - self.stack[-1].startElement(self.stack, name, a) - - def endElement(self, name): - """ - Handle endElement() events. Mostly this means handling any - accumulated element text. - """ - text = self.text.encode("ascii").strip() - self.text = "" - self.stack[-1].endElement(self.stack, name, text) - - @classmethod - def saxify(cls, elt): - """ - Create a one-off SAX parser, parse an ETree, return the result. - """ - self = cls() - lxml.sax.saxify(elt, self) - return self.result - - def create_top_level(self, name, attrs): - """ - Handle top-level PDU for this protocol. - """ - assert name == self.name and attrs["version"] == self.version - return self.pdu() - -class base_elt(object): - """ - Virtual base class for XML message elements. The left-right and - publication protocols use this. At least for now, the up-down - protocol does not, due to different design assumptions. - """ - - ## @var attributes - # XML attributes for this element. - attributes = () - - ## @var elements - # XML elements contained by this element. - elements = () - - ## @var booleans - # Boolean attributes (value "yes" or "no") for this element. - booleans = () - - def startElement(self, stack, name, attrs): - """ - Default startElement() handler: just process attributes. - """ - if name not in self.elements: - assert name == self.element_name, "Unexpected name %s, stack %s" % (name, stack) - self.read_attrs(attrs) - - def endElement(self, stack, name, text): - """ - Default endElement() handler: just pop the stack. - """ - assert name == self.element_name, "Unexpected name %s, stack %s" % (name, stack) - stack.pop() - - def toXML(self): - """ - Default toXML() element generator. - """ - return self.make_elt() - - def read_attrs(self, attrs): - """ - Template-driven attribute reader. - """ - for key in self.attributes: - val = attrs.get(key, None) - if isinstance(val, str) and val.isdigit() and not key.endswith("_handle"): - val = long(val) - setattr(self, key, val) - for key in self.booleans: - setattr(self, key, attrs.get(key, False)) - - def make_elt(self): - """ - XML element constructor. - """ - elt = lxml.etree.Element(self.xmlns + self.element_name, nsmap = self.nsmap) - for key in self.attributes: - val = getattr(self, key, None) - if val is not None: - elt.set(key, str(val)) - for key in self.booleans: - if getattr(self, key, False): - elt.set(key, "yes") - return elt - - def make_b64elt(self, elt, name, value): - """ - Constructor for Base64-encoded subelement. - """ - if value is not None and not value.empty(): - lxml.etree.SubElement(elt, self.xmlns + name, nsmap = self.nsmap).text = value.get_Base64() - - def __str__(self): - """ - Convert a base_elt object to string format. - """ - return lxml.etree.tostring(self.toXML(), pretty_print = True, encoding = "us-ascii") - - @classmethod - def make_pdu(cls, **kargs): - """ - Generic PDU constructor. - """ - self = cls() - for k, v in kargs.items(): - if isinstance(v, bool): - v = 1 if v else 0 - setattr(self, k, v) - return self - -class text_elt(base_elt): - """ - Virtual base class for XML message elements that contain text. - """ - - ## @var text_attribute - # Name of the class attribute that holds the text value. - text_attribute = None - - def endElement(self, stack, name, text): - """ - Extract text from parsed XML. - """ - base_elt.endElement(self, stack, name, text) - setattr(self, self.text_attribute, text) - - def toXML(self): - """ - Insert text into generated XML. - """ - elt = self.make_elt() - elt.text = getattr(self, self.text_attribute) or None - return elt - -class data_elt(base_elt): - """ - Virtual base class for PDUs that map to SQL objects. These objects - all implement the create/set/get/list/destroy action attribute. - """ - - def endElement(self, stack, name, text): - """ - Default endElement handler for SQL-based objects. This assumes - that sub-elements are Base64-encoded using the sql_template - mechanism. - """ - if name in self.elements: - elt_type = self.sql_template.map.get(name) - assert elt_type is not None, "Couldn't find element type for %s, stack %s" % (name, stack) - setattr(self, name, elt_type(Base64 = text)) - else: - assert name == self.element_name, "Unexpected name %s, stack %s" % (name, stack) - stack.pop() - - def toXML(self): - """ - Default element generator for SQL-based objects. This assumes - that sub-elements are Base64-encoded DER objects. - """ - elt = self.make_elt() - for i in self.elements: - self.make_b64elt(elt, i, getattr(self, i, None)) - return elt - - def make_reply(self, r_pdu = None): - """ - Construct a reply PDU. - """ - if r_pdu is None: - r_pdu = self.__class__() - self.make_reply_clone_hook(r_pdu) - handle_name = self.element_name + "_handle" - setattr(r_pdu, handle_name, getattr(self, handle_name, None)) - else: - self.make_reply_clone_hook(r_pdu) - for b in r_pdu.booleans: - setattr(r_pdu, b, False) - r_pdu.action = self.action - r_pdu.tag = self.tag - return r_pdu - - def make_reply_clone_hook(self, r_pdu): - """ - Overridable hook. - """ - pass - - def serve_fetch_one(self): - """ - Find the object on which a get, set, or destroy method should - operate. - """ - r = self.serve_fetch_one_maybe() - if r is None: - raise rpki.exceptions.NotFound - return r - - def serve_pre_save_hook(self, q_pdu, r_pdu, cb, eb): - """ - Overridable hook. - """ - cb() - - def serve_post_save_hook(self, q_pdu, r_pdu, cb, eb): - """ - Overridable hook. - """ - cb() - - def serve_create(self, r_msg, cb, eb): - """ - Handle a create action. - """ - - r_pdu = self.make_reply() - - def one(): - self.sql_store() - setattr(r_pdu, self.sql_template.index, getattr(self, self.sql_template.index)) - self.serve_post_save_hook(self, r_pdu, two, eb) - - def two(): - r_msg.append(r_pdu) - cb() - - oops = self.serve_fetch_one_maybe() - if oops is not None: - raise rpki.exceptions.DuplicateObject("Object already exists: %r[%r] %r[%r]" % (self, getattr(self, self.element_name + "_handle"), - oops, getattr(oops, oops.element_name + "_handle"))) - - self.serve_pre_save_hook(self, r_pdu, one, eb) - - def serve_set(self, r_msg, cb, eb): - """ - Handle a set action. - """ - - db_pdu = self.serve_fetch_one() - r_pdu = self.make_reply() - for a in db_pdu.sql_template.columns[1:]: - v = getattr(self, a, None) - if v is not None: - setattr(db_pdu, a, v) - db_pdu.sql_mark_dirty() - - def one(): - db_pdu.sql_store() - db_pdu.serve_post_save_hook(self, r_pdu, two, eb) - - def two(): - r_msg.append(r_pdu) - cb() - - db_pdu.serve_pre_save_hook(self, r_pdu, one, eb) - - def serve_get(self, r_msg, cb, eb): - """ - Handle a get action. - """ - r_pdu = self.serve_fetch_one() - self.make_reply(r_pdu) - r_msg.append(r_pdu) - cb() - - def serve_list(self, r_msg, cb, eb): - """ - Handle a list action for non-self objects. - """ - for r_pdu in self.serve_fetch_all(): - self.make_reply(r_pdu) - r_msg.append(r_pdu) - cb() - - def serve_destroy_hook(self, cb, eb): - """ - Overridable hook. - """ - cb() - - def serve_destroy(self, r_msg, cb, eb): - """ - Handle a destroy action. - """ - def done(): - db_pdu.sql_delete() - r_msg.append(self.make_reply()) - cb() - db_pdu = self.serve_fetch_one() - db_pdu.serve_destroy_hook(done, eb) - - def serve_dispatch(self, r_msg, cb, eb): - """ - Action dispatch handler. - """ - dispatch = { "create" : self.serve_create, - "set" : self.serve_set, - "get" : self.serve_get, - "list" : self.serve_list, - "destroy" : self.serve_destroy } - if self.action not in dispatch: - raise rpki.exceptions.BadQuery("Unexpected query: action %s" % self.action) - dispatch[self.action](r_msg, cb, eb) - - def unimplemented_control(self, *controls): - """ - Uniform handling for unimplemented control operations. - """ - unimplemented = [x for x in controls if getattr(self, x, False)] - if unimplemented: - raise rpki.exceptions.NotImplementedYet("Unimplemented control %s" % ", ".join(unimplemented)) - -class msg(list): - """ - Generic top-level PDU. - """ - - def startElement(self, stack, name, attrs): - """ - Handle top-level PDU. - """ - if name == "msg": - assert self.version == int(attrs["version"]) - self.type = attrs["type"] - else: - elt = self.pdus[name]() - self.append(elt) - stack.append(elt) - elt.startElement(stack, name, attrs) - - def endElement(self, stack, name, text): - """ - Handle top-level PDU. - """ - assert name == "msg", "Unexpected name %s, stack %s" % (name, stack) - assert len(stack) == 1 - stack.pop() - - def __str__(self): - """ - Convert msg object to string. - """ - return lxml.etree.tostring(self.toXML(), pretty_print = True, encoding = "us-ascii") - - def toXML(self): - """ - Generate top-level PDU. - """ - elt = lxml.etree.Element(self.xmlns + "msg", nsmap = self.nsmap, version = str(self.version), type = self.type) - elt.extend([i.toXML() for i in self]) - return elt - - @classmethod - def query(cls, *args): - """ - Create a query PDU. - """ - self = cls(args) - self.type = "query" - return self - - @classmethod - def reply(cls, *args): - """ - Create a reply PDU. - """ - self = cls(args) - self.type = "reply" - return self - - def is_query(self): - """ - Is this msg a query? - """ - return self.type == "query" - - def is_reply(self): - """ - Is this msg a reply? - """ - return self.type == "reply" diff --git a/schemas/relaxng/left-right-schema.rnc b/schemas/relaxng/left-right.rnc index 201f8ff0..b5ab56a7 100644 --- a/schemas/relaxng/left-right-schema.rnc +++ b/schemas/relaxng/left-right.rnc @@ -162,8 +162,8 @@ parent_payload = (attribute peer_contact_uri { uri }?, repository_handle?, attribute sender_name { up_down_name }?, attribute recipient_name { up_down_name }?, - element bpki_cms_cert { base64 }?, - element bpki_cms_glue { base64 }?) + element bpki_cert { base64 }?, + element bpki_glue { base64 }?) parent_query |= element parent { ctl_create, self_handle, parent_handle, parent_bool, parent_payload } parent_reply |= element parent { ctl_create, self_handle, parent_handle } @@ -206,6 +206,7 @@ repository_bool = attribute clear_replay_protection { "yes" }? repository_payload = (attribute peer_contact_uri { uri }?, bsc_handle?, + attribute rrdp_notification_uri { uri }?, element bpki_cert { base64 }?, element bpki_glue { base64 }?) @@ -271,7 +272,7 @@ list_ee_certificate_requests_reply = element list_ee_certificate_requests { attribute asn { asn_list }?, attribute ipv4 { ipv4_list }?, attribute ipv6 { ipv6_list }?, - attribute cn { xsd:string { maxLength="64" pattern="[\-0-9A-Za-z_ ]+" } }?, + attribute cn { xsd:string { maxLength="64" pattern="[\-0-9A-Za-z_ ]+" } }, attribute sn { xsd:string { maxLength="64" pattern="[0-9A-Fa-f]+" } }?, attribute eku { xsd:string { maxLength="512000" pattern="[.,0-9]+" } }?, element pkcs10 { base64 } diff --git a/schemas/relaxng/left-right-schema.rng b/schemas/relaxng/left-right.rng index c5596a2f..24804af2 100644 --- a/schemas/relaxng/left-right-schema.rng +++ b/schemas/relaxng/left-right.rng @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <!-- - $Id: left-right-schema.rnc 5902 2014-07-18 16:37:04Z sra $ + $Id: left-right.rnc 5981 2014-10-02 04:54:51Z sra $ RelaxNG schema for RPKI left-right protocol. @@ -546,12 +546,12 @@ </attribute> </optional> <optional> - <element name="bpki_cms_cert"> + <element name="bpki_cert"> <ref name="base64"/> </element> </optional> <optional> - <element name="bpki_cms_glue"> + <element name="bpki_glue"> <ref name="base64"/> </element> </optional> @@ -762,6 +762,11 @@ <ref name="bsc_handle"/> </optional> <optional> + <attribute name="rrdp_notification_uri"> + <ref name="uri"/> + </attribute> + </optional> + <optional> <element name="bpki_cert"> <ref name="base64"/> </element> @@ -961,14 +966,12 @@ <ref name="ipv6_list"/> </attribute> </optional> - <optional> - <attribute name="cn"> - <data type="string"> - <param name="maxLength">64</param> - <param name="pattern">[\-0-9A-Za-z_ ]+</param> - </data> - </attribute> - </optional> + <attribute name="cn"> + <data type="string"> + <param name="maxLength">64</param> + <param name="pattern">[\-0-9A-Za-z_ ]+</param> + </data> + </attribute> <optional> <attribute name="sn"> <data type="string"> diff --git a/schemas/relaxng/myrpki.rng b/schemas/relaxng/myrpki.rng index 8c7473eb..3beafe8f 100644 --- a/schemas/relaxng/myrpki.rng +++ b/schemas/relaxng/myrpki.rng @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <!-- - $Id: myrpki.rnc 5757 2014-04-05 22:42:12Z sra $ + $Id: myrpki.rnc 5876 2014-06-26 19:00:12Z sra $ RelaxNG schema for MyRPKI XML messages. diff --git a/schemas/relaxng/oob-setup.rnc b/schemas/relaxng/oob-setup.rnc new file mode 100644 index 00000000..3bd7a652 --- /dev/null +++ b/schemas/relaxng/oob-setup.rnc @@ -0,0 +1,68 @@ +# $Id: rpki-setup.rnc 3429 2015-10-14 23:46:50Z sra $ + +default namespace = "http://www.hactrn.net/uris/rpki/rpki-setup/" + +version = "1" + +base64 = xsd:base64Binary { maxLength="512000" } +handle = xsd:string { maxLength="255" pattern="[\-_A-Za-z0-9/]*" } +uri = xsd:anyURI { maxLength="4096" } +any = element * { attribute * { text }*, ( any | text )* } + +authorization_token = base64 +bpki_ta = base64 + +start |= element child_request { + attribute version { version }, + attribute child_handle { handle }, + element child_bpki_ta { bpki_ta } +} + +start |= element parent_response { + attribute version { version }, + attribute service_uri { uri }, + attribute child_handle { handle }, + attribute parent_handle { handle }, + element parent_bpki_ta { bpki_ta }, + element offer { empty }?, + element referral { + attribute referrer { handle }, + attribute contact_uri { uri }?, + authorization_token + }* +} + +start |= element publisher_request { + attribute version { version }, + attribute publisher_handle { handle }, + element publisher_bpki_ta { bpki_ta }, + element referral { + attribute referrer { handle }, + authorization_token + }* +} + +start |= element repository_response { + attribute version { version }, + attribute service_uri { uri }, + attribute publisher_handle { handle }, + attribute sia_base { uri }, + attribute rrdp_notification_uri { uri }?, + element repository_bpki_ta { bpki_ta } +} + +start |= element authorization { + attribute version { version }, + attribute authorized_sia_base { uri }, + bpki_ta +} + +start |= element error { + attribute version { version }, + attribute reason { + "syntax-error" | + "authentication-failure" | + "refused" + }, + any? +} diff --git a/schemas/relaxng/oob-setup.rng b/schemas/relaxng/oob-setup.rng new file mode 100644 index 00000000..00278047 --- /dev/null +++ b/schemas/relaxng/oob-setup.rng @@ -0,0 +1,168 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- $Id: rpki-setup.rnc 3429 2015-10-14 23:46:50Z sra $ --> +<grammar ns="http://www.hactrn.net/uris/rpki/rpki-setup/" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes"> + <define name="version"> + <value>1</value> + </define> + <define name="base64"> + <data type="base64Binary"> + <param name="maxLength">512000</param> + </data> + </define> + <define name="handle"> + <data type="string"> + <param name="maxLength">255</param> + <param name="pattern">[\-_A-Za-z0-9/]*</param> + </data> + </define> + <define name="uri"> + <data type="anyURI"> + <param name="maxLength">4096</param> + </data> + </define> + <define name="any"> + <element> + <anyName/> + <zeroOrMore> + <attribute> + <anyName/> + </attribute> + </zeroOrMore> + <zeroOrMore> + <choice> + <ref name="any"/> + <text/> + </choice> + </zeroOrMore> + </element> + </define> + <define name="authorization_token"> + <ref name="base64"/> + </define> + <define name="bpki_ta"> + <ref name="base64"/> + </define> + <start combine="choice"> + <element name="child_request"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="child_handle"> + <ref name="handle"/> + </attribute> + <element name="child_bpki_ta"> + <ref name="bpki_ta"/> + </element> + </element> + </start> + <start combine="choice"> + <element name="parent_response"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="service_uri"> + <ref name="uri"/> + </attribute> + <attribute name="child_handle"> + <ref name="handle"/> + </attribute> + <attribute name="parent_handle"> + <ref name="handle"/> + </attribute> + <element name="parent_bpki_ta"> + <ref name="bpki_ta"/> + </element> + <optional> + <element name="offer"> + <empty/> + </element> + </optional> + <zeroOrMore> + <element name="referral"> + <attribute name="referrer"> + <ref name="handle"/> + </attribute> + <optional> + <attribute name="contact_uri"> + <ref name="uri"/> + </attribute> + </optional> + <ref name="authorization_token"/> + </element> + </zeroOrMore> + </element> + </start> + <start combine="choice"> + <element name="publisher_request"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="publisher_handle"> + <ref name="handle"/> + </attribute> + <element name="publisher_bpki_ta"> + <ref name="bpki_ta"/> + </element> + <zeroOrMore> + <element name="referral"> + <attribute name="referrer"> + <ref name="handle"/> + </attribute> + <ref name="authorization_token"/> + </element> + </zeroOrMore> + </element> + </start> + <start combine="choice"> + <element name="repository_response"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="service_uri"> + <ref name="uri"/> + </attribute> + <attribute name="publisher_handle"> + <ref name="handle"/> + </attribute> + <attribute name="sia_base"> + <ref name="uri"/> + </attribute> + <optional> + <attribute name="rrdp_notification_uri"> + <ref name="uri"/> + </attribute> + </optional> + <element name="repository_bpki_ta"> + <ref name="bpki_ta"/> + </element> + </element> + </start> + <start combine="choice"> + <element name="authorization"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="authorized_sia_base"> + <ref name="uri"/> + </attribute> + <ref name="bpki_ta"/> + </element> + </start> + <start combine="choice"> + <element name="error"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="reason"> + <choice> + <value>syntax-error</value> + <value>authentication-failure</value> + <value>refused</value> + </choice> + </attribute> + <optional> + <ref name="any"/> + </optional> + </element> + </start> +</grammar> diff --git a/schemas/relaxng/publication-schema.rnc b/schemas/relaxng/publication-control.rnc index fdf38c9e..ac59c617 100644 --- a/schemas/relaxng/publication-schema.rnc +++ b/schemas/relaxng/publication-control.rnc @@ -19,7 +19,7 @@ # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -default namespace = "http://www.hactrn.net/uris/rpki/publication-spec/" +default namespace = "http://www.hactrn.net/uris/rpki/publication-control/" version = "1" @@ -32,12 +32,10 @@ start = element msg { } # PDUs allowed in a query -query_elt = ( config_query | client_query | certificate_query | crl_query | - manifest_query | roa_query | ghostbuster_query ) +query_elt = client_query # PDUs allowed in a reply -reply_elt = ( config_reply | client_reply | certificate_reply | crl_reply | - manifest_reply | roa_reply | ghostbuster_reply | report_error_reply ) +reply_elt = ( client_reply | report_error_reply ) # Tag attributes for bulk operations tag = attribute tag { xsd:token {maxLength="1024" } } @@ -58,17 +56,7 @@ uri = attribute uri { uri_t } # hierarchy delimiter. object_handle = xsd:string { maxLength="255" pattern="[\-_A-Za-z0-9/]+" } -# <config/> element (use restricted to repository operator) -# config_handle attribute, create, list, and destroy commands omitted deliberately, see code for details - -config_payload = (element bpki_crl { base64 }?) - -config_query |= element config { attribute action { "set" }, tag?, config_payload } -config_reply |= element config { attribute action { "set" }, tag? } -config_query |= element config { attribute action { "get" }, tag? } -config_reply |= element config { attribute action { "get" }, tag?, config_payload } - -# <client/> element (use restricted to repository operator) +# <client/> element client_handle = attribute client_handle { object_handle } @@ -87,41 +75,6 @@ client_reply |= element client { attribute action { "list" }, tag?, client_ha client_query |= element client { attribute action { "destroy" }, tag?, client_handle } client_reply |= element client { attribute action { "destroy" }, tag?, client_handle } -# <certificate/> element - -certificate_query |= element certificate { attribute action { "publish" }, tag?, uri, base64 } -certificate_reply |= element certificate { attribute action { "publish" }, tag?, uri } -certificate_query |= element certificate { attribute action { "withdraw" }, tag?, uri } -certificate_reply |= element certificate { attribute action { "withdraw" }, tag?, uri } - -# <crl/> element - -crl_query |= element crl { attribute action { "publish" }, tag?, uri, base64 } -crl_reply |= element crl { attribute action { "publish" }, tag?, uri } -crl_query |= element crl { attribute action { "withdraw" }, tag?, uri } -crl_reply |= element crl { attribute action { "withdraw" }, tag?, uri } - -# <manifest/> element - -manifest_query |= element manifest { attribute action { "publish" }, tag?, uri, base64 } -manifest_reply |= element manifest { attribute action { "publish" }, tag?, uri } -manifest_query |= element manifest { attribute action { "withdraw" }, tag?, uri } -manifest_reply |= element manifest { attribute action { "withdraw" }, tag?, uri } - -# <roa/> element - -roa_query |= element roa { attribute action { "publish" }, tag?, uri, base64 } -roa_reply |= element roa { attribute action { "publish" }, tag?, uri } -roa_query |= element roa { attribute action { "withdraw" }, tag?, uri } -roa_reply |= element roa { attribute action { "withdraw" }, tag?, uri } - -# <ghostbuster/> element - -ghostbuster_query |= element ghostbuster { attribute action { "publish" }, tag?, uri, base64 } -ghostbuster_reply |= element ghostbuster { attribute action { "publish" }, tag?, uri } -ghostbuster_query |= element ghostbuster { attribute action { "withdraw" }, tag?, uri } -ghostbuster_reply |= element ghostbuster { attribute action { "withdraw" }, tag?, uri } - # <report_error/> element error = xsd:token { maxLength="1024" } diff --git a/schemas/relaxng/publication-control.rng b/schemas/relaxng/publication-control.rng new file mode 100644 index 00000000..606deb53 --- /dev/null +++ b/schemas/relaxng/publication-control.rng @@ -0,0 +1,280 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + $Id: publication-control.rnc 5903 2014-07-18 17:08:13Z sra $ + + RelaxNG schema for RPKI publication protocol. + + Copyright (C) 2012- -2014 Dragon Research Labs ("DRL") + Portions copyright (C) 2009- -2011 Internet Systems Consortium ("ISC") + Portions copyright (C) 2007- -2008 American Registry for Internet Numbers ("ARIN") + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notices and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND DRL, ISC, AND ARIN DISCLAIM ALL + WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL, + ISC, OR ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR + CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, + NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION + WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +--> +<grammar ns="http://www.hactrn.net/uris/rpki/publication-control/" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes"> + <define name="version"> + <value>1</value> + </define> + <!-- Top level PDU --> + <start> + <element name="msg"> + <attribute name="version"> + <data type="positiveInteger"> + <param name="maxInclusive">1</param> + </data> + </attribute> + <choice> + <group> + <attribute name="type"> + <value>query</value> + </attribute> + <zeroOrMore> + <ref name="query_elt"/> + </zeroOrMore> + </group> + <group> + <attribute name="type"> + <value>reply</value> + </attribute> + <zeroOrMore> + <ref name="reply_elt"/> + </zeroOrMore> + </group> + </choice> + </element> + </start> + <!-- PDUs allowed in a query --> + <define name="query_elt"> + <ref name="client_query"/> + </define> + <!-- PDUs allowed in a reply --> + <define name="reply_elt"> + <choice> + <ref name="client_reply"/> + <ref name="report_error_reply"/> + </choice> + </define> + <!-- Tag attributes for bulk operations --> + <define name="tag"> + <attribute name="tag"> + <data type="token"> + <param name="maxLength">1024</param> + </data> + </attribute> + </define> + <!-- + Base64 encoded DER stuff + base64 = xsd:base64Binary { maxLength="512000" } + + Sadly, it turns out that CRLs can in fact get longer than this for an active CA. + Remove length limit for now, think about whether to put it back later. + --> + <define name="base64"> + <data type="base64Binary"/> + </define> + <!-- Publication URLs --> + <define name="uri_t"> + <data type="anyURI"> + <param name="maxLength">4096</param> + </data> + </define> + <define name="uri"> + <attribute name="uri"> + <ref name="uri_t"/> + </attribute> + </define> + <!-- + Handles on remote objects (replaces passing raw SQL IDs). NB: + Unlike the up-down protocol, handles in this protocol allow "/" as a + hierarchy delimiter. + --> + <define name="object_handle"> + <data type="string"> + <param name="maxLength">255</param> + <param name="pattern">[\-_A-Za-z0-9/]+</param> + </data> + </define> + <!-- <client/> element --> + <define name="client_handle"> + <attribute name="client_handle"> + <ref name="object_handle"/> + </attribute> + </define> + <define name="client_bool"> + <optional> + <attribute name="clear_replay_protection"> + <value>yes</value> + </attribute> + </optional> + </define> + <define name="client_payload"> + <optional> + <attribute name="base_uri"> + <ref name="uri_t"/> + </attribute> + </optional> + <optional> + <element name="bpki_cert"> + <ref name="base64"/> + </element> + </optional> + <optional> + <element name="bpki_glue"> + <ref name="base64"/> + </element> + </optional> + </define> + <define name="client_query" combine="choice"> + <element name="client"> + <attribute name="action"> + <value>create</value> + </attribute> + <optional> + <ref name="tag"/> + </optional> + <ref name="client_handle"/> + <ref name="client_bool"/> + <ref name="client_payload"/> + </element> + </define> + <define name="client_reply" combine="choice"> + <element name="client"> + <attribute name="action"> + <value>create</value> + </attribute> + <optional> + <ref name="tag"/> + </optional> + <ref name="client_handle"/> + </element> + </define> + <define name="client_query" combine="choice"> + <element name="client"> + <attribute name="action"> + <value>set</value> + </attribute> + <optional> + <ref name="tag"/> + </optional> + <ref name="client_handle"/> + <ref name="client_bool"/> + <ref name="client_payload"/> + </element> + </define> + <define name="client_reply" combine="choice"> + <element name="client"> + <attribute name="action"> + <value>set</value> + </attribute> + <optional> + <ref name="tag"/> + </optional> + <ref name="client_handle"/> + </element> + </define> + <define name="client_query" combine="choice"> + <element name="client"> + <attribute name="action"> + <value>get</value> + </attribute> + <optional> + <ref name="tag"/> + </optional> + <ref name="client_handle"/> + </element> + </define> + <define name="client_reply" combine="choice"> + <element name="client"> + <attribute name="action"> + <value>get</value> + </attribute> + <optional> + <ref name="tag"/> + </optional> + <ref name="client_handle"/> + <ref name="client_payload"/> + </element> + </define> + <define name="client_query" combine="choice"> + <element name="client"> + <attribute name="action"> + <value>list</value> + </attribute> + <optional> + <ref name="tag"/> + </optional> + </element> + </define> + <define name="client_reply" combine="choice"> + <element name="client"> + <attribute name="action"> + <value>list</value> + </attribute> + <optional> + <ref name="tag"/> + </optional> + <ref name="client_handle"/> + <ref name="client_payload"/> + </element> + </define> + <define name="client_query" combine="choice"> + <element name="client"> + <attribute name="action"> + <value>destroy</value> + </attribute> + <optional> + <ref name="tag"/> + </optional> + <ref name="client_handle"/> + </element> + </define> + <define name="client_reply" combine="choice"> + <element name="client"> + <attribute name="action"> + <value>destroy</value> + </attribute> + <optional> + <ref name="tag"/> + </optional> + <ref name="client_handle"/> + </element> + </define> + <!-- <report_error/> element --> + <define name="error"> + <data type="token"> + <param name="maxLength">1024</param> + </data> + </define> + <define name="report_error_reply"> + <element name="report_error"> + <optional> + <ref name="tag"/> + </optional> + <attribute name="error_code"> + <ref name="error"/> + </attribute> + <optional> + <data type="string"> + <param name="maxLength">512000</param> + </data> + </optional> + </element> + </define> +</grammar> +<!-- + Local Variables: + indent-tabs-mode: nil + comment-start: "# " + comment-start-skip: "#[ \t]*" + End: +--> diff --git a/schemas/relaxng/publication-schema.rng b/schemas/relaxng/publication-schema.rng deleted file mode 100644 index 482fa477..00000000 --- a/schemas/relaxng/publication-schema.rng +++ /dev/null @@ -1,577 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - $Id: publication-schema.rnc 5902 2014-07-18 16:37:04Z sra $ - - RelaxNG schema for RPKI publication protocol. - - Copyright (C) 2012- -2014 Dragon Research Labs ("DRL") - Portions copyright (C) 2009- -2011 Internet Systems Consortium ("ISC") - Portions copyright (C) 2007- -2008 American Registry for Internet Numbers ("ARIN") - - Permission to use, copy, modify, and distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notices and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND DRL, ISC, AND ARIN DISCLAIM ALL - WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL, - ISC, OR ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR - CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS - OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, - NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION - WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ---> -<grammar ns="http://www.hactrn.net/uris/rpki/publication-spec/" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes"> - <define name="version"> - <value>1</value> - </define> - <!-- Top level PDU --> - <start> - <element name="msg"> - <attribute name="version"> - <data type="positiveInteger"> - <param name="maxInclusive">1</param> - </data> - </attribute> - <choice> - <group> - <attribute name="type"> - <value>query</value> - </attribute> - <zeroOrMore> - <ref name="query_elt"/> - </zeroOrMore> - </group> - <group> - <attribute name="type"> - <value>reply</value> - </attribute> - <zeroOrMore> - <ref name="reply_elt"/> - </zeroOrMore> - </group> - </choice> - </element> - </start> - <!-- PDUs allowed in a query --> - <define name="query_elt"> - <choice> - <ref name="config_query"/> - <ref name="client_query"/> - <ref name="certificate_query"/> - <ref name="crl_query"/> - <ref name="manifest_query"/> - <ref name="roa_query"/> - <ref name="ghostbuster_query"/> - </choice> - </define> - <!-- PDUs allowed in a reply --> - <define name="reply_elt"> - <choice> - <ref name="config_reply"/> - <ref name="client_reply"/> - <ref name="certificate_reply"/> - <ref name="crl_reply"/> - <ref name="manifest_reply"/> - <ref name="roa_reply"/> - <ref name="ghostbuster_reply"/> - <ref name="report_error_reply"/> - </choice> - </define> - <!-- Tag attributes for bulk operations --> - <define name="tag"> - <attribute name="tag"> - <data type="token"> - <param name="maxLength">1024</param> - </data> - </attribute> - </define> - <!-- - Base64 encoded DER stuff - base64 = xsd:base64Binary { maxLength="512000" } - - Sadly, it turns out that CRLs can in fact get longer than this for an active CA. - Remove length limit for now, think about whether to put it back later. - --> - <define name="base64"> - <data type="base64Binary"/> - </define> - <!-- Publication URLs --> - <define name="uri_t"> - <data type="anyURI"> - <param name="maxLength">4096</param> - </data> - </define> - <define name="uri"> - <attribute name="uri"> - <ref name="uri_t"/> - </attribute> - </define> - <!-- - Handles on remote objects (replaces passing raw SQL IDs). NB: - Unlike the up-down protocol, handles in this protocol allow "/" as a - hierarchy delimiter. - --> - <define name="object_handle"> - <data type="string"> - <param name="maxLength">255</param> - <param name="pattern">[\-_A-Za-z0-9/]+</param> - </data> - </define> - <!-- - <config/> element (use restricted to repository operator) - config_handle attribute, create, list, and destroy commands omitted deliberately, see code for details - --> - <define name="config_payload"> - <optional> - <element name="bpki_crl"> - <ref name="base64"/> - </element> - </optional> - </define> - <define name="config_query" combine="choice"> - <element name="config"> - <attribute name="action"> - <value>set</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="config_payload"/> - </element> - </define> - <define name="config_reply" combine="choice"> - <element name="config"> - <attribute name="action"> - <value>set</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - </element> - </define> - <define name="config_query" combine="choice"> - <element name="config"> - <attribute name="action"> - <value>get</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - </element> - </define> - <define name="config_reply" combine="choice"> - <element name="config"> - <attribute name="action"> - <value>get</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="config_payload"/> - </element> - </define> - <!-- <client/> element (use restricted to repository operator) --> - <define name="client_handle"> - <attribute name="client_handle"> - <ref name="object_handle"/> - </attribute> - </define> - <define name="client_bool"> - <optional> - <attribute name="clear_replay_protection"> - <value>yes</value> - </attribute> - </optional> - </define> - <define name="client_payload"> - <optional> - <attribute name="base_uri"> - <ref name="uri_t"/> - </attribute> - </optional> - <optional> - <element name="bpki_cert"> - <ref name="base64"/> - </element> - </optional> - <optional> - <element name="bpki_glue"> - <ref name="base64"/> - </element> - </optional> - </define> - <define name="client_query" combine="choice"> - <element name="client"> - <attribute name="action"> - <value>create</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="client_handle"/> - <ref name="client_bool"/> - <ref name="client_payload"/> - </element> - </define> - <define name="client_reply" combine="choice"> - <element name="client"> - <attribute name="action"> - <value>create</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="client_handle"/> - </element> - </define> - <define name="client_query" combine="choice"> - <element name="client"> - <attribute name="action"> - <value>set</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="client_handle"/> - <ref name="client_bool"/> - <ref name="client_payload"/> - </element> - </define> - <define name="client_reply" combine="choice"> - <element name="client"> - <attribute name="action"> - <value>set</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="client_handle"/> - </element> - </define> - <define name="client_query" combine="choice"> - <element name="client"> - <attribute name="action"> - <value>get</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="client_handle"/> - </element> - </define> - <define name="client_reply" combine="choice"> - <element name="client"> - <attribute name="action"> - <value>get</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="client_handle"/> - <ref name="client_payload"/> - </element> - </define> - <define name="client_query" combine="choice"> - <element name="client"> - <attribute name="action"> - <value>list</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - </element> - </define> - <define name="client_reply" combine="choice"> - <element name="client"> - <attribute name="action"> - <value>list</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="client_handle"/> - <ref name="client_payload"/> - </element> - </define> - <define name="client_query" combine="choice"> - <element name="client"> - <attribute name="action"> - <value>destroy</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="client_handle"/> - </element> - </define> - <define name="client_reply" combine="choice"> - <element name="client"> - <attribute name="action"> - <value>destroy</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="client_handle"/> - </element> - </define> - <!-- <certificate/> element --> - <define name="certificate_query" combine="choice"> - <element name="certificate"> - <attribute name="action"> - <value>publish</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - <ref name="base64"/> - </element> - </define> - <define name="certificate_reply" combine="choice"> - <element name="certificate"> - <attribute name="action"> - <value>publish</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> - </define> - <define name="certificate_query" combine="choice"> - <element name="certificate"> - <attribute name="action"> - <value>withdraw</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> - </define> - <define name="certificate_reply" combine="choice"> - <element name="certificate"> - <attribute name="action"> - <value>withdraw</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> - </define> - <!-- <crl/> element --> - <define name="crl_query" combine="choice"> - <element name="crl"> - <attribute name="action"> - <value>publish</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - <ref name="base64"/> - </element> - </define> - <define name="crl_reply" combine="choice"> - <element name="crl"> - <attribute name="action"> - <value>publish</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> - </define> - <define name="crl_query" combine="choice"> - <element name="crl"> - <attribute name="action"> - <value>withdraw</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> - </define> - <define name="crl_reply" combine="choice"> - <element name="crl"> - <attribute name="action"> - <value>withdraw</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> - </define> - <!-- <manifest/> element --> - <define name="manifest_query" combine="choice"> - <element name="manifest"> - <attribute name="action"> - <value>publish</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - <ref name="base64"/> - </element> - </define> - <define name="manifest_reply" combine="choice"> - <element name="manifest"> - <attribute name="action"> - <value>publish</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> - </define> - <define name="manifest_query" combine="choice"> - <element name="manifest"> - <attribute name="action"> - <value>withdraw</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> - </define> - <define name="manifest_reply" combine="choice"> - <element name="manifest"> - <attribute name="action"> - <value>withdraw</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> - </define> - <!-- <roa/> element --> - <define name="roa_query" combine="choice"> - <element name="roa"> - <attribute name="action"> - <value>publish</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - <ref name="base64"/> - </element> - </define> - <define name="roa_reply" combine="choice"> - <element name="roa"> - <attribute name="action"> - <value>publish</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> - </define> - <define name="roa_query" combine="choice"> - <element name="roa"> - <attribute name="action"> - <value>withdraw</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> - </define> - <define name="roa_reply" combine="choice"> - <element name="roa"> - <attribute name="action"> - <value>withdraw</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> - </define> - <!-- <ghostbuster/> element --> - <define name="ghostbuster_query" combine="choice"> - <element name="ghostbuster"> - <attribute name="action"> - <value>publish</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - <ref name="base64"/> - </element> - </define> - <define name="ghostbuster_reply" combine="choice"> - <element name="ghostbuster"> - <attribute name="action"> - <value>publish</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> - </define> - <define name="ghostbuster_query" combine="choice"> - <element name="ghostbuster"> - <attribute name="action"> - <value>withdraw</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> - </define> - <define name="ghostbuster_reply" combine="choice"> - <element name="ghostbuster"> - <attribute name="action"> - <value>withdraw</value> - </attribute> - <optional> - <ref name="tag"/> - </optional> - <ref name="uri"/> - </element> - </define> - <!-- <report_error/> element --> - <define name="error"> - <data type="token"> - <param name="maxLength">1024</param> - </data> - </define> - <define name="report_error_reply"> - <element name="report_error"> - <optional> - <ref name="tag"/> - </optional> - <attribute name="error_code"> - <ref name="error"/> - </attribute> - <optional> - <data type="string"> - <param name="maxLength">512000</param> - </data> - </optional> - </element> - </define> -</grammar> -<!-- - Local Variables: - indent-tabs-mode: nil - comment-start: "# " - comment-start-skip: "#[ \t]*" - End: ---> diff --git a/schemas/relaxng/publication.rnc b/schemas/relaxng/publication.rnc new file mode 100644 index 00000000..f3d1f94e --- /dev/null +++ b/schemas/relaxng/publication.rnc @@ -0,0 +1,111 @@ +# $Id$ +# +# RelaxNG schema for RPKI publication protocol, from current I-D. +# +# Copyright (c) 2014 IETF Trust and the persons identified as authors +# of the code. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# * Neither the name of Internet Society, IETF or IETF Trust, nor the +# names of specific contributors, may be used to endorse or promote +# products derived from this software without specific prior written +# permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +default namespace = + "http://www.hactrn.net/uris/rpki/publication-spec/" + +# This is version 3 of the protocol. + +version = "3" + +# Top level PDU is either a query or a reply. + +start |= element msg { + attribute version { version }, + attribute type { "query" }, + query_elt* +} + +start |= element msg { + attribute version { version }, + attribute type { "reply" }, + reply_elt* +} + +# PDUs allowed in queries and replies. + +query_elt = publish_query | withdraw_query | list_query +reply_elt = publish_reply | withdraw_reply | list_reply | report_error_reply + +# Tag attributes for bulk operations. + +tag = attribute tag { xsd:token { maxLength="1024" } } + +# Base64 encoded DER stuff. + +base64 = xsd:base64Binary + +# Publication URIs. + +uri = attribute uri { xsd:anyURI { maxLength="4096" } } + +# Digest of objects being withdrawn + +hash = attribute hash { xsd:string { pattern = "[0-9a-fA-F]+" } } + +# Error codes. + +error = xsd:token { maxLength="1024" } + +# <publish/> element + +publish_query = element publish { tag?, uri, hash?, base64 } +publish_reply = element publish { tag?, uri } + +# <withdraw/> element + +withdraw_query = element withdraw { tag?, uri, hash } +withdraw_reply = element withdraw { tag?, uri } + +# <list/> element + +list_query = element list { tag? } +list_reply = element list { tag?, uri, hash } + +# <report_error/> element + +report_error_reply = element report_error { + tag?, + attribute error_code { error }, + xsd:string { maxLength="512000" }? +} + +# Local Variables: +# indent-tabs-mode: nil +# comment-start: "# " +# comment-start-skip: "#[ \t]*" +# End: diff --git a/schemas/relaxng/publication.rng b/schemas/relaxng/publication.rng new file mode 100644 index 00000000..5e72407e --- /dev/null +++ b/schemas/relaxng/publication.rng @@ -0,0 +1,201 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + $Id: publication.rnc 5896 2014-07-15 19:34:32Z sra $ + + RelaxNG schema for RPKI publication protocol, from current I-D. + + Copyright (c) 2014 IETF Trust and the persons identified as authors + of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Internet Society, IETF or IETF Trust, nor the + names of specific contributors, may be used to endorse or promote + products derived from this software without specific prior written + permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. +--> +<grammar ns="http://www.hactrn.net/uris/rpki/publication-spec/" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes"> + <!-- This is version 3 of the protocol. --> + <define name="version"> + <value>3</value> + </define> + <!-- Top level PDU is either a query or a reply. --> + <start combine="choice"> + <element name="msg"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="type"> + <value>query</value> + </attribute> + <zeroOrMore> + <ref name="query_elt"/> + </zeroOrMore> + </element> + </start> + <start combine="choice"> + <element name="msg"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="type"> + <value>reply</value> + </attribute> + <zeroOrMore> + <ref name="reply_elt"/> + </zeroOrMore> + </element> + </start> + <!-- PDUs allowed in queries and replies. --> + <define name="query_elt"> + <choice> + <ref name="publish_query"/> + <ref name="withdraw_query"/> + <ref name="list_query"/> + </choice> + </define> + <define name="reply_elt"> + <choice> + <ref name="publish_reply"/> + <ref name="withdraw_reply"/> + <ref name="list_reply"/> + <ref name="report_error_reply"/> + </choice> + </define> + <!-- Tag attributes for bulk operations. --> + <define name="tag"> + <attribute name="tag"> + <data type="token"> + <param name="maxLength">1024</param> + </data> + </attribute> + </define> + <!-- Base64 encoded DER stuff. --> + <define name="base64"> + <data type="base64Binary"/> + </define> + <!-- Publication URIs. --> + <define name="uri"> + <attribute name="uri"> + <data type="anyURI"> + <param name="maxLength">4096</param> + </data> + </attribute> + </define> + <!-- Digest of objects being withdrawn --> + <define name="hash"> + <attribute name="hash"> + <data type="string"> + <param name="pattern">[0-9a-fA-F]+</param> + </data> + </attribute> + </define> + <!-- Error codes. --> + <define name="error"> + <data type="token"> + <param name="maxLength">1024</param> + </data> + </define> + <!-- <publish/> element --> + <define name="publish_query"> + <element name="publish"> + <optional> + <ref name="tag"/> + </optional> + <ref name="uri"/> + <optional> + <ref name="hash"/> + </optional> + <ref name="base64"/> + </element> + </define> + <define name="publish_reply"> + <element name="publish"> + <optional> + <ref name="tag"/> + </optional> + <ref name="uri"/> + </element> + </define> + <!-- <withdraw/> element --> + <define name="withdraw_query"> + <element name="withdraw"> + <optional> + <ref name="tag"/> + </optional> + <ref name="uri"/> + <ref name="hash"/> + </element> + </define> + <define name="withdraw_reply"> + <element name="withdraw"> + <optional> + <ref name="tag"/> + </optional> + <ref name="uri"/> + </element> + </define> + <!-- <list/> element --> + <define name="list_query"> + <element name="list"> + <optional> + <ref name="tag"/> + </optional> + </element> + </define> + <define name="list_reply"> + <element name="list"> + <optional> + <ref name="tag"/> + </optional> + <ref name="uri"/> + <ref name="hash"/> + </element> + </define> + <!-- <report_error/> element --> + <define name="report_error_reply"> + <element name="report_error"> + <optional> + <ref name="tag"/> + </optional> + <attribute name="error_code"> + <ref name="error"/> + </attribute> + <optional> + <data type="string"> + <param name="maxLength">512000</param> + </data> + </optional> + </element> + </define> +</grammar> +<!-- + Local Variables: + indent-tabs-mode: nil + comment-start: "# " + comment-start-skip: "#[ \t]*" + End: +--> diff --git a/schemas/relaxng/router-certificate-schema.rnc b/schemas/relaxng/router-certificate.rnc index 8cc325ce..8cc325ce 100644 --- a/schemas/relaxng/router-certificate-schema.rnc +++ b/schemas/relaxng/router-certificate.rnc diff --git a/schemas/relaxng/router-certificate-schema.rng b/schemas/relaxng/router-certificate.rng index 90b50107..9352ed76 100644 --- a/schemas/relaxng/router-certificate-schema.rng +++ b/schemas/relaxng/router-certificate.rng @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <!-- - $Id: router-certificate-schema.rnc 5757 2014-04-05 22:42:12Z sra $ + $Id: router-certificate.rnc 5881 2014-07-03 16:55:02Z sra $ RelaxNG schema for BGPSEC router certificate interchange format. diff --git a/schemas/relaxng/rrdp.rnc b/schemas/relaxng/rrdp.rnc new file mode 100644 index 00000000..7809abdd --- /dev/null +++ b/schemas/relaxng/rrdp.rnc @@ -0,0 +1,81 @@ +# $Id$ +# +# RelaxNG schema for RPKI Repository Delta Protocol (RRDP). +# +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +default namespace = "http://www.ripe.net/rpki/rrdp" + +version = xsd:positiveInteger { maxInclusive="1" } +serial = xsd:nonNegativeInteger +uri = xsd:anyURI +uuid = xsd:string { pattern = "[\-0-9a-fA-F]+" } +hash = xsd:string { pattern = "[0-9a-fA-F]+" } +base64 = xsd:base64Binary + +# Notification file: lists current snapshots and deltas + +start |= element notification { + attribute version { version }, + attribute session_id { uuid }, + attribute serial { serial }, + element snapshot { + attribute uri { uri }, + attribute hash { hash } + }, + element delta { + attribute serial { serial }, + attribute uri { uri }, + attribute hash { hash } + }* +} + +# Snapshot segment: think DNS AXFR. + +start |= element snapshot { + attribute version { version }, + attribute session_id { uuid }, + attribute serial { serial }, + element publish { + attribute uri { uri }, + base64 + }* +} + +# Delta segment: think DNS IXFR. + +start |= element delta { + attribute version { version }, + attribute session_id { uuid }, + attribute serial { serial }, + delta_element+ +} + +delta_element |= element publish { + attribute uri { uri }, + attribute hash { hash }?, + base64 +} + +delta_element |= element withdraw { + attribute uri { uri }, + attribute hash { hash } +} + +# Local Variables: +# indent-tabs-mode: nil +# comment-start: "# " +# comment-start-skip: "#[ \t]*" +# End: diff --git a/schemas/relaxng/rrdp.rng b/schemas/relaxng/rrdp.rng new file mode 100644 index 00000000..7d2fde9c --- /dev/null +++ b/schemas/relaxng/rrdp.rng @@ -0,0 +1,150 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + $Id: rrdp.rnc 6010 2014-11-08 18:01:58Z sra $ + + RelaxNG schema for RPKI Repository Delta Protocol (RRDP). + + Copyright (C) 2014 Dragon Research Labs ("DRL") + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH + REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, + INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + PERFORMANCE OF THIS SOFTWARE. +--> +<grammar ns="http://www.ripe.net/rpki/rrdp" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes"> + <define name="version"> + <data type="positiveInteger"> + <param name="maxInclusive">1</param> + </data> + </define> + <define name="serial"> + <data type="nonNegativeInteger"/> + </define> + <define name="uri"> + <data type="anyURI"/> + </define> + <define name="uuid"> + <data type="string"> + <param name="pattern">[\-0-9a-fA-F]+</param> + </data> + </define> + <define name="hash"> + <data type="string"> + <param name="pattern">[0-9a-fA-F]+</param> + </data> + </define> + <define name="base64"> + <data type="base64Binary"/> + </define> + <!-- Notification file: lists current snapshots and deltas --> + <start combine="choice"> + <element name="notification"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="session_id"> + <ref name="uuid"/> + </attribute> + <attribute name="serial"> + <ref name="serial"/> + </attribute> + <element name="snapshot"> + <attribute name="uri"> + <ref name="uri"/> + </attribute> + <attribute name="hash"> + <ref name="hash"/> + </attribute> + </element> + <zeroOrMore> + <element name="delta"> + <attribute name="serial"> + <ref name="serial"/> + </attribute> + <attribute name="uri"> + <ref name="uri"/> + </attribute> + <attribute name="hash"> + <ref name="hash"/> + </attribute> + </element> + </zeroOrMore> + </element> + </start> + <!-- Snapshot segment: think DNS AXFR. --> + <start combine="choice"> + <element name="snapshot"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="session_id"> + <ref name="uuid"/> + </attribute> + <attribute name="serial"> + <ref name="serial"/> + </attribute> + <zeroOrMore> + <element name="publish"> + <attribute name="uri"> + <ref name="uri"/> + </attribute> + <ref name="base64"/> + </element> + </zeroOrMore> + </element> + </start> + <!-- Delta segment: think DNS IXFR. --> + <start combine="choice"> + <element name="delta"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="session_id"> + <ref name="uuid"/> + </attribute> + <attribute name="serial"> + <ref name="serial"/> + </attribute> + <oneOrMore> + <ref name="delta_element"/> + </oneOrMore> + </element> + </start> + <define name="delta_element" combine="choice"> + <element name="publish"> + <attribute name="uri"> + <ref name="uri"/> + </attribute> + <optional> + <attribute name="hash"> + <ref name="hash"/> + </attribute> + </optional> + <ref name="base64"/> + </element> + </define> + <define name="delta_element" combine="choice"> + <element name="withdraw"> + <attribute name="uri"> + <ref name="uri"/> + </attribute> + <attribute name="hash"> + <ref name="hash"/> + </attribute> + </element> + </define> +</grammar> +<!-- + Local Variables: + indent-tabs-mode: nil + comment-start: "# " + comment-start-skip: "#[ \t]*" + End: +--> diff --git a/schemas/relaxng/up-down-schema.rnc b/schemas/relaxng/up-down.rnc index a603b8fe..a603b8fe 100644 --- a/schemas/relaxng/up-down-schema.rnc +++ b/schemas/relaxng/up-down.rnc diff --git a/schemas/relaxng/up-down-schema.rng b/schemas/relaxng/up-down.rng index 89235b7e..a0fc0514 100644 --- a/schemas/relaxng/up-down-schema.rng +++ b/schemas/relaxng/up-down.rng @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <!-- - $Id: up-down-schema.rnc 5757 2014-04-05 22:42:12Z sra $ + $Id: up-down.rnc 5881 2014-07-03 16:55:02Z sra $ RelaxNG schema for the up-down protocol, extracted from RFC 6492. diff --git a/schemas/sql/pubd.sql b/schemas/sql/pubd.sql index 3a58ec00..2a0e2851 100644 --- a/schemas/sql/pubd.sql +++ b/schemas/sql/pubd.sql @@ -1,47 +1,36 @@ -- $Id$ --- Copyright (C) 2009--2010 Internet Systems Consortium ("ISC") +-- Copyright (C) 2012--2014 Dragon Research Labs ("DRL") +-- Portions copyright (C) 2009--2010 Internet Systems Consortium ("ISC") +-- Portions copyright (C) 2008 American Registry for Internet Numbers ("ARIN") -- -- Permission to use, copy, modify, and distribute this software for any -- purpose with or without fee is hereby granted, provided that the above --- copyright notice and this permission notice appear in all copies. +-- copyright notices and this permission notice appear in all copies. -- --- THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH --- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY --- AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, --- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM --- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE --- OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR --- PERFORMANCE OF THIS SOFTWARE. - --- Copyright (C) 2008 American Registry for Internet Numbers ("ARIN") --- --- Permission to use, copy, modify, and distribute this software for any --- purpose with or without fee is hereby granted, provided that the above --- copyright notice and this permission notice appear in all copies. --- --- THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH --- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY --- AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, --- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM --- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE --- OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR --- PERFORMANCE OF THIS SOFTWARE. +-- THE SOFTWARE IS PROVIDED "AS IS" AND DRL, ISC, AND ARIN DISCLAIM ALL +-- WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +-- WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL, +-- ISC, OR ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +-- CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +-- OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +-- NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION +-- WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -- SQL objects needed by pubd.py. --- The config table is weird because we're really only using it --- to store one BPKI CRL, but putting this here lets us use a lot of --- existing machinery and the alternatives are whacky in other ways. +-- Old tables that should just be flushed if present at all. -DROP TABLE IF EXISTS client; DROP TABLE IF EXISTS config; +DROP TABLE IF EXISTS snapshot; -CREATE TABLE config ( - config_id SERIAL NOT NULL, - bpki_crl LONGBLOB, - PRIMARY KEY (config_id) -) ENGINE=InnoDB; +-- DROP TABLE commands must be in correct (reverse dependency) order +-- to satisfy FOREIGN KEY constraints. + +DROP TABLE IF EXISTS object; +DROP TABLE IF EXISTS delta; +DROP TABLE IF EXISTS session; +DROP TABLE IF EXISTS client; CREATE TABLE client ( client_id SERIAL NOT NULL, @@ -54,6 +43,43 @@ CREATE TABLE client ( UNIQUE (client_handle) ) ENGINE=InnoDB; +CREATE TABLE session ( + session_id SERIAL NOT NULL, + uuid VARCHAR(36) NOT NULL, + serial BIGINT UNSIGNED NOT NULL, + snapshot LONGTEXT, + hash CHAR(64), + PRIMARY KEY (session_id), + UNIQUE (uuid) +) ENGINE=InnoDB; + +CREATE TABLE delta ( + delta_id SERIAL NOT NULL, + serial BIGINT UNSIGNED NOT NULL, + xml LONGTEXT NOT NULL, + hash CHAR(64) NOT NULL, + expires DATETIME NOT NULL, + session_id BIGINT UNSIGNED NOT NULL, + PRIMARY KEY (delta_id), + CONSTRAINT delta_session_id + FOREIGN KEY (session_id) REFERENCES session (session_id) ON DELETE CASCADE +) ENGINE=InnoDB; + +CREATE TABLE object ( + object_id SERIAL NOT NULL, + uri VARCHAR(255) NOT NULL, + der LONGBLOB NOT NULL, + hash CHAR(64) NOT NULL, + client_id BIGINT UNSIGNED NOT NULL, + session_id BIGINT UNSIGNED NOT NULL, + PRIMARY KEY (object_id), + CONSTRAINT object_client_id + FOREIGN KEY (client_id) REFERENCES client (client_id) ON DELETE CASCADE, + CONSTRAINT object_session_id + FOREIGN KEY (session_id) REFERENCES session (session_id) ON DELETE CASCADE, + UNIQUE (session_id, hash) +) ENGINE=InnoDB; + -- Local Variables: -- indent-tabs-mode: nil -- End: diff --git a/schemas/sql/rpkid.sql b/schemas/sql/rpkid.sql index ad0c39b0..14499091 100644 --- a/schemas/sql/rpkid.sql +++ b/schemas/sql/rpkid.sql @@ -1,32 +1,21 @@ -- $Id$ --- Copyright (C) 2009--2011 Internet Systems Consortium ("ISC") +-- Copyright (C) 2012--2014 Dragon Research Labs ("DRL") +-- Portions copyright (C) 2009--2011 Internet Systems Consortium ("ISC") +-- Portions copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") -- -- Permission to use, copy, modify, and distribute this software for any -- purpose with or without fee is hereby granted, provided that the above --- copyright notice and this permission notice appear in all copies. +-- copyright notices and this permission notice appear in all copies. -- --- THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH --- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY --- AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, --- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM --- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE --- OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR --- PERFORMANCE OF THIS SOFTWARE. - --- Copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") --- --- Permission to use, copy, modify, and distribute this software for any --- purpose with or without fee is hereby granted, provided that the above --- copyright notice and this permission notice appear in all copies. --- --- THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH --- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY --- AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, --- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM --- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE --- OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR --- PERFORMANCE OF THIS SOFTWARE. +-- THE SOFTWARE IS PROVIDED "AS IS" AND DRL, ISC, AND ARIN DISCLAIM ALL +-- WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +-- WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL, +-- ISC, OR ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +-- CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +-- OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +-- NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION +-- WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -- SQL objects needed by the RPKI engine (rpkid.py). @@ -78,6 +67,7 @@ CREATE TABLE repository ( repository_id SERIAL NOT NULL, repository_handle VARCHAR(255) NOT NULL, peer_contact_uri TEXT, + rrdp_notification_uri TEXT, bpki_cert LONGBLOB, bpki_glue LONGBLOB, last_cms_timestamp DATETIME, @@ -94,8 +84,8 @@ CREATE TABLE repository ( CREATE TABLE parent ( parent_id SERIAL NOT NULL, parent_handle VARCHAR(255) NOT NULL, - bpki_cms_cert LONGBLOB, - bpki_cms_glue LONGBLOB, + bpki_cert LONGBLOB, + bpki_glue LONGBLOB, peer_contact_uri TEXT, sia_base TEXT, sender_name TEXT, @@ -58,15 +58,17 @@ if autoconf.RP_TARGET == "rp": "rpki.POW", "rpki.rtr", "rpki.irdb", + "rpki.pubdb", + "rpki.rpkidb", "rpki.gui", "rpki.gui.app", "rpki.gui.cacheview", "rpki.gui.api", "rpki.gui.routeview"], - ext_modules = [Extension("rpki.POW._POW", ["ext/POW.c"], - extra_compile_args = autoconf.CFLAGS.split(), - extra_link_args = (autoconf.LDFLAGS + " " + - autoconf.LIBS).split())], + ext_modules = [Extension("rpki.POW._POW", ["ext/POW.c"], + include_dirs = [cflag[2:] for cflag in autoconf.CFLAGS.split() if cflag.startswith("-I")], + extra_compile_args = [cflag for cflag in autoconf.CFLAGS.split() if not cflag.startswith("-I")], + extra_link_args = autoconf.LDFLAGS.split() + autoconf.LIBS.split())], package_data = {"rpki.gui.app" : ["migrations/*.py", "static/*/*", |