From b46deb1417dc3596e9ac9fe2fe8cc0b7f42457e7 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Mon, 26 Oct 2015 06:29:00 +0000 Subject: "Any programmer who fails to comply with the standard naming, formatting, or commenting conventions should be shot. If it so happens that it is inconvenient to shoot him, then he is to be politely requested to recode his program in adherence to the above standard." -- Michael Spier, Digital Equipment Corporation svn path=/branches/tk705/; revision=6152 --- buildtools/build-freebsd-ports.py | 126 +- buildtools/build-ubuntu-ports.py | 4 +- buildtools/defstack.py | 26 +- buildtools/make-rcynic-script.py | 4 +- buildtools/make-relaxng.py | 14 +- buildtools/make-sql-schemas.py | 18 +- buildtools/make-version.py | 38 +- buildtools/pull-doc-from-wiki.py | 188 +- buildtools/pylint.rc | 2 +- buildtools/rpki-pbuilder.py | 24 +- ca/irbe_cli | 422 ++-- ca/irdbd | 4 +- ca/pubd | 4 +- ca/rootd | 4 +- ca/rpki-confgen | 455 ++-- ca/rpki-sql-backup | 14 +- ca/rpki-sql-setup | 378 +-- ca/rpki-start-servers | 36 +- ca/rpkic | 4 +- ca/rpkid | 4 +- ca/rpkigui-apache-conf-gen | 672 ++--- ca/rpkigui-import-routes | 1 - ca/tests/bgpsec-yaml.py | 56 +- ca/tests/myrpki-xml-parse-test.py | 74 +- ca/tests/old_irdbd.py | 4 +- ca/tests/smoketest.py | 2196 ++++++++--------- ca/tests/sql-cleaner.py | 32 +- ca/tests/sql-dumper.py | 20 +- ca/tests/test-rrdp.py | 114 +- ca/tests/testpoke.py | 118 +- ca/tests/xml-parse-test.py | 108 +- ca/tests/yamlconf.py | 1426 +++++------ ca/tests/yamltest.py | 1606 ++++++------ potpourri/analyze-rcynic-history.py | 428 ++-- potpourri/analyze-transition.py | 102 +- potpourri/apnic-to-csv.py | 38 +- potpourri/arin-to-csv.py | 64 +- potpourri/cross_certify.py | 20 +- potpourri/csvgrep.py | 56 +- potpourri/expand-roa-prefixes.py | 76 +- potpourri/extract-key.py | 18 +- potpourri/fakerootd.py | 11 +- potpourri/format-application-x-rpki.py | 126 +- potpourri/gc_summary.py | 158 +- potpourri/generate-ripe-root-cert.py | 32 +- potpourri/gski.py | 4 +- potpourri/guess-roas-from-routeviews.py | 36 +- potpourri/iana-to-csv.py | 56 +- potpourri/missing-oids.py | 32 +- potpourri/object-dates.py | 38 +- potpourri/print-profile.py | 2 +- potpourri/rcynic-diff.py | 150 +- potpourri/ripe-asns-to-csv.py | 132 +- potpourri/ripe-to-csv.py | 154 +- potpourri/roa-to-irr.py | 236 +- potpourri/rrd-rcynic-history.py | 288 +-- potpourri/rrdp-fetch.py | 28 +- potpourri/show-key-identifiers.py | 60 +- potpourri/show-tracking-data.py | 18 +- potpourri/signed-object-dates.py | 26 +- potpourri/testbed-rootcert.py | 14 +- potpourri/translate-handles.py | 18 +- potpourri/upgrade-add-ghostbusters.py | 32 +- potpourri/validation-status-sql.py | 333 ++- potpourri/whack-ripe-asns.py | 46 +- potpourri/whack-ripe-prefixes.py | 66 +- potpourri/x509-dot.py | 156 +- rp/rcynic/rcynic-cron | 90 +- rp/rcynic/rcynic-html | 932 +++---- rp/rcynic/rcynic-svn | 106 +- rp/rcynic/rcynic-text | 144 +- rp/rcynic/rpki-torrent.py | 1166 ++++----- rp/rcynic/validation_status | 16 +- rp/rpki-rtr/rpki-rtr | 4 +- rp/utils/find_roa | 202 +- rp/utils/hashdir | 50 +- rp/utils/print_roa | 70 +- rp/utils/print_rpki_manifest | 34 +- rp/utils/scan_roas | 44 +- rp/utils/scan_routercerts | 28 +- rp/utils/uri | 70 +- rpki/adns.py | 594 ++--- rpki/cli.py | 402 +-- rpki/config.py | 412 ++-- rpki/csv_utils.py | 156 +- rpki/daemonize.py | 88 +- rpki/django_settings/common.py | 2 +- rpki/exceptions.py | 154 +- rpki/fields.py | 204 +- rpki/gui/app/forms.py | 198 +- rpki/gui/app/views.py | 78 +- rpki/gui/cacheview/tests.py | 1 - rpki/http_simple.py | 150 +- rpki/ipaddrs.py | 142 +- rpki/irdb/models.py | 704 +++--- rpki/irdb/router.py | 128 +- rpki/irdb/zookeeper.py | 2945 +++++++++++----------- rpki/irdbd.py | 360 +-- rpki/left_right.py | 10 +- rpki/log.py | 372 +-- rpki/myrpki.py | 4 +- rpki/mysql_import.py | 8 +- rpki/oids.py | 22 +- rpki/old_irdbd.py | 486 ++-- rpki/pubd.py | 484 ++-- rpki/pubdb/models.py | 476 ++-- rpki/publication.py | 54 +- rpki/publication_control.py | 42 +- rpki/rcynic.py | 434 ++-- rpki/relaxng.py | 76 +- rpki/relaxng_parser.py | 32 +- rpki/resource_set.py | 1968 +++++++-------- rpki/rootd.py | 810 +++--- rpki/rpkic.py | 1433 ++++++----- rpki/rpkid.py | 1218 +++++----- rpki/rpkid_tasks.py | 912 +++---- rpki/rpkidb/models.py | 4060 +++++++++++++++---------------- rpki/rtr/bgpdump.py | 482 ++-- rpki/rtr/channels.py | 364 +-- rpki/rtr/client.py | 816 +++---- rpki/rtr/generator.py | 948 ++++---- rpki/rtr/main.py | 110 +- rpki/rtr/pdus.py | 960 ++++---- rpki/rtr/server.py | 872 +++---- rpki/sundial.py | 456 ++-- rpki/up_down.py | 140 +- rpki/x509.py | 3474 +++++++++++++------------- setup.py | 172 +- setup_extensions.py | 130 +- 129 files changed, 21376 insertions(+), 21373 deletions(-) diff --git a/buildtools/build-freebsd-ports.py b/buildtools/build-freebsd-ports.py index b4031302..bf0b2c47 100644 --- a/buildtools/build-freebsd-ports.py +++ b/buildtools/build-freebsd-ports.py @@ -2,11 +2,11 @@ # # Copyright (C) 2014 Dragon Research Labs ("DRL") # Portions copyright (C) 2012-2013 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notices and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL # WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR @@ -33,9 +33,9 @@ import argparse import subprocess def check_dir(s): - if not os.path.isdir(s): - raise argparse.ArgumentTypeError("%r is not a directory" % s) - return s + if not os.path.isdir(s): + raise argparse.ArgumentTypeError("%r is not a directory" % s) + return s parser = argparse.ArgumentParser(description = __doc__, formatter_class = argparse.ArgumentDefaultsHelpFormatter) @@ -57,15 +57,15 @@ args = parser.parse_args() svnversion = subprocess.check_output(("svnversion", "-c", args.svndir)).strip().split(":")[-1] if args.local_dist: - svnversion = svnversion.translate(None, "M") + svnversion = svnversion.translate(None, "M") if not svnversion.isdigit(): - sys.exit("Sources don't look pristine, not building (%r)" % svnversion) + sys.exit("Sources don't look pristine, not building (%r)" % svnversion) branch = os.path.basename(args.svndir.rstrip(os.path.sep)) if branch != "trunk" and (branch[:2] != "tk" or not branch[2:].isdigit()): - sys.exit("Could not parse branch from working directory name, not building (%r)" % branch) + sys.exit("Could not parse branch from working directory name, not building (%r)" % branch) version = "0." + svnversion tarname = "rpki-%s-r%s" % (branch, svnversion) @@ -74,39 +74,39 @@ tarball = tarname + ".tar.xz" portsdir_old = args.portsdir + ".old" if os.path.isdir(portsdir_old): - shutil.rmtree(portsdir_old) + shutil.rmtree(portsdir_old) if os.path.isdir(args.portsdir): - os.rename(args.portsdir, portsdir_old) + os.rename(args.portsdir, portsdir_old) shutil.copytree(os.path.join(args.svndir, "buildtools", "freebsd-skeleton"), args.portsdir) if args.local_dist: - subprocess.check_call(("svn", "export", args.svndir, os.path.join(args.portsdir, tarname))) - for fn, fmt in (("VERSION", "%s\n"), ("rpki/version.py", "VERSION = \"%s\"\n")): - with open(os.path.join(args.portsdir, tarname, fn), "w") as f: - f.write(fmt % version) - subprocess.check_call(("tar", "cJvvf", tarball, tarname), cwd = args.portsdir) - shutil.rmtree(os.path.join(args.portsdir, tarname)) + subprocess.check_call(("svn", "export", args.svndir, os.path.join(args.portsdir, tarname))) + for fn, fmt in (("VERSION", "%s\n"), ("rpki/version.py", "VERSION = \"%s\"\n")): + with open(os.path.join(args.portsdir, tarname, fn), "w") as f: + f.write(fmt % version) + subprocess.check_call(("tar", "cJvvf", tarball, tarname), cwd = args.portsdir) + shutil.rmtree(os.path.join(args.portsdir, tarname)) elif os.path.exists(os.path.join(portsdir_old, tarball)): - os.link(os.path.join(portsdir_old, tarball), os.path.join(args.portsdir, tarball)) + os.link(os.path.join(portsdir_old, tarball), os.path.join(args.portsdir, tarball)) elif os.path.exists(os.path.join("/usr/ports/distfiles", tarball)): - shutil.copy(os.path.join("/usr/ports/distfiles", tarball), os.path.join(args.portsdir, tarball)) + shutil.copy(os.path.join("/usr/ports/distfiles", tarball), os.path.join(args.portsdir, tarball)) if os.path.isdir(portsdir_old): - shutil.rmtree(portsdir_old) + shutil.rmtree(portsdir_old) if args.make_package or args.local_dist: - pkgdir = os.path.join(args.portsdir, "packages") - os.mkdir(pkgdir) + pkgdir = os.path.join(args.portsdir, "packages") + os.mkdir(pkgdir) py_lib = re.compile(r"^lib/python\d+\.\d+") py_sitelib = re.compile(r"^lib/python\d+\.\d+/site-packages") if args.local_dist: - master_site = "file://" + args.portsdir + "/" + master_site = "file://" + args.portsdir + "/" else: - master_site = "http://download.rpki.net/" + master_site = "http://download.rpki.net/" formatdict = dict(SVNVERSION = svnversion, SVNBRANCH = branch, MASTER_SITE = master_site) @@ -114,43 +114,43 @@ keepdirs = ("usr", "etc", "bin", "var", "lib", "libexec", "sbin", "share", "etc/ for port in ("rpki-rp", "rpki-ca"): - base = os.path.join(args.portsdir, port) - stage = os.path.join(base, "work", "stage") - fn = os.path.join(args.portsdir, port, "Makefile") - with open(fn, "r") as f: - template = f.read() - with open(fn, "w") as f: - f.write(template % formatdict) - - subprocess.check_call(("make", "makesum", "stage", "DISTDIR=" + args.portsdir, "NO_DEPENDS=yes"), - cwd = base) - - with open(os.path.join(base, "pkg-plist"), "w") as f: - usr_local = None - for dirpath, dirnames, filenames in os.walk(stage, topdown = False): - dn = dirpath[len(stage)+1:] - if dn.startswith("usr/local"): - if not usr_local and usr_local is not None: - f.write("@cwd\n") - usr_local = True - dn = dn[len("usr/local/"):] - dn = py_sitelib.sub("%%PYTHON_SITELIBDIR%%", dn) - if dn == "etc/rc.d": - continue - else: - if usr_local: - f.write("@cwd /\n") - usr_local = False - for fn in filenames: - f.write(os.path.join(dn, fn) + "\n") - if dn and dn not in keepdirs and not py_lib.match(dn): - f.write("@dirrm %s\n" % dn) - - if args.make_package or args.local_dist: - subprocess.check_call(("make", "clean", "package", "DISTDIR=" + args.portsdir, "PKGREPOSITORY=" + pkgdir), cwd = base) - - if not args.no_clean: - subprocess.check_call(("make", "clean"), cwd = base) - - if not args.no_tarball and not args.no_clean: - subprocess.check_call(("tar", "czf", "%s-port.tgz" % port, port), cwd = args.portsdir) + base = os.path.join(args.portsdir, port) + stage = os.path.join(base, "work", "stage") + fn = os.path.join(args.portsdir, port, "Makefile") + with open(fn, "r") as f: + template = f.read() + with open(fn, "w") as f: + f.write(template % formatdict) + + subprocess.check_call(("make", "makesum", "stage", "DISTDIR=" + args.portsdir, "NO_DEPENDS=yes"), + cwd = base) + + with open(os.path.join(base, "pkg-plist"), "w") as f: + usr_local = None + for dirpath, dirnames, filenames in os.walk(stage, topdown = False): + dn = dirpath[len(stage)+1:] + if dn.startswith("usr/local"): + if not usr_local and usr_local is not None: + f.write("@cwd\n") + usr_local = True + dn = dn[len("usr/local/"):] + dn = py_sitelib.sub("%%PYTHON_SITELIBDIR%%", dn) + if dn == "etc/rc.d": + continue + else: + if usr_local: + f.write("@cwd /\n") + usr_local = False + for fn in filenames: + f.write(os.path.join(dn, fn) + "\n") + if dn and dn not in keepdirs and not py_lib.match(dn): + f.write("@dirrm %s\n" % dn) + + if args.make_package or args.local_dist: + subprocess.check_call(("make", "clean", "package", "DISTDIR=" + args.portsdir, "PKGREPOSITORY=" + pkgdir), cwd = base) + + if not args.no_clean: + subprocess.check_call(("make", "clean"), cwd = base) + + if not args.no_tarball and not args.no_clean: + subprocess.check_call(("tar", "czf", "%s-port.tgz" % port, port), cwd = args.portsdir) diff --git a/buildtools/build-ubuntu-ports.py b/buildtools/build-ubuntu-ports.py index 0a326da8..19f61f6d 100644 --- a/buildtools/build-ubuntu-ports.py +++ b/buildtools/build-ubuntu-ports.py @@ -2,11 +2,11 @@ # # Copyright (C) 2014 Dragon Research Labs ("DRL") # Portions copyright (C) 2013 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notices and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL # WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR diff --git a/buildtools/defstack.py b/buildtools/defstack.py index 757516f3..b3df0777 100644 --- a/buildtools/defstack.py +++ b/buildtools/defstack.py @@ -8,11 +8,11 @@ # code with code maintained by humans, so "nasty" is a relative term. # # Copyright (C) 2011-2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -68,7 +68,7 @@ template = ''' ''' if len(sys.argv) < 2: - sys.exit("Usage: %s source.c [source.c ...]" % sys.argv[0]) + sys.exit("Usage: %s source.c [source.c ...]" % sys.argv[0]) splitter = re.compile("[() \t]+").split @@ -76,16 +76,16 @@ token = None for line in fileinput.input(): - if token is None: - path = fileinput.filename().split(os.path.sep) - path = os.path.join(path[-2], path[-1]) if len(path) > 1 else path[-1] - token = "".join(c if c.isalnum() else "_" for c in path.upper()) - sys.stdout.write(header.replace("%", token)) + if token is None: + path = fileinput.filename().split(os.path.sep) + path = os.path.join(path[-2], path[-1]) if len(path) > 1 else path[-1] + token = "".join(c if c.isalnum() else "_" for c in path.upper()) + sys.stdout.write(header.replace("%", token)) - if "DECLARE_STACK_OF" in line: - words = splitter(line) - if len(words) > 1 and words[0] == "DECLARE_STACK_OF": - sys.stdout.write(template.replace("%", words[1])) + if "DECLARE_STACK_OF" in line: + words = splitter(line) + if len(words) > 1 and words[0] == "DECLARE_STACK_OF": + sys.stdout.write(template.replace("%", words[1])) if token is not None: - sys.stdout.write(footer.replace("%", token)) + sys.stdout.write(footer.replace("%", token)) diff --git a/buildtools/make-rcynic-script.py b/buildtools/make-rcynic-script.py index 94fb6f32..fdfb3d6b 100644 --- a/buildtools/make-rcynic-script.py +++ b/buildtools/make-rcynic-script.py @@ -24,8 +24,8 @@ sys.stdout.write('''\ ''' % os.environ) for k, v in os.environ.iteritems(): - if k.startswith("AC_") and k != "AC_PYTHON_INTERPRETER": - sys.stdout.write("%s = '''%s'''\n" % (k.lower(), v)) + if k.startswith("AC_") and k != "AC_PYTHON_INTERPRETER": + sys.stdout.write("%s = '''%s'''\n" % (k.lower(), v)) sys.stdout.write('''\ diff --git a/buildtools/make-relaxng.py b/buildtools/make-relaxng.py index d540fa9a..3d239e8a 100644 --- a/buildtools/make-relaxng.py +++ b/buildtools/make-relaxng.py @@ -1,5 +1,5 @@ # $Id$ -# +# # Copyright (C) 2014 Dragon Research Labs ("DRL") # Portions copyright (C) 2009--2012 Internet Systems Consortium ("ISC") # Portions copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") @@ -40,13 +40,13 @@ del RelaxNGParser """ def symbol(s): - for suffix in (".rng", "-schema"): - if s.endswith(suffix): - s = s[:-len(suffix)] - return s.replace("-", "_") + for suffix in (".rng", "-schema"): + if s.endswith(suffix): + s = s[:-len(suffix)] + return s.replace("-", "_") sys.stdout.write(header) for fn in sys.argv[1:]: - with open(fn, "r") as f: - sys.stdout.write(format % dict(name = symbol(fn), rng = f.read())) + with open(fn, "r") as f: + sys.stdout.write(format % dict(name = symbol(fn), rng = f.read())) sys.stdout.write(footer) diff --git a/buildtools/make-sql-schemas.py b/buildtools/make-sql-schemas.py index 0df775c2..051f17e8 100644 --- a/buildtools/make-sql-schemas.py +++ b/buildtools/make-sql-schemas.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2009-2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -13,13 +13,13 @@ # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE # OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR # PERFORMANCE OF THIS SOFTWARE. -# +# # Portions copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -47,6 +47,6 @@ format_2 = """\ print format_1 for name in schemas: - print format_2 % { - "name" : name, - "sql" : open(name + ".sql").read() } + print format_2 % { + "name" : name, + "sql" : open(name + ".sql").read() } diff --git a/buildtools/make-version.py b/buildtools/make-version.py index a73a89ab..09d43801 100644 --- a/buildtools/make-version.py +++ b/buildtools/make-version.py @@ -2,11 +2,11 @@ # $Id$ # Copyright (C) 2013 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -37,33 +37,33 @@ import sys unknown = "Unknown" try: - v = subprocess.Popen(("svnversion", "-c"), stdout = subprocess.PIPE).communicate()[0] - err = None + v = subprocess.Popen(("svnversion", "-c"), stdout = subprocess.PIPE).communicate()[0] + err = None except Exception, e: - v = unknown - err = e + v = unknown + err = e if any(s in v for s in ("Unversioned", "Uncommitted", unknown)): - v = unknown + v = unknown else: - v = "0." + v.strip().split(":")[-1].translate(None, "SMP") + v = "0." + v.strip().split(":")[-1].translate(None, "SMP") try: - old = open("VERSION", "r").read().strip() + old = open("VERSION", "r").read().strip() except: - old = None + old = None if err is not None and (old is None or old == unknown): - sys.stderr.write("Warning: No saved version and svnversion failed: %s\n" % err) + sys.stderr.write("Warning: No saved version and svnversion failed: %s\n" % err) if v == unknown: - if old is not None and old != unknown: - v = old - else: - sys.stderr.write("Warning: Could not determine software version\n") + if old is not None and old != unknown: + v = old + else: + sys.stderr.write("Warning: Could not determine software version\n") if old is None or v != old: - with open("rpki/version.py", "w") as f: - f.write("VERSION = \"%s\"\n" % v) - with open("VERSION", "w") as f: - f.write(v + "\n") + with open("rpki/version.py", "w") as f: + f.write("VERSION = \"%s\"\n" % v) + with open("VERSION", "w") as f: + f.write(v + "\n") diff --git a/buildtools/pull-doc-from-wiki.py b/buildtools/pull-doc-from-wiki.py index e3b61b53..5995823a 100644 --- a/buildtools/pull-doc-from-wiki.py +++ b/buildtools/pull-doc-from-wiki.py @@ -1,12 +1,12 @@ # $Id$ -# +# # Copyright (C) 2014 Dragon Research Labs ("DRL") # Portions copyright (C) 2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notices and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL # WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR @@ -45,94 +45,94 @@ import tempfile def main(): - base = "https://trac.rpki.net" - - parser = argparse.ArgumentParser(description = __doc__) - parser.add_argument("-b", "--base_url", - default = base, - help = "base URL for documentation web site") - parser.add_argument("-t", "--toc", - default = base + "/wiki/doc/RPKI/TOC", - help = "table of contents URL") - parser.add_argument("-d", "--directory", - default = ".", - help = "output directory") - parser.add_argument("-p", "--pdf_file", - default = "manual.pdf", - help = "output PDF file") - parser.add_argument("-r", "--html2textrc", - default = os.path.join(os.path.dirname(sys.argv[0]), "html2textrc"), - help = "html2textrc rules file") - args = parser.parse_args() - - urls = str(xsl_get_toc(lxml.etree.parse(urllib.urlopen(args.toc)).getroot(), - basename = repr(args.base_url))).splitlines() - - assert all(urlparse.urlparse(url).path.startswith("/wiki/") for url in urls) - - htmldoc = subprocess.Popen( - ("htmldoc", "--book", "--title", "--outfile", args.pdf_file, "--format", "pdf", - "--firstpage", "p1", "--size", "Universal", "--no-duplex", - "--fontsize", "11.0", "--fontspacing", "1.1", "--headfootsize", "11.0", - "--headingfont", "Helvetica", "--bodyfont", "Times", "--headfootfont", "Helvetica-Oblique", - "-"), stdin = subprocess.PIPE) - - lxml.etree.ElementTree(xml_title).write(htmldoc.stdin) - - png_fns = [] - - for url in urls: - path = urlparse.urlparse(url).path - page = xsl_get_page(lxml.etree.parse(urllib.urlopen(url)).getroot(), - basename = repr(args.base_url), - path = repr(path)) - - for img in page.xpath("//img | //object | //embed"): - attr = "data" if img.tag == "object" else "src" - img_url = img.get(attr) - if img_url.endswith(".svg"): - #sys.stderr.write("Converting %s to PNG\n" % img_url) - png_fd, png_fn = tempfile.mkstemp(suffix = ".png") - subprocess.Popen(("svg2png", "-h", "700", "-w", "600", "-", "-"), - stdout = png_fd, - stdin = subprocess.PIPE).communicate(urllib.urlopen(img_url).read()) - os.close(png_fd) - img.set(attr, png_fn) - png_fns.append(png_fn) - - page.write(htmldoc.stdin) - - html2text = subprocess.Popen(("html2text", "-rcfile", args.html2textrc, "-nobs", "-ascii"), - stdin = subprocess.PIPE, - stdout = subprocess.PIPE) - page.write(html2text.stdin) - html2text.stdin.close() - lines = html2text.stdout.readlines() - html2text.stdout.close() - html2text.wait() - - while lines and lines[0].isspace(): - del lines[0] - - fn = os.path.join(args.directory, path[len("/wiki/"):].replace("/", ".")) - f = open(fn, "w") - want_blank = False - for line in lines: - blank = line.isspace() - if want_blank and not blank: - f.write("\n") - if not blank: - f.write(line) - want_blank = blank - f.close() - sys.stderr.write("Wrote %s\n" % fn) - - htmldoc.stdin.close() - htmldoc.wait() - sys.stderr.write("Wrote %s\n" % args.pdf_file) - - for png_fn in png_fns: - os.unlink(png_fn) + base = "https://trac.rpki.net" + + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument("-b", "--base_url", + default = base, + help = "base URL for documentation web site") + parser.add_argument("-t", "--toc", + default = base + "/wiki/doc/RPKI/TOC", + help = "table of contents URL") + parser.add_argument("-d", "--directory", + default = ".", + help = "output directory") + parser.add_argument("-p", "--pdf_file", + default = "manual.pdf", + help = "output PDF file") + parser.add_argument("-r", "--html2textrc", + default = os.path.join(os.path.dirname(sys.argv[0]), "html2textrc"), + help = "html2textrc rules file") + args = parser.parse_args() + + urls = str(xsl_get_toc(lxml.etree.parse(urllib.urlopen(args.toc)).getroot(), + basename = repr(args.base_url))).splitlines() + + assert all(urlparse.urlparse(url).path.startswith("/wiki/") for url in urls) + + htmldoc = subprocess.Popen( + ("htmldoc", "--book", "--title", "--outfile", args.pdf_file, "--format", "pdf", + "--firstpage", "p1", "--size", "Universal", "--no-duplex", + "--fontsize", "11.0", "--fontspacing", "1.1", "--headfootsize", "11.0", + "--headingfont", "Helvetica", "--bodyfont", "Times", "--headfootfont", "Helvetica-Oblique", + "-"), stdin = subprocess.PIPE) + + lxml.etree.ElementTree(xml_title).write(htmldoc.stdin) + + png_fns = [] + + for url in urls: + path = urlparse.urlparse(url).path + page = xsl_get_page(lxml.etree.parse(urllib.urlopen(url)).getroot(), + basename = repr(args.base_url), + path = repr(path)) + + for img in page.xpath("//img | //object | //embed"): + attr = "data" if img.tag == "object" else "src" + img_url = img.get(attr) + if img_url.endswith(".svg"): + #sys.stderr.write("Converting %s to PNG\n" % img_url) + png_fd, png_fn = tempfile.mkstemp(suffix = ".png") + subprocess.Popen(("svg2png", "-h", "700", "-w", "600", "-", "-"), + stdout = png_fd, + stdin = subprocess.PIPE).communicate(urllib.urlopen(img_url).read()) + os.close(png_fd) + img.set(attr, png_fn) + png_fns.append(png_fn) + + page.write(htmldoc.stdin) + + html2text = subprocess.Popen(("html2text", "-rcfile", args.html2textrc, "-nobs", "-ascii"), + stdin = subprocess.PIPE, + stdout = subprocess.PIPE) + page.write(html2text.stdin) + html2text.stdin.close() + lines = html2text.stdout.readlines() + html2text.stdout.close() + html2text.wait() + + while lines and lines[0].isspace(): + del lines[0] + + fn = os.path.join(args.directory, path[len("/wiki/"):].replace("/", ".")) + f = open(fn, "w") + want_blank = False + for line in lines: + blank = line.isspace() + if want_blank and not blank: + f.write("\n") + if not blank: + f.write(line) + want_blank = blank + f.close() + sys.stderr.write("Wrote %s\n" % fn) + + htmldoc.stdin.close() + htmldoc.wait() + sys.stderr.write("Wrote %s\n" % args.pdf_file) + + for png_fn in png_fns: + os.unlink(png_fn) # HTMLDOC title page. At some point we might want to generate this # dynamically as an ElementTree, but static content will do for the @@ -188,7 +188,7 @@ xsl_get_toc = lxml.etree.XSLT(lxml.etree.XML('''\ # we care, and this seems to work. # # Original author's explanation: -# +# # The rather convoluted XPath expression for selecting the following # sibling aaa nodes which are merged with the current one: # @@ -321,12 +321,12 @@ xsl_get_page = lxml.etree.XSLT(lxml.etree.XML('''\ - + - self.version: - self.cur.execute("DELETE FROM upgrade_version") - self.cur.execute("INSERT upgrade_version (version, updated) VALUES (%s, %s)", (v, datetime.datetime.now())) - self.db.commit() - log("Updated %s to %s" % (self.name, v)) + upgrade_version_table_schema = """ + CREATE TABLE upgrade_version ( + version TEXT NOT NULL, + updated DATETIME NOT NULL + ) ENGINE=InnoDB + """ + + def __init__(self, name): + self.name = name + self.database = cfg.get("sql-database", section = name) + self.username = cfg.get("sql-username", section = name) + self.password = cfg.get("sql-password", section = name) + self.db = None + self.cur = None + + def open(self): + self.db = MySQLdb.connect(db = self.database, user = self.username, passwd = self.password) + self.db.autocommit(False) + self.cur = self.db.cursor() + + def close(self): + if self.cur is not None: + self.cur.close() + self.cur = None + if self.db is not None: + self.db.commit() + self.db.close() + self.db = None + + @property + def exists_and_accessible(self): + try: + MySQLdb.connect(db = self.database, user = self.username, passwd = self.password).close() + except: # pylint: disable=W0702 + return False + else: + return True + + @property + def version(self): + try: + self.cur.execute("SELECT version FROM upgrade_version") + v = self.cur.fetchone() + return Version(None if v is None else v[0]) + except _mysql_exceptions.ProgrammingError, e: + if e.args[0] != ER_NO_SUCH_TABLE: + raise + log("Creating upgrade_version table in %s" % self.name) + self.cur.execute(self.upgrade_version_table_schema) + return Version(None) + + @version.setter + def version(self, v): + if v > self.version: + self.cur.execute("DELETE FROM upgrade_version") + self.cur.execute("INSERT upgrade_version (version, updated) VALUES (%s, %s)", (v, datetime.datetime.now())) + self.db.commit() + log("Updated %s to %s" % (self.name, v)) class Version(object): - """ - A version number. This is a class in its own right to force the - comparision and string I/O behavior we want. - """ + """ + A version number. This is a class in its own right to force the + comparision and string I/O behavior we want. + """ - def __init__(self, v): - if v is None: - v = "0.0" - self.v = tuple(v.lower().split(".")) + def __init__(self, v): + if v is None: + v = "0.0" + self.v = tuple(v.lower().split(".")) - def __str__(self): - return ".".join(self.v) + def __str__(self): + return ".".join(self.v) - def __cmp__(self, other): - return cmp(self.v, other.v) + def __cmp__(self, other): + return cmp(self.v, other.v) class Upgrade(object): - """ - One upgrade script. Really, just its filename and the Version - object we parse from its filename, we don't need to read the script - itself except when applying it, but we do need to sort all the - available upgrade scripts into version order. - """ - - @classmethod - def load_all(cls, name, dn): - g = os.path.join(dn, "upgrade-%s-to-*.py" % name) - for fn in glob.iglob(g): - yield cls(g, fn) - - def __init__(self, g, fn): - head, sep, tail = g.partition("*") # pylint: disable=W0612 - self.fn = fn - self.version = Version(fn[len(head):-len(tail)]) - - def __cmp__(self, other): - return cmp(self.version, other.version) - - def apply(self, db): - # db is an argument here primarily so the script we exec can get at it - log("Applying %s to %s" % (self.fn, db.name)) - with open(self.fn, "r") as f: - exec f # pylint: disable=W0122 + """ + One upgrade script. Really, just its filename and the Version + object we parse from its filename, we don't need to read the script + itself except when applying it, but we do need to sort all the + available upgrade scripts into version order. + """ + + @classmethod + def load_all(cls, name, dn): + g = os.path.join(dn, "upgrade-%s-to-*.py" % name) + for fn in glob.iglob(g): + yield cls(g, fn) + + def __init__(self, g, fn): + head, sep, tail = g.partition("*") # pylint: disable=W0612 + self.fn = fn + self.version = Version(fn[len(head):-len(tail)]) + + def __cmp__(self, other): + return cmp(self.version, other.version) + + def apply(self, db): + # db is an argument here primarily so the script we exec can get at it + log("Applying %s to %s" % (self.fn, db.name)) + with open(self.fn, "r") as f: + exec f # pylint: disable=W0122 def do_drop(name): - db = UserDB(name) - if db.database in root.databases: - log("DROP DATABASE %s" % db.database) - root.cur.execute("DROP DATABASE %s" % db.database) - root.db.commit() + db = UserDB(name) + if db.database in root.databases: + log("DROP DATABASE %s" % db.database) + root.cur.execute("DROP DATABASE %s" % db.database) + root.db.commit() def do_create(name): - db = UserDB(name) - log("CREATE DATABASE %s" % db.database) - root.cur.execute("CREATE DATABASE %s" % db.database) - log("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY ###" % (db.database, db.username)) - root.cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % (db.database, db.username), - (db.password,)) - root.db.commit() - db.open() - db.version = current_version - db.close() + db = UserDB(name) + log("CREATE DATABASE %s" % db.database) + root.cur.execute("CREATE DATABASE %s" % db.database) + log("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY ###" % (db.database, db.username)) + root.cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % (db.database, db.username), + (db.password,)) + root.db.commit() + db.open() + db.version = current_version + db.close() def do_script_drop(name): - db = UserDB(name) - print "DROP DATABASE IF EXISTS %s;" % db.database + db = UserDB(name) + print "DROP DATABASE IF EXISTS %s;" % db.database def do_drop_and_create(name): - do_drop(name) - do_create(name) + do_drop(name) + do_create(name) def do_fix_grants(name): - db = UserDB(name) - if not db.exists_and_accessible: - log("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY ###" % (db.database, db.username)) - root.cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % (db.database, db.username), - (db.password,)) - root.db.commit() + db = UserDB(name) + if not db.exists_and_accessible: + log("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY ###" % (db.database, db.username)) + root.cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % (db.database, db.username), + (db.password,)) + root.db.commit() def do_create_if_missing(name): - db = UserDB(name) - if not db.exists_and_accessible: - do_create(name) + db = UserDB(name) + if not db.exists_and_accessible: + do_create(name) def do_apply_upgrades(name): - upgrades = sorted(Upgrade.load_all(name, args.upgrade_scripts)) - if upgrades: - db = UserDB(name) - db.open() - log("Current version of %s is %s" % (db.name, db.version)) - for upgrade in upgrades: - if upgrade.version > db.version: - upgrade.apply(db) - db.version = upgrade.version - db.version = current_version - db.close() + upgrades = sorted(Upgrade.load_all(name, args.upgrade_scripts)) + if upgrades: + db = UserDB(name) + db.open() + log("Current version of %s is %s" % (db.name, db.version)) + for upgrade in upgrades: + if upgrade.version > db.version: + upgrade.apply(db) + db.version = upgrade.version + db.version = current_version + db.close() def log(text): - if args.verbose: - print "#", text + if args.verbose: + print "#", text parser = argparse.ArgumentParser(description = """\ Automated setup of all SQL stuff used by the RPKI CA tools. Pulls @@ -291,13 +291,13 @@ parser.set_defaults(dispatch = do_create_if_missing) args = parser.parse_args() try: - cfg = rpki.config.parser(set_filename = args.config, section = "myrpki") - root = RootDB(args.mysql_defaults) - current_version = Version(rpki.version.VERSION) - for program_name in ("irdbd", "rpkid", "pubd"): - if cfg.getboolean("start_" + program_name, False): - args.dispatch(program_name) - root.close() + cfg = rpki.config.parser(set_filename = args.config, section = "myrpki") + root = RootDB(args.mysql_defaults) + current_version = Version(rpki.version.VERSION) + for program_name in ("irdbd", "rpkid", "pubd"): + if cfg.getboolean("start_" + program_name, False): + args.dispatch(program_name) + root.close() except Exception, e: - #raise - sys.exit(str(e)) + #raise + sys.exit(str(e)) diff --git a/ca/rpki-start-servers b/ca/rpki-start-servers index f1f70aa8..e01f4f9b 100755 --- a/ca/rpki-start-servers +++ b/ca/rpki-start-servers @@ -38,9 +38,9 @@ os.environ["TZ"] = "UTC" time.tzset() def non_negative_integer(s): - if int(s) < 0: - raise ValueError - return s + if int(s) < 0: + raise ValueError + return s parser = argparse.ArgumentParser(description = __doc__) parser.add_argument("-c", "--config", @@ -67,21 +67,21 @@ args = parser.parse_args() cfg = rpki.config.parser(set_filename = args.config, section = "myrpki") def run(name, old_flag = None): - if cfg.getboolean("start_" + name, cfg.getboolean("run_" + name if old_flag is None else old_flag, False)): - # pylint: disable=E1103 - log_file = os.path.join(args.log_directory, name + ".log") - cmd = (os.path.join(rpki.autoconf.libexecdir, name), "--log-level", args.log_level) - if args.log_file: - cmd += ("--log-file", log_file) - elif args.log_rotating_file_kbytes: - cmd += ("--log-rotating-file", log_file, args.log_rotating_file_kbytes, args.log_backup_count) - elif args.log_rotating_file_hours: - cmd += ("--log-timed-rotating-file", log_file, args.log_rotating_file_hours, args.log_backup_count) - else: - cmd += ("--log-syslog", args.log_syslog) - proc = subprocess.Popen(cmd) - if proc.wait() != 0: - sys.exit("Problem starting %s, pid %s" % (name, proc.pid)) + if cfg.getboolean("start_" + name, cfg.getboolean("run_" + name if old_flag is None else old_flag, False)): + # pylint: disable=E1103 + log_file = os.path.join(args.log_directory, name + ".log") + cmd = (os.path.join(rpki.autoconf.libexecdir, name), "--log-level", args.log_level) + if args.log_file: + cmd += ("--log-file", log_file) + elif args.log_rotating_file_kbytes: + cmd += ("--log-rotating-file", log_file, args.log_rotating_file_kbytes, args.log_backup_count) + elif args.log_rotating_file_hours: + cmd += ("--log-timed-rotating-file", log_file, args.log_rotating_file_hours, args.log_backup_count) + else: + cmd += ("--log-syslog", args.log_syslog) + proc = subprocess.Popen(cmd) + if proc.wait() != 0: + sys.exit("Problem starting %s, pid %s" % (name, proc.pid)) run("irdbd", "run_rpkid") run("rpkid") diff --git a/ca/rpkic b/ca/rpkic index 333a5eb7..3c3c7d99 100755 --- a/ca/rpkic +++ b/ca/rpkic @@ -17,5 +17,5 @@ # PERFORMANCE OF THIS SOFTWARE. if __name__ == "__main__": - import rpki.rpkic - rpki.rpkic.main() + import rpki.rpkic + rpki.rpkic.main() diff --git a/ca/rpkid b/ca/rpkid index a4cc6cd3..f63e441e 100755 --- a/ca/rpkid +++ b/ca/rpkid @@ -17,5 +17,5 @@ # PERFORMANCE OF THIS SOFTWARE. if __name__ == "__main__": - import rpki.rpkid - rpki.rpkid.main() + import rpki.rpkid + rpki.rpkid.main() diff --git a/ca/rpkigui-apache-conf-gen b/ca/rpkigui-apache-conf-gen index 8ac9c94a..a6a58c2c 100755 --- a/ca/rpkigui-apache-conf-gen +++ b/ca/rpkigui-apache-conf-gen @@ -188,370 +188,370 @@ NameVirtualHost *:443 ''' def Guess(args): - """ - Guess what platform this is and dispatch to platform constructor. - """ - - system = platform.system() - if system == "FreeBSD": - return FreeBSD(args) - if system == "Darwin": - return Darwin(args) - if system == "Linux": - distro = platform.linux_distribution()[0].lower() - if distro == "debian": - return Debian(args) - if distro == "ubuntu": - return Ubuntu(args) - if distro in ("fedora", "centos"): - return Redhat(args) - raise NotImplementedError("Can't guess what platform this is, sorry") + """ + Guess what platform this is and dispatch to platform constructor. + """ + + system = platform.system() + if system == "FreeBSD": + return FreeBSD(args) + if system == "Darwin": + return Darwin(args) + if system == "Linux": + distro = platform.linux_distribution()[0].lower() + if distro == "debian": + return Debian(args) + if distro == "ubuntu": + return Ubuntu(args) + if distro in ("fedora", "centos"): + return Redhat(args) + raise NotImplementedError("Can't guess what platform this is, sorry") class Platform(object): - """ - Abstract base class representing an operating system platform. - """ - - apache_cer = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.cer") - apache_key = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.key") - - apache_conf = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.conf") - apache_conf_sample = apache_conf + ".sample" - - apache_conf_preface = "" - - def __init__(self, args): - self.args = args - self.log("RPKI Apache configuration: platform \"%s\", action \"%s\"" % ( - self.__class__.__name__, args.action)) - getattr(self, args.action)() - - def log(self, msg): - if self.args.verbose: - print msg - - def run(self, *cmd, **kwargs): - self.log("Running %s" % " ".join(cmd)) - subprocess.check_call(cmd, **kwargs) - - req_cmd = ("openssl", "req", "-new", - "-config", "/dev/stdin", - "-out", "/dev/stdout", - "-keyout", apache_key, - "-newkey", "rsa:2048") - - x509_cmd = ("openssl", "x509", "-req", "-sha256", - "-signkey", apache_key, - "-in", "/dev/stdin", - "-out", apache_cer, - "-days", "3650") - - req_conf = '''\ - [req] - default_bits = 2048 - default_md = sha256 - distinguished_name = req_dn - prompt = no - encrypt_key = no - [req_dn] - CN = %s - ''' % fqdn - - def unlink(self, fn, silent = False): - if os.path.lexists(fn): - if not silent: - self.log("Removing %s" % fn) - os.unlink(fn) - elif not silent: - self.log("Would have removed %s if it existed" % fn) - - def del_certs(self, silent = False): - self.unlink(self.apache_cer, silent) - self.unlink(self.apache_key, silent) - - def add_certs(self): - if os.path.exists(self.apache_cer) and os.path.exists(self.apache_key): - return - self.del_certs() - req = subprocess.Popen(self.req_cmd, - stdin = subprocess.PIPE, - stdout = subprocess.PIPE, - stderr = open("/dev/null", "w")) - x509 = subprocess.Popen(self.x509_cmd, - stdin = req.stdout, - stderr = open("/dev/null", "w")) - req.stdin.write(self.req_conf) - req.stdin.close() - if req.wait(): - raise subprocess.CalledProcessError(req.returncode, self.req_cmd) - if x509.wait(): - raise subprocess.CalledProcessError(x509.returncode, self.x509_cmd) - self.log("Created %s and %s, chmoding %s" % ( - self.apache_cer, self.apache_key, self.apache_key)) - os.chmod(self.apache_key, 0600) - - _vhost = None - - @property - def vhost(self): - if self._vhost is None: - allow = allow_22_template if self.args.apache_version <= 22 else allow_24_template - self._vhost = vhost_template % dict(rpki.autoconf.__dict__, fqdn = fqdn, allow = allow) - return self._vhost - - @property - def name_virtual_host(self): - return name_virtual_host_template if self.args.apache_version <= 22 else "" - - @property - def too_complex(self): - return textwrap.dedent('''\ - # It looks like you already have HTTPS enabled in your - # Apache configuration, which makes your configuration too - # complex for us to enable support for the RPKI GUI automatically. - # - # To enable support, take a look at %s - # and copy what you need from that file into %s, - # paying attention to the comments which mark the bits that - # you might (or might not) need to change or omit, depending - # on the details of your particular Apache configuration. - ''' % (self.apache_conf_sample, self.apache_conf)) - - def install(self): - with open(self.apache_conf_sample, "w") as f: - self.log("Writing %s" % f.name) - f.write(self.apache_conf_preface) - f.write(self.name_virtual_host) - f.write(self.vhost) - if not os.path.exists(self.apache_conf): - self.unlink(self.apache_conf) - with open(self.apache_conf, "w") as f: - self.log("Writing %s" % f.name) - if self.test_url("https://%s/" % fqdn): - f.write(self.too_complex) - sys.stdout.write(self.too_complex) - else: - if not self.test_tcp("localhost", 443): + """ + Abstract base class representing an operating system platform. + """ + + apache_cer = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.cer") + apache_key = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.key") + + apache_conf = os.path.join(rpki.autoconf.sysconfdir, "rpki", "apache.conf") + apache_conf_sample = apache_conf + ".sample" + + apache_conf_preface = "" + + def __init__(self, args): + self.args = args + self.log("RPKI Apache configuration: platform \"%s\", action \"%s\"" % ( + self.__class__.__name__, args.action)) + getattr(self, args.action)() + + def log(self, msg): + if self.args.verbose: + print msg + + def run(self, *cmd, **kwargs): + self.log("Running %s" % " ".join(cmd)) + subprocess.check_call(cmd, **kwargs) + + req_cmd = ("openssl", "req", "-new", + "-config", "/dev/stdin", + "-out", "/dev/stdout", + "-keyout", apache_key, + "-newkey", "rsa:2048") + + x509_cmd = ("openssl", "x509", "-req", "-sha256", + "-signkey", apache_key, + "-in", "/dev/stdin", + "-out", apache_cer, + "-days", "3650") + + req_conf = '''\ + [req] + default_bits = 2048 + default_md = sha256 + distinguished_name = req_dn + prompt = no + encrypt_key = no + [req_dn] + CN = %s + ''' % fqdn + + def unlink(self, fn, silent = False): + if os.path.lexists(fn): + if not silent: + self.log("Removing %s" % fn) + os.unlink(fn) + elif not silent: + self.log("Would have removed %s if it existed" % fn) + + def del_certs(self, silent = False): + self.unlink(self.apache_cer, silent) + self.unlink(self.apache_key, silent) + + def add_certs(self): + if os.path.exists(self.apache_cer) and os.path.exists(self.apache_key): + return + self.del_certs() + req = subprocess.Popen(self.req_cmd, + stdin = subprocess.PIPE, + stdout = subprocess.PIPE, + stderr = open("/dev/null", "w")) + x509 = subprocess.Popen(self.x509_cmd, + stdin = req.stdout, + stderr = open("/dev/null", "w")) + req.stdin.write(self.req_conf) + req.stdin.close() + if req.wait(): + raise subprocess.CalledProcessError(req.returncode, self.req_cmd) + if x509.wait(): + raise subprocess.CalledProcessError(x509.returncode, self.x509_cmd) + self.log("Created %s and %s, chmoding %s" % ( + self.apache_cer, self.apache_key, self.apache_key)) + os.chmod(self.apache_key, 0600) + + _vhost = None + + @property + def vhost(self): + if self._vhost is None: + allow = allow_22_template if self.args.apache_version <= 22 else allow_24_template + self._vhost = vhost_template % dict(rpki.autoconf.__dict__, fqdn = fqdn, allow = allow) + return self._vhost + + @property + def name_virtual_host(self): + return name_virtual_host_template if self.args.apache_version <= 22 else "" + + @property + def too_complex(self): + return textwrap.dedent('''\ + # It looks like you already have HTTPS enabled in your + # Apache configuration, which makes your configuration too + # complex for us to enable support for the RPKI GUI automatically. + # + # To enable support, take a look at %s + # and copy what you need from that file into %s, + # paying attention to the comments which mark the bits that + # you might (or might not) need to change or omit, depending + # on the details of your particular Apache configuration. + ''' % (self.apache_conf_sample, self.apache_conf)) + + def install(self): + with open(self.apache_conf_sample, "w") as f: + self.log("Writing %s" % f.name) f.write(self.apache_conf_preface) f.write(self.name_virtual_host) - f.write(self.vhost) - if not os.path.exists(self.apache_conf_target): - self.unlink(self.apache_conf_target) - self.log("Symlinking %s to %s" % ( - self.apache_conf_target, self.apache_conf)) - os.symlink(self.apache_conf, self.apache_conf_target) - self.add_certs() - self.enable() - self.restart() - - def enable(self): - pass - - def disable(self): - pass - - def remove(self): - try: - same = open(self.apache_conf, "r").read() == open(self.apache_conf_sample, "r").read() - except: # pylint: disable=W0702 - same = False - self.unlink(self.apache_conf_sample) - if same: - self.unlink(self.apache_conf) - self.unlink(self.apache_conf_target) - self.disable() - self.restart() - - def purge(self): - self.remove() - self.unlink(self.apache_conf) - self.del_certs() - - @staticmethod - def test_url(url = "https://localhost/"): - try: - urllib2.urlopen(url).close() - except IOError: - return False - else: - return True - - @staticmethod - def test_tcp(host = "localhost", port = 443, family = socket.AF_UNSPEC, proto = socket.SOCK_STREAM): - try: - addrinfo = socket.getaddrinfo(host, port, family, proto) - except socket.error: - return False - for af, socktype, proto, canon, sa in addrinfo: # pylint: disable=W0612 - try: - s = socket.socket(af, socktype, proto) - s.connect(sa) - s.close() - except socket.error: - continue - else: - return True - return False + f.write(self.vhost) + if not os.path.exists(self.apache_conf): + self.unlink(self.apache_conf) + with open(self.apache_conf, "w") as f: + self.log("Writing %s" % f.name) + if self.test_url("https://%s/" % fqdn): + f.write(self.too_complex) + sys.stdout.write(self.too_complex) + else: + if not self.test_tcp("localhost", 443): + f.write(self.apache_conf_preface) + f.write(self.name_virtual_host) + f.write(self.vhost) + if not os.path.exists(self.apache_conf_target): + self.unlink(self.apache_conf_target) + self.log("Symlinking %s to %s" % ( + self.apache_conf_target, self.apache_conf)) + os.symlink(self.apache_conf, self.apache_conf_target) + self.add_certs() + self.enable() + self.restart() + + def enable(self): + pass + + def disable(self): + pass + + def remove(self): + try: + same = open(self.apache_conf, "r").read() == open(self.apache_conf_sample, "r").read() + except: # pylint: disable=W0702 + same = False + self.unlink(self.apache_conf_sample) + if same: + self.unlink(self.apache_conf) + self.unlink(self.apache_conf_target) + self.disable() + self.restart() + + def purge(self): + self.remove() + self.unlink(self.apache_conf) + self.del_certs() + + @staticmethod + def test_url(url = "https://localhost/"): + try: + urllib2.urlopen(url).close() + except IOError: + return False + else: + return True + + @staticmethod + def test_tcp(host = "localhost", port = 443, family = socket.AF_UNSPEC, proto = socket.SOCK_STREAM): + try: + addrinfo = socket.getaddrinfo(host, port, family, proto) + except socket.error: + return False + for af, socktype, proto, canon, sa in addrinfo: # pylint: disable=W0612 + try: + s = socket.socket(af, socktype, proto) + s.connect(sa) + s.close() + except socket.error: + continue + else: + return True + return False class FreeBSD(Platform): - """ - FreeBSD. - """ + """ + FreeBSD. + """ - # On FreeBSD we have to ask httpd what version it is before we know - # where to put files or what to call the service. In FreeBSD's makefiles, - # this value is called APACHE_VERSION, and is calculated thusly: - # - # httpd -V | sed -ne 's/^Server version: Apache\/\([0-9]\)\.\([0-9]*\).*/\1\2/p' + # On FreeBSD we have to ask httpd what version it is before we know + # where to put files or what to call the service. In FreeBSD's makefiles, + # this value is called APACHE_VERSION, and is calculated thusly: + # + # httpd -V | sed -ne 's/^Server version: Apache\/\([0-9]\)\.\([0-9]*\).*/\1\2/p' - _apache_name = None + _apache_name = None - @property - def apache_name(self): - if self._apache_name is None: - self._apache_name = "apache%s" % self.args.apache_version - return self._apache_name + @property + def apache_name(self): + if self._apache_name is None: + self._apache_name = "apache%s" % self.args.apache_version + return self._apache_name - @property - def apache_conf_target(self): - return "/usr/local/etc/%s/Includes/rpki.conf" % self.apache_name + @property + def apache_conf_target(self): + return "/usr/local/etc/%s/Includes/rpki.conf" % self.apache_name - apache_conf_preface = textwrap.dedent('''\ - # These directives tell Apache to listen on the HTTPS port - # and to enable name-based virtual hosting. If you already - # have HTTPS enabled elsewhere in your configuration, you may - # need to remove these. + apache_conf_preface = textwrap.dedent('''\ + # These directives tell Apache to listen on the HTTPS port + # and to enable name-based virtual hosting. If you already + # have HTTPS enabled elsewhere in your configuration, you may + # need to remove these. - Listen [::]:443 - Listen 0.0.0.0:443 - ''') + Listen [::]:443 + Listen 0.0.0.0:443 + ''') - def restart(self): - self.run("service", self.apache_name, "restart") + def restart(self): + self.run("service", self.apache_name, "restart") class Debian(Platform): - """ - Debian and related platforms like Ubuntu. - """ + """ + Debian and related platforms like Ubuntu. + """ - @property - def distribution_version(self): - return tuple(int(v) for v in platform.linux_distribution()[1].split(".")) + @property + def distribution_version(self): + return tuple(int(v) for v in platform.linux_distribution()[1].split(".")) - # As of Wheezy, Debian still wants the configuration filename - # without the .conf suffix. Keep an eye on this, as it may change - # in future releases. - # - @property - def apache_conf_target(self): - return "/etc/apache2/sites-available/rpki" - - snake_oil_cer = "/etc/ssl/certs/ssl-cert-snakeoil.pem" - snake_oil_key = "/etc/ssl/private/ssl-cert-snakeoil.key" - - def add_certs(self): - if not os.path.exists(self.snake_oil_cer) or not os.path.exists(self.snake_oil_key): - return Platform.add_certs(self) - if not os.path.exists(self.apache_cer): - self.unlink(self.apache_cer) - os.symlink(self.snake_oil_cer, self.apache_cer) - if not os.path.exists(self.apache_key): - self.unlink(self.apache_key) - os.symlink(self.snake_oil_key, self.apache_key) - - def enable(self): - self.run("a2enmod", "ssl") - self.run("a2enmod", "expires") - self.run("a2ensite", "rpki") + # As of Wheezy, Debian still wants the configuration filename + # without the .conf suffix. Keep an eye on this, as it may change + # in future releases. # - # In light of BREACH and CRIME attacks, mod_deflate is looking - # like a bad idea, so make sure it's off. - self.run("a2dismod", "deflate") - - def disable(self): - self.run("a2dissite", "rpki") - - def restart(self): - self.run("service", "apache2", "restart") + @property + def apache_conf_target(self): + return "/etc/apache2/sites-available/rpki" + + snake_oil_cer = "/etc/ssl/certs/ssl-cert-snakeoil.pem" + snake_oil_key = "/etc/ssl/private/ssl-cert-snakeoil.key" + + def add_certs(self): + if not os.path.exists(self.snake_oil_cer) or not os.path.exists(self.snake_oil_key): + return Platform.add_certs(self) + if not os.path.exists(self.apache_cer): + self.unlink(self.apache_cer) + os.symlink(self.snake_oil_cer, self.apache_cer) + if not os.path.exists(self.apache_key): + self.unlink(self.apache_key) + os.symlink(self.snake_oil_key, self.apache_key) + + def enable(self): + self.run("a2enmod", "ssl") + self.run("a2enmod", "expires") + self.run("a2ensite", "rpki") + # + # In light of BREACH and CRIME attacks, mod_deflate is looking + # like a bad idea, so make sure it's off. + self.run("a2dismod", "deflate") + + def disable(self): + self.run("a2dissite", "rpki") + + def restart(self): + self.run("service", "apache2", "restart") class Ubuntu(Debian): - # On Ubuntu, the filename must end in .conf on Trusty and must not - # end in .conf on Precise. - @property - def apache_conf_target(self): - if self.distribution_version >= (14, 0): - return "/etc/apache2/sites-available/rpki.conf" - else: - return "/etc/apache2/sites-available/rpki" + # On Ubuntu, the filename must end in .conf on Trusty and must not + # end in .conf on Precise. + @property + def apache_conf_target(self): + if self.distribution_version >= (14, 0): + return "/etc/apache2/sites-available/rpki.conf" + else: + return "/etc/apache2/sites-available/rpki" class NIY(Platform): - def __init__(self, args): - super(NIY, self).__init__(args) - raise NotImplementedError("Platform %s not implemented yet, sorry" % self.__class__.__name__) + def __init__(self, args): + super(NIY, self).__init__(args) + raise NotImplementedError("Platform %s not implemented yet, sorry" % self.__class__.__name__) class Redhat(NIY): - """ - Redhat family of Linux distributions (Fedora, CentOS). - """ + """ + Redhat family of Linux distributions (Fedora, CentOS). + """ class Darwin(NIY): - """ - Mac OS X (aka Darwin). - """ + """ + Mac OS X (aka Darwin). + """ def main(): - """ - Generate and (de)install configuration suitable for using Apache httpd - to drive the RPKI web interface under WSGI. - """ - - parser = argparse.ArgumentParser(description = __doc__) - group1 = parser.add_mutually_exclusive_group() - group2 = parser.add_mutually_exclusive_group() - - parser.add_argument("-v", "--verbose", - help = "whistle while you work", action = "store_true") - parser.add_argument("--apache-version", - help = "Apache version (default " + rpki.autoconf.APACHE_VERSION + ")", - type = int, default = rpki.autoconf.APACHE_VERSION) - - group1.add_argument("--freebsd", - help = "configure for FreeBSD", - action = "store_const", dest = "platform", const = FreeBSD) - group1.add_argument("--debian", - help = "configure for Debian", - action = "store_const", dest = "platform", const = Debian) - group1.add_argument("--ubuntu", - help = "configure for Ubuntu", - action = "store_const", dest = "platform", const = Ubuntu) - group1.add_argument("--redhat", "--fedora", "--centos", - help = "configure for Redhat/Fedora/CentOS", - action = "store_const", dest = "platform", const = Redhat) - group1.add_argument("--macosx", "--darwin", - help = "configure for Mac OS X (Darwin)", - action = "store_const", dest = "platform", const = Darwin) - group1.add_argument("--guess", - help = "guess which platform configuration to use", - action = "store_const", dest = "platform", const = Guess) - - group2.add_argument("-i", "--install", - help = "install configuration", - action = "store_const", dest = "action", const = "install") - group2.add_argument("-r", "--remove", "--deinstall", "--uninstall", - help = "remove configuration", - action = "store_const", dest = "action", const = "remove") - group2.add_argument("-P", "--purge", - help = "remove configuration with extreme prejudice", - action = "store_const", dest = "action", const = "purge") - - parser.set_defaults(platform = Guess, action = "install") - args = parser.parse_args() - - try: - args.platform(args) - except Exception, e: - sys.exit(str(e)) + """ + Generate and (de)install configuration suitable for using Apache httpd + to drive the RPKI web interface under WSGI. + """ + + parser = argparse.ArgumentParser(description = __doc__) + group1 = parser.add_mutually_exclusive_group() + group2 = parser.add_mutually_exclusive_group() + + parser.add_argument("-v", "--verbose", + help = "whistle while you work", action = "store_true") + parser.add_argument("--apache-version", + help = "Apache version (default " + rpki.autoconf.APACHE_VERSION + ")", + type = int, default = rpki.autoconf.APACHE_VERSION) + + group1.add_argument("--freebsd", + help = "configure for FreeBSD", + action = "store_const", dest = "platform", const = FreeBSD) + group1.add_argument("--debian", + help = "configure for Debian", + action = "store_const", dest = "platform", const = Debian) + group1.add_argument("--ubuntu", + help = "configure for Ubuntu", + action = "store_const", dest = "platform", const = Ubuntu) + group1.add_argument("--redhat", "--fedora", "--centos", + help = "configure for Redhat/Fedora/CentOS", + action = "store_const", dest = "platform", const = Redhat) + group1.add_argument("--macosx", "--darwin", + help = "configure for Mac OS X (Darwin)", + action = "store_const", dest = "platform", const = Darwin) + group1.add_argument("--guess", + help = "guess which platform configuration to use", + action = "store_const", dest = "platform", const = Guess) + + group2.add_argument("-i", "--install", + help = "install configuration", + action = "store_const", dest = "action", const = "install") + group2.add_argument("-r", "--remove", "--deinstall", "--uninstall", + help = "remove configuration", + action = "store_const", dest = "action", const = "remove") + group2.add_argument("-P", "--purge", + help = "remove configuration with extreme prejudice", + action = "store_const", dest = "action", const = "purge") + + parser.set_defaults(platform = Guess, action = "install") + args = parser.parse_args() + + try: + args.platform(args) + except Exception, e: + sys.exit(str(e)) if __name__ == "__main__": - main() + main() diff --git a/ca/rpkigui-import-routes b/ca/rpkigui-import-routes index 0fbe0126..fb8e381e 100755 --- a/ca/rpkigui-import-routes +++ b/ca/rpkigui-import-routes @@ -110,4 +110,3 @@ automatically.""") except Exception as e: logging.exception(e) sys.exit(1) - diff --git a/ca/tests/bgpsec-yaml.py b/ca/tests/bgpsec-yaml.py index d33184bf..500d2b9d 100755 --- a/ca/tests/bgpsec-yaml.py +++ b/ca/tests/bgpsec-yaml.py @@ -30,28 +30,28 @@ root = "Root" class Kid(object): - def __init__(self, i): - self.name = "ISP-%03d" % i - self.ipv4 = "10.%d.0.0/16" % i - self.asn = i - self.router_id = i * 10000 - - @property - def declare(self): - return dict(name = self.name, - ipv4 = self.ipv4, - asn = self.asn, - hosted_by = root, - roa_request = [dict(asn = self.asn, ipv4 = self.ipv4)], - router_cert = [dict(asn = self.asn, router_id = self.router_id)]) - - @property - def del_routercert(self): - return dict(name = self.name, router_cert_del = [dict(asn = self.asn, router_id = self.router_id)]) - - @property - def add_routercert(self): - return dict(name = self.name, router_cert_add = [dict(asn = self.asn, router_id = self.router_id)]) + def __init__(self, i): + self.name = "ISP-%03d" % i + self.ipv4 = "10.%d.0.0/16" % i + self.asn = i + self.router_id = i * 10000 + + @property + def declare(self): + return dict(name = self.name, + ipv4 = self.ipv4, + asn = self.asn, + hosted_by = root, + roa_request = [dict(asn = self.asn, ipv4 = self.ipv4)], + router_cert = [dict(asn = self.asn, router_id = self.router_id)]) + + @property + def del_routercert(self): + return dict(name = self.name, router_cert_del = [dict(asn = self.asn, router_id = self.router_id)]) + + @property + def add_routercert(self): + return dict(name = self.name, router_cert_add = [dict(asn = self.asn, router_id = self.router_id)]) kids = [Kid(n + 1) for n in xrange(200)] @@ -72,14 +72,14 @@ docs.append([shell_first, gym = kids[50:70] for kid in gym: - docs.append([shell_next, - kid.del_routercert, - sleeper]) + docs.append([shell_next, + kid.del_routercert, + sleeper]) for kid in gym: - docs.append([shell_next, - kid.add_routercert, - sleeper]) + docs.append([shell_next, + kid.add_routercert, + sleeper]) print '''\ # This configuration was generated by a script. Edit at your own risk. diff --git a/ca/tests/myrpki-xml-parse-test.py b/ca/tests/myrpki-xml-parse-test.py index 9db7ec57..d915ea5b 100644 --- a/ca/tests/myrpki-xml-parse-test.py +++ b/ca/tests/myrpki-xml-parse-test.py @@ -25,77 +25,77 @@ relaxng = lxml.etree.RelaxNG(file = "myrpki.rng") tree = lxml.etree.parse("myrpki.xml").getroot() if False: - print lxml.etree.tostring(tree, pretty_print = True, encoding = "us-ascii", xml_declaration = True) + print lxml.etree.tostring(tree, pretty_print = True, encoding = "us-ascii", xml_declaration = True) relaxng.assertValid(tree) def showitems(y): - if False: - for k, v in y.items(): - if v: - print " ", k, v + if False: + for k, v in y.items(): + if v: + print " ", k, v def tag(t): - return "{http://www.hactrn.net/uris/rpki/myrpki/}" + t + return "{http://www.hactrn.net/uris/rpki/myrpki/}" + t print "My handle:", tree.get("handle") print "Children:" for x in tree.getiterator(tag("child")): - print " ", x - print " Handle:", x.get("handle") - print " ASNS: ", rpki.resource_set.resource_set_as(x.get("asns")) - print " IPv4: ", rpki.resource_set.resource_set_ipv4(x.get("v4")) - print " Valid: ", x.get("valid_until") - showitems(x) + print " ", x + print " Handle:", x.get("handle") + print " ASNS: ", rpki.resource_set.resource_set_as(x.get("asns")) + print " IPv4: ", rpki.resource_set.resource_set_ipv4(x.get("v4")) + print " Valid: ", x.get("valid_until") + showitems(x) print print "ROA requests:" for x in tree.getiterator(tag("roa_request")): - print " ", x - print " ASN: ", x.get("asn") - print " IPv4:", rpki.resource_set.roa_prefix_set_ipv4(x.get("v4")) - print " IPv6:", rpki.resource_set.roa_prefix_set_ipv6(x.get("v6")) - showitems(x) + print " ", x + print " ASN: ", x.get("asn") + print " IPv4:", rpki.resource_set.roa_prefix_set_ipv4(x.get("v4")) + print " IPv6:", rpki.resource_set.roa_prefix_set_ipv6(x.get("v6")) + showitems(x) print def showpem(label, b64, kind): - cmd = ("openssl", kind, "-noout", "-text", "-inform", "DER") - if kind == "x509": - cmd += ("-certopt", "no_pubkey,no_sigdump") - p = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE) - text = p.communicate(input = base64.b64decode(b64))[0] - if p.returncode != 0: - raise subprocess.CalledProcessError(returncode = p.returncode, cmd = cmd) - print label, text + cmd = ("openssl", kind, "-noout", "-text", "-inform", "DER") + if kind == "x509": + cmd += ("-certopt", "no_pubkey,no_sigdump") + p = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE) + text = p.communicate(input = base64.b64decode(b64))[0] + if p.returncode != 0: + raise subprocess.CalledProcessError(returncode = p.returncode, cmd = cmd) + print label, text for x in tree.getiterator(tag("child")): - cert = x.findtext(tag("bpki_certificate")) - if cert: - showpem("Child", cert, "x509") + cert = x.findtext(tag("bpki_certificate")) + if cert: + showpem("Child", cert, "x509") for x in tree.getiterator(tag("parent")): - print "Parent URI:", x.get("service_uri") - cert = x.findtext(tag("bpki_certificate")) - if cert: - showpem("Parent", cert, "x509") + print "Parent URI:", x.get("service_uri") + cert = x.findtext(tag("bpki_certificate")) + if cert: + showpem("Parent", cert, "x509") ca = tree.findtext(tag("bpki_ca_certificate")) if ca: - showpem("CA", ca, "x509") + showpem("CA", ca, "x509") bsc = tree.findtext(tag("bpki_bsc_certificate")) if bsc: - showpem("BSC EE", bsc, "x509") + showpem("BSC EE", bsc, "x509") repo = tree.findtext(tag("bpki_repository_certificate")) if repo: - showpem("Repository", repo, "x509") + showpem("Repository", repo, "x509") req = tree.findtext(tag("bpki_bsc_pkcs10")) if req: - showpem("BSC EE", req, "req") + showpem("BSC EE", req, "req") crl = tree.findtext(tag("bpki_crl")) if crl: - showpem("CA", crl, "crl") + showpem("CA", crl, "crl") diff --git a/ca/tests/old_irdbd.py b/ca/tests/old_irdbd.py index d66e683e..d26c3476 100644 --- a/ca/tests/old_irdbd.py +++ b/ca/tests/old_irdbd.py @@ -15,5 +15,5 @@ # PERFORMANCE OF THIS SOFTWARE. if __name__ == "__main__": - import rpki.old_irdbd - rpki.old_irdbd.main() + import rpki.old_irdbd + rpki.old_irdbd.main() diff --git a/ca/tests/smoketest.py b/ca/tests/smoketest.py index 3960981a..9d82c640 100644 --- a/ca/tests/smoketest.py +++ b/ca/tests/smoketest.py @@ -76,14 +76,14 @@ yaml_script = [y for y in yaml.safe_load_all(args.yaml_file)] # Define port allocator early, so we can use it while reading config def allocate_port(): - """ - Allocate a TCP port number. - """ + """ + Allocate a TCP port number. + """ - global base_port - p = base_port - base_port += 1 - return p + global base_port + p = base_port + base_port += 1 + return p # Most filenames in the following are relative to the working directory. @@ -139,14 +139,14 @@ pubd_last_cms_time = None ecdsa_params = None class CantRekeyYAMLLeaf(Exception): - """ - Can't rekey YAML leaf. - """ + """ + Can't rekey YAML leaf. + """ class CouldntIssueBSCEECertificate(Exception): - """ - Couldn't issue BSC EE certificate - """ + """ + Couldn't issue BSC EE certificate + """ sql_conversions = MySQLdb.converters.conversions.copy() sql_conversions.update({ @@ -154,202 +154,202 @@ sql_conversions.update({ MySQLdb.converters.FIELD_TYPE.DATETIME : rpki.sundial.datetime.DateTime_or_None }) def main(): - """ - Main program. - """ - - rpki.log.init(smoketest_name, argparse.Namespace(log_level = logging.DEBUG, - log_handler = lambda: logging.StreamHandler(sys.stdout))) - logger.info("Starting") - - rpki.http.http_client.timeout = rpki.sundial.timedelta(hours = 1) - - pubd_process = None - rootd_process = None - rsyncd_process = None - - rpki_sql = mangle_sql(rpki_sql_file) - irdb_sql = mangle_sql(irdb_sql_file) - pubd_sql = mangle_sql(pub_sql_file) - - logger.info("Initializing test directory") - - # Connect to test directory, creating it if necessary - try: - os.chdir(smoketest_dir) - except OSError: - os.makedirs(smoketest_dir) - os.chdir(smoketest_dir) - - # Now that we're in the right directory, we can figure out whether - # we have a private openssl executable to use - global prog_openssl - if not os.path.exists(prog_openssl): - prog_openssl = "openssl" - - # Discard everything but keys, which take a while to generate. - # Apparently os.walk() can't tell the difference between directories - # and symlinks to directories, so we have to handle both. - for root, dirs, files in os.walk(".", topdown = False): - for fn in files: - if not fn.endswith(".key"): - os.remove(os.path.join(root, fn)) - for d in dirs: - try: - os.rmdir(os.path.join(root, d)) - except OSError, e: - if e.errno == errno.ENOTDIR: - os.remove(os.path.join(root, d)) - else: - raise - - logger.info("Reading master YAML configuration") - y = yaml_script.pop(0) + """ + Main program. + """ - logger.info("Constructing internal allocation database") - db = allocation_db(y) + rpki.log.init(smoketest_name, argparse.Namespace(log_level = logging.DEBUG, + log_handler = lambda: logging.StreamHandler(sys.stdout))) + logger.info("Starting") - logger.info("Constructing BPKI keys and certs for rootd") - setup_bpki_cert_chain(rootd_name, ee = ("RPKI",)) + rpki.http.http_client.timeout = rpki.sundial.timedelta(hours = 1) - logger.info("Constructing BPKI keys and certs for pubd") - setup_bpki_cert_chain(pubd_name, ee = ("PUBD", "IRBE")) + pubd_process = None + rootd_process = None + rsyncd_process = None + rpki_sql = mangle_sql(rpki_sql_file) + irdb_sql = mangle_sql(irdb_sql_file) + pubd_sql = mangle_sql(pub_sql_file) - for a in db: - a.setup_bpki_certs() + logger.info("Initializing test directory") - setup_publication(pubd_sql, db.root.irdb_db_name) - setup_rootd(db.root, y.get("rootd", {}), db) - setup_rsyncd() - setup_rcynic() + # Connect to test directory, creating it if necessary + try: + os.chdir(smoketest_dir) + except OSError: + os.makedirs(smoketest_dir) + os.chdir(smoketest_dir) + + # Now that we're in the right directory, we can figure out whether + # we have a private openssl executable to use + global prog_openssl + if not os.path.exists(prog_openssl): + prog_openssl = "openssl" + + # Discard everything but keys, which take a while to generate. + # Apparently os.walk() can't tell the difference between directories + # and symlinks to directories, so we have to handle both. + for root, dirs, files in os.walk(".", topdown = False): + for fn in files: + if not fn.endswith(".key"): + os.remove(os.path.join(root, fn)) + for d in dirs: + try: + os.rmdir(os.path.join(root, d)) + except OSError, e: + if e.errno == errno.ENOTDIR: + os.remove(os.path.join(root, d)) + else: + raise + + logger.info("Reading master YAML configuration") + y = yaml_script.pop(0) + + logger.info("Constructing internal allocation database") + db = allocation_db(y) + + logger.info("Constructing BPKI keys and certs for rootd") + setup_bpki_cert_chain(rootd_name, ee = ("RPKI",)) + + logger.info("Constructing BPKI keys and certs for pubd") + setup_bpki_cert_chain(pubd_name, ee = ("PUBD", "IRBE")) + + + for a in db: + a.setup_bpki_certs() + + setup_publication(pubd_sql, db.root.irdb_db_name) + setup_rootd(db.root, y.get("rootd", {}), db) + setup_rsyncd() + setup_rcynic() - for a in db.engines: - a.setup_conf_file() - a.setup_sql(rpki_sql, irdb_sql) - a.sync_sql() + for a in db.engines: + a.setup_conf_file() + a.setup_sql(rpki_sql, irdb_sql) + a.sync_sql() - try: + try: - logger.info("Starting rootd") - rootd_process = subprocess.Popen((prog_python, prog_rootd, "--foreground", "--log-stdout", "--log-level", "debug"), - env = dict(os.environ, RPKI_CONF = rootd_name + ".conf")) + logger.info("Starting rootd") + rootd_process = subprocess.Popen((prog_python, prog_rootd, "--foreground", "--log-stdout", "--log-level", "debug"), + env = dict(os.environ, RPKI_CONF = rootd_name + ".conf")) - logger.info("Starting pubd") - pubd_process = subprocess.Popen((prog_python, prog_pubd, "--foreground", "--log-stdout", "--log-level", "debug") + - (("-p", pubd_name + ".prof") if args.profile else ()), - env = dict(os.environ, RPKI_CONF = pubd_name + ".conf")) + logger.info("Starting pubd") + pubd_process = subprocess.Popen((prog_python, prog_pubd, "--foreground", "--log-stdout", "--log-level", "debug") + + (("-p", pubd_name + ".prof") if args.profile else ()), + env = dict(os.environ, RPKI_CONF = pubd_name + ".conf")) - logger.info("Starting rsyncd") - rsyncd_process = subprocess.Popen((prog_rsyncd, "--daemon", "--no-detach", "--config", rsyncd_name + ".conf")) + logger.info("Starting rsyncd") + rsyncd_process = subprocess.Popen((prog_rsyncd, "--daemon", "--no-detach", "--config", rsyncd_name + ".conf")) - # Start rpkid and irdbd instances - for a in db.engines: - a.run_daemons() + # Start rpkid and irdbd instances + for a in db.engines: + a.run_daemons() - # From this point on we'll be running event-driven, so the rest of - # the code until final exit is all closures. + # From this point on we'll be running event-driven, so the rest of + # the code until final exit is all closures. - def start(): - rpki.async.iterator(db.engines, create_rpki_objects, create_pubd_objects) + def start(): + rpki.async.iterator(db.engines, create_rpki_objects, create_pubd_objects) - def create_rpki_objects(iterator, a): - a.create_rpki_objects(iterator) + def create_rpki_objects(iterator, a): + a.create_rpki_objects(iterator) - def create_pubd_objects(): - call_pubd([rpki.publication_control.client_elt.make_pdu(action = "create", - client_handle = db.root.client_handle + "-" + rootd_name, - base_uri = rootd_sia, - bpki_cert = cross_certify(rootd_name + "-TA", pubd_name + "-TA"))], - cb = lambda ignored: yaml_loop()) + def create_pubd_objects(): + call_pubd([rpki.publication_control.client_elt.make_pdu(action = "create", + client_handle = db.root.client_handle + "-" + rootd_name, + base_uri = rootd_sia, + bpki_cert = cross_certify(rootd_name + "-TA", pubd_name + "-TA"))], + cb = lambda ignored: yaml_loop()) - def yaml_loop(): + def yaml_loop(): - # This is probably where we should be updating expired BPKI - # objects, particular CRLs + # This is probably where we should be updating expired BPKI + # objects, particular CRLs - logger.info("Running cron for all RPKI engines") - rpki.async.iterator(db.engines, run_cron, run_yaml) + logger.info("Running cron for all RPKI engines") + rpki.async.iterator(db.engines, run_cron, run_yaml) - def run_cron(iterator, a): - a.run_cron(iterator) + def run_cron(iterator, a): + a.run_cron(iterator) - def run_yaml(): + def run_yaml(): - # Run rcynic to check results - run_rcynic() + # Run rcynic to check results + run_rcynic() - # Apply next delta if we have one; otherwise, we're done. - if yaml_script: - logger.info("Applying deltas") - db.apply_delta(yaml_script.pop(0), apply_delta_done) - else: - logger.info("No more deltas to apply, done") - rpki.async.exit_event_loop() + # Apply next delta if we have one; otherwise, we're done. + if yaml_script: + logger.info("Applying deltas") + db.apply_delta(yaml_script.pop(0), apply_delta_done) + else: + logger.info("No more deltas to apply, done") + rpki.async.exit_event_loop() - def apply_delta_done(): + def apply_delta_done(): - # Resync IRDBs - for a in db.engines: - a.sync_sql() + # Resync IRDBs + for a in db.engines: + a.sync_sql() - # Loop until we run out of control YAML - yaml_loop() + # Loop until we run out of control YAML + yaml_loop() - logger.info("Sleeping %d seconds while daemons start up", startup_delay) - rpki.async.timer(start).set(rpki.sundial.timedelta(seconds = startup_delay)) - rpki.async.event_loop() + logger.info("Sleeping %d seconds while daemons start up", startup_delay) + rpki.async.timer(start).set(rpki.sundial.timedelta(seconds = startup_delay)) + rpki.async.event_loop() - # At this point we have gone into event-driven code. + # At this point we have gone into event-driven code. - logger.info("Event loop exited normally") + logger.info("Event loop exited normally") - except Exception, e: - logger.exception("Event loop exited with an exception") + except Exception, e: + logger.exception("Event loop exited with an exception") - finally: - logger.info("Cleaning up") - for a in db.engines: - a.kill_daemons() - for proc, name in ((rootd_process, "rootd"), - (pubd_process, "pubd"), - (rsyncd_process, "rsyncd")): - # pylint: disable=E1103 - if proc is not None and proc.poll() is None: - logger.info("Killing %s, pid %s", name, proc.pid) - try: - proc.terminate() - except OSError: - pass - if proc is not None: - logger.info("Daemon %s, pid %s exited with code %s", name, proc.pid, proc.wait()) + finally: + logger.info("Cleaning up") + for a in db.engines: + a.kill_daemons() + for proc, name in ((rootd_process, "rootd"), + (pubd_process, "pubd"), + (rsyncd_process, "rsyncd")): + # pylint: disable=E1103 + if proc is not None and proc.poll() is None: + logger.info("Killing %s, pid %s", name, proc.pid) + try: + proc.terminate() + except OSError: + pass + if proc is not None: + logger.info("Daemon %s, pid %s exited with code %s", name, proc.pid, proc.wait()) def cmd_sleep(cb, interval): - """ - Set an alarm, then wait for it to go off. - """ + """ + Set an alarm, then wait for it to go off. + """ - howlong = rpki.sundial.timedelta.parse(interval) - logger.info("Sleeping %r", howlong) - rpki.async.timer(cb).set(howlong) + howlong = rpki.sundial.timedelta.parse(interval) + logger.info("Sleeping %r", howlong) + rpki.async.timer(cb).set(howlong) def cmd_shell(cb, *cmd): - """ - Run a shell command. - """ + """ + Run a shell command. + """ - cmd = " ".join(cmd) - status = subprocess.call(cmd, shell = True) - logger.info("Shell command returned status %d", status) - cb() + cmd = " ".join(cmd) + status = subprocess.call(cmd, shell = True) + logger.info("Shell command returned status %d", status) + cb() def cmd_echo(cb, *words): - """ - Echo some text to the log. - """ + """ + Echo some text to the log. + """ - logger.info(" ".join(words)) - cb() + logger.info(" ".join(words)) + cb() ## @var cmds # Dispatch table for commands embedded in delta sections @@ -359,1019 +359,1019 @@ cmds = { "sleep" : cmd_sleep, "echo" : cmd_echo } class roa_request(object): - """ - Representation for a roa_request object. - """ - - def __init__(self, asn, ipv4, ipv6): - self.asn = asn - self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None - self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None - - def __eq__(self, other): - return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 - - def __hash__(self): - v4 = tuple(self.v4) if self.v4 is not None else None - v6 = tuple(self.v6) if self.v6 is not None else None - return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - - def __str__(self): - if self.v4 and self.v6: s = str(self.v4) + "," + str(self.v6) - elif self.v4: s = str(self.v4) - else: s = str(self.v6) - return "%s: %s" % (self.asn, s) - - @classmethod - def parse(cls, yaml): - return cls(yaml.get("asn"), yaml.get("ipv4"), yaml.get("ipv6")) - -class router_cert(object): - """ - Representation for a router_cert object. - """ - - _ecparams = None - _keypair = None - _pkcs10 = None - _gski = None - - @classmethod - def ecparams(cls): - if cls._ecparams is None: - cls._ecparams = rpki.x509.KeyParams.generateEC() - return cls._ecparams - - def __init__(self, asn, router_id): - self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) - self.router_id = router_id - self.cn = "ROUTER-%08x" % self.asn[0].min - self.sn = "%08x" % self.router_id - self.eku = rpki.oids.id_kp_bgpsec_router - - @property - def keypair(self): - if self._keypair is None: - self._keypair = rpki.x509.ECDSA.generate(self.ecparams()) - return self._keypair - - @property - def pkcs10(self): - if self._pkcs10 is None: - self._pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) - return self._pkcs10 - - @property - def gski(self): - if self._gski is None: - self._gski = self.pkcs10.gSKI() - return self._gski - - def __eq__(self, other): - return self.asn == other.asn and self.sn == other.sn - - def __hash__(self): - return tuple(self.asn).__hash__() + self.cn.__hash__() + self.sn.__hash__() - - def __str__(self): - return "%s: %s,%s: %s" % (self.asn, self.cn, self.sn, self.gski) - - @classmethod - def parse(cls, yaml): - return cls(yaml.get("asn"), yaml.get("router_id")) - -class allocation_db(list): - """ - Representation of all the entities and allocations in the test - system. Almost everything is generated out of this database. - """ - - def __init__(self, yaml): - """ - Initialize database from the (first) YAML document. """ - - list.__init__(self) - self.root = allocation(yaml, self) - assert self.root.is_root - if self.root.crl_interval is None: - self.root.crl_interval = rpki.sundial.timedelta.parse(cfg.get("crl_interval", "1d")).convert_to_seconds() - if self.root.regen_margin is None: - self.root.regen_margin = rpki.sundial.timedelta.parse(cfg.get("regen_margin", "1d")).convert_to_seconds() - for a in self: - if a.sia_base is None: - a.sia_base = (rootd_sia + "root/trunk/" if a.is_root else a.parent.sia_base) + a.name + "/" - if a.base.valid_until is None: - a.base.valid_until = a.parent.base.valid_until - if a.crl_interval is None: - a.crl_interval = a.parent.crl_interval - if a.regen_margin is None: - a.regen_margin = a.parent.regen_margin - a.client_handle = "/".join(a.sia_base.split("/")[4:]).rstrip("/") - self.root.closure() - self.map = dict((a.name, a) for a in self) - self.engines = [a for a in self if a.is_engine] - for i, a in enumerate(self.engines): - a.set_engine_number(i) - for a in self: - if a.is_hosted: - a.hosted_by = self.map[a.hosted_by] - a.hosted_by.hosts.append(a) - assert a.is_twig, "%s is not twig" % a.name - assert not a.hosted_by.is_hosted, "%s is hosted by a hosted entity" % a.name - - def apply_delta(self, delta, cb): + Representation for a roa_request object. """ - Apply a delta or run a command. - """ - - def loop(iterator, d): - if isinstance(d, str): - c = d.split() - cmds[c[0]](iterator, *c[1:]) - else: - self.map[d["name"]].apply_delta(d, iterator) - def done(): - self.root.closure() - cb() + def __init__(self, asn, ipv4, ipv6): + self.asn = asn + self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None + self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None - if delta is None: - cb() - else: - rpki.async.iterator(delta, loop, done) + def __eq__(self, other): + return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 - def dump(self): - """ - Print content of the database. - """ + def __hash__(self): + v4 = tuple(self.v4) if self.v4 is not None else None + v6 = tuple(self.v6) if self.v6 is not None else None + return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - for a in self: - print a + def __str__(self): + if self.v4 and self.v6: s = str(self.v4) + "," + str(self.v6) + elif self.v4: s = str(self.v4) + else: s = str(self.v6) + return "%s: %s" % (self.asn, s) -class allocation(object): - - parent = None - irdb_db_name = None - irdb_port = None - rpki_db_name = None - rpki_port = None - crl_interval = None - regen_margin = None - last_cms_time = None - rpkid_process = None - irdbd_process = None - - def __init__(self, yaml, db, parent = None): - """ - Initialize one entity and insert it into the database. - """ + @classmethod + def parse(cls, yaml): + return cls(yaml.get("asn"), yaml.get("ipv4"), yaml.get("ipv6")) - db.append(self) - self.name = yaml["name"] - self.parent = parent - self.kids = [allocation(k, db, self) for k in yaml.get("kids", ())] - valid_until = None - if "valid_until" in yaml: - valid_until = rpki.sundial.datetime.from_datetime(yaml.get("valid_until")) - if valid_until is None and "valid_for" in yaml: - valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(yaml["valid_for"]) - self.base = rpki.resource_set.resource_bag( - asn = rpki.resource_set.resource_set_as(yaml.get("asn")), - v4 = rpki.resource_set.resource_set_ipv4(yaml.get("ipv4")), - v6 = rpki.resource_set.resource_set_ipv6(yaml.get("ipv6")), - valid_until = valid_until) - self.sia_base = yaml.get("sia_base") - if "crl_interval" in yaml: - self.crl_interval = rpki.sundial.timedelta.parse(yaml["crl_interval"]).convert_to_seconds() - if "regen_margin" in yaml: - self.regen_margin = rpki.sundial.timedelta.parse(yaml["regen_margin"]).convert_to_seconds() - self.roa_requests = [roa_request.parse(y) for y in yaml.get("roa_request", yaml.get("route_origin", ()))] - for r in self.roa_requests: - if r.v4: - self.base.v4 |= r.v4.to_resource_set() - if r.v6: - self.base.v6 |= r.v6.to_resource_set() - self.router_certs = [router_cert.parse(y) for y in yaml.get("router_cert", ())] - for r in self.router_certs: - self.base.asn |= r.asn - self.hosted_by = yaml.get("hosted_by") - self.extra_conf = yaml.get("extra_conf", []) - self.hosts = [] - - def closure(self): +class router_cert(object): """ - Compute the transitive resource closure. + Representation for a router_cert object. """ - resources = self.base - for kid in self.kids: - resources |= kid.closure() - self.resources = resources - return resources + _ecparams = None + _keypair = None + _pkcs10 = None + _gski = None + + @classmethod + def ecparams(cls): + if cls._ecparams is None: + cls._ecparams = rpki.x509.KeyParams.generateEC() + return cls._ecparams + + def __init__(self, asn, router_id): + self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) + self.router_id = router_id + self.cn = "ROUTER-%08x" % self.asn[0].min + self.sn = "%08x" % self.router_id + self.eku = rpki.oids.id_kp_bgpsec_router + + @property + def keypair(self): + if self._keypair is None: + self._keypair = rpki.x509.ECDSA.generate(self.ecparams()) + return self._keypair + + @property + def pkcs10(self): + if self._pkcs10 is None: + self._pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) + return self._pkcs10 + + @property + def gski(self): + if self._gski is None: + self._gski = self.pkcs10.gSKI() + return self._gski + + def __eq__(self, other): + return self.asn == other.asn and self.sn == other.sn + + def __hash__(self): + return tuple(self.asn).__hash__() + self.cn.__hash__() + self.sn.__hash__() + + def __str__(self): + return "%s: %s,%s: %s" % (self.asn, self.cn, self.sn, self.gski) + + @classmethod + def parse(cls, yaml): + return cls(yaml.get("asn"), yaml.get("router_id")) - def apply_delta(self, yaml, cb): +class allocation_db(list): """ - Apply deltas to this entity. + Representation of all the entities and allocations in the test + system. Almost everything is generated out of this database. """ - logger.info("Applying delta: %s", yaml) - - def loop(iterator, kv): - if kv[0] == "name": - iterator() - else: - getattr(self, "apply_" + kv[0])(kv[1], iterator) - - rpki.async.iterator(yaml.items(), loop, cb) - - def apply_add_as(self, text, cb): - self.base.asn |= rpki.resource_set.resource_set_as(text) - cb() - - def apply_add_v4(self, text, cb): - self.base.v4 |= rpki.resource_set.resource_set_ipv4(text) - cb() - - def apply_add_v6(self, text, cb): - self.base.v6 |= rpki.resource_set.resource_set_ipv6(text) - cb() - - def apply_sub_as(self, text, cb): - self.base.asn |= rpki.resource_set.resource_set_as(text) - cb() + def __init__(self, yaml): + """ + Initialize database from the (first) YAML document. + """ + + list.__init__(self) + self.root = allocation(yaml, self) + assert self.root.is_root + if self.root.crl_interval is None: + self.root.crl_interval = rpki.sundial.timedelta.parse(cfg.get("crl_interval", "1d")).convert_to_seconds() + if self.root.regen_margin is None: + self.root.regen_margin = rpki.sundial.timedelta.parse(cfg.get("regen_margin", "1d")).convert_to_seconds() + for a in self: + if a.sia_base is None: + a.sia_base = (rootd_sia + "root/trunk/" if a.is_root else a.parent.sia_base) + a.name + "/" + if a.base.valid_until is None: + a.base.valid_until = a.parent.base.valid_until + if a.crl_interval is None: + a.crl_interval = a.parent.crl_interval + if a.regen_margin is None: + a.regen_margin = a.parent.regen_margin + a.client_handle = "/".join(a.sia_base.split("/")[4:]).rstrip("/") + self.root.closure() + self.map = dict((a.name, a) for a in self) + self.engines = [a for a in self if a.is_engine] + for i, a in enumerate(self.engines): + a.set_engine_number(i) + for a in self: + if a.is_hosted: + a.hosted_by = self.map[a.hosted_by] + a.hosted_by.hosts.append(a) + assert a.is_twig, "%s is not twig" % a.name + assert not a.hosted_by.is_hosted, "%s is hosted by a hosted entity" % a.name + + def apply_delta(self, delta, cb): + """ + Apply a delta or run a command. + """ + + def loop(iterator, d): + if isinstance(d, str): + c = d.split() + cmds[c[0]](iterator, *c[1:]) + else: + self.map[d["name"]].apply_delta(d, iterator) + + def done(): + self.root.closure() + cb() + + if delta is None: + cb() + else: + rpki.async.iterator(delta, loop, done) - def apply_sub_v4(self, text, cb): - self.base.v4 |= rpki.resource_set.resource_set_ipv4(text) - cb() + def dump(self): + """ + Print content of the database. + """ - def apply_sub_v6(self, text, cb): - self.base.v6 |= rpki.resource_set.resource_set_ipv6(text) - cb() + for a in self: + print a - def apply_valid_until(self, stamp, cb): - self.base.valid_until = rpki.sundial.datetime.from_datetime(stamp) - cb() +class allocation(object): - def apply_valid_for(self, text, cb): - self.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(text) - cb() + parent = None + irdb_db_name = None + irdb_port = None + rpki_db_name = None + rpki_port = None + crl_interval = None + regen_margin = None + last_cms_time = None + rpkid_process = None + irdbd_process = None + + def __init__(self, yaml, db, parent = None): + """ + Initialize one entity and insert it into the database. + """ + + db.append(self) + self.name = yaml["name"] + self.parent = parent + self.kids = [allocation(k, db, self) for k in yaml.get("kids", ())] + valid_until = None + if "valid_until" in yaml: + valid_until = rpki.sundial.datetime.from_datetime(yaml.get("valid_until")) + if valid_until is None and "valid_for" in yaml: + valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(yaml["valid_for"]) + self.base = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as(yaml.get("asn")), + v4 = rpki.resource_set.resource_set_ipv4(yaml.get("ipv4")), + v6 = rpki.resource_set.resource_set_ipv6(yaml.get("ipv6")), + valid_until = valid_until) + self.sia_base = yaml.get("sia_base") + if "crl_interval" in yaml: + self.crl_interval = rpki.sundial.timedelta.parse(yaml["crl_interval"]).convert_to_seconds() + if "regen_margin" in yaml: + self.regen_margin = rpki.sundial.timedelta.parse(yaml["regen_margin"]).convert_to_seconds() + self.roa_requests = [roa_request.parse(y) for y in yaml.get("roa_request", yaml.get("route_origin", ()))] + for r in self.roa_requests: + if r.v4: + self.base.v4 |= r.v4.to_resource_set() + if r.v6: + self.base.v6 |= r.v6.to_resource_set() + self.router_certs = [router_cert.parse(y) for y in yaml.get("router_cert", ())] + for r in self.router_certs: + self.base.asn |= r.asn + self.hosted_by = yaml.get("hosted_by") + self.extra_conf = yaml.get("extra_conf", []) + self.hosts = [] + + def closure(self): + """ + Compute the transitive resource closure. + """ + + resources = self.base + for kid in self.kids: + resources |= kid.closure() + self.resources = resources + return resources + + def apply_delta(self, yaml, cb): + """ + Apply deltas to this entity. + """ + + logger.info("Applying delta: %s", yaml) + + def loop(iterator, kv): + if kv[0] == "name": + iterator() + else: + getattr(self, "apply_" + kv[0])(kv[1], iterator) + + rpki.async.iterator(yaml.items(), loop, cb) + + def apply_add_as(self, text, cb): + self.base.asn |= rpki.resource_set.resource_set_as(text) + cb() + + def apply_add_v4(self, text, cb): + self.base.v4 |= rpki.resource_set.resource_set_ipv4(text) + cb() + + def apply_add_v6(self, text, cb): + self.base.v6 |= rpki.resource_set.resource_set_ipv6(text) + cb() + + def apply_sub_as(self, text, cb): + self.base.asn |= rpki.resource_set.resource_set_as(text) + cb() + + def apply_sub_v4(self, text, cb): + self.base.v4 |= rpki.resource_set.resource_set_ipv4(text) + cb() + + def apply_sub_v6(self, text, cb): + self.base.v6 |= rpki.resource_set.resource_set_ipv6(text) + cb() + + def apply_valid_until(self, stamp, cb): + self.base.valid_until = rpki.sundial.datetime.from_datetime(stamp) + cb() + + def apply_valid_for(self, text, cb): + self.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(text) + cb() + + def apply_valid_add(self, text, cb): + self.base.valid_until += rpki.sundial.timedelta.parse(text) + cb() + + def apply_valid_sub(self, text, cb): + self.base.valid_until -= rpki.sundial.timedelta.parse(text) + cb() + + def apply_roa_request_add(self, yaml, cb): + for y in yaml: + r = roa_request.parse(y) + if r not in self.roa_requests: + self.roa_requests.append(r) + cb() + + def apply_roa_request_del(self, yaml, cb): + for y in yaml: + r = roa_request.parse(y) + if r in self.roa_requests: + self.roa_requests.remove(r) + cb() + + def apply_router_cert_add(self, yaml, cb): + for y in yaml: + r = router_cert.parse(y) + if r not in self.router_certs: + self.router_certs.append(r) + cb() + + def apply_router_cert_del(self, yaml, cb): + for y in yaml: + r = router_cert.parse(y) + if r in self.router_certs: + self.router_certs.remove(r) + cb() + + def apply_rekey(self, target, cb): + + def done(e): + if isinstance(e, Exception): + logger.exception("Exception while rekeying %s", self.name) + raise e + cb() + + if target is None: + logger.info("Rekeying %s", self.name) + self.call_rpkid([rpki.left_right.self_elt.make_pdu( + action = "set", self_handle = self.name, rekey = "yes")], cb = done) + else: + logger.info("Rekeying %s %s", self.name, target) + self.call_rpkid([rpki.left_right.parent_elt.make_pdu( + action = "set", self_handle = self.name, parent_handle = target, rekey = "yes")], cb = done) + + def apply_revoke(self, target, cb): + + def done(e): + if isinstance(e, Exception): + logger.exception("Exception while revoking %s", self.name) + raise e + cb() + + if target is None: + logger.info("Revoking %s", self.name) + self.call_rpkid([rpki.left_right.self_elt.make_pdu( + action = "set", self_handle = self.name, revoke = "yes")], cb = done) + else: + logger.info("Revoking %s %s", self.name, target) + self.call_rpkid([rpki.left_right.parent_elt.make_pdu( + action = "set", self_handle = self.name, parent_handle = target, revoke = "yes")], cb = done) + + def __str__(self): + s = self.name + "\n" + if self.resources.asn: s += " ASN: %s\n" % self.resources.asn + if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 + if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 + if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) + if self.parent: s += " Up: %s\n" % self.parent.name + if self.sia_base: s += " SIA: %s\n" % self.sia_base + return s + "Until: %s\n" % self.resources.valid_until + + + @property + def is_root(self): + return self.parent is None + + @property + def is_twig(self): + return not self.is_root + + @property + def is_hosted(self): + return self.hosted_by is not None + + @property + def is_engine(self): + return not self.is_hosted + + def set_engine_number(self, n): + """ + Set the engine number for this entity. + """ + + self.irdb_db_name = "irdb%d" % n + self.irdb_port = allocate_port() + self.rpki_db_name = "rpki%d" % n + self.rpki_port = allocate_port() + + def get_rpki_port(self): + """ + Get rpki port to use for this entity. + """ + + if self.is_hosted: + assert self.hosted_by.rpki_port is not None + return self.hosted_by.rpki_port + else: + assert self.rpki_port is not None + return self.rpki_port + + def setup_bpki_certs(self): + """ + Create BPKI certificates for this entity. + """ + + logger.info("Constructing BPKI keys and certs for %s", self.name) + setup_bpki_cert_chain(name = self.name, + ee = ("RPKI", "IRDB", "IRBE"), + ca = ("SELF",)) + self.rpkid_ta = rpki.x509.X509(PEM_file = self.name + "-TA.cer") + self.irbe_key = rpki.x509.RSA( PEM_file = self.name + "-IRBE.key") + self.irbe_cert = rpki.x509.X509(PEM_file = self.name + "-IRBE.cer") + self.rpkid_cert = rpki.x509.X509(PEM_file = self.name + "-RPKI.cer") + + def setup_conf_file(self): + """ + Write config files for this entity. + """ + + logger.info("Writing config files for %s", self.name) + assert self.rpki_port is not None + d = dict(my_name = self.name, + irdb_db_name = self.irdb_db_name, + irdb_db_pass = irdb_db_pass, + irdb_port = self.irdb_port, + rpki_db_name = self.rpki_db_name, + rpki_db_pass = rpki_db_pass, + rpki_port = self.rpki_port) + f = open(self.name + ".conf", "w") + f.write(conf_fmt_1 % d) + for line in self.extra_conf: + f.write(line + "\n") + f.close() + + def setup_sql(self, rpki_sql, irdb_sql): + """ + Set up this entity's IRDB. + """ + + logger.info("Setting up MySQL for %s", self.name) + db = MySQLdb.connect(user = "rpki", db = self.rpki_db_name, passwd = rpki_db_pass, + conv = sql_conversions) + cur = db.cursor() + db.autocommit(True) + for sql in rpki_sql: + try: + cur.execute(sql) + except: + if "DROP TABLE IF EXISTS" not in sql.upper(): + raise + db.close() + db = MySQLdb.connect(user = "irdb", db = self.irdb_db_name, passwd = irdb_db_pass, + conv = sql_conversions) + cur = db.cursor() + db.autocommit(True) + for sql in irdb_sql: + try: + cur.execute(sql) + except: + if "DROP TABLE IF EXISTS" not in sql.upper(): + raise + for s in [self] + self.hosts: + for kid in s.kids: + cur.execute("INSERT registrant (registrant_handle, registry_handle, valid_until) VALUES (%s, %s, %s)", + (kid.name, s.name, kid.resources.valid_until)) + db.close() + + def sync_sql(self): + """ + Whack this entity's IRDB to match our master database. We do this + once during setup, then do it again every time we apply a delta to + this entity. + """ + + logger.info("Updating MySQL data for IRDB %s", self.name) + db = MySQLdb.connect(user = "irdb", db = self.irdb_db_name, passwd = irdb_db_pass, + conv = sql_conversions) + cur = db.cursor() + db.autocommit(True) + cur.execute("DELETE FROM registrant_asn") + cur.execute("DELETE FROM registrant_net") + cur.execute("DELETE FROM roa_request_prefix") + cur.execute("DELETE FROM roa_request") + cur.execute("DELETE FROM ee_certificate_asn") + cur.execute("DELETE FROM ee_certificate_net") + cur.execute("DELETE FROM ee_certificate") + + for s in [self] + self.hosts: + for kid in s.kids: + cur.execute("SELECT registrant_id FROM registrant WHERE registrant_handle = %s AND registry_handle = %s", + (kid.name, s.name)) + registrant_id = cur.fetchone()[0] + for as_range in kid.resources.asn: + cur.execute("INSERT registrant_asn (start_as, end_as, registrant_id) VALUES (%s, %s, %s)", + (as_range.min, as_range.max, registrant_id)) + for v4_range in kid.resources.v4: + cur.execute("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 4, %s)", + (v4_range.min, v4_range.max, registrant_id)) + for v6_range in kid.resources.v6: + cur.execute("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 6, %s)", + (v6_range.min, v6_range.max, registrant_id)) + cur.execute("UPDATE registrant SET valid_until = %s WHERE registrant_id = %s", + (kid.resources.valid_until, registrant_id)) + for r in s.roa_requests: + cur.execute("INSERT roa_request (self_handle, asn) VALUES (%s, %s)", + (s.name, r.asn)) + roa_request_id = cur.lastrowid + for version, prefix_set in ((4, r.v4), (6, r.v6)): + if prefix_set: + cur.executemany("INSERT roa_request_prefix " + "(roa_request_id, prefix, prefixlen, max_prefixlen, version) " + "VALUES (%s, %s, %s, %s, %s)", + ((roa_request_id, x.prefix, x.prefixlen, x.max_prefixlen, version) + for x in prefix_set)) + for r in s.router_certs: + cur.execute("INSERT ee_certificate (self_handle, pkcs10, gski, cn, sn, eku, valid_until) " + "VALUES (%s, %s, %s, %s, %s, %s, %s)", + (s.name, r.pkcs10.get_DER(), r.gski, r.cn, r.sn, r.eku, s.resources.valid_until)) + ee_certificate_id = cur.lastrowid + cur.executemany("INSERT ee_certificate_asn (ee_certificate_id, start_as, end_as) VALUES (%s, %s, %s)", + ((ee_certificate_id, a.min, a.max) for a in r.asn)) + db.close() + + def run_daemons(self): + """ + Run daemons for this entity. + """ + + logger.info("Running daemons for %s", self.name) + env = dict(os.environ, RPKI_CONF = self.name + ".conf") + self.rpkid_process = subprocess.Popen((prog_python, prog_rpkid, "--foreground", "--log-stdout", "--log-level", "debug") + + (("--profile", self.name + ".prof") if args.profile else ()), + env = env) + self.irdbd_process = subprocess.Popen((prog_python, prog_irdbd, "--foreground", "--log-stdout", "--log-level", "debug"), + env = env) + + def kill_daemons(self): + """ + Kill daemons for this entity. + """ + + # pylint: disable=E1103 + for proc, name in ((self.rpkid_process, "rpkid"), + (self.irdbd_process, "irdbd")): + if proc is not None and proc.poll() is None: + logger.info("Killing daemon %s pid %s for %s", name, proc.pid, self.name) + try: + proc.terminate() + except OSError: + pass + if proc is not None: + logger.info("Daemon %s pid %s for %s exited with code %s", + name, proc.pid, self.name, proc.wait()) + + def call_rpkid(self, pdus, cb): + """ + Send a left-right message to this entity's RPKI daemon and return + the response. + + If this entity is hosted (does not run its own RPKI daemon), all + of this happens with the hosting RPKI daemon. + """ + + logger.info("Calling rpkid for %s", self.name) + + if self.is_hosted: + logger.info("rpkid %s is hosted by rpkid %s, switching", self.name, self.hosted_by.name) + self = self.hosted_by + assert not self.is_hosted + + assert isinstance(pdus, (list, tuple)) + assert self.rpki_port is not None + + q_msg = rpki.left_right.msg.query(*pdus) + q_cms = rpki.left_right.cms_msg_saxify() + q_der = q_cms.wrap(q_msg, self.irbe_key, self.irbe_cert) + q_url = "http://localhost:%d/left-right" % self.rpki_port + + logger.debug(q_cms.pretty_print_content()) + + def done(r_der): + logger.info("Callback from rpkid %s", self.name) + r_cms = rpki.left_right.cms_msg_saxify(DER = r_der) + r_msg = r_cms.unwrap((self.rpkid_ta, self.rpkid_cert)) + self.last_cms_time = r_cms.check_replay(self.last_cms_time, q_url) + logger.debug(r_cms.pretty_print_content()) + assert r_msg.is_reply + for r_pdu in r_msg: + assert not isinstance(r_pdu, rpki.left_right.report_error_elt) + cb(r_msg) + + def lose(e): + raise + + rpki.http.client( + url = q_url, + msg = q_der, + callback = done, + errback = lose) + + def cross_certify(self, certificant, reverse = False): + """ + Cross-certify and return the resulting certificate. + """ + + if reverse: + certifier = certificant + certificant = self.name + "-SELF" + else: + certifier = self.name + "-SELF" + return cross_certify(certificant, certifier) + + def create_rpki_objects(self, cb): + """ + Create RPKI engine objects for this engine. + + Root node of the engine tree is special, it too has a parent but + that one is the magic self-signed micro engine. + + The rest of this is straightforward. There are a lot of objects + to create, but we can do batch them all into one honking PDU, then + issue one more PDU to set BSC EE certificates based on the PKCS + #10 requests we get back when we tell rpkid to generate BSC keys. + """ + + assert not self.is_hosted + + selves = [self] + self.hosts + + rpkid_pdus = [] + pubd_pdus = [] + + for i, s in enumerate(selves): + logger.info("Creating RPKI objects for [%d] %s", i, s.name) + + rpkid_pdus.append(rpki.left_right.self_elt.make_pdu( + action = "create", + self_handle = s.name, + crl_interval = s.crl_interval, + regen_margin = s.regen_margin, + bpki_cert = (s.cross_certify(s.hosted_by.name + "-TA", reverse = True) + if s.is_hosted else + rpki.x509.X509(Auto_file = s.name + "-SELF.cer")))) + + rpkid_pdus.append(rpki.left_right.bsc_elt.make_pdu( + action = "create", + self_handle = s.name, + bsc_handle = "b", + generate_keypair = True)) + + pubd_pdus.append(rpki.publication_control.client_elt.make_pdu( + action = "create", + client_handle = s.client_handle, + base_uri = s.sia_base, + bpki_cert = s.cross_certify(pubd_name + "-TA", reverse = True))) + + rpkid_pdus.append(rpki.left_right.repository_elt.make_pdu( + action = "create", + self_handle = s.name, + bsc_handle = "b", + repository_handle = "r", + bpki_cert = s.cross_certify(pubd_name + "-TA"), + peer_contact_uri = "http://localhost:%d/client/%s" % (pubd_port, s.client_handle))) + + for k in s.kids: + rpkid_pdus.append(rpki.left_right.child_elt.make_pdu( + action = "create", + self_handle = s.name, + child_handle = k.name, + bsc_handle = "b", + bpki_cert = s.cross_certify(k.name + "-SELF"))) + + if s.is_root: + rootd_cert = s.cross_certify(rootd_name + "-TA") + rpkid_pdus.append(rpki.left_right.parent_elt.make_pdu( + action = "create", + self_handle = s.name, + parent_handle = "rootd", + bsc_handle = "b", + repository_handle = "r", + sia_base = s.sia_base, + bpki_cert = rootd_cert, + sender_name = s.name, + recipient_name = "rootd", + peer_contact_uri = "http://localhost:%s/" % rootd_port)) + else: + rpkid_pdus.append(rpki.left_right.parent_elt.make_pdu( + action = "create", + self_handle = s.name, + parent_handle = s.parent.name, + bsc_handle = "b", + repository_handle = "r", + sia_base = s.sia_base, + bpki_cert = s.cross_certify(s.parent.name + "-SELF"), + sender_name = s.name, + recipient_name = s.parent.name, + peer_contact_uri = "http://localhost:%s/up-down/%s/%s" % (s.parent.get_rpki_port(), + s.parent.name, s.name))) + + def one(): + call_pubd(pubd_pdus, cb = two) + + def two(vals): + self.call_rpkid(rpkid_pdus, cb = three) + + def three(vals): + + bsc_dict = dict((b.self_handle, b) for b in vals if isinstance(b, rpki.left_right.bsc_elt)) + + bsc_pdus = [] + + for s in selves: + b = bsc_dict[s.name] + + logger.info("Issuing BSC EE cert for %s", s.name) + cmd = (prog_openssl, "x509", "-req", "-sha256", "-extfile", s.name + "-RPKI.conf", + "-extensions", "req_x509_ext", "-days", "30", + "-CA", s.name + "-SELF.cer", "-CAkey", s.name + "-SELF.key", "-CAcreateserial", "-text") + signer = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE) + signed = signer.communicate(input = b.pkcs10_request.get_PEM()) + if not signed[0]: + logger.warning(signed[1]) + raise CouldntIssueBSCEECertificate("Couldn't issue BSC EE certificate") + s.bsc_ee = rpki.x509.X509(PEM = signed[0]) + s.bsc_crl = rpki.x509.CRL(PEM_file = s.name + "-SELF.crl") + logger.info("BSC EE cert for %s SKI %s", s.name, s.bsc_ee.hSKI()) + + bsc_pdus.append(rpki.left_right.bsc_elt.make_pdu( + action = "set", + self_handle = s.name, + bsc_handle = "b", + signing_cert = s.bsc_ee, + signing_cert_crl = s.bsc_crl)) + + self.call_rpkid(bsc_pdus, cb = four) + + def four(vals): + cb() + + one() + + def setup_yaml_leaf(self): + """ + Generate certificates and write YAML scripts for leaf nodes. + + We're cheating a bit here: properly speaking, we can't generate + issue or revoke requests without knowing the class, which is + generated on the fly, but at the moment the test case is + simplistic enough that the class will always be "1", so we just + wire in that value for now. + + Well, ok, we just broke that assumption. Now we do something even + nastier, just to eke a bit more life out of this kludge. This + really needs to be rewritten, but it may require a different tool + than testpoke. + """ + + if not os.path.exists(self.name + ".key"): + logger.info("Generating RPKI key for %s", self.name) + subprocess.check_call((prog_openssl, "genrsa", "-out", self.name + ".key", "2048" ), + stdout = subprocess.PIPE, stderr = subprocess.STDOUT) + ski = rpki.x509.RSA(PEM_file = self.name + ".key").gSKI() + + if self.parent.is_hosted: + parent_host = self.parent.hosted_by.name + else: + parent_host = self.parent.name - def apply_valid_add(self, text, cb): - self.base.valid_until += rpki.sundial.timedelta.parse(text) - cb() + self.cross_certify(self.parent.name + "-SELF") + self.cross_certify(parent_host + "-TA") - def apply_valid_sub(self, text, cb): - self.base.valid_until -= rpki.sundial.timedelta.parse(text) - cb() + def run_cron(self, cb): + """ + Trigger cron run for this engine. + """ - def apply_roa_request_add(self, yaml, cb): - for y in yaml: - r = roa_request.parse(y) - if r not in self.roa_requests: - self.roa_requests.append(r) - cb() + logger.info("Running cron for %s", self.name) - def apply_roa_request_del(self, yaml, cb): - for y in yaml: - r = roa_request.parse(y) - if r in self.roa_requests: - self.roa_requests.remove(r) - cb() + assert self.rpki_port is not None - def apply_router_cert_add(self, yaml, cb): - for y in yaml: - r = router_cert.parse(y) - if r not in self.router_certs: - self.router_certs.append(r) - cb() + def done(result): + assert result == "OK", 'Expected "OK" result from cronjob, got %r' % result + cb() - def apply_router_cert_del(self, yaml, cb): - for y in yaml: - r = router_cert.parse(y) - if r in self.router_certs: - self.router_certs.remove(r) - cb() + rpki.http.client( + url = "http://localhost:%d/cronjob" % self.rpki_port, + msg = "Run cron now, please", + callback = done, + errback = done) - def apply_rekey(self, target, cb): - - def done(e): - if isinstance(e, Exception): - logger.exception("Exception while rekeying %s", self.name) - raise e - cb() - - if target is None: - logger.info("Rekeying %s", self.name) - self.call_rpkid([rpki.left_right.self_elt.make_pdu( - action = "set", self_handle = self.name, rekey = "yes")], cb = done) - else: - logger.info("Rekeying %s %s", self.name, target) - self.call_rpkid([rpki.left_right.parent_elt.make_pdu( - action = "set", self_handle = self.name, parent_handle = target, rekey = "yes")], cb = done) - - def apply_revoke(self, target, cb): - - def done(e): - if isinstance(e, Exception): - logger.exception("Exception while revoking %s", self.name) - raise e - cb() - - if target is None: - logger.info("Revoking %s", self.name) - self.call_rpkid([rpki.left_right.self_elt.make_pdu( - action = "set", self_handle = self.name, revoke = "yes")], cb = done) - else: - logger.info("Revoking %s %s", self.name, target) - self.call_rpkid([rpki.left_right.parent_elt.make_pdu( - action = "set", self_handle = self.name, parent_handle = target, revoke = "yes")], cb = done) - - def __str__(self): - s = self.name + "\n" - if self.resources.asn: s += " ASN: %s\n" % self.resources.asn - if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 - if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 - if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) - if self.parent: s += " Up: %s\n" % self.parent.name - if self.sia_base: s += " SIA: %s\n" % self.sia_base - return s + "Until: %s\n" % self.resources.valid_until - - - @property - def is_root(self): - return self.parent is None - - @property - def is_twig(self): - return not self.is_root - - @property - def is_hosted(self): - return self.hosted_by is not None - - @property - def is_engine(self): - return not self.is_hosted - - def set_engine_number(self, n): - """ - Set the engine number for this entity. - """ + def run_yaml(self): + """ + Run YAML scripts for this leaf entity. Since we're not bothering + to check the class list returned by the list command, the issue + command may fail, so we treat failure of the list command as an + error, but only issue a warning when issue fails. + """ - self.irdb_db_name = "irdb%d" % n - self.irdb_port = allocate_port() - self.rpki_db_name = "rpki%d" % n - self.rpki_port = allocate_port() + logger.info("Running YAML for %s", self.name) + subprocess.check_call((prog_python, prog_poke, "-y", self.name + ".yaml", "-r", "list")) + if subprocess.call((prog_python, prog_poke, "-y", self.name + ".yaml", "-r", "issue")) != 0: + logger.warning("YAML issue command failed for %s, continuing", self.name) - def get_rpki_port(self): +def setup_bpki_cert_chain(name, ee = (), ca = ()): """ - Get rpki port to use for this entity. + Build a set of BPKI certificates. """ - if self.is_hosted: - assert self.hosted_by.rpki_port is not None - return self.hosted_by.rpki_port - else: - assert self.rpki_port is not None - return self.rpki_port + s = "exec >/dev/null 2>&1\n" + #s = "set -x\n" + for kind in ("TA",) + ee + ca: + d = dict(name = name, + kind = kind, + ca = "false" if kind in ee else "true", + openssl = prog_openssl) + f = open("%(name)s-%(kind)s.conf" % d, "w") + f.write(bpki_cert_fmt_1 % d) + f.close() + if not os.path.exists("%(name)s-%(kind)s.key" % d): + s += bpki_cert_fmt_2 % d + s += bpki_cert_fmt_3 % d + d = dict(name = name, + openssl = prog_openssl) + s += bpki_cert_fmt_4 % d + for kind in ee + ca: + d["kind"] = kind + s += bpki_cert_fmt_5 % d + for kind in ("TA",) + ca: + d["kind"] = kind + s += bpki_cert_fmt_6 % d + subprocess.check_call(s, shell = True) - def setup_bpki_certs(self): +def setup_rootd(rpkid, rootd_yaml, db): """ - Create BPKI certificates for this entity. + Write the config files for rootd. """ - logger.info("Constructing BPKI keys and certs for %s", self.name) - setup_bpki_cert_chain(name = self.name, - ee = ("RPKI", "IRDB", "IRBE"), - ca = ("SELF",)) - self.rpkid_ta = rpki.x509.X509(PEM_file = self.name + "-TA.cer") - self.irbe_key = rpki.x509.RSA( PEM_file = self.name + "-IRBE.key") - self.irbe_cert = rpki.x509.X509(PEM_file = self.name + "-IRBE.cer") - self.rpkid_cert = rpki.x509.X509(PEM_file = self.name + "-RPKI.cer") + rpkid.cross_certify(rootd_name + "-TA", reverse = True) + cross_certify(pubd_name + "-TA", rootd_name + "-TA") + logger.info("Writing config files for %s", rootd_name) + d = dict(rootd_name = rootd_name, + rootd_port = rootd_port, + rpkid_name = rpkid.name, + pubd_name = pubd_name, + rootd_sia = rootd_sia, + rsyncd_dir = rsyncd_dir, + openssl = prog_openssl, + lifetime = rootd_yaml.get("lifetime", "30d"), + pubd_port = pubd_port, + rootd_handle = db.root.client_handle + "-" + rootd_name) + f = open(rootd_name + ".conf", "w") + f.write(rootd_fmt_1 % d) + f.close() + s = "exec >/dev/null 2>&1\n" + #s = "set -x\n" + if not os.path.exists("root.key"): + s += rootd_fmt_2 % d + s += rootd_fmt_3 % d + subprocess.check_call(s, shell = True) - def setup_conf_file(self): +def setup_rcynic(): """ - Write config files for this entity. + Write the config file for rcynic. """ - logger.info("Writing config files for %s", self.name) - assert self.rpki_port is not None - d = dict(my_name = self.name, - irdb_db_name = self.irdb_db_name, - irdb_db_pass = irdb_db_pass, - irdb_port = self.irdb_port, - rpki_db_name = self.rpki_db_name, - rpki_db_pass = rpki_db_pass, - rpki_port = self.rpki_port) - f = open(self.name + ".conf", "w") - f.write(conf_fmt_1 % d) - for line in self.extra_conf: - f.write(line + "\n") + logger.info("Config file for rcynic") + d = dict(rcynic_name = rcynic_name, + rootd_name = rootd_name, + rootd_sia = rootd_sia) + f = open(rcynic_name + ".conf", "w") + f.write(rcynic_fmt_1 % d) f.close() - def setup_sql(self, rpki_sql, irdb_sql): +def setup_rsyncd(): """ - Set up this entity's IRDB. + Write the config file for rsyncd. """ - logger.info("Setting up MySQL for %s", self.name) - db = MySQLdb.connect(user = "rpki", db = self.rpki_db_name, passwd = rpki_db_pass, - conv = sql_conversions) - cur = db.cursor() - db.autocommit(True) - for sql in rpki_sql: - try: - cur.execute(sql) - except: - if "DROP TABLE IF EXISTS" not in sql.upper(): - raise - db.close() - db = MySQLdb.connect(user = "irdb", db = self.irdb_db_name, passwd = irdb_db_pass, - conv = sql_conversions) - cur = db.cursor() - db.autocommit(True) - for sql in irdb_sql: - try: - cur.execute(sql) - except: - if "DROP TABLE IF EXISTS" not in sql.upper(): - raise - for s in [self] + self.hosts: - for kid in s.kids: - cur.execute("INSERT registrant (registrant_handle, registry_handle, valid_until) VALUES (%s, %s, %s)", - (kid.name, s.name, kid.resources.valid_until)) - db.close() + logger.info("Config file for rsyncd") + d = dict(rsyncd_name = rsyncd_name, + rsyncd_port = rsyncd_port, + rsyncd_module = rsyncd_module, + rsyncd_dir = rsyncd_dir) + f = open(rsyncd_name + ".conf", "w") + f.write(rsyncd_fmt_1 % d) + f.close() - def sync_sql(self): +def setup_publication(pubd_sql, irdb_db_name): """ - Whack this entity's IRDB to match our master database. We do this - once during setup, then do it again every time we apply a delta to - this entity. + Set up publication daemon. """ - logger.info("Updating MySQL data for IRDB %s", self.name) - db = MySQLdb.connect(user = "irdb", db = self.irdb_db_name, passwd = irdb_db_pass, + logger.info("Configure publication daemon") + publication_dir = os.getcwd() + "/publication" + assert rootd_sia.startswith("rsync://") + global rsyncd_dir + rsyncd_dir = publication_dir + "/".join(rootd_sia.split("/")[4:]) + if not rsyncd_dir.endswith("/"): + rsyncd_dir += "/" + os.makedirs(rsyncd_dir + "root/trunk") + db = MySQLdb.connect(db = pubd_db_name, user = pubd_db_user, passwd = pubd_db_pass, conv = sql_conversions) cur = db.cursor() db.autocommit(True) - cur.execute("DELETE FROM registrant_asn") - cur.execute("DELETE FROM registrant_net") - cur.execute("DELETE FROM roa_request_prefix") - cur.execute("DELETE FROM roa_request") - cur.execute("DELETE FROM ee_certificate_asn") - cur.execute("DELETE FROM ee_certificate_net") - cur.execute("DELETE FROM ee_certificate") - - for s in [self] + self.hosts: - for kid in s.kids: - cur.execute("SELECT registrant_id FROM registrant WHERE registrant_handle = %s AND registry_handle = %s", - (kid.name, s.name)) - registrant_id = cur.fetchone()[0] - for as_range in kid.resources.asn: - cur.execute("INSERT registrant_asn (start_as, end_as, registrant_id) VALUES (%s, %s, %s)", - (as_range.min, as_range.max, registrant_id)) - for v4_range in kid.resources.v4: - cur.execute("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 4, %s)", - (v4_range.min, v4_range.max, registrant_id)) - for v6_range in kid.resources.v6: - cur.execute("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 6, %s)", - (v6_range.min, v6_range.max, registrant_id)) - cur.execute("UPDATE registrant SET valid_until = %s WHERE registrant_id = %s", - (kid.resources.valid_until, registrant_id)) - for r in s.roa_requests: - cur.execute("INSERT roa_request (self_handle, asn) VALUES (%s, %s)", - (s.name, r.asn)) - roa_request_id = cur.lastrowid - for version, prefix_set in ((4, r.v4), (6, r.v6)): - if prefix_set: - cur.executemany("INSERT roa_request_prefix " - "(roa_request_id, prefix, prefixlen, max_prefixlen, version) " - "VALUES (%s, %s, %s, %s, %s)", - ((roa_request_id, x.prefix, x.prefixlen, x.max_prefixlen, version) - for x in prefix_set)) - for r in s.router_certs: - cur.execute("INSERT ee_certificate (self_handle, pkcs10, gski, cn, sn, eku, valid_until) " - "VALUES (%s, %s, %s, %s, %s, %s, %s)", - (s.name, r.pkcs10.get_DER(), r.gski, r.cn, r.sn, r.eku, s.resources.valid_until)) - ee_certificate_id = cur.lastrowid - cur.executemany("INSERT ee_certificate_asn (ee_certificate_id, start_as, end_as) VALUES (%s, %s, %s)", - ((ee_certificate_id, a.min, a.max) for a in r.asn)) + for sql in pubd_sql: + try: + cur.execute(sql) + except: + if "DROP TABLE IF EXISTS" not in sql.upper(): + raise db.close() + d = dict(pubd_name = pubd_name, + pubd_port = pubd_port, + pubd_db_name = pubd_db_name, + pubd_db_user = pubd_db_user, + pubd_db_pass = pubd_db_pass, + pubd_dir = rsyncd_dir, + irdb_db_name = irdb_db_name, + irdb_db_pass = irdb_db_pass) + f = open(pubd_name + ".conf", "w") + f.write(pubd_fmt_1 % d) + f.close() + global pubd_ta + global pubd_irbe_key + global pubd_irbe_cert + global pubd_pubd_cert + pubd_ta = rpki.x509.X509(Auto_file = pubd_name + "-TA.cer") + pubd_irbe_key = rpki.x509.RSA( Auto_file = pubd_name + "-IRBE.key") + pubd_irbe_cert = rpki.x509.X509(Auto_file = pubd_name + "-IRBE.cer") + pubd_pubd_cert = rpki.x509.X509(Auto_file = pubd_name + "-PUBD.cer") - def run_daemons(self): - """ - Run daemons for this entity. - """ - - logger.info("Running daemons for %s", self.name) - env = dict(os.environ, RPKI_CONF = self.name + ".conf") - self.rpkid_process = subprocess.Popen((prog_python, prog_rpkid, "--foreground", "--log-stdout", "--log-level", "debug") + - (("--profile", self.name + ".prof") if args.profile else ()), - env = env) - self.irdbd_process = subprocess.Popen((prog_python, prog_irdbd, "--foreground", "--log-stdout", "--log-level", "debug"), - env = env) - - def kill_daemons(self): - """ - Kill daemons for this entity. - """ - - # pylint: disable=E1103 - for proc, name in ((self.rpkid_process, "rpkid"), - (self.irdbd_process, "irdbd")): - if proc is not None and proc.poll() is None: - logger.info("Killing daemon %s pid %s for %s", name, proc.pid, self.name) - try: - proc.terminate() - except OSError: - pass - if proc is not None: - logger.info("Daemon %s pid %s for %s exited with code %s", - name, proc.pid, self.name, proc.wait()) - - def call_rpkid(self, pdus, cb): +def call_pubd(pdus, cb): """ - Send a left-right message to this entity's RPKI daemon and return + Send a publication control message to publication daemon and return the response. - - If this entity is hosted (does not run its own RPKI daemon), all - of this happens with the hosting RPKI daemon. """ - logger.info("Calling rpkid for %s", self.name) - - if self.is_hosted: - logger.info("rpkid %s is hosted by rpkid %s, switching", self.name, self.hosted_by.name) - self = self.hosted_by - assert not self.is_hosted - - assert isinstance(pdus, (list, tuple)) - assert self.rpki_port is not None - - q_msg = rpki.left_right.msg.query(*pdus) - q_cms = rpki.left_right.cms_msg_saxify() - q_der = q_cms.wrap(q_msg, self.irbe_key, self.irbe_cert) - q_url = "http://localhost:%d/left-right" % self.rpki_port + logger.info("Calling pubd") + q_msg = rpki.publication_control.msg.query(*pdus) + q_cms = rpki.publication_control.cms_msg_saxify() + q_der = q_cms.wrap(q_msg, pubd_irbe_key, pubd_irbe_cert) + q_url = "http://localhost:%d/control" % pubd_port logger.debug(q_cms.pretty_print_content()) - def done(r_der): - logger.info("Callback from rpkid %s", self.name) - r_cms = rpki.left_right.cms_msg_saxify(DER = r_der) - r_msg = r_cms.unwrap((self.rpkid_ta, self.rpkid_cert)) - self.last_cms_time = r_cms.check_replay(self.last_cms_time, q_url) - logger.debug(r_cms.pretty_print_content()) - assert r_msg.is_reply - for r_pdu in r_msg: - assert not isinstance(r_pdu, rpki.left_right.report_error_elt) - cb(r_msg) + def call_pubd_cb(r_der): + global pubd_last_cms_time + r_cms = rpki.publication_control.cms_msg_saxify(DER = r_der) + r_msg = r_cms.unwrap((pubd_ta, pubd_pubd_cert)) + pubd_last_cms_time = r_cms.check_replay(pubd_last_cms_time, q_url) + logger.debug(r_cms.pretty_print_content()) + assert r_msg.is_reply + for r_pdu in r_msg: + r_pdu.raise_if_error() + cb(r_msg) - def lose(e): - raise + def call_pubd_eb(e): + logger.exception("Problem calling pubd") rpki.http.client( - url = q_url, - msg = q_der, - callback = done, - errback = lose) + url = q_url, + msg = q_der, + callback = call_pubd_cb, + errback = call_pubd_eb) + - def cross_certify(self, certificant, reverse = False): +def cross_certify(certificant, certifier): """ Cross-certify and return the resulting certificate. """ - if reverse: - certifier = certificant - certificant = self.name + "-SELF" - else: - certifier = self.name + "-SELF" - return cross_certify(certificant, certifier) - - def create_rpki_objects(self, cb): - """ - Create RPKI engine objects for this engine. + certfile = certifier + "-" + certificant + ".cer" - Root node of the engine tree is special, it too has a parent but - that one is the magic self-signed micro engine. - - The rest of this is straightforward. There are a lot of objects - to create, but we can do batch them all into one honking PDU, then - issue one more PDU to set BSC EE certificates based on the PKCS - #10 requests we get back when we tell rpkid to generate BSC keys. - """ + logger.info("Cross certifying %s into %s's BPKI (%s)", certificant, certifier, certfile) - assert not self.is_hosted - - selves = [self] + self.hosts - - rpkid_pdus = [] - pubd_pdus = [] - - for i, s in enumerate(selves): - logger.info("Creating RPKI objects for [%d] %s", i, s.name) - - rpkid_pdus.append(rpki.left_right.self_elt.make_pdu( - action = "create", - self_handle = s.name, - crl_interval = s.crl_interval, - regen_margin = s.regen_margin, - bpki_cert = (s.cross_certify(s.hosted_by.name + "-TA", reverse = True) - if s.is_hosted else - rpki.x509.X509(Auto_file = s.name + "-SELF.cer")))) - - rpkid_pdus.append(rpki.left_right.bsc_elt.make_pdu( - action = "create", - self_handle = s.name, - bsc_handle = "b", - generate_keypair = True)) - - pubd_pdus.append(rpki.publication_control.client_elt.make_pdu( - action = "create", - client_handle = s.client_handle, - base_uri = s.sia_base, - bpki_cert = s.cross_certify(pubd_name + "-TA", reverse = True))) - - rpkid_pdus.append(rpki.left_right.repository_elt.make_pdu( - action = "create", - self_handle = s.name, - bsc_handle = "b", - repository_handle = "r", - bpki_cert = s.cross_certify(pubd_name + "-TA"), - peer_contact_uri = "http://localhost:%d/client/%s" % (pubd_port, s.client_handle))) - - for k in s.kids: - rpkid_pdus.append(rpki.left_right.child_elt.make_pdu( - action = "create", - self_handle = s.name, - child_handle = k.name, - bsc_handle = "b", - bpki_cert = s.cross_certify(k.name + "-SELF"))) - - if s.is_root: - rootd_cert = s.cross_certify(rootd_name + "-TA") - rpkid_pdus.append(rpki.left_right.parent_elt.make_pdu( - action = "create", - self_handle = s.name, - parent_handle = "rootd", - bsc_handle = "b", - repository_handle = "r", - sia_base = s.sia_base, - bpki_cert = rootd_cert, - sender_name = s.name, - recipient_name = "rootd", - peer_contact_uri = "http://localhost:%s/" % rootd_port)) - else: - rpkid_pdus.append(rpki.left_right.parent_elt.make_pdu( - action = "create", - self_handle = s.name, - parent_handle = s.parent.name, - bsc_handle = "b", - repository_handle = "r", - sia_base = s.sia_base, - bpki_cert = s.cross_certify(s.parent.name + "-SELF"), - sender_name = s.name, - recipient_name = s.parent.name, - peer_contact_uri = "http://localhost:%s/up-down/%s/%s" % (s.parent.get_rpki_port(), - s.parent.name, s.name))) - - def one(): - call_pubd(pubd_pdus, cb = two) - - def two(vals): - self.call_rpkid(rpkid_pdus, cb = three) - - def three(vals): - - bsc_dict = dict((b.self_handle, b) for b in vals if isinstance(b, rpki.left_right.bsc_elt)) - - bsc_pdus = [] - - for s in selves: - b = bsc_dict[s.name] - - logger.info("Issuing BSC EE cert for %s", s.name) - cmd = (prog_openssl, "x509", "-req", "-sha256", "-extfile", s.name + "-RPKI.conf", - "-extensions", "req_x509_ext", "-days", "30", - "-CA", s.name + "-SELF.cer", "-CAkey", s.name + "-SELF.key", "-CAcreateserial", "-text") - signer = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE) - signed = signer.communicate(input = b.pkcs10_request.get_PEM()) - if not signed[0]: - logger.warning(signed[1]) - raise CouldntIssueBSCEECertificate("Couldn't issue BSC EE certificate") - s.bsc_ee = rpki.x509.X509(PEM = signed[0]) - s.bsc_crl = rpki.x509.CRL(PEM_file = s.name + "-SELF.crl") - logger.info("BSC EE cert for %s SKI %s", s.name, s.bsc_ee.hSKI()) - - bsc_pdus.append(rpki.left_right.bsc_elt.make_pdu( - action = "set", - self_handle = s.name, - bsc_handle = "b", - signing_cert = s.bsc_ee, - signing_cert_crl = s.bsc_crl)) - - self.call_rpkid(bsc_pdus, cb = four) - - def four(vals): - cb() - - one() - - def setup_yaml_leaf(self): - """ - Generate certificates and write YAML scripts for leaf nodes. - - We're cheating a bit here: properly speaking, we can't generate - issue or revoke requests without knowing the class, which is - generated on the fly, but at the moment the test case is - simplistic enough that the class will always be "1", so we just - wire in that value for now. - - Well, ok, we just broke that assumption. Now we do something even - nastier, just to eke a bit more life out of this kludge. This - really needs to be rewritten, but it may require a different tool - than testpoke. - """ + child = rpki.x509.X509(Auto_file = certificant + ".cer") + parent = rpki.x509.X509(Auto_file = certifier + ".cer") + keypair = rpki.x509.RSA(Auto_file = certifier + ".key") + serial_file = certifier + ".srl" - if not os.path.exists(self.name + ".key"): - logger.info("Generating RPKI key for %s", self.name) - subprocess.check_call((prog_openssl, "genrsa", "-out", self.name + ".key", "2048" ), - stdout = subprocess.PIPE, stderr = subprocess.STDOUT) - ski = rpki.x509.RSA(PEM_file = self.name + ".key").gSKI() + now = rpki.sundial.now() + notAfter = now + rpki.sundial.timedelta(days = 30) - if self.parent.is_hosted: - parent_host = self.parent.hosted_by.name - else: - parent_host = self.parent.name + try: + with open(serial_file, "r") as f: + serial = int(f.read().splitlines()[0], 16) + except IOError: + serial = 1 - self.cross_certify(self.parent.name + "-SELF") - self.cross_certify(parent_host + "-TA") + x = parent.bpki_cross_certify( + keypair = keypair, + source_cert = child, + serial = serial, + notAfter = notAfter, + now = now) - def run_cron(self, cb): - """ - Trigger cron run for this engine. - """ + with open(serial_file, "w") as f: + f.write("%02x\n" % (serial + 1)) - logger.info("Running cron for %s", self.name) + with open(certfile, "w") as f: + f.write(x.get_PEM()) - assert self.rpki_port is not None + logger.debug("Cross certified %s:", certfile) + logger.debug(" Issuer %s [%s]", x.getIssuer(), x.hAKI()) + logger.debug(" Subject %s [%s]", x.getSubject(), x.hSKI()) + return x - def done(result): - assert result == "OK", 'Expected "OK" result from cronjob, got %r' % result - cb() - - rpki.http.client( - url = "http://localhost:%d/cronjob" % self.rpki_port, - msg = "Run cron now, please", - callback = done, - errback = done) +last_rcynic_run = None - def run_yaml(self): +def run_rcynic(): """ - Run YAML scripts for this leaf entity. Since we're not bothering - to check the class list returned by the list command, the issue - command may fail, so we treat failure of the list command as an - error, but only issue a warning when issue fails. + Run rcynic to see whether what was published makes sense. """ - logger.info("Running YAML for %s", self.name) - subprocess.check_call((prog_python, prog_poke, "-y", self.name + ".yaml", "-r", "list")) - if subprocess.call((prog_python, prog_poke, "-y", self.name + ".yaml", "-r", "issue")) != 0: - logger.warning("YAML issue command failed for %s, continuing", self.name) + logger.info("Running rcynic") + env = os.environ.copy() + env["TZ"] = "" + global last_rcynic_run + if int(time.time()) == last_rcynic_run: + time.sleep(1) + subprocess.check_call((prog_rcynic, "-c", rcynic_name + ".conf"), env = env) + subprocess.call(rcynic_stats, shell = True, env = env) + last_rcynic_run = int(time.time()) + os.link("%s.xml" % rcynic_name, "%s.%s.xml" % (rcynic_name, last_rcynic_run)) -def setup_bpki_cert_chain(name, ee = (), ca = ()): - """ - Build a set of BPKI certificates. - """ +def mangle_sql(filename): + """ + Mangle an SQL file into a sequence of SQL statements. + """ - s = "exec >/dev/null 2>&1\n" - #s = "set -x\n" - for kind in ("TA",) + ee + ca: - d = dict(name = name, - kind = kind, - ca = "false" if kind in ee else "true", - openssl = prog_openssl) - f = open("%(name)s-%(kind)s.conf" % d, "w") - f.write(bpki_cert_fmt_1 % d) + words = [] + f = open(filename) + for line in f: + words.extend(line.partition("--")[0].split()) f.close() - if not os.path.exists("%(name)s-%(kind)s.key" % d): - s += bpki_cert_fmt_2 % d - s += bpki_cert_fmt_3 % d - d = dict(name = name, - openssl = prog_openssl) - s += bpki_cert_fmt_4 % d - for kind in ee + ca: - d["kind"] = kind - s += bpki_cert_fmt_5 % d - for kind in ("TA",) + ca: - d["kind"] = kind - s += bpki_cert_fmt_6 % d - subprocess.check_call(s, shell = True) - -def setup_rootd(rpkid, rootd_yaml, db): - """ - Write the config files for rootd. - """ - - rpkid.cross_certify(rootd_name + "-TA", reverse = True) - cross_certify(pubd_name + "-TA", rootd_name + "-TA") - logger.info("Writing config files for %s", rootd_name) - d = dict(rootd_name = rootd_name, - rootd_port = rootd_port, - rpkid_name = rpkid.name, - pubd_name = pubd_name, - rootd_sia = rootd_sia, - rsyncd_dir = rsyncd_dir, - openssl = prog_openssl, - lifetime = rootd_yaml.get("lifetime", "30d"), - pubd_port = pubd_port, - rootd_handle = db.root.client_handle + "-" + rootd_name) - f = open(rootd_name + ".conf", "w") - f.write(rootd_fmt_1 % d) - f.close() - s = "exec >/dev/null 2>&1\n" - #s = "set -x\n" - if not os.path.exists("root.key"): - s += rootd_fmt_2 % d - s += rootd_fmt_3 % d - subprocess.check_call(s, shell = True) - -def setup_rcynic(): - """ - Write the config file for rcynic. - """ - - logger.info("Config file for rcynic") - d = dict(rcynic_name = rcynic_name, - rootd_name = rootd_name, - rootd_sia = rootd_sia) - f = open(rcynic_name + ".conf", "w") - f.write(rcynic_fmt_1 % d) - f.close() - -def setup_rsyncd(): - """ - Write the config file for rsyncd. - """ - - logger.info("Config file for rsyncd") - d = dict(rsyncd_name = rsyncd_name, - rsyncd_port = rsyncd_port, - rsyncd_module = rsyncd_module, - rsyncd_dir = rsyncd_dir) - f = open(rsyncd_name + ".conf", "w") - f.write(rsyncd_fmt_1 % d) - f.close() - -def setup_publication(pubd_sql, irdb_db_name): - """ - Set up publication daemon. - """ - - logger.info("Configure publication daemon") - publication_dir = os.getcwd() + "/publication" - assert rootd_sia.startswith("rsync://") - global rsyncd_dir - rsyncd_dir = publication_dir + "/".join(rootd_sia.split("/")[4:]) - if not rsyncd_dir.endswith("/"): - rsyncd_dir += "/" - os.makedirs(rsyncd_dir + "root/trunk") - db = MySQLdb.connect(db = pubd_db_name, user = pubd_db_user, passwd = pubd_db_pass, - conv = sql_conversions) - cur = db.cursor() - db.autocommit(True) - for sql in pubd_sql: - try: - cur.execute(sql) - except: - if "DROP TABLE IF EXISTS" not in sql.upper(): - raise - db.close() - d = dict(pubd_name = pubd_name, - pubd_port = pubd_port, - pubd_db_name = pubd_db_name, - pubd_db_user = pubd_db_user, - pubd_db_pass = pubd_db_pass, - pubd_dir = rsyncd_dir, - irdb_db_name = irdb_db_name, - irdb_db_pass = irdb_db_pass) - f = open(pubd_name + ".conf", "w") - f.write(pubd_fmt_1 % d) - f.close() - global pubd_ta - global pubd_irbe_key - global pubd_irbe_cert - global pubd_pubd_cert - pubd_ta = rpki.x509.X509(Auto_file = pubd_name + "-TA.cer") - pubd_irbe_key = rpki.x509.RSA( Auto_file = pubd_name + "-IRBE.key") - pubd_irbe_cert = rpki.x509.X509(Auto_file = pubd_name + "-IRBE.cer") - pubd_pubd_cert = rpki.x509.X509(Auto_file = pubd_name + "-PUBD.cer") - -def call_pubd(pdus, cb): - """ - Send a publication control message to publication daemon and return - the response. - """ - - logger.info("Calling pubd") - q_msg = rpki.publication_control.msg.query(*pdus) - q_cms = rpki.publication_control.cms_msg_saxify() - q_der = q_cms.wrap(q_msg, pubd_irbe_key, pubd_irbe_cert) - q_url = "http://localhost:%d/control" % pubd_port - - logger.debug(q_cms.pretty_print_content()) - - def call_pubd_cb(r_der): - global pubd_last_cms_time - r_cms = rpki.publication_control.cms_msg_saxify(DER = r_der) - r_msg = r_cms.unwrap((pubd_ta, pubd_pubd_cert)) - pubd_last_cms_time = r_cms.check_replay(pubd_last_cms_time, q_url) - logger.debug(r_cms.pretty_print_content()) - assert r_msg.is_reply - for r_pdu in r_msg: - r_pdu.raise_if_error() - cb(r_msg) - - def call_pubd_eb(e): - logger.exception("Problem calling pubd") - - rpki.http.client( - url = q_url, - msg = q_der, - callback = call_pubd_cb, - errback = call_pubd_eb) - - -def cross_certify(certificant, certifier): - """ - Cross-certify and return the resulting certificate. - """ - - certfile = certifier + "-" + certificant + ".cer" - - logger.info("Cross certifying %s into %s's BPKI (%s)", certificant, certifier, certfile) - - child = rpki.x509.X509(Auto_file = certificant + ".cer") - parent = rpki.x509.X509(Auto_file = certifier + ".cer") - keypair = rpki.x509.RSA(Auto_file = certifier + ".key") - serial_file = certifier + ".srl" - - now = rpki.sundial.now() - notAfter = now + rpki.sundial.timedelta(days = 30) - - try: - with open(serial_file, "r") as f: - serial = int(f.read().splitlines()[0], 16) - except IOError: - serial = 1 - - x = parent.bpki_cross_certify( - keypair = keypair, - source_cert = child, - serial = serial, - notAfter = notAfter, - now = now) - - with open(serial_file, "w") as f: - f.write("%02x\n" % (serial + 1)) - - with open(certfile, "w") as f: - f.write(x.get_PEM()) - - logger.debug("Cross certified %s:", certfile) - logger.debug(" Issuer %s [%s]", x.getIssuer(), x.hAKI()) - logger.debug(" Subject %s [%s]", x.getSubject(), x.hSKI()) - return x - -last_rcynic_run = None - -def run_rcynic(): - """ - Run rcynic to see whether what was published makes sense. - """ - - logger.info("Running rcynic") - env = os.environ.copy() - env["TZ"] = "" - global last_rcynic_run - if int(time.time()) == last_rcynic_run: - time.sleep(1) - subprocess.check_call((prog_rcynic, "-c", rcynic_name + ".conf"), env = env) - subprocess.call(rcynic_stats, shell = True, env = env) - last_rcynic_run = int(time.time()) - os.link("%s.xml" % rcynic_name, "%s.%s.xml" % (rcynic_name, last_rcynic_run)) - -def mangle_sql(filename): - """ - Mangle an SQL file into a sequence of SQL statements. - """ - - words = [] - f = open(filename) - for line in f: - words.extend(line.partition("--")[0].split()) - f.close() - return " ".join(words).strip(";").split(";") + return " ".join(words).strip(";").split(";") bpki_cert_fmt_1 = '''\ [req] diff --git a/ca/tests/sql-cleaner.py b/ca/tests/sql-cleaner.py index 828100a4..c518b77b 100644 --- a/ca/tests/sql-cleaner.py +++ b/ca/tests/sql-cleaner.py @@ -25,27 +25,27 @@ cfg = rpki.config.parser(section = "yamltest", allow_missing = True) for name in ("rpkid", "irdbd", "pubd"): - username = cfg.get("%s_sql_username" % name, name[:4]) - password = cfg.get("%s_sql_password" % name, "fnord") + username = cfg.get("%s_sql_username" % name, name[:4]) + password = cfg.get("%s_sql_password" % name, "fnord") - db = MySQLdb.connect(user = username, passwd = password) - cur = db.cursor() + db = MySQLdb.connect(user = username, passwd = password) + cur = db.cursor() - cur.execute("SHOW DATABASES") + cur.execute("SHOW DATABASES") - databases = [r[0] for r in cur.fetchall() if r[0][:4] == name[:4] and r[0][4:].isdigit()] + databases = [r[0] for r in cur.fetchall() if r[0][:4] == name[:4] and r[0][4:].isdigit()] - for database in databases: + for database in databases: - cur.execute("USE " + database) + cur.execute("USE " + database) - cur.execute("SHOW TABLES") - tables = [r[0] for r in cur.fetchall()] + cur.execute("SHOW TABLES") + tables = [r[0] for r in cur.fetchall()] - cur.execute("SET foreign_key_checks = 0") - for table in tables: - cur.execute("DROP TABLE %s" % table) - cur.execute("SET foreign_key_checks = 1") + cur.execute("SET foreign_key_checks = 0") + for table in tables: + cur.execute("DROP TABLE %s" % table) + cur.execute("SET foreign_key_checks = 1") - cur.close() - db.close() + cur.close() + db.close() diff --git a/ca/tests/sql-dumper.py b/ca/tests/sql-dumper.py index d0fe3489..af24f2d4 100644 --- a/ca/tests/sql-dumper.py +++ b/ca/tests/sql-dumper.py @@ -26,18 +26,18 @@ cfg = rpki.config.parser(section = "yamltest", allow_missing = True) for name in ("rpkid", "irdbd", "pubd"): - username = cfg.get("%s_sql_username" % name, name[:4]) - password = cfg.get("%s_sql_password" % name, "fnord") + username = cfg.get("%s_sql_username" % name, name[:4]) + password = cfg.get("%s_sql_password" % name, "fnord") - cmd = ["mysqldump", "-u", username, "-p" + password, "--databases"] + cmd = ["mysqldump", "-u", username, "-p" + password, "--databases"] - db = MySQLdb.connect(user = username, passwd = password) - cur = db.cursor() + db = MySQLdb.connect(user = username, passwd = password) + cur = db.cursor() - cur.execute("SHOW DATABASES") - cmd.extend(r[0] for r in cur.fetchall() if r[0][:4] == name[:4] and r[0][4:].isdigit()) + cur.execute("SHOW DATABASES") + cmd.extend(r[0] for r in cur.fetchall() if r[0][:4] == name[:4] and r[0][4:].isdigit()) - cur.close() - db.close() + cur.close() + db.close() - subprocess.check_call(cmd, stdout = open("backup.%s.sql" % name, "w")) + subprocess.check_call(cmd, stdout = open("backup.%s.sql" % name, "w")) diff --git a/ca/tests/test-rrdp.py b/ca/tests/test-rrdp.py index 1a9db929..97797444 100755 --- a/ca/tests/test-rrdp.py +++ b/ca/tests/test-rrdp.py @@ -38,22 +38,22 @@ parser.add_argument("--dry-run", action = "store_true") args = parser.parse_args() def log(msg): - sys.stdout.write(msg + "\n") - sys.stdout.flush() + sys.stdout.write(msg + "\n") + sys.stdout.flush() def run(*argv): - log("Running: " + " ".join(argv)) - if not args.dry_run: - subprocess.check_call(argv) + log("Running: " + " ".join(argv)) + if not args.dry_run: + subprocess.check_call(argv) def dataglob(pattern): - return glob.iglob(os.path.join(("smoketest.dir" if args.use_smoketest else "yamltest.dir/RIR"), pattern)) + return glob.iglob(os.path.join(("smoketest.dir" if args.use_smoketest else "yamltest.dir/RIR"), pattern)) def snapshot_to_serial(fn): - return int(os.path.splitext(os.path.basename(fn))[0]) + return int(os.path.splitext(os.path.basename(fn))[0]) def delta_to_serial(fn): - return int(os.path.splitext(os.path.basename(fn))[0]) + return int(os.path.splitext(os.path.basename(fn))[0]) top = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "..", "..")) @@ -62,62 +62,62 @@ rcynic = os.path.join(top, "rp/rcynic/rcynic") rcynic_text = os.path.join(top, "rp/rcynic/rcynic-text") with open("rcynic-rrdp.conf", "w") as f: - f.write(textwrap.dedent('''# Automatically generated for RRDP tests, do not edit. - [rcynic] - xml-summary = rcynic.xml - jitter = 0 - use-links = yes - use-syslog = no - use-stderr = yes - log-level = log_debug - run-rsync = no - ''')) - if args.use_smoketest: - f.write("trust-anchor = smoketest.dir/root.cer\n") - else: - f.write("trust-anchor = yamltest.dir/RIR/publication/RIR-root/root.cer\n") + f.write(textwrap.dedent('''# Automatically generated for RRDP tests, do not edit. + [rcynic] + xml-summary = rcynic.xml + jitter = 0 + use-links = yes + use-syslog = no + use-stderr = yes + log-level = log_debug + run-rsync = no + ''')) + if args.use_smoketest: + f.write("trust-anchor = smoketest.dir/root.cer\n") + else: + f.write("trust-anchor = yamltest.dir/RIR/publication/RIR-root/root.cer\n") if args.skip_daemons: - log("--skip-daemons specified, so running neither smoketest nor yamltest") + log("--skip-daemons specified, so running neither smoketest nor yamltest") elif args.use_smoketest: - run("python", "smoketest.py", args.yaml_file) + run("python", "smoketest.py", args.yaml_file) else: - run("python", "sql-cleaner.py") - class GotSIGUSR1(Exception): - pass - def handle_sigusr1(signum, frame): - raise GotSIGUSR1 - old_sigusr1 = signal.signal(signal.SIGUSR1, handle_sigusr1) - cmd = ("python", "yamltest.py", args.yaml_file, "--notify-when-startup-complete", str(os.getpid())) - log("Running: " + " ".join(cmd)) - yamltest = subprocess.Popen(cmd) - log("Waiting for SIGUSR1 from yamltest") - try: - while True: - signal.pause() - except GotSIGUSR1: - signal.signal(signal.SIGUSR1, old_sigusr1) - log("Sleeping %s" % args.delay) - time.sleep(args.delay) - yamltest.terminate() + run("python", "sql-cleaner.py") + class GotSIGUSR1(Exception): + pass + def handle_sigusr1(signum, frame): + raise GotSIGUSR1 + old_sigusr1 = signal.signal(signal.SIGUSR1, handle_sigusr1) + cmd = ("python", "yamltest.py", args.yaml_file, "--notify-when-startup-complete", str(os.getpid())) + log("Running: " + " ".join(cmd)) + yamltest = subprocess.Popen(cmd) + log("Waiting for SIGUSR1 from yamltest") + try: + while True: + signal.pause() + except GotSIGUSR1: + signal.signal(signal.SIGUSR1, old_sigusr1) + log("Sleeping %s" % args.delay) + time.sleep(args.delay) + yamltest.terminate() snapshots = dict((snapshot_to_serial(fn), fn) for fn in dataglob("rrdp-publication/*/snapshot/*.xml")) deltas = dict((delta_to_serial(fn), fn) for fn in dataglob("rrdp-publication/*/deltas/*.xml")) for snapshot in sorted(snapshots): - time.sleep(1) - run("rm", "-rf", "rcynic-data") - run(rrdp_test_tool, snapshots[snapshot]) - run(rcynic, "-c", "rcynic-rrdp.conf") - run(rcynic_text, "rcynic.xml") - - for delta in sorted(deltas): - if delta > snapshot: - time.sleep(1) - run(rrdp_test_tool, deltas[delta]) - run(rcynic, "-c", "rcynic-rrdp.conf") - run(rcynic_text, "rcynic.xml") - - if not args.exhaustive: - break + time.sleep(1) + run("rm", "-rf", "rcynic-data") + run(rrdp_test_tool, snapshots[snapshot]) + run(rcynic, "-c", "rcynic-rrdp.conf") + run(rcynic_text, "rcynic.xml") + + for delta in sorted(deltas): + if delta > snapshot: + time.sleep(1) + run(rrdp_test_tool, deltas[delta]) + run(rcynic, "-c", "rcynic-rrdp.conf") + run(rcynic_text, "rcynic.xml") + + if not args.exhaustive: + break diff --git a/ca/tests/testpoke.py b/ca/tests/testpoke.py index 68f967b9..60cc5690 100644 --- a/ca/tests/testpoke.py +++ b/ca/tests/testpoke.py @@ -58,80 +58,80 @@ yaml_data = yaml.load(args.yaml) yaml_cmd = args.request if yaml_cmd is None and len(yaml_data["requests"]) == 1: - yaml_cmd = yaml_data["requests"].keys()[0] + yaml_cmd = yaml_data["requests"].keys()[0] yaml_req = yaml_data["requests"][yaml_cmd] def get_PEM(name, cls, y = yaml_data): - if name in y: - return cls(PEM = y[name]) - if name + "-file" in y: - return cls(PEM_file = y[name + "-file"]) - return None + if name in y: + return cls(PEM = y[name]) + if name + "-file" in y: + return cls(PEM_file = y[name + "-file"]) + return None def get_PEM_chain(name, cert = None): - chain = [] - if cert is not None: - chain.append(cert) - if name in yaml_data: - chain.extend(rpki.x509.X509(PEM = x) for x in yaml_data[name]) - elif name + "-file" in yaml_data: - chain.extend(rpki.x509.X509(PEM_file = x) for x in yaml_data[name + "-file"]) - return chain + chain = [] + if cert is not None: + chain.append(cert) + if name in yaml_data: + chain.extend(rpki.x509.X509(PEM = x) for x in yaml_data[name]) + elif name + "-file" in yaml_data: + chain.extend(rpki.x509.X509(PEM_file = x) for x in yaml_data[name + "-file"]) + return chain def query_up_down(q_pdu): - q_msg = rpki.up_down.message_pdu.make_query( - payload = q_pdu, - sender = yaml_data["sender-id"], - recipient = yaml_data["recipient-id"]) - q_der = rpki.up_down.cms_msg_saxify().wrap(q_msg, cms_key, cms_certs, cms_crl) - - def done(r_der): - global last_cms_timestamp - r_cms = rpki.up_down.cms_msg_saxify(DER = r_der) - r_msg = r_cms.unwrap([cms_ta] + cms_ca_certs) - last_cms_timestamp = r_cms.check_replay(last_cms_timestamp) - print r_cms.pretty_print_content() - try: - r_msg.payload.check_response() - except (rpki.async.ExitNow, SystemExit): - raise - except Exception, e: - fail(e) - - rpki.http.want_persistent_client = False - - rpki.http.client( - msg = q_der, - url = yaml_data["posturl"], - callback = done, - errback = fail, - content_type = rpki.up_down.content_type) + q_msg = rpki.up_down.message_pdu.make_query( + payload = q_pdu, + sender = yaml_data["sender-id"], + recipient = yaml_data["recipient-id"]) + q_der = rpki.up_down.cms_msg_saxify().wrap(q_msg, cms_key, cms_certs, cms_crl) + + def done(r_der): + global last_cms_timestamp + r_cms = rpki.up_down.cms_msg_saxify(DER = r_der) + r_msg = r_cms.unwrap([cms_ta] + cms_ca_certs) + last_cms_timestamp = r_cms.check_replay(last_cms_timestamp) + print r_cms.pretty_print_content() + try: + r_msg.payload.check_response() + except (rpki.async.ExitNow, SystemExit): + raise + except Exception, e: + fail(e) + + rpki.http.want_persistent_client = False + + rpki.http.client( + msg = q_der, + url = yaml_data["posturl"], + callback = done, + errback = fail, + content_type = rpki.up_down.content_type) def do_list(): - query_up_down(rpki.up_down.list_pdu()) + query_up_down(rpki.up_down.list_pdu()) def do_issue(): - q_pdu = rpki.up_down.issue_pdu() - req_key = get_PEM("cert-request-key", rpki.x509.RSA, yaml_req) or cms_key - q_pdu.class_name = yaml_req["class"] - q_pdu.pkcs10 = rpki.x509.PKCS10.create( - keypair = req_key, - is_ca = True, - caRepository = yaml_req["sia"][0], - rpkiManifest = yaml_req["sia"][0] + req_key.gSKI() + ".mft") - query_up_down(q_pdu) + q_pdu = rpki.up_down.issue_pdu() + req_key = get_PEM("cert-request-key", rpki.x509.RSA, yaml_req) or cms_key + q_pdu.class_name = yaml_req["class"] + q_pdu.pkcs10 = rpki.x509.PKCS10.create( + keypair = req_key, + is_ca = True, + caRepository = yaml_req["sia"][0], + rpkiManifest = yaml_req["sia"][0] + req_key.gSKI() + ".mft") + query_up_down(q_pdu) def do_revoke(): - q_pdu = rpki.up_down.revoke_pdu() - q_pdu.class_name = yaml_req["class"] - q_pdu.ski = yaml_req["ski"] - query_up_down(q_pdu) + q_pdu = rpki.up_down.revoke_pdu() + q_pdu.class_name = yaml_req["class"] + q_pdu.ski = yaml_req["ski"] + query_up_down(q_pdu) dispatch = { "list" : do_list, "issue" : do_issue, "revoke" : do_revoke } def fail(e): # pylint: disable=W0621 - sys.exit("Testpoke failed: %s" % e) + sys.exit("Testpoke failed: %s" % e) cms_ta = get_PEM("cms-ca-cert", rpki.x509.X509) cms_cert = get_PEM("cms-cert", rpki.x509.X509) @@ -143,7 +143,7 @@ cms_ca_certs = get_PEM_chain("cms-ca-certs") last_cms_timestamp = None try: - dispatch[yaml_req["type"]]() - rpki.async.event_loop() + dispatch[yaml_req["type"]]() + rpki.async.event_loop() except Exception, e: - fail(e) + fail(e) diff --git a/ca/tests/xml-parse-test.py b/ca/tests/xml-parse-test.py index 90c80775..f24d5683 100644 --- a/ca/tests/xml-parse-test.py +++ b/ca/tests/xml-parse-test.py @@ -39,70 +39,70 @@ import rpki.relaxng verbose = False def test(fileglob, rng, parser, encoding, tester = None): - files = glob.glob(fileglob) - files.sort() - for f in files: - print "" - elt_in = lxml.etree.parse(f).getroot() - if verbose: - print "" - print lxml.etree.tostring(elt_in, pretty_print = True, encoding = encoding, xml_declaration = True) - rng.assertValid(elt_in) - parsed = parser.fromXML(elt_in) - elt_out = parsed.toXML() - if verbose: - print "" - print lxml.etree.tostring(elt_out, pretty_print = True, encoding = encoding, xml_declaration = True) - rng.assertValid(elt_out) - if tester: - tester(elt_in, elt_out, parsed) - if verbose: - print + files = glob.glob(fileglob) + files.sort() + for f in files: + print "" + elt_in = lxml.etree.parse(f).getroot() + if verbose: + print "" + print lxml.etree.tostring(elt_in, pretty_print = True, encoding = encoding, xml_declaration = True) + rng.assertValid(elt_in) + parsed = parser.fromXML(elt_in) + elt_out = parsed.toXML() + if verbose: + print "" + print lxml.etree.tostring(elt_out, pretty_print = True, encoding = encoding, xml_declaration = True) + rng.assertValid(elt_out) + if tester: + tester(elt_in, elt_out, parsed) + if verbose: + print def pprint(pairs): - if verbose: - for thing, name in pairs: - if thing is not None: - print "[%s]" % name - print thing.get_POW().pprint() + if verbose: + for thing, name in pairs: + if thing is not None: + print "[%s]" % name + print thing.get_POW().pprint() def ud_tester(elt_in, elt_out, msg): - assert isinstance(msg, rpki.up_down.message_pdu) - if isinstance(msg.payload, rpki.up_down.list_response_pdu): - for c in msg.payload.classes: - pprint([(c.certs[i].cert, ("%s certificate #%d" % (c.class_name, i))) for i in xrange(len(c.certs))] + [(c.issuer, ("%s issuer" % c.class_name))]) + assert isinstance(msg, rpki.up_down.message_pdu) + if isinstance(msg.payload, rpki.up_down.list_response_pdu): + for c in msg.payload.classes: + pprint([(c.certs[i].cert, ("%s certificate #%d" % (c.class_name, i))) for i in xrange(len(c.certs))] + [(c.issuer, ("%s issuer" % c.class_name))]) def lr_tester(elt_in, elt_out, msg): - assert isinstance(msg, rpki.left_right.msg) - for obj in msg: - if isinstance(obj, rpki.left_right.self_elt): - pprint(((obj.bpki_cert, "BPKI cert"), - (obj.bpki_glue, "BPKI glue"))) - if isinstance(obj, rpki.left_right.bsc_elt): - pprint(((obj.signing_cert, "Signing certificate"), - (obj.signing_cert_crl, "Signing certificate CRL"))) - # (obj.pkcs10_request, "PKCS #10 request") - if isinstance(obj, rpki.left_right.parent_elt): - pprint(((obj.bpki_cert, "BPKI certificate"), - (obj.bpki_glue, "BPKI glue"))) - if isinstance(obj, (rpki.left_right.child_elt, rpki.left_right.repository_elt)): - pprint(((obj.bpki_cert, "BPKI certificate"), - (obj.bpki_glue, "BPKI glue"))) + assert isinstance(msg, rpki.left_right.msg) + for obj in msg: + if isinstance(obj, rpki.left_right.self_elt): + pprint(((obj.bpki_cert, "BPKI cert"), + (obj.bpki_glue, "BPKI glue"))) + if isinstance(obj, rpki.left_right.bsc_elt): + pprint(((obj.signing_cert, "Signing certificate"), + (obj.signing_cert_crl, "Signing certificate CRL"))) + # (obj.pkcs10_request, "PKCS #10 request") + if isinstance(obj, rpki.left_right.parent_elt): + pprint(((obj.bpki_cert, "BPKI certificate"), + (obj.bpki_glue, "BPKI glue"))) + if isinstance(obj, (rpki.left_right.child_elt, rpki.left_right.repository_elt)): + pprint(((obj.bpki_cert, "BPKI certificate"), + (obj.bpki_glue, "BPKI glue"))) def pp_tester(elt_in, elt_out, msg): - assert isinstance(msg, rpki.publication.msg) - for obj in msg: - if isinstance(obj, rpki.publication.publish_elt): - pprint(((obj.payload, "Publish object"),)) - if isinstance(obj, rpki.publication.withdraw_elt): - pprint(((None, "Withdraw object"),)) + assert isinstance(msg, rpki.publication.msg) + for obj in msg: + if isinstance(obj, rpki.publication.publish_elt): + pprint(((obj.payload, "Publish object"),)) + if isinstance(obj, rpki.publication.withdraw_elt): + pprint(((None, "Withdraw object"),)) def pc_tester(elt_in, elt_out, msg): - assert isinstance(msg, rpki.publication_control.msg) - for obj in msg: - if isinstance(obj, rpki.publication_control.client_elt): - pprint(((obj.bpki_cert, "BPKI cert"), - (obj.bpki_glue, "BPKI glue"))) + assert isinstance(msg, rpki.publication_control.msg) + for obj in msg: + if isinstance(obj, rpki.publication_control.client_elt): + pprint(((obj.bpki_cert, "BPKI cert"), + (obj.bpki_glue, "BPKI glue"))) test(fileglob = "up-down-protocol-samples/*.xml", rng = rpki.relaxng.up_down, diff --git a/ca/tests/yamlconf.py b/ca/tests/yamlconf.py index 13456377..52c4da26 100644 --- a/ca/tests/yamlconf.py +++ b/ca/tests/yamlconf.py @@ -75,787 +75,787 @@ config_overrides = { "pubd_sql_username" : "pubd", "pubd_sql_password" : "fnord" } def cleanpath(*names): - return os.path.normpath(os.path.join(*names)) + return os.path.normpath(os.path.join(*names)) this_dir = os.getcwd() test_dir = None rpki_conf = None class roa_request(object): - """ - Representation of a ROA request. - """ - - def __init__(self, asn, ipv4, ipv6): - self.asn = asn - self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None - self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None - - def __eq__(self, other): - return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 - - def __hash__(self): - v4 = tuple(self.v4) if self.v4 is not None else None - v6 = tuple(self.v6) if self.v6 is not None else None - return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - - def __str__(self): - if self.v4 and self.v6: - return "%s: %s,%s" % (self.asn, self.v4, self.v6) - else: - return "%s: %s" % (self.asn, self.v4 or self.v6) + """ + Representation of a ROA request. + """ - @classmethod - def parse(cls, y): - return cls(y.get("asn"), y.get("ipv4"), y.get("ipv6")) + def __init__(self, asn, ipv4, ipv6): + self.asn = asn + self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None + self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None -class router_cert(object): - """ - Representation for a router_cert object. - """ + def __eq__(self, other): + return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 - _ecparams = None + def __hash__(self): + v4 = tuple(self.v4) if self.v4 is not None else None + v6 = tuple(self.v6) if self.v6 is not None else None + return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - @classmethod - def ecparams(cls): - if cls._ecparams is None: - cls._ecparams = rpki.x509.KeyParams.generateEC() - return cls._ecparams + def __str__(self): + if self.v4 and self.v6: + return "%s: %s,%s" % (self.asn, self.v4, self.v6) + else: + return "%s: %s" % (self.asn, self.v4 or self.v6) - def __init__(self, asn, router_id): - self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) - self.router_id = router_id - self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True) - self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) - self.gski = self.pkcs10.gSKI() + @classmethod + def parse(cls, y): + return cls(y.get("asn"), y.get("ipv4"), y.get("ipv6")) - def __eq__(self, other): - return self.asn == other.asn and self.router_id == other.router_id and self.gski == other.gski +class router_cert(object): + """ + Representation for a router_cert object. + """ - def __hash__(self): - return tuple(self.asn).__hash__() + self.router_id.__hash__() + self.gski.__hash__() + _ecparams = None - def __str__(self): - return "%s: %s: %s" % (self.asn, self.router_id, self.gski) + @classmethod + def ecparams(cls): + if cls._ecparams is None: + cls._ecparams = rpki.x509.KeyParams.generateEC() + return cls._ecparams - @classmethod - def parse(cls, yaml): - return cls(yaml.get("asn"), yaml.get("router_id")) + def __init__(self, asn, router_id): + self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) + self.router_id = router_id + self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True) + self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) + self.gski = self.pkcs10.gSKI() + def __eq__(self, other): + return self.asn == other.asn and self.router_id == other.router_id and self.gski == other.gski -class allocation_db(list): - """ - Allocation database. - """ - - def __init__(self, y): - list.__init__(self) - self.root = allocation(y, self) - assert self.root.is_root - if self.root.crl_interval is None: - self.root.crl_interval = 60 * 60 - if self.root.regen_margin is None: - self.root.regen_margin = 24 * 60 * 60 - if self.root.base.valid_until is None: - self.root.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 2) - for a in self: - if a.base.valid_until is None: - a.base.valid_until = a.parent.base.valid_until - if a.crl_interval is None: - a.crl_interval = a.parent.crl_interval - if a.regen_margin is None: - a.regen_margin = a.parent.regen_margin - self.root.closure() - self.map = dict((a.name, a) for a in self) - for a in self: - if a.is_hosted: - a.hosted_by = self.map[a.hosted_by] - a.hosted_by.hosts.append(a) - assert not a.is_root and not a.hosted_by.is_hosted - - def dump(self): - for a in self: - a.dump() + def __hash__(self): + return tuple(self.asn).__hash__() + self.router_id.__hash__() + self.gski.__hash__() + def __str__(self): + return "%s: %s: %s" % (self.asn, self.router_id, self.gski) -class allocation(object): - """ - One entity in our allocation database. Every entity in the database - is assumed to hold resources. Entities that don't have the - hosted_by property run their own copies of rpkid, irdbd, and pubd. - """ - - base_port = 4400 - base_engine = -1 - parent = None - crl_interval = None - regen_margin = None - engine = -1 - rpkid_port = 4404 - irdbd_port = 4403 - pubd_port = 4402 - rootd_port = 4401 - rsync_port = 873 - - @classmethod - def allocate_port(cls): - cls.base_port += 1 - return cls.base_port - - @classmethod - def allocate_engine(cls): - cls.base_engine += 1 - return cls.base_engine - - def __init__(self, y, db, parent = None): - db.append(self) - self.name = y["name"] - self.parent = parent - self.kids = [allocation(k, db, self) for k in y.get("kids", ())] - valid_until = None - if "valid_until" in y: - valid_until = rpki.sundial.datetime.from_datetime(y.get("valid_until")) - if valid_until is None and "valid_for" in y: - valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(y["valid_for"]) - self.base = rpki.resource_set.resource_bag( - asn = rpki.resource_set.resource_set_as(y.get("asn")), - v4 = rpki.resource_set.resource_set_ipv4(y.get("ipv4")), - v6 = rpki.resource_set.resource_set_ipv6(y.get("ipv6")), - valid_until = valid_until) - if "crl_interval" in y: - self.crl_interval = rpki.sundial.timedelta.parse(y["crl_interval"]).convert_to_seconds() - if "regen_margin" in y: - self.regen_margin = rpki.sundial.timedelta.parse(y["regen_margin"]).convert_to_seconds() - if "ghostbusters" in y: - self.ghostbusters = y.get("ghostbusters") - elif "ghostbuster" in y: - self.ghostbusters = [y.get("ghostbuster")] - else: - self.ghostbusters = [] - self.roa_requests = [roa_request.parse(r) for r in y.get("roa_request", ())] - self.router_certs = [router_cert.parse(r) for r in y.get("router_cert", ())] - for r in self.roa_requests: - if r.v4: - self.base.v4 |= r.v4.to_resource_set() - if r.v6: - self.base.v6 |= r.v6.to_resource_set() - for r in self.router_certs: - self.base.asn |= r.asn - self.hosted_by = y.get("hosted_by") - self.hosts = [] - if not self.is_hosted: - self.engine = self.allocate_engine() - if loopback and not self.is_hosted: - self.rpkid_port = self.allocate_port() - self.irdbd_port = self.allocate_port() - if loopback and self.runs_pubd: - self.pubd_port = self.allocate_port() - self.rsync_port = self.allocate_port() - if loopback and self.is_root: - self.rootd_port = self.allocate_port() - - def closure(self): - resources = self.base - for kid in self.kids: - resources |= kid.closure() - self.resources = resources - return resources - - @property - def hostname(self): - if loopback: - return "localhost" - elif dns_suffix: - return self.name + "." + dns_suffix.lstrip(".") - else: - return self.name + @classmethod + def parse(cls, yaml): + return cls(yaml.get("asn"), yaml.get("router_id")) - @property - def rsync_server(self): - if loopback: - return "%s:%s" % (self.pubd.hostname, self.pubd.rsync_port) - else: - return self.pubd.hostname - def dump(self): - if not quiet: - print str(self) - - def __str__(self): - s = self.name + ":\n" - if self.resources.asn: s += " ASNs: %s\n" % self.resources.asn - if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 - if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 - if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) - if self.parent: s += " Up: %s\n" % self.parent.name - if self.is_hosted: s += " Host: %s\n" % self.hosted_by.name - if self.hosts: s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts) - for r in self.roa_requests: s += " ROA: %s\n" % r - if not self.is_hosted: s += " IPort: %s\n" % self.irdbd_port - if self.runs_pubd: s += " PPort: %s\n" % self.pubd_port - if not self.is_hosted: s += " RPort: %s\n" % self.rpkid_port - if self.runs_pubd: s += " SPort: %s\n" % self.rsync_port - if self.is_root: s += " TPort: %s\n" % self.rootd_port - return s + " Until: %s\n" % self.resources.valid_until - - @property - def is_root(self): - return self.parent is None - - @property - def is_hosted(self): - return self.hosted_by is not None - - @property - def runs_pubd(self): - return self.is_root or not (self.is_hosted or only_one_pubd) - - def path(self, *names): - return cleanpath(test_dir, self.host.name, *names) - - def csvout(self, fn): - path = self.path(fn) - if not quiet: - print "Writing", path - return rpki.csv_utils.csv_writer(path) - - def up_down_url(self): - return "http://%s:%d/up-down/%s/%s" % (self.parent.host.hostname, - self.parent.host.rpkid_port, - self.parent.name, - self.name) - - def dump_asns(self, fn): - with self.csvout(fn) as f: - for k in self.kids: - f.writerows((k.name, a) for a in k.resources.asn) - - def dump_prefixes(self, fn): - with self.csvout(fn) as f: - for k in self.kids: - f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) - - def dump_roas(self, fn): - with self.csvout(fn) as f: - for g1, r in enumerate(self.roa_requests): - f.writerows((p, r.asn, "G%08d%08d" % (g1, g2)) - for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ()))) - - def dump_ghostbusters(self, fn): - if self.ghostbusters: - path = self.path(fn) - if not quiet: - print "Writing", path - with open(path, "w") as f: - for i, g in enumerate(self.ghostbusters): - if i > 0: - f.write("\n") - f.write(g) - - def dump_router_certificates(self, fn): - if self.router_certs: - path = self.path(fn) - if not quiet: - print "Writing", path - xmlns = rpki.relaxng.router_certificate.xmlns - xml = lxml.etree.Element(xmlns + "router_certificate_requests", - version = rpki.relaxng.router_certificate.version, - nsmap = rpki.relaxng.router_certificate.nsmap) - for r in self.router_certs: - x = lxml.etree.SubElement(xml, xmlns + "router_certificate_request", - router_id = str(r.router_id), - asn = str(r.asn), - valid_until = str(self.resources.valid_until)) - x.text = r.pkcs10.get_Base64() - rpki.relaxng.router_certificate.assertValid(xml) - lxml.etree.ElementTree(xml).write(path, pretty_print = True) - - @property - def pubd(self): - s = self - while not s.runs_pubd: - s = s.parent - return s - - @property - def client_handle(self): - path = [] - s = self - if not flat_publication: - while not s.runs_pubd: - path.append(s) - s = s.parent - path.append(s) - return ".".join(i.name for i in reversed(path)) - - @property - def host(self): - return self.hosted_by or self - - @property - def publication_base_directory(self): - if not loopback and publication_base is not None: - return publication_base - else: - return self.path("publication") +class allocation_db(list): + """ + Allocation database. + """ + + def __init__(self, y): + list.__init__(self) + self.root = allocation(y, self) + assert self.root.is_root + if self.root.crl_interval is None: + self.root.crl_interval = 60 * 60 + if self.root.regen_margin is None: + self.root.regen_margin = 24 * 60 * 60 + if self.root.base.valid_until is None: + self.root.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 2) + for a in self: + if a.base.valid_until is None: + a.base.valid_until = a.parent.base.valid_until + if a.crl_interval is None: + a.crl_interval = a.parent.crl_interval + if a.regen_margin is None: + a.regen_margin = a.parent.regen_margin + self.root.closure() + self.map = dict((a.name, a) for a in self) + for a in self: + if a.is_hosted: + a.hosted_by = self.map[a.hosted_by] + a.hosted_by.hosts.append(a) + assert not a.is_root and not a.hosted_by.is_hosted + + def dump(self): + for a in self: + a.dump() - @property - def publication_root_directory(self): - if not loopback and publication_root is not None: - return publication_root - else: - return self.path("publication.root") - - def dump_conf(self): - - r = dict( - handle = self.name, - run_rpkid = str(not self.is_hosted), - run_pubd = str(self.runs_pubd), - run_rootd = str(self.is_root), - irdbd_sql_username = "irdb", - rpkid_sql_username = "rpki", - rpkid_server_host = self.hostname, - rpkid_server_port = str(self.rpkid_port), - irdbd_server_host = "localhost", - irdbd_server_port = str(self.irdbd_port), - rootd_server_port = str(self.rootd_port), - pubd_sql_username = "pubd", - pubd_server_host = self.pubd.hostname, - pubd_server_port = str(self.pubd.pubd_port), - publication_rsync_server = self.rsync_server) - - if loopback: - r.update( - irdbd_sql_database = self.irdb_name, - rpkid_sql_database = "rpki%d" % self.engine, - pubd_sql_database = "pubd%d" % self.engine, - bpki_servers_directory = self.path(), - publication_base_directory = self.publication_base_directory) - - r.update(config_overrides) - - with open(self.path("rpki.conf"), "w") as f: - f.write("# Automatically generated, do not edit\n") - if not quiet: - print "Writing", f.name - - section = None - for line in open(rpki_conf): - m = section_regexp.match(line) - if m: - section = m.group(1) - m = variable_regexp.match(line) - option = m.group(1) if m and section == "myrpki" else None - if option and option in r: - line = "%s = %s\n" % (option, r[option]) - f.write(line) - - def dump_rsyncd(self): - lines = [] - if self.runs_pubd: - lines.extend(( - "# Automatically generated, do not edit", - "port = %d" % self.rsync_port, - "address = %s" % self.hostname, - "log file = rsyncd.log", - "read only = yes", - "use chroot = no", - "[rpki]", - "path = %s" % self.publication_base_directory, - "comment = RPKI test")) - if self.is_root: - assert self.runs_pubd - lines.extend(( - "[root]", - "path = %s" % self.publication_root_directory, - "comment = RPKI test root")) - if lines: - with open(self.path("rsyncd.conf"), "w") as f: - if not quiet: - print "Writing", f.name - f.writelines(line + "\n" for line in lines) - - @property - def irdb_name(self): - return "irdb%d" % self.host.engine - - @property - def irdb(self): - prior_name = self.zoo.handle - return rpki.irdb.database( - self.irdb_name, - on_entry = lambda: self.zoo.reset_identity(self.name), - on_exit = lambda: self.zoo.reset_identity(prior_name)) - - def syncdb(self): - import django.core.management - assert not self.is_hosted - django.core.management.call_command( - "syncdb", - verbosity = 0, - database = self.irdb_name, - migrate = True, - load_initial_data = False, - interactive = False) - - def hire_zookeeper(self): - assert not self.is_hosted - self._zoo = rpki.irdb.Zookeeper( - cfg = rpki.config.parser(filename = self.path("rpki.conf")), - logstream = None if quiet else sys.stdout) - - @property - def zoo(self): - return self.host._zoo - - def dump_root(self): - - assert self.is_root and not self.is_hosted - - root_resources = rpki.resource_set.resource_bag( - asn = rpki.resource_set.resource_set_as("0-4294967295"), - v4 = rpki.resource_set.resource_set_ipv4("0.0.0.0/0"), - v6 = rpki.resource_set.resource_set_ipv6("::/0")) - - root_key = rpki.x509.RSA.generate(quiet = True) - - root_uri = "rsync://%s/rpki/" % self.rsync_server - - root_sia = (root_uri, root_uri + "root.mft", None, rpki.publication.rrdp_sia_uri_kludge) - - root_cert = rpki.x509.X509.self_certify( - keypair = root_key, - subject_key = root_key.get_public(), - serial = 1, - sia = root_sia, - notAfter = rpki.sundial.now() + rpki.sundial.timedelta(days = 365), - resources = root_resources) - - with open(self.path("root.cer"), "wb") as f: - f.write(root_cert.get_DER()) - - with open(self.path("root.key"), "wb") as f: - f.write(root_key.get_DER()) - - with open(cleanpath(test_dir, "root.tal"), "w") as f: - f.write("rsync://%s/root/root.cer\n\n" % self.rsync_server) - f.write(root_key.get_public().get_Base64()) - - def mkdir(self, *path): - path = self.path(*path) - if not quiet: - print "Creating directory", path - os.makedirs(path) - def dump_sql(self): - if not self.is_hosted: - with open(self.path("rpkid.sql"), "w") as f: +class allocation(object): + """ + One entity in our allocation database. Every entity in the database + is assumed to hold resources. Entities that don't have the + hosted_by property run their own copies of rpkid, irdbd, and pubd. + """ + + base_port = 4400 + base_engine = -1 + parent = None + crl_interval = None + regen_margin = None + engine = -1 + rpkid_port = 4404 + irdbd_port = 4403 + pubd_port = 4402 + rootd_port = 4401 + rsync_port = 873 + + @classmethod + def allocate_port(cls): + cls.base_port += 1 + return cls.base_port + + @classmethod + def allocate_engine(cls): + cls.base_engine += 1 + return cls.base_engine + + def __init__(self, y, db, parent = None): + db.append(self) + self.name = y["name"] + self.parent = parent + self.kids = [allocation(k, db, self) for k in y.get("kids", ())] + valid_until = None + if "valid_until" in y: + valid_until = rpki.sundial.datetime.from_datetime(y.get("valid_until")) + if valid_until is None and "valid_for" in y: + valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(y["valid_for"]) + self.base = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as(y.get("asn")), + v4 = rpki.resource_set.resource_set_ipv4(y.get("ipv4")), + v6 = rpki.resource_set.resource_set_ipv6(y.get("ipv6")), + valid_until = valid_until) + if "crl_interval" in y: + self.crl_interval = rpki.sundial.timedelta.parse(y["crl_interval"]).convert_to_seconds() + if "regen_margin" in y: + self.regen_margin = rpki.sundial.timedelta.parse(y["regen_margin"]).convert_to_seconds() + if "ghostbusters" in y: + self.ghostbusters = y.get("ghostbusters") + elif "ghostbuster" in y: + self.ghostbusters = [y.get("ghostbuster")] + else: + self.ghostbusters = [] + self.roa_requests = [roa_request.parse(r) for r in y.get("roa_request", ())] + self.router_certs = [router_cert.parse(r) for r in y.get("router_cert", ())] + for r in self.roa_requests: + if r.v4: + self.base.v4 |= r.v4.to_resource_set() + if r.v6: + self.base.v6 |= r.v6.to_resource_set() + for r in self.router_certs: + self.base.asn |= r.asn + self.hosted_by = y.get("hosted_by") + self.hosts = [] + if not self.is_hosted: + self.engine = self.allocate_engine() + if loopback and not self.is_hosted: + self.rpkid_port = self.allocate_port() + self.irdbd_port = self.allocate_port() + if loopback and self.runs_pubd: + self.pubd_port = self.allocate_port() + self.rsync_port = self.allocate_port() + if loopback and self.is_root: + self.rootd_port = self.allocate_port() + + def closure(self): + resources = self.base + for kid in self.kids: + resources |= kid.closure() + self.resources = resources + return resources + + @property + def hostname(self): + if loopback: + return "localhost" + elif dns_suffix: + return self.name + "." + dns_suffix.lstrip(".") + else: + return self.name + + @property + def rsync_server(self): + if loopback: + return "%s:%s" % (self.pubd.hostname, self.pubd.rsync_port) + else: + return self.pubd.hostname + + def dump(self): if not quiet: - print "Writing", f.name - f.write(rpki.sql_schemas.rpkid) - if self.runs_pubd: - with open(self.path("pubd.sql"), "w") as f: + print str(self) + + def __str__(self): + s = self.name + ":\n" + if self.resources.asn: s += " ASNs: %s\n" % self.resources.asn + if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 + if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 + if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) + if self.parent: s += " Up: %s\n" % self.parent.name + if self.is_hosted: s += " Host: %s\n" % self.hosted_by.name + if self.hosts: s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts) + for r in self.roa_requests: s += " ROA: %s\n" % r + if not self.is_hosted: s += " IPort: %s\n" % self.irdbd_port + if self.runs_pubd: s += " PPort: %s\n" % self.pubd_port + if not self.is_hosted: s += " RPort: %s\n" % self.rpkid_port + if self.runs_pubd: s += " SPort: %s\n" % self.rsync_port + if self.is_root: s += " TPort: %s\n" % self.rootd_port + return s + " Until: %s\n" % self.resources.valid_until + + @property + def is_root(self): + return self.parent is None + + @property + def is_hosted(self): + return self.hosted_by is not None + + @property + def runs_pubd(self): + return self.is_root or not (self.is_hosted or only_one_pubd) + + def path(self, *names): + return cleanpath(test_dir, self.host.name, *names) + + def csvout(self, fn): + path = self.path(fn) if not quiet: - print "Writing", f.name - f.write(rpki.sql_schemas.pubd) - if not self.is_hosted: - username = config_overrides["irdbd_sql_username"] - password = config_overrides["irdbd_sql_password"] - cmd = ("mysqldump", "-u", username, "-p" + password, self.irdb_name) - with open(self.path("irdbd.sql"), "w") as f: + print "Writing", path + return rpki.csv_utils.csv_writer(path) + + def up_down_url(self): + return "http://%s:%d/up-down/%s/%s" % (self.parent.host.hostname, + self.parent.host.rpkid_port, + self.parent.name, + self.name) + + def dump_asns(self, fn): + with self.csvout(fn) as f: + for k in self.kids: + f.writerows((k.name, a) for a in k.resources.asn) + + def dump_prefixes(self, fn): + with self.csvout(fn) as f: + for k in self.kids: + f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) + + def dump_roas(self, fn): + with self.csvout(fn) as f: + for g1, r in enumerate(self.roa_requests): + f.writerows((p, r.asn, "G%08d%08d" % (g1, g2)) + for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ()))) + + def dump_ghostbusters(self, fn): + if self.ghostbusters: + path = self.path(fn) + if not quiet: + print "Writing", path + with open(path, "w") as f: + for i, g in enumerate(self.ghostbusters): + if i > 0: + f.write("\n") + f.write(g) + + def dump_router_certificates(self, fn): + if self.router_certs: + path = self.path(fn) + if not quiet: + print "Writing", path + xmlns = rpki.relaxng.router_certificate.xmlns + xml = lxml.etree.Element(xmlns + "router_certificate_requests", + version = rpki.relaxng.router_certificate.version, + nsmap = rpki.relaxng.router_certificate.nsmap) + for r in self.router_certs: + x = lxml.etree.SubElement(xml, xmlns + "router_certificate_request", + router_id = str(r.router_id), + asn = str(r.asn), + valid_until = str(self.resources.valid_until)) + x.text = r.pkcs10.get_Base64() + rpki.relaxng.router_certificate.assertValid(xml) + lxml.etree.ElementTree(xml).write(path, pretty_print = True) + + @property + def pubd(self): + s = self + while not s.runs_pubd: + s = s.parent + return s + + @property + def client_handle(self): + path = [] + s = self + if not flat_publication: + while not s.runs_pubd: + path.append(s) + s = s.parent + path.append(s) + return ".".join(i.name for i in reversed(path)) + + @property + def host(self): + return self.hosted_by or self + + @property + def publication_base_directory(self): + if not loopback and publication_base is not None: + return publication_base + else: + return self.path("publication") + + @property + def publication_root_directory(self): + if not loopback and publication_root is not None: + return publication_root + else: + return self.path("publication.root") + + def dump_conf(self): + + r = dict( + handle = self.name, + run_rpkid = str(not self.is_hosted), + run_pubd = str(self.runs_pubd), + run_rootd = str(self.is_root), + irdbd_sql_username = "irdb", + rpkid_sql_username = "rpki", + rpkid_server_host = self.hostname, + rpkid_server_port = str(self.rpkid_port), + irdbd_server_host = "localhost", + irdbd_server_port = str(self.irdbd_port), + rootd_server_port = str(self.rootd_port), + pubd_sql_username = "pubd", + pubd_server_host = self.pubd.hostname, + pubd_server_port = str(self.pubd.pubd_port), + publication_rsync_server = self.rsync_server) + + if loopback: + r.update( + irdbd_sql_database = self.irdb_name, + rpkid_sql_database = "rpki%d" % self.engine, + pubd_sql_database = "pubd%d" % self.engine, + bpki_servers_directory = self.path(), + publication_base_directory = self.publication_base_directory) + + r.update(config_overrides) + + with open(self.path("rpki.conf"), "w") as f: + f.write("# Automatically generated, do not edit\n") + if not quiet: + print "Writing", f.name + + section = None + for line in open(rpki_conf): + m = section_regexp.match(line) + if m: + section = m.group(1) + m = variable_regexp.match(line) + option = m.group(1) if m and section == "myrpki" else None + if option and option in r: + line = "%s = %s\n" % (option, r[option]) + f.write(line) + + def dump_rsyncd(self): + lines = [] + if self.runs_pubd: + lines.extend(( + "# Automatically generated, do not edit", + "port = %d" % self.rsync_port, + "address = %s" % self.hostname, + "log file = rsyncd.log", + "read only = yes", + "use chroot = no", + "[rpki]", + "path = %s" % self.publication_base_directory, + "comment = RPKI test")) + if self.is_root: + assert self.runs_pubd + lines.extend(( + "[root]", + "path = %s" % self.publication_root_directory, + "comment = RPKI test root")) + if lines: + with open(self.path("rsyncd.conf"), "w") as f: + if not quiet: + print "Writing", f.name + f.writelines(line + "\n" for line in lines) + + @property + def irdb_name(self): + return "irdb%d" % self.host.engine + + @property + def irdb(self): + prior_name = self.zoo.handle + return rpki.irdb.database( + self.irdb_name, + on_entry = lambda: self.zoo.reset_identity(self.name), + on_exit = lambda: self.zoo.reset_identity(prior_name)) + + def syncdb(self): + import django.core.management + assert not self.is_hosted + django.core.management.call_command( + "syncdb", + verbosity = 0, + database = self.irdb_name, + migrate = True, + load_initial_data = False, + interactive = False) + + def hire_zookeeper(self): + assert not self.is_hosted + self._zoo = rpki.irdb.Zookeeper( + cfg = rpki.config.parser(filename = self.path("rpki.conf")), + logstream = None if quiet else sys.stdout) + + @property + def zoo(self): + return self.host._zoo + + def dump_root(self): + + assert self.is_root and not self.is_hosted + + root_resources = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as("0-4294967295"), + v4 = rpki.resource_set.resource_set_ipv4("0.0.0.0/0"), + v6 = rpki.resource_set.resource_set_ipv6("::/0")) + + root_key = rpki.x509.RSA.generate(quiet = True) + + root_uri = "rsync://%s/rpki/" % self.rsync_server + + root_sia = (root_uri, root_uri + "root.mft", None, rpki.publication.rrdp_sia_uri_kludge) + + root_cert = rpki.x509.X509.self_certify( + keypair = root_key, + subject_key = root_key.get_public(), + serial = 1, + sia = root_sia, + notAfter = rpki.sundial.now() + rpki.sundial.timedelta(days = 365), + resources = root_resources) + + with open(self.path("root.cer"), "wb") as f: + f.write(root_cert.get_DER()) + + with open(self.path("root.key"), "wb") as f: + f.write(root_key.get_DER()) + + with open(cleanpath(test_dir, "root.tal"), "w") as f: + f.write("rsync://%s/root/root.cer\n\n" % self.rsync_server) + f.write(root_key.get_public().get_Base64()) + + def mkdir(self, *path): + path = self.path(*path) if not quiet: - print "Writing", f.name - subprocess.check_call(cmd, stdout = f) + print "Creating directory", path + os.makedirs(path) + + def dump_sql(self): + if not self.is_hosted: + with open(self.path("rpkid.sql"), "w") as f: + if not quiet: + print "Writing", f.name + f.write(rpki.sql_schemas.rpkid) + if self.runs_pubd: + with open(self.path("pubd.sql"), "w") as f: + if not quiet: + print "Writing", f.name + f.write(rpki.sql_schemas.pubd) + if not self.is_hosted: + username = config_overrides["irdbd_sql_username"] + password = config_overrides["irdbd_sql_password"] + cmd = ("mysqldump", "-u", username, "-p" + password, self.irdb_name) + with open(self.path("irdbd.sql"), "w") as f: + if not quiet: + print "Writing", f.name + subprocess.check_call(cmd, stdout = f) def pre_django_sql_setup(needed): - username = config_overrides["irdbd_sql_username"] - password = config_overrides["irdbd_sql_password"] - - # If we have the MySQL root password, just blow away and recreate - # the required databases. Otherwise, check for missing databases, - # then blow away all tables in the required databases. In either - # case, we assume that the Django syncdb code will populate - # databases as necessary, all we need to do here is provide empty - # databases for the Django code to fill in. + username = config_overrides["irdbd_sql_username"] + password = config_overrides["irdbd_sql_password"] + + # If we have the MySQL root password, just blow away and recreate + # the required databases. Otherwise, check for missing databases, + # then blow away all tables in the required databases. In either + # case, we assume that the Django syncdb code will populate + # databases as necessary, all we need to do here is provide empty + # databases for the Django code to fill in. + + if mysql_rootpass is not None: + if mysql_rootpass: + db = MySQLdb.connect(user = mysql_rootuser, passwd = mysql_rootpass) + else: + db = MySQLdb.connect(user = mysql_rootuser) + cur = db.cursor() + for database in needed: + try: + cur.execute("DROP DATABASE IF EXISTS %s" % database) + except: + pass + cur.execute("CREATE DATABASE %s" % database) + cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % ( + database, username), (password,)) - if mysql_rootpass is not None: - if mysql_rootpass: - db = MySQLdb.connect(user = mysql_rootuser, passwd = mysql_rootpass) else: - db = MySQLdb.connect(user = mysql_rootuser) - cur = db.cursor() - for database in needed: - try: - cur.execute("DROP DATABASE IF EXISTS %s" % database) - except: - pass - cur.execute("CREATE DATABASE %s" % database) - cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % ( - database, username), (password,)) - - else: - db = MySQLdb.connect(user = username, passwd = password) - cur = db.cursor() - cur.execute("SHOW DATABASES") - existing = set(r[0] for r in cur.fetchall()) - if needed - existing: - sys.stderr.write("The following databases are missing:\n") - for database in sorted(needed - existing): - sys.stderr.write(" %s\n" % database) - sys.stderr.write("Please create them manually or put MySQL root password in my config file\n") - sys.exit("Missing databases and MySQL root password not known, can't continue") - for database in needed: - db.select_db(database) - cur.execute("SHOW TABLES") - tables = [r[0] for r in cur.fetchall()] - cur.execute("SET foreign_key_checks = 0") - for table in tables: - cur.execute("DROP TABLE %s" % table) - cur.execute("SET foreign_key_checks = 1") - - cur.close() - db.commit() - db.close() + db = MySQLdb.connect(user = username, passwd = password) + cur = db.cursor() + cur.execute("SHOW DATABASES") + existing = set(r[0] for r in cur.fetchall()) + if needed - existing: + sys.stderr.write("The following databases are missing:\n") + for database in sorted(needed - existing): + sys.stderr.write(" %s\n" % database) + sys.stderr.write("Please create them manually or put MySQL root password in my config file\n") + sys.exit("Missing databases and MySQL root password not known, can't continue") + for database in needed: + db.select_db(database) + cur.execute("SHOW TABLES") + tables = [r[0] for r in cur.fetchall()] + cur.execute("SET foreign_key_checks = 0") + for table in tables: + cur.execute("DROP TABLE %s" % table) + cur.execute("SET foreign_key_checks = 1") + + cur.close() + db.commit() + db.close() class timestamp(object): - def __init__(self, *args): - self.count = 0 - self.start = self.tick = rpki.sundial.now() + def __init__(self, *args): + self.count = 0 + self.start = self.tick = rpki.sundial.now() - def __call__(self, *args): - now = rpki.sundial.now() - if not quiet: - print "[Count %s last %s total %s now %s]" % ( - self.count, now - self.tick, now - self.start, now) - self.tick = now - self.count += 1 + def __call__(self, *args): + now = rpki.sundial.now() + if not quiet: + print "[Count %s last %s total %s now %s]" % ( + self.count, now - self.tick, now - self.start, now) + self.tick = now + self.count += 1 def main(): - global flat_publication - global config_overrides - global only_one_pubd - global loopback - global dns_suffix - global mysql_rootuser - global mysql_rootpass - global yaml_file - global test_dir - global rpki_conf - global publication_base - global publication_root - global quiet - - os.environ["TZ"] = "UTC" - time.tzset() - - parser = argparse.ArgumentParser(description = "yamlconf") - parser.add_argument("-c", "--config", help = "configuration file") - parser.add_argument("--dns_suffix", - help = "DNS suffix to add to hostnames") - parser.add_argument("-l", "--loopback", action = "store_true", - help = "Configure for use with yamltest on localhost") - parser.add_argument("-f", "--flat_publication", action = "store_true", - help = "Use flat publication model") - parser.add_argument("-q", "--quiet", action = "store_true", - help = "Work more quietly") - parser.add_argument("--profile", - help = "Filename for profile output") - parser.add_argument("yaml_file", type = argparse.FileType("r"), - help = "YAML file describing network to build") - args = parser.parse_args() - - dns_suffix = args.dns_suffix - loopback = args.loopback - flat_publication = args.flat_publication - quiet = args.quiet - yaml_file = args.yaml_file - - rpki.log.init("yamlconf", argparse.Namespace(log_level = logging.DEBUG, - log_handler = lambda: logging.StreamHandler(sys.stdout))) - - # Allow optional config file for this tool to override default - # passwords: this is mostly so that I can show a complete working - # example without publishing my own server's passwords. - - cfg = rpki.config.parser(set_filename = args.config, section = "yamlconf", allow_missing = True) - try: - cfg.set_global_flags() - except: - pass - - # Use of "yamltest.dir" is deliberate: intent is for what we write to - # be usable with "yamltest --skip_config". - - only_one_pubd = cfg.getboolean("only_one_pubd", True) - test_dir = cfg.get("test_directory", cleanpath(this_dir, "yamltest.dir")) - rpki_conf = cfg.get("rpki_conf", cleanpath(this_dir, "..", "examples/rpki.conf")) - mysql_rootuser = cfg.get("mysql_rootuser", "root") - - try: - mysql_rootpass = cfg.get("mysql_rootpass") - except: - pass - - try: - publication_base = cfg.get("publication_base") - except: - pass - - try: - publication_root = cfg.get("publication_root") - except: - pass - - for k in ("rpkid_sql_password", "irdbd_sql_password", "pubd_sql_password", - "rpkid_sql_username", "irdbd_sql_username", "pubd_sql_username"): - if cfg.has_option(k): - config_overrides[k] = cfg.get(k) - - if args.profile: - import cProfile - prof = cProfile.Profile() + global flat_publication + global config_overrides + global only_one_pubd + global loopback + global dns_suffix + global mysql_rootuser + global mysql_rootpass + global yaml_file + global test_dir + global rpki_conf + global publication_base + global publication_root + global quiet + + os.environ["TZ"] = "UTC" + time.tzset() + + parser = argparse.ArgumentParser(description = "yamlconf") + parser.add_argument("-c", "--config", help = "configuration file") + parser.add_argument("--dns_suffix", + help = "DNS suffix to add to hostnames") + parser.add_argument("-l", "--loopback", action = "store_true", + help = "Configure for use with yamltest on localhost") + parser.add_argument("-f", "--flat_publication", action = "store_true", + help = "Use flat publication model") + parser.add_argument("-q", "--quiet", action = "store_true", + help = "Work more quietly") + parser.add_argument("--profile", + help = "Filename for profile output") + parser.add_argument("yaml_file", type = argparse.FileType("r"), + help = "YAML file describing network to build") + args = parser.parse_args() + + dns_suffix = args.dns_suffix + loopback = args.loopback + flat_publication = args.flat_publication + quiet = args.quiet + yaml_file = args.yaml_file + + rpki.log.init("yamlconf", argparse.Namespace(log_level = logging.DEBUG, + log_handler = lambda: logging.StreamHandler(sys.stdout))) + + # Allow optional config file for this tool to override default + # passwords: this is mostly so that I can show a complete working + # example without publishing my own server's passwords. + + cfg = rpki.config.parser(set_filename = args.config, section = "yamlconf", allow_missing = True) try: - prof.runcall(body) - finally: - prof.dump_stats(args.profile) - if not quiet: - print - print "Dumped profile data to %s" % args.profile - else: - body() + cfg.set_global_flags() + except: + pass -def body(): + # Use of "yamltest.dir" is deliberate: intent is for what we write to + # be usable with "yamltest --skip_config". + + only_one_pubd = cfg.getboolean("only_one_pubd", True) + test_dir = cfg.get("test_directory", cleanpath(this_dir, "yamltest.dir")) + rpki_conf = cfg.get("rpki_conf", cleanpath(this_dir, "..", "examples/rpki.conf")) + mysql_rootuser = cfg.get("mysql_rootuser", "root") - global rpki + try: + mysql_rootpass = cfg.get("mysql_rootpass") + except: + pass - ts = timestamp() + try: + publication_base = cfg.get("publication_base") + except: + pass - for root, dirs, files in os.walk(test_dir, topdown = False): - for fn in files: - os.unlink(os.path.join(root, fn)) - for d in dirs: - os.rmdir(os.path.join(root, d)) + try: + publication_root = cfg.get("publication_root") + except: + pass - if not quiet: - print - print "Reading YAML", yaml_file.name + for k in ("rpkid_sql_password", "irdbd_sql_password", "pubd_sql_password", + "rpkid_sql_username", "irdbd_sql_username", "pubd_sql_username"): + if cfg.has_option(k): + config_overrides[k] = cfg.get(k) + + if args.profile: + import cProfile + prof = cProfile.Profile() + try: + prof.runcall(body) + finally: + prof.dump_stats(args.profile) + if not quiet: + print + print "Dumped profile data to %s" % args.profile + else: + body() - db = allocation_db(yaml.safe_load_all(yaml_file).next()) +def body(): - # Show what we loaded + global rpki - #db.dump() + ts = timestamp() - # Do pre-Django SQL setup + for root, dirs, files in os.walk(test_dir, topdown = False): + for fn in files: + os.unlink(os.path.join(root, fn)) + for d in dirs: + os.rmdir(os.path.join(root, d)) - pre_django_sql_setup(set(d.irdb_name for d in db if not d.is_hosted)) + if not quiet: + print + print "Reading YAML", yaml_file.name - # Now ready for fun with multiple databases in Django! - # - # https://docs.djangoproject.com/en/1.4/topics/db/multi-db/ - # https://docs.djangoproject.com/en/1.4/topics/db/sql/ - # - # This program's use of the ORM is sufficiently different that it's - # not worth straining to use rpki.django_settings, so we just use - # Django's settings API directly. + db = allocation_db(yaml.safe_load_all(yaml_file).next()) - database_template = { - "ENGINE" : "django.db.backends.mysql", - "USER" : config_overrides["irdbd_sql_username"], - "PASSWORD" : config_overrides["irdbd_sql_password"], - "HOST" : "", - "PORT" : "", - "OPTIONS" : { "init_command": "SET storage_engine=INNODB" }} + # Show what we loaded - databases = dict((d.irdb_name, dict(database_template, NAME = d.irdb_name)) - for d in db if not d.is_hosted) + #db.dump() - databases["default"] = databases[db.root.irdb_name] + # Do pre-Django SQL setup - import django - django.setup() + pre_django_sql_setup(set(d.irdb_name for d in db if not d.is_hosted)) - from django.conf import settings + # Now ready for fun with multiple databases in Django! + # + # https://docs.djangoproject.com/en/1.4/topics/db/multi-db/ + # https://docs.djangoproject.com/en/1.4/topics/db/sql/ + # + # This program's use of the ORM is sufficiently different that it's + # not worth straining to use rpki.django_settings, so we just use + # Django's settings API directly. - settings.configure( - DATABASES = databases, - DATABASE_ROUTERS = ["rpki.irdb.router.DBContextRouter"], - INSTALLED_APPS = ["rpki.irdb"]) + database_template = { + "ENGINE" : "django.db.backends.mysql", + "USER" : config_overrides["irdbd_sql_username"], + "PASSWORD" : config_overrides["irdbd_sql_password"], + "HOST" : "", + "PORT" : "", + "OPTIONS" : { "init_command": "SET storage_engine=INNODB" }} - import rpki.irdb + databases = dict((d.irdb_name, dict(database_template, NAME = d.irdb_name)) + for d in db if not d.is_hosted) - rpki.irdb.models.ca_certificate_lifetime = rpki.sundial.timedelta(days = 3652 * 2) - rpki.irdb.models.ee_certificate_lifetime = rpki.sundial.timedelta(days = 3652) + databases["default"] = databases[db.root.irdb_name] - ts() + import django + django.setup() - for d in db: - if not quiet: - print - print "Configuring", d.name - - if not d.is_hosted: - d.mkdir() - if d.runs_pubd: - d.mkdir("publication") - if d.is_root: - d.mkdir("publication.root") - - if not d.is_hosted: - d.dump_conf() - d.dump_rsyncd() - - d.dump_asns("%s.asns.csv" % d.name) - d.dump_prefixes("%s.prefixes.csv" % d.name) - d.dump_roas("%s.roas.csv" % d.name) - d.dump_ghostbusters("%s.ghostbusters.vcard" % d.name) - d.dump_router_certificates("%s.routercerts.xml" % d.name) - - if not d.is_hosted: - if not quiet: - print "Initializing SQL" - d.syncdb() - if not quiet: - print "Hiring zookeeper" - d.hire_zookeeper() - - with d.irdb: - if not quiet: - print "Creating identity" - x = d.zoo.initialize() - - if d.is_root: - if not quiet: - print "Creating RPKI root certificate and TAL" - d.dump_root() - x = d.zoo.configure_rootd() + from django.conf import settings - else: - with d.parent.irdb: - x = d.parent.zoo.configure_child(x.file)[0] - x = d.zoo.configure_parent(x.file)[0] + settings.configure( + DATABASES = databases, + DATABASE_ROUTERS = ["rpki.irdb.router.DBContextRouter"], + INSTALLED_APPS = ["rpki.irdb"]) - with d.pubd.irdb: - x = d.pubd.zoo.configure_publication_client(x.file, flat = flat_publication)[0] - d.zoo.configure_repository(x.file) + import rpki.irdb - if loopback and not d.is_hosted: - with d.irdb: - d.zoo.write_bpki_files() + rpki.irdb.models.ca_certificate_lifetime = rpki.sundial.timedelta(days = 3652 * 2) + rpki.irdb.models.ee_certificate_lifetime = rpki.sundial.timedelta(days = 3652) ts() - if not loopback: - if not quiet: - print for d in db: - d.dump_sql() + if not quiet: + print + print "Configuring", d.name + + if not d.is_hosted: + d.mkdir() + if d.runs_pubd: + d.mkdir("publication") + if d.is_root: + d.mkdir("publication.root") + + if not d.is_hosted: + d.dump_conf() + d.dump_rsyncd() + + d.dump_asns("%s.asns.csv" % d.name) + d.dump_prefixes("%s.prefixes.csv" % d.name) + d.dump_roas("%s.roas.csv" % d.name) + d.dump_ghostbusters("%s.ghostbusters.vcard" % d.name) + d.dump_router_certificates("%s.routercerts.xml" % d.name) + + if not d.is_hosted: + if not quiet: + print "Initializing SQL" + d.syncdb() + if not quiet: + print "Hiring zookeeper" + d.hire_zookeeper() + + with d.irdb: + if not quiet: + print "Creating identity" + x = d.zoo.initialize() + + if d.is_root: + if not quiet: + print "Creating RPKI root certificate and TAL" + d.dump_root() + x = d.zoo.configure_rootd() + + else: + with d.parent.irdb: + x = d.parent.zoo.configure_child(x.file)[0] + x = d.zoo.configure_parent(x.file)[0] + + with d.pubd.irdb: + x = d.pubd.zoo.configure_publication_client(x.file, flat = flat_publication)[0] + d.zoo.configure_repository(x.file) + + if loopback and not d.is_hosted: + with d.irdb: + d.zoo.write_bpki_files() + + ts() + + if not loopback: + if not quiet: + print + for d in db: + d.dump_sql() if __name__ == "__main__": - main() + main() diff --git a/ca/tests/yamltest.py b/ca/tests/yamltest.py index a26fefb9..f82cc192 100644 --- a/ca/tests/yamltest.py +++ b/ca/tests/yamltest.py @@ -65,11 +65,11 @@ section_regexp = re.compile(r"\s*\[\s*(.+?)\s*\]\s*$") variable_regexp = re.compile(r"\s*([-a-zA-Z0-9_]+)\s*=\s*(.+?)\s*$") def cleanpath(*names): - """ - Construct normalized pathnames. - """ + """ + Construct normalized pathnames. + """ - return os.path.normpath(os.path.join(*names)) + return os.path.normpath(os.path.join(*names)) # Pathnames for various things we need @@ -85,652 +85,652 @@ prog_rootd = cleanpath(ca_dir, "rootd") prog_rpki_manage = cleanpath(ca_dir, "rpki-manage") class roa_request(object): - """ - Representation of a ROA request. - """ - - def __init__(self, asn, ipv4, ipv6): - self.asn = asn - self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None - self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None - - def __eq__(self, other): - return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 - - def __hash__(self): - v4 = tuple(self.v4) if self.v4 is not None else None - v6 = tuple(self.v6) if self.v6 is not None else None - return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - - def __str__(self): - if self.v4 and self.v6: - return "%s: %s,%s" % (self.asn, self.v4, self.v6) - else: - return "%s: %s" % (self.asn, self.v4 or self.v6) - - @classmethod - def parse(cls, y): - """ - Parse a ROA request from YAML format. - """ - - return cls(y.get("asn"), y.get("ipv4"), y.get("ipv6")) - - -class router_cert(object): - """ - Representation for a router_cert object. - """ - - _ecparams = None - - @classmethod - def ecparams(cls): - if cls._ecparams is None: - cls._ecparams = rpki.x509.KeyParams.generateEC() - return cls._ecparams - - def __init__(self, asn, router_id): - self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) - self.router_id = router_id - self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True) - self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) - self.gski = self.pkcs10.gSKI() - - def __eq__(self, other): - return self.asn == other.asn and self.router_id == other.router_id and self.gski == other.gski - - def __hash__(self): - return tuple(self.asn).__hash__() + self.router_id.__hash__() + self.gski.__hash__() - - def __str__(self): - return "%s: %s: %s" % (self.asn, self.router_id, self.gski) - - @classmethod - def parse(cls, yaml): - return cls(yaml.get("asn"), yaml.get("router_id")) - -class allocation_db(list): - """ - Our allocation database. - """ - - def __init__(self, yaml): - list.__init__(self) - self.root = allocation(yaml, self) - assert self.root.is_root and not any(a.is_root for a in self if a is not self.root) and self[0] is self.root - if self.root.crl_interval is None: - self.root.crl_interval = 60 * 60 - if self.root.regen_margin is None: - self.root.regen_margin = 24 * 60 * 60 - if self.root.base.valid_until is None: - self.root.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 2) - for a in self: - if a.base.valid_until is None: - a.base.valid_until = a.parent.base.valid_until - if a.crl_interval is None: - a.crl_interval = a.parent.crl_interval - if a.regen_margin is None: - a.regen_margin = a.parent.regen_margin - self.root.closure() - self.map = dict((a.name, a) for a in self) - for a in self: - if a.is_hosted: - a.hosted_by = self.map[a.hosted_by] - a.hosted_by.hosts.append(a) - assert not a.is_root and not a.hosted_by.is_hosted - - def dump(self): """ - Show contents of allocation database. + Representation of a ROA request. """ - for a in self: - a.dump() + def __init__(self, asn, ipv4, ipv6): + self.asn = asn + self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None + self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None + def __eq__(self, other): + return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 -class allocation(object): - """ - One entity in our allocation database. Every entity in the database - is assumed to hold resources, so needs at least rpkic services. - Entities that don't have the hosted_by property run their own copies - of rpkid, irdbd, and pubd, so they also need myirbe services. - """ - - base_port = None - parent = None - crl_interval = None - regen_margin = None - rootd_port = None - engine = -1 - rpkid_port = -1 - irdbd_port = -1 - pubd_port = -1 - rsync_port = -1 - rootd_port = -1 - rrdp_port = -1 - rpkic_counter = 0L - - @classmethod - def allocate_port(cls): - """ - Allocate a TCP port. - """ - - cls.base_port += 1 - return cls.base_port - - base_engine = -1 - - @classmethod - def allocate_engine(cls): - """ - Allocate an engine number, mostly used to construct SQL database - names. - """ - - cls.base_engine += 1 - return cls.base_engine - - def __init__(self, yaml, db, parent = None): - db.append(self) - self.name = yaml["name"] - self.parent = parent - self.kids = [allocation(k, db, self) for k in yaml.get("kids", ())] - valid_until = None - if "valid_until" in yaml: - valid_until = rpki.sundial.datetime.from_datetime(yaml.get("valid_until")) - if valid_until is None and "valid_for" in yaml: - valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(yaml["valid_for"]) - self.base = rpki.resource_set.resource_bag( - asn = rpki.resource_set.resource_set_as(yaml.get("asn")), - v4 = rpki.resource_set.resource_set_ipv4(yaml.get("ipv4")), - v6 = rpki.resource_set.resource_set_ipv6(yaml.get("ipv6")), - valid_until = valid_until) - if "crl_interval" in yaml: - self.crl_interval = rpki.sundial.timedelta.parse(yaml["crl_interval"]).convert_to_seconds() - if "regen_margin" in yaml: - self.regen_margin = rpki.sundial.timedelta.parse(yaml["regen_margin"]).convert_to_seconds() - self.roa_requests = [roa_request.parse(y) for y in yaml.get("roa_request", yaml.get("route_origin", ()))] - self.router_certs = [router_cert.parse(y) for y in yaml.get("router_cert", ())] - if "ghostbusters" in yaml: - self.ghostbusters = yaml.get("ghostbusters") - elif "ghostbuster" in yaml: - self.ghostbusters = [yaml.get("ghostbuster")] - else: - self.ghostbusters = [] - for r in self.roa_requests: - if r.v4: - self.base.v4 |= r.v4.to_resource_set() - if r.v6: - self.base.v6 |= r.v6.to_resource_set() - for r in self.router_certs: - self.base.asn |= r.asn - self.hosted_by = yaml.get("hosted_by") - self.hosts = [] - if not self.is_hosted: - self.engine = self.allocate_engine() - self.rpkid_port = self.allocate_port() - self.irdbd_port = self.allocate_port() - if self.runs_pubd: - self.pubd_port = self.allocate_port() - self.rsync_port = self.allocate_port() - self.rrdp_port = self.allocate_port() - if self.is_root: - self.rootd_port = self.allocate_port() - - def closure(self): - """ - Compute resource closure of this node and its children, to avoid a - lot of tedious (and error-prone) duplication in the YAML file. - """ - - resources = self.base - for kid in self.kids: - resources |= kid.closure() - self.resources = resources - return resources - - def dump(self): - """ - Show content of this allocation node. - """ - - print str(self) - - def __str__(self): - s = self.name + ":\n" - if self.resources.asn: s += " ASNs: %s\n" % self.resources.asn - if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 - if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 - if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) - if self.parent: s += " Up: %s\n" % self.parent.name - if self.is_hosted: s += " Host: %s\n" % self.hosted_by.name - if self.hosts: s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts) - for r in self.roa_requests: s += " ROA: %s\n" % r - if not self.is_hosted: s += " IPort: %s\n" % self.irdbd_port - if self.runs_pubd: s += " PPort: %s\n" % self.pubd_port - if not self.is_hosted: s += " RPort: %s\n" % self.rpkid_port - if self.runs_pubd: s += " SPort: %s\n" % self.rsync_port - if self.is_root: s += " TPort: %s\n" % self.rootd_port - return s + " Until: %s\n" % self.resources.valid_until - - @property - def is_root(self): - """ - Is this the root node? - """ + def __hash__(self): + v4 = tuple(self.v4) if self.v4 is not None else None + v6 = tuple(self.v6) if self.v6 is not None else None + return self.asn.__hash__() + v4.__hash__() + v6.__hash__() - return self.parent is None - - @property - def is_hosted(self): - """ - Is this entity hosted? - """ - - return self.hosted_by is not None - - @property - def runs_pubd(self): - """ - Does this entity run a pubd? - """ + def __str__(self): + if self.v4 and self.v6: + return "%s: %s,%s" % (self.asn, self.v4, self.v6) + else: + return "%s: %s" % (self.asn, self.v4 or self.v6) - return self.is_root or not (self.is_hosted or only_one_pubd) + @classmethod + def parse(cls, y): + """ + Parse a ROA request from YAML format. + """ - def path(self, *names): - """ - Construct pathnames in this entity's test directory. - """ + return cls(y.get("asn"), y.get("ipv4"), y.get("ipv6")) - return cleanpath(test_dir, self.host.name, *names) - def csvout(self, fn): +class router_cert(object): """ - Open and log a CSV output file. + Representation for a router_cert object. """ - path = self.path(fn) - print "Writing", path - return rpki.csv_utils.csv_writer(path) + _ecparams = None - def up_down_url(self): - """ - Construct service URL for this node's parent. - """ + @classmethod + def ecparams(cls): + if cls._ecparams is None: + cls._ecparams = rpki.x509.KeyParams.generateEC() + return cls._ecparams - return "http://localhost:%d/up-down/%s/%s" % (self.parent.host.rpkid_port, - self.parent.name, - self.name) + def __init__(self, asn, router_id): + self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) + self.router_id = router_id + self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True) + self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) + self.gski = self.pkcs10.gSKI() - def dump_asns(self): - """ - Write Autonomous System Numbers CSV file. - """ + def __eq__(self, other): + return self.asn == other.asn and self.router_id == other.router_id and self.gski == other.gski - fn = "%s.asns.csv" % d.name - if not args.skip_config: - with self.csvout(fn) as f: - for k in self.kids: - f.writerows((k.name, a) for a in k.resources.asn) - if not args.stop_after_config: - self.run_rpkic("load_asns", fn) + def __hash__(self): + return tuple(self.asn).__hash__() + self.router_id.__hash__() + self.gski.__hash__() - def dump_prefixes(self): - """ - Write prefixes CSV file. - """ + def __str__(self): + return "%s: %s: %s" % (self.asn, self.router_id, self.gski) - fn = "%s.prefixes.csv" % d.name - if not args.skip_config: - with self.csvout(fn) as f: - for k in self.kids: - f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) - if not args.stop_after_config: - self.run_rpkic("load_prefixes", fn) + @classmethod + def parse(cls, yaml): + return cls(yaml.get("asn"), yaml.get("router_id")) - def dump_roas(self): - """ - Write ROA CSV file. - """ - - fn = "%s.roas.csv" % d.name - if not args.skip_config: - with self.csvout(fn) as f: - for g1, r in enumerate(self.roa_requests): - f.writerows((p, r.asn, "G%08d%08d" % (g1, g2)) - for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ()))) - if not args.stop_after_config: - self.run_rpkic("load_roa_requests", fn) - - def dump_ghostbusters(self): - """ - Write Ghostbusters vCard file. +class allocation_db(list): """ + Our allocation database. + """ + + def __init__(self, yaml): + list.__init__(self) + self.root = allocation(yaml, self) + assert self.root.is_root and not any(a.is_root for a in self if a is not self.root) and self[0] is self.root + if self.root.crl_interval is None: + self.root.crl_interval = 60 * 60 + if self.root.regen_margin is None: + self.root.regen_margin = 24 * 60 * 60 + if self.root.base.valid_until is None: + self.root.base.valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 2) + for a in self: + if a.base.valid_until is None: + a.base.valid_until = a.parent.base.valid_until + if a.crl_interval is None: + a.crl_interval = a.parent.crl_interval + if a.regen_margin is None: + a.regen_margin = a.parent.regen_margin + self.root.closure() + self.map = dict((a.name, a) for a in self) + for a in self: + if a.is_hosted: + a.hosted_by = self.map[a.hosted_by] + a.hosted_by.hosts.append(a) + assert not a.is_root and not a.hosted_by.is_hosted + + def dump(self): + """ + Show contents of allocation database. + """ + + for a in self: + a.dump() - if self.ghostbusters: - fn = "%s.ghostbusters.vcard" % d.name - if not args.skip_config: - path = self.path(fn) - print "Writing", path - with open(path, "w") as f: - f.write("\n".join(self.ghostbusters)) - if not args.stop_after_config: - self.run_rpkic("load_ghostbuster_requests", fn) - def dump_router_certificates(self): - """ - Write EE certificates (router certificates, etc). +class allocation(object): """ + One entity in our allocation database. Every entity in the database + is assumed to hold resources, so needs at least rpkic services. + Entities that don't have the hosted_by property run their own copies + of rpkid, irdbd, and pubd, so they also need myirbe services. + """ + + base_port = None + parent = None + crl_interval = None + regen_margin = None + rootd_port = None + engine = -1 + rpkid_port = -1 + irdbd_port = -1 + pubd_port = -1 + rsync_port = -1 + rootd_port = -1 + rrdp_port = -1 + rpkic_counter = 0L + + @classmethod + def allocate_port(cls): + """ + Allocate a TCP port. + """ + + cls.base_port += 1 + return cls.base_port + + base_engine = -1 + + @classmethod + def allocate_engine(cls): + """ + Allocate an engine number, mostly used to construct SQL database + names. + """ + + cls.base_engine += 1 + return cls.base_engine + + def __init__(self, yaml, db, parent = None): + db.append(self) + self.name = yaml["name"] + self.parent = parent + self.kids = [allocation(k, db, self) for k in yaml.get("kids", ())] + valid_until = None + if "valid_until" in yaml: + valid_until = rpki.sundial.datetime.from_datetime(yaml.get("valid_until")) + if valid_until is None and "valid_for" in yaml: + valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(yaml["valid_for"]) + self.base = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as(yaml.get("asn")), + v4 = rpki.resource_set.resource_set_ipv4(yaml.get("ipv4")), + v6 = rpki.resource_set.resource_set_ipv6(yaml.get("ipv6")), + valid_until = valid_until) + if "crl_interval" in yaml: + self.crl_interval = rpki.sundial.timedelta.parse(yaml["crl_interval"]).convert_to_seconds() + if "regen_margin" in yaml: + self.regen_margin = rpki.sundial.timedelta.parse(yaml["regen_margin"]).convert_to_seconds() + self.roa_requests = [roa_request.parse(y) for y in yaml.get("roa_request", yaml.get("route_origin", ()))] + self.router_certs = [router_cert.parse(y) for y in yaml.get("router_cert", ())] + if "ghostbusters" in yaml: + self.ghostbusters = yaml.get("ghostbusters") + elif "ghostbuster" in yaml: + self.ghostbusters = [yaml.get("ghostbuster")] + else: + self.ghostbusters = [] + for r in self.roa_requests: + if r.v4: + self.base.v4 |= r.v4.to_resource_set() + if r.v6: + self.base.v6 |= r.v6.to_resource_set() + for r in self.router_certs: + self.base.asn |= r.asn + self.hosted_by = yaml.get("hosted_by") + self.hosts = [] + if not self.is_hosted: + self.engine = self.allocate_engine() + self.rpkid_port = self.allocate_port() + self.irdbd_port = self.allocate_port() + if self.runs_pubd: + self.pubd_port = self.allocate_port() + self.rsync_port = self.allocate_port() + self.rrdp_port = self.allocate_port() + if self.is_root: + self.rootd_port = self.allocate_port() + + def closure(self): + """ + Compute resource closure of this node and its children, to avoid a + lot of tedious (and error-prone) duplication in the YAML file. + """ + + resources = self.base + for kid in self.kids: + resources |= kid.closure() + self.resources = resources + return resources + + def dump(self): + """ + Show content of this allocation node. + """ + + print str(self) + + def __str__(self): + s = self.name + ":\n" + if self.resources.asn: s += " ASNs: %s\n" % self.resources.asn + if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 + if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 + if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) + if self.parent: s += " Up: %s\n" % self.parent.name + if self.is_hosted: s += " Host: %s\n" % self.hosted_by.name + if self.hosts: s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts) + for r in self.roa_requests: s += " ROA: %s\n" % r + if not self.is_hosted: s += " IPort: %s\n" % self.irdbd_port + if self.runs_pubd: s += " PPort: %s\n" % self.pubd_port + if not self.is_hosted: s += " RPort: %s\n" % self.rpkid_port + if self.runs_pubd: s += " SPort: %s\n" % self.rsync_port + if self.is_root: s += " TPort: %s\n" % self.rootd_port + return s + " Until: %s\n" % self.resources.valid_until + + @property + def is_root(self): + """ + Is this the root node? + """ + + return self.parent is None + + @property + def is_hosted(self): + """ + Is this entity hosted? + """ + + return self.hosted_by is not None + + @property + def runs_pubd(self): + """ + Does this entity run a pubd? + """ + + return self.is_root or not (self.is_hosted or only_one_pubd) + + def path(self, *names): + """ + Construct pathnames in this entity's test directory. + """ + + return cleanpath(test_dir, self.host.name, *names) + + def csvout(self, fn): + """ + Open and log a CSV output file. + """ - if self.router_certs: - fn = "%s.routercerts.xml" % d.name - if not args.skip_config: path = self.path(fn) print "Writing", path - xmlns = rpki.relaxng.router_certificate.xmlns - xml = lxml.etree.Element(xmlns + "router_certificate_requests", - version = rpki.relaxng.router_certificate.version, - nsmap = rpki.relaxng.router_certificate.nsmap) - for r in self.router_certs: - x = lxml.etree.SubElement(xml, xmlns + "router_certificate_request", - router_id = str(r.router_id), - asn = str(r.asn), - valid_until = str(self.resources.valid_until)) - x.text = r.pkcs10.get_Base64() - rpki.relaxng.router_certificate.assertValid(xml) - lxml.etree.ElementTree(xml).write(path, pretty_print = True) - if not args.stop_after_config: - self.run_rpkic("add_router_certificate_request", fn) - if not args.skip_config and args.store_router_private_keys: - path = self.path("%s.routercerts.keys" % d.name) - print "Writing", path - with open(path, "w") as f: - for r in self.router_certs: - f.write(r.keypair.get_PEM()) - - @property - def pubd(self): - """ - Walk up tree until we find somebody who runs pubd. - """ - - s = self - while not s.runs_pubd: - s = s.parent - return s - - @property - def client_handle(self): - """ - Work out what pubd configure_publication_client will call us. - """ - - path = [] - s = self - if not args.flat_publication: - while not s.runs_pubd: + return rpki.csv_utils.csv_writer(path) + + def up_down_url(self): + """ + Construct service URL for this node's parent. + """ + + return "http://localhost:%d/up-down/%s/%s" % (self.parent.host.rpkid_port, + self.parent.name, + self.name) + + def dump_asns(self): + """ + Write Autonomous System Numbers CSV file. + """ + + fn = "%s.asns.csv" % d.name + if not args.skip_config: + with self.csvout(fn) as f: + for k in self.kids: + f.writerows((k.name, a) for a in k.resources.asn) + if not args.stop_after_config: + self.run_rpkic("load_asns", fn) + + def dump_prefixes(self): + """ + Write prefixes CSV file. + """ + + fn = "%s.prefixes.csv" % d.name + if not args.skip_config: + with self.csvout(fn) as f: + for k in self.kids: + f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) + if not args.stop_after_config: + self.run_rpkic("load_prefixes", fn) + + def dump_roas(self): + """ + Write ROA CSV file. + """ + + fn = "%s.roas.csv" % d.name + if not args.skip_config: + with self.csvout(fn) as f: + for g1, r in enumerate(self.roa_requests): + f.writerows((p, r.asn, "G%08d%08d" % (g1, g2)) + for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ()))) + if not args.stop_after_config: + self.run_rpkic("load_roa_requests", fn) + + def dump_ghostbusters(self): + """ + Write Ghostbusters vCard file. + """ + + if self.ghostbusters: + fn = "%s.ghostbusters.vcard" % d.name + if not args.skip_config: + path = self.path(fn) + print "Writing", path + with open(path, "w") as f: + f.write("\n".join(self.ghostbusters)) + if not args.stop_after_config: + self.run_rpkic("load_ghostbuster_requests", fn) + + def dump_router_certificates(self): + """ + Write EE certificates (router certificates, etc). + """ + + if self.router_certs: + fn = "%s.routercerts.xml" % d.name + if not args.skip_config: + path = self.path(fn) + print "Writing", path + xmlns = rpki.relaxng.router_certificate.xmlns + xml = lxml.etree.Element(xmlns + "router_certificate_requests", + version = rpki.relaxng.router_certificate.version, + nsmap = rpki.relaxng.router_certificate.nsmap) + for r in self.router_certs: + x = lxml.etree.SubElement(xml, xmlns + "router_certificate_request", + router_id = str(r.router_id), + asn = str(r.asn), + valid_until = str(self.resources.valid_until)) + x.text = r.pkcs10.get_Base64() + rpki.relaxng.router_certificate.assertValid(xml) + lxml.etree.ElementTree(xml).write(path, pretty_print = True) + if not args.stop_after_config: + self.run_rpkic("add_router_certificate_request", fn) + if not args.skip_config and args.store_router_private_keys: + path = self.path("%s.routercerts.keys" % d.name) + print "Writing", path + with open(path, "w") as f: + for r in self.router_certs: + f.write(r.keypair.get_PEM()) + + @property + def pubd(self): + """ + Walk up tree until we find somebody who runs pubd. + """ + + s = self + while not s.runs_pubd: + s = s.parent + return s + + @property + def client_handle(self): + """ + Work out what pubd configure_publication_client will call us. + """ + + path = [] + s = self + if not args.flat_publication: + while not s.runs_pubd: + path.append(s) + s = s.parent path.append(s) - s = s.parent - path.append(s) - return ".".join(i.name for i in reversed(path)) - - @property - def host(self): - return self.hosted_by or self - - def dump_conf(self): - """ - Write configuration file for OpenSSL and RPKI tools. - """ - - r = dict( - handle = self.name, - run_rpkid = str(not self.is_hosted), - run_pubd = str(self.runs_pubd), - run_rootd = str(self.is_root), - rpkid_server_host = "localhost", - rpkid_server_port = str(self.rpkid_port), - irdbd_server_host = "localhost", - irdbd_server_port = str(self.irdbd_port), - rootd_server_port = str(self.rootd_port), - pubd_server_host = "localhost", - pubd_server_port = str(self.pubd.pubd_port), - publication_rsync_server = "localhost:%s" % self.pubd.rsync_port, - publication_rrdp_notification_uri = "http://localhost:%s/rrdp/notify.xml" % self.pubd.rrdp_port, - bpki_servers_directory = self.path(), - publication_base_directory = self.path("publication"), - rrdp_publication_base_directory = self.path("rrdp-publication"), - shared_sql_engine = args.sql_engine, - shared_sql_password = "fnord", - irdbd_sql_username = "irdb", - rpkid_sql_username = "rpki", - pubd_sql_username = "pubd") - - if args.sql_engine == "sqlite3": - r.update( - irdbd_sql_database = self.path("irdb.sqlite3"), - rpkid_sql_database = self.path("rpkidb.sqlite3"), - pubd_sql_database = self.path("pubdb.sqlite3")) - else: - r.update( - irdbd_sql_database = "irdb%d" % self.engine, - rpkid_sql_database = "rpki%d" % self.engine, - pubd_sql_database = "pubd%d" % self.engine) - - r.update(config_overrides) - - with open(self.path("rpki.conf"), "w") as f: - f.write("# Automatically generated, do not edit\n") - print "Writing", f.name - - section = None - for line in open(cleanpath(ca_dir, "examples/rpki.conf")): - m = section_regexp.match(line) - if m: - section = m.group(1) - m = variable_regexp.match(line) - option = m.group(1) if m and section == "myrpki" else None - if option and option in r: - line = "%s = %s\n" % (option, r[option]) - f.write(line) - - def dump_rsyncd(self): - """ - Write rsyncd configuration file. - """ - - if self.runs_pubd: - with open(self.path("rsyncd.conf"), "w") as f: - print "Writing", f.name - f.writelines(s + "\n" for s in - ("# Automatically generated, do not edit", - "port = %d" % self.rsync_port, - "address = localhost", - "[rpki]", - "log file = rsyncd.log", - "read only = yes", - "use chroot = no", - "path = %s" % self.path("publication"), - "comment = RPKI test", - "[root]", - "log file = rsyncd_root.log", - "read only = yes", - "use chroot = no", - "path = %s" % self.path("publication.root"), - "comment = RPKI test root")) - - @classmethod - def next_rpkic_counter(cls): - cls.rpkic_counter += 10000 - return str(cls.rpkic_counter) - - def run_rpkic(self, *argv): - """ - Run rpkic for this entity. - """ - - cmd = [prog_rpkic, "-i", self.name] - if args.profile: - cmd.append("--profile") - cmd.append(self.path("rpkic.%s.prof" % rpki.sundial.now())) - cmd.extend(str(a) for a in argv if a is not None) - print 'Running "%s"' % " ".join(cmd) - env = dict(os.environ, - YAMLTEST_RPKIC_COUNTER = self.next_rpkic_counter(), - RPKI_CONF = self.path("rpki.conf")) - subprocess.check_call(cmd, cwd = self.host.path(), env = env) - - def syncdb(self): - """ - Run whatever Django ORM commands are necessary to set up the - database this week. - """ - - # Fork a sub-process for each syncdb/migrate run, because it's - # easier than figuring out how to change Django settings after - # initialization. - - def sync_settings(settings, verbosity = 1): - - if verbosity > 0: - print "Running Django setup for", self.name - - pid = os.fork() - - if pid == 0: - logging.getLogger().setLevel(logging.WARNING) - - os.environ.update(RPKI_CONF = self.path("rpki.conf"), - DJANGO_SETTINGS_MODULE = "rpki.django_settings." + settings) - - import django - django.setup() - - import django.core.management - django.core.management.call_command("migrate", verbosity = verbosity, no_color = True, - load_initial_data = False, interactive = False) - - if settings in ("gui", "irdb"): - from django.contrib.auth.models import User - User.objects.create_superuser("root", "root@example.org", "fnord") - - sys.exit(0) - - elif os.waitpid(pid, 0)[1]: - raise RuntimeError("Django setup failed for %s %s" % (self.name, settings)) - - for settings in ("rpkid", "pubd", "gui"): - sync_settings(settings) - - def run_python_daemon(self, prog): - """ - Start a Python daemon and return a subprocess.Popen object - representing the running daemon. - """ - - basename = os.path.splitext(os.path.basename(prog))[0] - cmd = [prog, "--foreground", "--log-level", "debug", - "--log-file", self.path(basename + ".log")] - if args.profile and basename != "rootd": - cmd.extend(( - "--profile", self.path(basename + ".prof"))) - env = dict(os.environ, RPKI_CONF = self.path("rpki.conf")) - p = subprocess.Popen(cmd, cwd = self.path(), env = env) - print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p) - return p - - def run_rpkid(self): - """ - Run rpkid. - """ - - return self.run_python_daemon(prog_rpkid) - - def run_irdbd(self): - """ - Run irdbd. - """ - - return self.run_python_daemon(prog_irdbd) - - def run_pubd(self): - """ - Run pubd. - """ - - return self.run_python_daemon(prog_pubd) - - def run_rootd(self): - """ - Run rootd. - """ - - return self.run_python_daemon(prog_rootd) - - def run_rsyncd(self): - """ - Run rsyncd. - """ - - p = subprocess.Popen(("rsync", "--daemon", "--no-detach", "--config", "rsyncd.conf"), - cwd = self.path()) - print "Running rsyncd for %s: pid %d process %r" % (self.name, p.pid, p) - return p + return ".".join(i.name for i in reversed(path)) + + @property + def host(self): + return self.hosted_by or self + + def dump_conf(self): + """ + Write configuration file for OpenSSL and RPKI tools. + """ + + r = dict( + handle = self.name, + run_rpkid = str(not self.is_hosted), + run_pubd = str(self.runs_pubd), + run_rootd = str(self.is_root), + rpkid_server_host = "localhost", + rpkid_server_port = str(self.rpkid_port), + irdbd_server_host = "localhost", + irdbd_server_port = str(self.irdbd_port), + rootd_server_port = str(self.rootd_port), + pubd_server_host = "localhost", + pubd_server_port = str(self.pubd.pubd_port), + publication_rsync_server = "localhost:%s" % self.pubd.rsync_port, + publication_rrdp_notification_uri = "http://localhost:%s/rrdp/notify.xml" % self.pubd.rrdp_port, + bpki_servers_directory = self.path(), + publication_base_directory = self.path("publication"), + rrdp_publication_base_directory = self.path("rrdp-publication"), + shared_sql_engine = args.sql_engine, + shared_sql_password = "fnord", + irdbd_sql_username = "irdb", + rpkid_sql_username = "rpki", + pubd_sql_username = "pubd") + + if args.sql_engine == "sqlite3": + r.update( + irdbd_sql_database = self.path("irdb.sqlite3"), + rpkid_sql_database = self.path("rpkidb.sqlite3"), + pubd_sql_database = self.path("pubdb.sqlite3")) + else: + r.update( + irdbd_sql_database = "irdb%d" % self.engine, + rpkid_sql_database = "rpki%d" % self.engine, + pubd_sql_database = "pubd%d" % self.engine) + + r.update(config_overrides) + + with open(self.path("rpki.conf"), "w") as f: + f.write("# Automatically generated, do not edit\n") + print "Writing", f.name + + section = None + for line in open(cleanpath(ca_dir, "examples/rpki.conf")): + m = section_regexp.match(line) + if m: + section = m.group(1) + m = variable_regexp.match(line) + option = m.group(1) if m and section == "myrpki" else None + if option and option in r: + line = "%s = %s\n" % (option, r[option]) + f.write(line) + + def dump_rsyncd(self): + """ + Write rsyncd configuration file. + """ + + if self.runs_pubd: + with open(self.path("rsyncd.conf"), "w") as f: + print "Writing", f.name + f.writelines(s + "\n" for s in + ("# Automatically generated, do not edit", + "port = %d" % self.rsync_port, + "address = localhost", + "[rpki]", + "log file = rsyncd.log", + "read only = yes", + "use chroot = no", + "path = %s" % self.path("publication"), + "comment = RPKI test", + "[root]", + "log file = rsyncd_root.log", + "read only = yes", + "use chroot = no", + "path = %s" % self.path("publication.root"), + "comment = RPKI test root")) + + @classmethod + def next_rpkic_counter(cls): + cls.rpkic_counter += 10000 + return str(cls.rpkic_counter) + + def run_rpkic(self, *argv): + """ + Run rpkic for this entity. + """ + + cmd = [prog_rpkic, "-i", self.name] + if args.profile: + cmd.append("--profile") + cmd.append(self.path("rpkic.%s.prof" % rpki.sundial.now())) + cmd.extend(str(a) for a in argv if a is not None) + print 'Running "%s"' % " ".join(cmd) + env = dict(os.environ, + YAMLTEST_RPKIC_COUNTER = self.next_rpkic_counter(), + RPKI_CONF = self.path("rpki.conf")) + subprocess.check_call(cmd, cwd = self.host.path(), env = env) + + def syncdb(self): + """ + Run whatever Django ORM commands are necessary to set up the + database this week. + """ + + # Fork a sub-process for each syncdb/migrate run, because it's + # easier than figuring out how to change Django settings after + # initialization. + + def sync_settings(settings, verbosity = 1): + + if verbosity > 0: + print "Running Django setup for", self.name + + pid = os.fork() + + if pid == 0: + logging.getLogger().setLevel(logging.WARNING) + + os.environ.update(RPKI_CONF = self.path("rpki.conf"), + DJANGO_SETTINGS_MODULE = "rpki.django_settings." + settings) + + import django + django.setup() + + import django.core.management + django.core.management.call_command("migrate", verbosity = verbosity, no_color = True, + load_initial_data = False, interactive = False) + + if settings in ("gui", "irdb"): + from django.contrib.auth.models import User + User.objects.create_superuser("root", "root@example.org", "fnord") + + sys.exit(0) + + elif os.waitpid(pid, 0)[1]: + raise RuntimeError("Django setup failed for %s %s" % (self.name, settings)) + + for settings in ("rpkid", "pubd", "gui"): + sync_settings(settings) + + def run_python_daemon(self, prog): + """ + Start a Python daemon and return a subprocess.Popen object + representing the running daemon. + """ + + basename = os.path.splitext(os.path.basename(prog))[0] + cmd = [prog, "--foreground", "--log-level", "debug", + "--log-file", self.path(basename + ".log")] + if args.profile and basename != "rootd": + cmd.extend(( + "--profile", self.path(basename + ".prof"))) + env = dict(os.environ, RPKI_CONF = self.path("rpki.conf")) + p = subprocess.Popen(cmd, cwd = self.path(), env = env) + print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p) + return p + + def run_rpkid(self): + """ + Run rpkid. + """ + + return self.run_python_daemon(prog_rpkid) + + def run_irdbd(self): + """ + Run irdbd. + """ + + return self.run_python_daemon(prog_irdbd) + + def run_pubd(self): + """ + Run pubd. + """ + + return self.run_python_daemon(prog_pubd) + + def run_rootd(self): + """ + Run rootd. + """ + + return self.run_python_daemon(prog_rootd) + + def run_rsyncd(self): + """ + Run rsyncd. + """ + + p = subprocess.Popen(("rsync", "--daemon", "--no-detach", "--config", "rsyncd.conf"), + cwd = self.path()) + print "Running rsyncd for %s: pid %d process %r" % (self.name, p.pid, p) + return p - def run_gui(self): - """ - Start an instance of the RPKI GUI under the Django test server and - return a subprocess.Popen object representing the running daemon. - """ + def run_gui(self): + """ + Start an instance of the RPKI GUI under the Django test server and + return a subprocess.Popen object representing the running daemon. + """ - port = 8000 + self.engine - cmd = (prog_rpki_manage, "runserver", str(port)) - env = dict(os.environ, - RPKI_CONF = self.path("rpki.conf"), - RPKI_DJANGO_DEBUG = "yes", - ALLOW_PLAIN_HTTP_FOR_TESTING = "I solemnly swear that I am not running this in production") - p = subprocess.Popen(cmd, cwd = self.path(), env = env, - stdout = open(self.path("gui.log"), "w"), stderr = subprocess.STDOUT) - print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p) - return p + port = 8000 + self.engine + cmd = (prog_rpki_manage, "runserver", str(port)) + env = dict(os.environ, + RPKI_CONF = self.path("rpki.conf"), + RPKI_DJANGO_DEBUG = "yes", + ALLOW_PLAIN_HTTP_FOR_TESTING = "I solemnly swear that I am not running this in production") + p = subprocess.Popen(cmd, cwd = self.path(), env = env, + stdout = open(self.path("gui.log"), "w"), stderr = subprocess.STDOUT) + print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p) + return p def create_root_certificate(db_root): - print "Creating rootd RPKI root certificate" + print "Creating rootd RPKI root certificate" - root_resources = rpki.resource_set.resource_bag( - asn = rpki.resource_set.resource_set_as("0-4294967295"), - v4 = rpki.resource_set.resource_set_ipv4("0.0.0.0/0"), - v6 = rpki.resource_set.resource_set_ipv6("::/0")) + root_resources = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as("0-4294967295"), + v4 = rpki.resource_set.resource_set_ipv4("0.0.0.0/0"), + v6 = rpki.resource_set.resource_set_ipv6("::/0")) - root_key = rpki.x509.RSA.generate(quiet = True) + root_key = rpki.x509.RSA.generate(quiet = True) - root_uri = "rsync://localhost:%d/rpki/%s-root/root" % (db_root.pubd.rsync_port, db_root.name) + root_uri = "rsync://localhost:%d/rpki/%s-root/root" % (db_root.pubd.rsync_port, db_root.name) - rrdp_uri = "http://localhost:%s/rrdp/notify.xml" % db.root.pubd.rrdp_port + rrdp_uri = "http://localhost:%s/rrdp/notify.xml" % db.root.pubd.rrdp_port - root_sia = (root_uri + "/", root_uri + "/root.mft", None, rrdp_uri) + root_sia = (root_uri + "/", root_uri + "/root.mft", None, rrdp_uri) - root_cert = rpki.x509.X509.self_certify( - keypair = root_key, - subject_key = root_key.get_public(), - serial = 1, - sia = root_sia, - notAfter = rpki.sundial.now() + rpki.sundial.timedelta(days = 365), - resources = root_resources) + root_cert = rpki.x509.X509.self_certify( + keypair = root_key, + subject_key = root_key.get_public(), + serial = 1, + sia = root_sia, + notAfter = rpki.sundial.now() + rpki.sundial.timedelta(days = 365), + resources = root_resources) - with open(db_root.path("root.cer"), "wb") as f: - f.write(root_cert.get_DER()) + with open(db_root.path("root.cer"), "wb") as f: + f.write(root_cert.get_DER()) - with open(db_root.path("root.key"), "wb") as f: - f.write(root_key.get_DER()) + with open(db_root.path("root.key"), "wb") as f: + f.write(root_key.get_DER()) - with open(os.path.join(test_dir, "root.tal"), "w") as f: - f.write(root_uri + ".cer\n\n") - f.write(root_key.get_public().get_Base64()) + with open(os.path.join(test_dir, "root.tal"), "w") as f: + f.write(root_uri + ".cer\n\n") + f.write(root_key.get_public().get_Base64()) logger = logging.getLogger(__name__) @@ -772,247 +772,247 @@ args = parser.parse_args() try: - if args.pidfile is not None: - with open(args.pidfile, "w") as f: - print "Writing pidfile", f.name - f.write("%s\n" % os.getpid()) + if args.pidfile is not None: + with open(args.pidfile, "w") as f: + print "Writing pidfile", f.name + f.write("%s\n" % os.getpid()) - rpki.log.init("yamltest", argparse.Namespace(log_level = logging.DEBUG, - log_handler = lambda: logging.StreamHandler(sys.stdout))) + rpki.log.init("yamltest", argparse.Namespace(log_level = logging.DEBUG, + log_handler = lambda: logging.StreamHandler(sys.stdout))) - # Allow optional config file for this tool to override default - # passwords: this is mostly so that I can show a complete working - # example without publishing my own server's passwords. + # Allow optional config file for this tool to override default + # passwords: this is mostly so that I can show a complete working + # example without publishing my own server's passwords. - cfg = rpki.config.parser(set_filename = args.config, section = "yamltest", allow_missing = True) + cfg = rpki.config.parser(set_filename = args.config, section = "yamltest", allow_missing = True) - only_one_pubd = cfg.getboolean("only_one_pubd", True) - allocation.base_port = cfg.getint("base_port", 4400) + only_one_pubd = cfg.getboolean("only_one_pubd", True) + allocation.base_port = cfg.getint("base_port", 4400) - config_overrides = dict( - (k, cfg.get(k)) - for k in ("rpkid_sql_password", "irdbd_sql_password", "pubd_sql_password", - "rpkid_sql_username", "irdbd_sql_username", "pubd_sql_username") - if cfg.has_option(k)) + config_overrides = dict( + (k, cfg.get(k)) + for k in ("rpkid_sql_password", "irdbd_sql_password", "pubd_sql_password", + "rpkid_sql_username", "irdbd_sql_username", "pubd_sql_username") + if cfg.has_option(k)) - # Start clean, maybe + # Start clean, maybe - if not args.skip_config: - for root, dirs, files in os.walk(test_dir, topdown = False): - for fn in files: - os.unlink(os.path.join(root, fn)) - for d in dirs: - os.rmdir(os.path.join(root, d)) + if not args.skip_config: + for root, dirs, files in os.walk(test_dir, topdown = False): + for fn in files: + os.unlink(os.path.join(root, fn)) + for d in dirs: + os.rmdir(os.path.join(root, d)) - # Read first YAML doc in file and process as compact description of - # test layout and resource allocations. Ignore subsequent YAML docs, - # they're for smoketest.py, not this script. + # Read first YAML doc in file and process as compact description of + # test layout and resource allocations. Ignore subsequent YAML docs, + # they're for smoketest.py, not this script. - db = allocation_db(yaml.safe_load_all(args.yaml_file).next()) + db = allocation_db(yaml.safe_load_all(args.yaml_file).next()) - # Show what we loaded + # Show what we loaded - #db.dump() + #db.dump() - if args.skip_config: + if args.skip_config: - print "Skipping pre-daemon configuration, assuming you already did that" + print "Skipping pre-daemon configuration, assuming you already did that" - else: + else: - # Set up each entity in our test, create publication directories, - # and initialize server BPKI. + # Set up each entity in our test, create publication directories, + # and initialize server BPKI. - for d in db: - if not d.is_hosted: - print "Initializing", d.name - os.makedirs(d.path()) - d.dump_conf() - if d.runs_pubd: - os.makedirs(d.path("publication")) - d.dump_rsyncd() - if d.is_root: - os.makedirs(d.path("publication.root")) - d.syncdb() - d.run_rpkic("initialize_server_bpki") - print + for d in db: + if not d.is_hosted: + print "Initializing", d.name + os.makedirs(d.path()) + d.dump_conf() + if d.runs_pubd: + os.makedirs(d.path("publication")) + d.dump_rsyncd() + if d.is_root: + os.makedirs(d.path("publication.root")) + d.syncdb() + d.run_rpkic("initialize_server_bpki") + print - # Initialize resource holding BPKI and generate self-descriptor - # for each entity. + # Initialize resource holding BPKI and generate self-descriptor + # for each entity. - for d in db: - d.run_rpkic("create_identity", d.name) + for d in db: + d.run_rpkic("create_identity", d.name) - # Create RPKI root certificate. + # Create RPKI root certificate. - create_root_certificate(db.root) + create_root_certificate(db.root) - # Set up rootd. + # Set up rootd. - db.root.run_rpkic("configure_root") + db.root.run_rpkic("configure_root") - # From here on we need to pay attention to initialization order. We - # used to do all the pre-configure_daemons stuff before running any - # of the daemons, but that doesn't work right in hosted cases, so we - # have to interleave configuration with starting daemons, just as - # one would in the real world for this sort of thing. + # From here on we need to pay attention to initialization order. We + # used to do all the pre-configure_daemons stuff before running any + # of the daemons, but that doesn't work right in hosted cases, so we + # have to interleave configuration with starting daemons, just as + # one would in the real world for this sort of thing. - progs = [] + progs = [] - try: + try: - for d in db: + for d in db: - if not d.is_hosted: - print - print "Running daemons for", d.name - if d.is_root: - progs.append(d.run_rootd()) - progs.append(d.run_irdbd()) - progs.append(d.run_rpkid()) - if d.runs_pubd: - progs.append(d.run_pubd()) - progs.append(d.run_rsyncd()) - if args.run_gui: - progs.append(d.run_gui()) + if not d.is_hosted: + print + print "Running daemons for", d.name + if d.is_root: + progs.append(d.run_rootd()) + progs.append(d.run_irdbd()) + progs.append(d.run_rpkid()) + if d.runs_pubd: + progs.append(d.run_pubd()) + progs.append(d.run_rsyncd()) + if args.run_gui: + progs.append(d.run_gui()) - if args.synchronize or not args.skip_config: + if args.synchronize or not args.skip_config: - print - print "Giving daemons time to start up" - time.sleep(20) - assert all(p.poll() is None for p in progs) + print + print "Giving daemons time to start up" + time.sleep(20) + assert all(p.poll() is None for p in progs) - if args.skip_config: + if args.skip_config: - print - print "Skipping configure_*, you'll have to do that yourself if needed" + print + print "Skipping configure_*, you'll have to do that yourself if needed" - else: + else: + + for d in db: + + print + print "Configuring", d.name + print + if d.is_root: + assert not d.is_hosted + d.run_rpkic("configure_publication_client", + "--flat" if args.flat_publication else None, + d.path("%s.%s.repository-request.xml" % (d.name, d.name))) + print + d.run_rpkic("configure_repository", + d.path("%s.repository-response.xml" % d.client_handle)) + print + else: + d.parent.run_rpkic("configure_child", + "--valid_until", d.resources.valid_until, + d.path("%s.identity.xml" % d.name)) + print + d.run_rpkic("configure_parent", + d.parent.path("%s.%s.parent-response.xml" % (d.parent.name, d.name))) + print + d.pubd.run_rpkic("configure_publication_client", + "--flat" if args.flat_publication else None, + d.path("%s.%s.repository-request.xml" % (d.name, d.parent.name))) + print + d.run_rpkic("configure_repository", + d.pubd.path("%s.repository-response.xml" % d.client_handle)) + print + + print + print "Done with initial configuration" + print + + if args.synchronize: + print + print "Synchronizing" + print + for d in db: + if not d.is_hosted: + d.run_rpkic("synchronize") + + if args.synchronize or not args.skip_config: + print + print "Loading CSV files" + print + for d in db: + d.dump_asns() + d.dump_prefixes() + d.dump_roas() + d.dump_ghostbusters() + d.dump_router_certificates() - for d in db: + if args.run_gui: + print + print 'GUI user "root", password "fnord"' + for d in db: + if not d.is_hosted: + url = "http://127.0.0.1:%d/rpki/" % (8000 + d.engine) + print "GUI URL", url, "for", d.name + if args.browser: + if d is db.root: + webbrowser.open_new(url) + else: + webbrowser.open_new_tab(url) + time.sleep(2) + + # Wait until something terminates. + + if not args.stop_after_config or args.keep_going: + if args.notify_when_startup_complete: + print + print "Sending SIGUSR1 to process", args.notify_when_startup_complete + os.kill(args.notify_when_startup_complete, signal.SIGUSR1) + print + print "Waiting for daemons to exit" + signal.signal(signal.SIGCHLD, lambda *dont_care: None) + while (any(p.poll() is None for p in progs) + if args.keep_going else + all(p.poll() is None for p in progs)): + signal.pause() + + finally: print - print "Configuring", d.name + print "Shutting down" print - if d.is_root: - assert not d.is_hosted - d.run_rpkic("configure_publication_client", - "--flat" if args.flat_publication else None, - d.path("%s.%s.repository-request.xml" % (d.name, d.name))) - print - d.run_rpkic("configure_repository", - d.path("%s.repository-response.xml" % d.client_handle)) - print + + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + + if args.profile: + how_long = 300 else: - d.parent.run_rpkic("configure_child", - "--valid_until", d.resources.valid_until, - d.path("%s.identity.xml" % d.name)) - print - d.run_rpkic("configure_parent", - d.parent.path("%s.%s.parent-response.xml" % (d.parent.name, d.name))) - print - d.pubd.run_rpkic("configure_publication_client", - "--flat" if args.flat_publication else None, - d.path("%s.%s.repository-request.xml" % (d.name, d.parent.name))) - print - d.run_rpkic("configure_repository", - d.pubd.path("%s.repository-response.xml" % d.client_handle)) - print - - print - print "Done with initial configuration" - print - - if args.synchronize: - print - print "Synchronizing" - print - for d in db: - if not d.is_hosted: - d.run_rpkic("synchronize") - - if args.synchronize or not args.skip_config: - print - print "Loading CSV files" - print - for d in db: - d.dump_asns() - d.dump_prefixes() - d.dump_roas() - d.dump_ghostbusters() - d.dump_router_certificates() - - if args.run_gui: - print - print 'GUI user "root", password "fnord"' - for d in db: - if not d.is_hosted: - url = "http://127.0.0.1:%d/rpki/" % (8000 + d.engine) - print "GUI URL", url, "for", d.name - if args.browser: - if d is db.root: - webbrowser.open_new(url) - else: - webbrowser.open_new_tab(url) - time.sleep(2) - - # Wait until something terminates. - - if not args.stop_after_config or args.keep_going: - if args.notify_when_startup_complete: - print - print "Sending SIGUSR1 to process", args.notify_when_startup_complete - os.kill(args.notify_when_startup_complete, signal.SIGUSR1) - print - print "Waiting for daemons to exit" - signal.signal(signal.SIGCHLD, lambda *dont_care: None) - while (any(p.poll() is None for p in progs) - if args.keep_going else - all(p.poll() is None for p in progs)): - signal.pause() - - finally: - - print - print "Shutting down" - print - - signal.signal(signal.SIGCHLD, signal.SIG_DFL) - - if args.profile: - how_long = 300 - else: - how_long = 30 + how_long = 30 - how_often = how_long / 2 + how_often = how_long / 2 - for i in xrange(how_long): - if i % how_often == 0: - for p in progs: - if p.poll() is None: - print "Politely nudging pid %d" % p.pid - p.terminate() - print - if all(p.poll() is not None for p in progs): - break - time.sleep(1) + for i in xrange(how_long): + if i % how_often == 0: + for p in progs: + if p.poll() is None: + print "Politely nudging pid %d" % p.pid + p.terminate() + print + if all(p.poll() is not None for p in progs): + break + time.sleep(1) - for p in progs: - if p.poll() is None: - print "Pulling the plug on pid %d" % p.pid - p.kill() + for p in progs: + if p.poll() is None: + print "Pulling the plug on pid %d" % p.pid + p.kill() - for p in progs: - print "Program pid %d %r returned %d" % (p.pid, p, p.wait()) + for p in progs: + print "Program pid %d %r returned %d" % (p.pid, p, p.wait()) except Exception, e: - print "Blowing out on exception", str(e) - raise + print "Blowing out on exception", str(e) + raise finally: - if args.pidfile is not None and os.path.exists(args.pidfile): - os.unlink(args.pidfile) + if args.pidfile is not None and os.path.exists(args.pidfile): + os.unlink(args.pidfile) # Local Variables: # indent-tabs-mode: nil diff --git a/potpourri/analyze-rcynic-history.py b/potpourri/analyze-rcynic-history.py index 648538cc..c0836ab2 100644 --- a/potpourri/analyze-rcynic-history.py +++ b/potpourri/analyze-rcynic-history.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2011-2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -35,201 +35,201 @@ from xml.etree.cElementTree import (ElementTree as ElementTree, fromstring as ElementTreeFromString) def parse_utc(s): - return datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%SZ") + return datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%SZ") class Rsync_History(object): - """ - An Rsync_History object represents one rsync connection. - """ + """ + An Rsync_History object represents one rsync connection. + """ - def __init__(self, elt): - self.error = elt.get("error") - self.uri = elt.text.strip() - self.hostname = urlparse.urlparse(self.uri).hostname or None - self.elapsed = parse_utc(elt.get("finished")) - parse_utc(elt.get("started")) + def __init__(self, elt): + self.error = elt.get("error") + self.uri = elt.text.strip() + self.hostname = urlparse.urlparse(self.uri).hostname or None + self.elapsed = parse_utc(elt.get("finished")) - parse_utc(elt.get("started")) class Host(object): - """ - A host object represents all the data collected for one host. Note - that it (usually) contains a list of all the sessions in which this - host appears. - """ - - def __init__(self, hostname, session_id): - self.hostname = hostname - self.session_id = session_id - self.elapsed = datetime.timedelta(0) - self.connection_count = 0 - self.dead_connections = 0 - self.uris = set() - self.total_connection_time = datetime.timedelta(0) - - def add_rsync_history(self, h): - self.connection_count += 1 - self.elapsed += h.elapsed - self.dead_connections += int(h.error is not None) - self.total_connection_time += h.elapsed - - def add_uri(self, u): - self.uris.add(u) - - def finalize(self): - self.object_count = len(self.uris) - del self.uris - - @property - def failed(self): - return 1 if self.dead_connections else 0 - - @property - def seconds_per_object(self): - if self.failed: - return None - else: - return float(self.elapsed.days * 24 * 60 * 60 + - self.elapsed.seconds + - self.elapsed.microseconds / 10**6) / float(self.object_count) - - @property - def objects_per_connection(self): - if self.failed: - return None - else: - return float(self.object_count) / float(self.connection_count) - - @property - def average_connection_time(self): - return float(self.total_connection_time.days * 24 * 60 * 60 + - self.total_connection_time.seconds + - self.total_connection_time.microseconds / 10**6) / float(self.connection_count) - - class Format(object): - - def __init__(self, attr, title, fmt, ylabel = ""): - self.attr = attr - self.title = title - self.width = len(title) - int("%" in fmt) - self.fmt = "%%%d%s" % (self.width, fmt) - self.oops = "*" * self.width - self.ylabel = ylabel - - def __call__(self, obj): - try: - value = getattr(obj, self.attr) - return None if value is None else self.fmt % value - except ZeroDivisionError: - return self.oops - - format = (Format("connection_count", "Connections", "d", "Connections To Repository (Per Session)"), - Format("object_count", "Objects", "d", "Objects In Repository (Distinct URIs Per Session)"), - Format("objects_per_connection", "Objects/Connection", ".3f", "Objects In Repository / Connections To Repository"), - Format("seconds_per_object", "Seconds/Object", ".3f", "Seconds To Transfer / Object (Average Per Session)"), - Format("failure_rate_running", "Failure Rate", ".3f%%", "Sessions With Failed Connections Within Last %d Hours" % window_hours), - Format("average_connection_time", "Average Connection", ".3f", "Seconds / Connection (Average Per Session)"), - Format("hostname", "Hostname", "s")) - - format_dict = dict((fmt.attr, fmt) for fmt in format) - - def format_field(self, name): - result = self.format_dict[name](self) - return None if result is None else result.strip() + """ + A host object represents all the data collected for one host. Note + that it (usually) contains a list of all the sessions in which this + host appears. + """ + + def __init__(self, hostname, session_id): + self.hostname = hostname + self.session_id = session_id + self.elapsed = datetime.timedelta(0) + self.connection_count = 0 + self.dead_connections = 0 + self.uris = set() + self.total_connection_time = datetime.timedelta(0) + + def add_rsync_history(self, h): + self.connection_count += 1 + self.elapsed += h.elapsed + self.dead_connections += int(h.error is not None) + self.total_connection_time += h.elapsed + + def add_uri(self, u): + self.uris.add(u) + + def finalize(self): + self.object_count = len(self.uris) + del self.uris + + @property + def failed(self): + return 1 if self.dead_connections else 0 + + @property + def seconds_per_object(self): + if self.failed: + return None + else: + return float(self.elapsed.days * 24 * 60 * 60 + + self.elapsed.seconds + + self.elapsed.microseconds / 10**6) / float(self.object_count) + + @property + def objects_per_connection(self): + if self.failed: + return None + else: + return float(self.object_count) / float(self.connection_count) + + @property + def average_connection_time(self): + return float(self.total_connection_time.days * 24 * 60 * 60 + + self.total_connection_time.seconds + + self.total_connection_time.microseconds / 10**6) / float(self.connection_count) + + class Format(object): + + def __init__(self, attr, title, fmt, ylabel = ""): + self.attr = attr + self.title = title + self.width = len(title) - int("%" in fmt) + self.fmt = "%%%d%s" % (self.width, fmt) + self.oops = "*" * self.width + self.ylabel = ylabel + + def __call__(self, obj): + try: + value = getattr(obj, self.attr) + return None if value is None else self.fmt % value + except ZeroDivisionError: + return self.oops + + format = (Format("connection_count", "Connections", "d", "Connections To Repository (Per Session)"), + Format("object_count", "Objects", "d", "Objects In Repository (Distinct URIs Per Session)"), + Format("objects_per_connection", "Objects/Connection", ".3f", "Objects In Repository / Connections To Repository"), + Format("seconds_per_object", "Seconds/Object", ".3f", "Seconds To Transfer / Object (Average Per Session)"), + Format("failure_rate_running", "Failure Rate", ".3f%%", "Sessions With Failed Connections Within Last %d Hours" % window_hours), + Format("average_connection_time", "Average Connection", ".3f", "Seconds / Connection (Average Per Session)"), + Format("hostname", "Hostname", "s")) + + format_dict = dict((fmt.attr, fmt) for fmt in format) + + def format_field(self, name): + result = self.format_dict[name](self) + return None if result is None else result.strip() class Session(dict): - """ - A session corresponds to one XML file. This is a dictionary of Host - objects, keyed by hostname. - """ - - def __init__(self, session_id, msg_key): - self.session_id = session_id - self.msg_key = msg_key - self.date = parse_utc(session_id) - self.calculated_failure_history = False - - @property - def hostnames(self): - return set(self.iterkeys()) - - def get_plot_row(self, name, hostnames): - return (self.session_id,) + tuple(self[h].format_field(name) if h in self else "" for h in hostnames) - - def add_rsync_history(self, h): - if h.hostname not in self: - self[h.hostname] = Host(h.hostname, self.session_id) - self[h.hostname].add_rsync_history(h) - - def add_uri(self, u): - h = urlparse.urlparse(u).hostname - if h and h in self: - self[h].add_uri(u) - - def finalize(self): - for h in self.itervalues(): - h.finalize() - - def calculate_failure_history(self, sessions): - start = self.date - datetime.timedelta(hours = window_hours) - sessions = tuple(s for s in sessions if s.date <= self.date and s.date > start) - for hostname, h in self.iteritems(): - i = n = 0 - for s in sessions: - if hostname in s: - i += s[hostname].failed - n += 1 - h.failure_rate_running = float(100 * i) / n - self.calculated_failure_history = True + """ + A session corresponds to one XML file. This is a dictionary of Host + objects, keyed by hostname. + """ + + def __init__(self, session_id, msg_key): + self.session_id = session_id + self.msg_key = msg_key + self.date = parse_utc(session_id) + self.calculated_failure_history = False + + @property + def hostnames(self): + return set(self.iterkeys()) + + def get_plot_row(self, name, hostnames): + return (self.session_id,) + tuple(self[h].format_field(name) if h in self else "" for h in hostnames) + + def add_rsync_history(self, h): + if h.hostname not in self: + self[h.hostname] = Host(h.hostname, self.session_id) + self[h.hostname].add_rsync_history(h) + + def add_uri(self, u): + h = urlparse.urlparse(u).hostname + if h and h in self: + self[h].add_uri(u) + + def finalize(self): + for h in self.itervalues(): + h.finalize() + + def calculate_failure_history(self, sessions): + start = self.date - datetime.timedelta(hours = window_hours) + sessions = tuple(s for s in sessions if s.date <= self.date and s.date > start) + for hostname, h in self.iteritems(): + i = n = 0 + for s in sessions: + if hostname in s: + i += s[hostname].failed + n += 1 + h.failure_rate_running = float(100 * i) / n + self.calculated_failure_history = True def plotter(f, hostnames, field, logscale = False): - plotlines = sorted(session.get_plot_row(field, hostnames) for session in sessions) - title = Host.format_dict[field].title - ylabel = Host.format_dict[field].ylabel - n = len(hostnames) + 1 - assert all(n == len(plotline) for plotline in plotlines) - if "%%" in Host.format_dict[field].fmt: - f.write('set format y "%.0f%%"\n') - else: - f.write('set format y\n') - if logscale: - f.write("set logscale y\n") - else: - f.write("unset logscale y\n") - f.write(""" - set xdata time - set timefmt '%Y-%m-%dT%H:%M:%SZ' - #set format x '%m/%d' - #set format x '%b%d' - #set format x '%Y-%m-%d' - set format x '%Y-%m' - #set title '""" + title + """' - set ylabel '""" + ylabel + """' - plot""" + ",".join(" '-' using 1:2 with linespoints pointinterval 500 title '%s'" % h for h in hostnames) + "\n") - for i in xrange(1, n): - for plotline in plotlines: - if plotline[i] is not None: - f.write("%s %s\n" % (plotline[0], plotline[i].rstrip("%"))) - f.write("e\n") + plotlines = sorted(session.get_plot_row(field, hostnames) for session in sessions) + title = Host.format_dict[field].title + ylabel = Host.format_dict[field].ylabel + n = len(hostnames) + 1 + assert all(n == len(plotline) for plotline in plotlines) + if "%%" in Host.format_dict[field].fmt: + f.write('set format y "%.0f%%"\n') + else: + f.write('set format y\n') + if logscale: + f.write("set logscale y\n") + else: + f.write("unset logscale y\n") + f.write(""" + set xdata time + set timefmt '%Y-%m-%dT%H:%M:%SZ' + #set format x '%m/%d' + #set format x '%b%d' + #set format x '%Y-%m-%d' + set format x '%Y-%m' + #set title '""" + title + """' + set ylabel '""" + ylabel + """' + plot""" + ",".join(" '-' using 1:2 with linespoints pointinterval 500 title '%s'" % h for h in hostnames) + "\n") + for i in xrange(1, n): + for plotline in plotlines: + if plotline[i] is not None: + f.write("%s %s\n" % (plotline[0], plotline[i].rstrip("%"))) + f.write("e\n") def plot_hosts(hostnames, fields): - for field in fields: - for logscale in (False, True): - gnuplot = subprocess.Popen(("gnuplot",), stdin = subprocess.PIPE) - gnuplot.stdin.write("set terminal pdf\n") - gnuplot.stdin.write("set output '%s/%s-%s.pdf'\n" % (outdir, field, "log" if logscale else "linear")) - plotter(gnuplot.stdin, hostnames, field, logscale = logscale) - gnuplot.stdin.close() - gnuplot.wait() + for field in fields: + for logscale in (False, True): + gnuplot = subprocess.Popen(("gnuplot",), stdin = subprocess.PIPE) + gnuplot.stdin.write("set terminal pdf\n") + gnuplot.stdin.write("set output '%s/%s-%s.pdf'\n" % (outdir, field, "log" if logscale else "linear")) + plotter(gnuplot.stdin, hostnames, field, logscale = logscale) + gnuplot.stdin.close() + gnuplot.wait() outdir = "images" if not os.path.exists(outdir): - os.makedirs(outdir) + os.makedirs(outdir) mb = mailbox.Maildir("/u/sra/rpki/rcynic-xml", factory = None, create = False) if sys.platform == "darwin": # Sigh - shelf = shelve.open("rcynic-xml", "c") + shelf = shelve.open("rcynic-xml", "c") else: - shelf = shelve.open("rcynic-xml.db", "c") + shelf = shelve.open("rcynic-xml.db", "c") sessions = [] @@ -237,55 +237,55 @@ latest = None parsed = 0 for i, key in enumerate(mb.iterkeys(), 1): - sys.stderr.write("\r%s %d/%d/%d..." % ("|\\-/"[i & 3], parsed, i, len(mb))) - - if key in shelf: - session = shelf[key] - - else: - sys.stderr.write("%s..." % key) - assert not mb[key].is_multipart() - input = ElementTreeFromString(mb[key].get_payload()) - date = input.get("date") - sys.stderr.write("%s..." % date) - session = Session(date, key) - for elt in input.findall("rsync_history"): - session.add_rsync_history(Rsync_History(elt)) - for elt in input.findall("validation_status"): - if elt.get("generation") == "current": - session.add_uri(elt.text.strip()) - session.finalize() - shelf[key] = session - parsed += 1 - - sessions.append(session) - if latest is None or session.session_id > latest.session_id: - latest = session + sys.stderr.write("\r%s %d/%d/%d..." % ("|\\-/"[i & 3], parsed, i, len(mb))) + + if key in shelf: + session = shelf[key] + + else: + sys.stderr.write("%s..." % key) + assert not mb[key].is_multipart() + input = ElementTreeFromString(mb[key].get_payload()) + date = input.get("date") + sys.stderr.write("%s..." % date) + session = Session(date, key) + for elt in input.findall("rsync_history"): + session.add_rsync_history(Rsync_History(elt)) + for elt in input.findall("validation_status"): + if elt.get("generation") == "current": + session.add_uri(elt.text.strip()) + session.finalize() + shelf[key] = session + parsed += 1 + + sessions.append(session) + if latest is None or session.session_id > latest.session_id: + latest = session sys.stderr.write("\n") shelf.sync() for session in sessions: - if not getattr(session, "calculated_failure_history", False): - session.calculate_failure_history(sessions) - shelf[session.msg_key] = session + if not getattr(session, "calculated_failure_history", False): + session.calculate_failure_history(sessions) + shelf[session.msg_key] = session if plot_all_hosts: - hostnames = sorted(reduce(lambda x, y: x | y, - (s.hostnames for s in sessions), - set())) + hostnames = sorted(reduce(lambda x, y: x | y, + (s.hostnames for s in sessions), + set())) else: - hostnames = ("rpki.apnic.net", "rpki.ripe.net", "repository.lacnic.net", "rpki.afrinic.net", "rpki.arin.net", - #"localcert.ripe.net", "arin.rpki.net", "repo0.rpki.net", "rgnet.rpki.net", - "ca0.rpki.net") + hostnames = ("rpki.apnic.net", "rpki.ripe.net", "repository.lacnic.net", "rpki.afrinic.net", "rpki.arin.net", + #"localcert.ripe.net", "arin.rpki.net", "repo0.rpki.net", "rgnet.rpki.net", + "ca0.rpki.net") plot_hosts(hostnames, [fmt.attr for fmt in Host.format if fmt.attr != "hostname"]) if latest is not None: - f = open("rcynic.xml", "wb") - f.write(mb[latest.msg_key].get_payload()) - f.close() + f = open("rcynic.xml", "wb") + f.write(mb[latest.msg_key].get_payload()) + f.close() shelf.close() diff --git a/potpourri/analyze-transition.py b/potpourri/analyze-transition.py index e2125dfb..9f7928dc 100644 --- a/potpourri/analyze-transition.py +++ b/potpourri/analyze-transition.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2012 Internet Systems Consortium, Inc. ("ISC") -# +# # Permission to use, copy, modify, and/or distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -21,44 +21,44 @@ Compare rcynic.xml files, tell the user what became invalid, and why. import sys try: - from lxml.etree import ElementTree + from lxml.etree import ElementTree except ImportError: - from xml.etree.ElementTree import ElementTree + from xml.etree.ElementTree import ElementTree class Object(object): - def __init__(self, session, uri): - self.session = session - self.uri = uri - self.labels = [] + def __init__(self, session, uri): + self.session = session + self.uri = uri + self.labels = [] - def add(self, label): - self.labels.append(label) + def add(self, label): + self.labels.append(label) - def __cmp__(self, other): - return cmp(self.labels, other.labels) + def __cmp__(self, other): + return cmp(self.labels, other.labels) - @property - def accepted(self): - return "object_accepted" in self.labels + @property + def accepted(self): + return "object_accepted" in self.labels class Session(dict): - def __init__(self, name): - self.name = name - tree = ElementTree(file = name) - labels = tuple((elt.tag.strip(), elt.text.strip()) for elt in tree.find("labels")) - self.labels = tuple(pair[0] for pair in labels) - self.descrs = dict(labels) - self.date = tree.getroot().get("date") - for elt in tree.findall("validation_status"): - status = elt.get("status") - uri = elt.text.strip() - if status.startswith("rsync_transfer_") or elt.get("generation") != "current": - continue - if uri not in self: - self[uri] = Object(self, uri) - self[uri].add(status) + def __init__(self, name): + self.name = name + tree = ElementTree(file = name) + labels = tuple((elt.tag.strip(), elt.text.strip()) for elt in tree.find("labels")) + self.labels = tuple(pair[0] for pair in labels) + self.descrs = dict(labels) + self.date = tree.getroot().get("date") + for elt in tree.findall("validation_status"): + status = elt.get("status") + uri = elt.text.strip() + if status.startswith("rsync_transfer_") or elt.get("generation") != "current": + continue + if uri not in self: + self[uri] = Object(self, uri) + self[uri].add(status) skip_labels = frozenset(("object_accepted", "object_rejected")) @@ -66,23 +66,23 @@ old_db = new_db = None for arg in sys.argv[1:]: - old_db = new_db - new_db = Session(arg) - if old_db is None: - continue - - old_uris = frozenset(old_db) - new_uris = frozenset(new_db) - - for uri in sorted(old_uris - new_uris): - print new_db.date, uri, "dropped" - - for uri in sorted(old_uris & new_uris): - old = old_db[uri] - new = new_db[uri] - if old.accepted and not new.accepted: - print new_db.date, uri, "invalid" - labels = frozenset(new.labels) - frozenset(old.labels) - skip_labels - for label in new.labels: - if label in labels: - print " ", new_db.descrs[label] + old_db = new_db + new_db = Session(arg) + if old_db is None: + continue + + old_uris = frozenset(old_db) + new_uris = frozenset(new_db) + + for uri in sorted(old_uris - new_uris): + print new_db.date, uri, "dropped" + + for uri in sorted(old_uris & new_uris): + old = old_db[uri] + new = new_db[uri] + if old.accepted and not new.accepted: + print new_db.date, uri, "invalid" + labels = frozenset(new.labels) - frozenset(old.labels) - skip_labels + for label in new.labels: + if label in labels: + print " ", new_db.descrs[label] diff --git a/potpourri/apnic-to-csv.py b/potpourri/apnic-to-csv.py index 62293a51..83f5388b 100644 --- a/potpourri/apnic-to-csv.py +++ b/potpourri/apnic-to-csv.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2010-2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -27,29 +27,29 @@ prefixes = csv_writer("prefixes.csv") for line in open("delegated-apnic-extended-latest"): - line = line.rstrip() + line = line.rstrip() - if not line.startswith("apnic|") or line.endswith("|summary"): - continue + if not line.startswith("apnic|") or line.endswith("|summary"): + continue - try: - registry, cc, rectype, start, value, date, status, opaque_id = line.split("|") - except ValueError: - continue + try: + registry, cc, rectype, start, value, date, status, opaque_id = line.split("|") + except ValueError: + continue - if not opaque_id: - continue + if not opaque_id: + continue - assert registry == "apnic" + assert registry == "apnic" - if rectype == "asn": - asns.writerow((opaque_id, "%s-%s" % (start, int(start) + int(value) - 1))) + if rectype == "asn": + asns.writerow((opaque_id, "%s-%s" % (start, int(start) + int(value) - 1))) - elif rectype == "ipv4": - prefixes.writerow((opaque_id, "%s-%s" % (start, v4addr(v4addr(start) + long(value) - 1)))) + elif rectype == "ipv4": + prefixes.writerow((opaque_id, "%s-%s" % (start, v4addr(v4addr(start) + long(value) - 1)))) - elif rectype == "ipv6": - prefixes.writerow((opaque_id, "%s/%s" % (start, value))) + elif rectype == "ipv6": + prefixes.writerow((opaque_id, "%s/%s" % (start, value))) asns.close() prefixes.close() diff --git a/potpourri/arin-to-csv.py b/potpourri/arin-to-csv.py index a4e7ffc3..a4b7f285 100644 --- a/potpourri/arin-to-csv.py +++ b/potpourri/arin-to-csv.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2009-2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -34,9 +34,9 @@ import lxml.etree from rpki.csv_utils import csv_writer def ns(tag): - return "{http://www.arin.net/bulkwhois/core/v1}" + tag + return "{http://www.arin.net/bulkwhois/core/v1}" + tag -tag_asn = ns("asn") +tag_asn = ns("asn") tag_net = ns("net") tag_org = ns("org") tag_poc = ns("poc") @@ -49,12 +49,12 @@ tag_startAsNumber = ns("startAsNumber") tag_endAsNumber = ns("endAsNumber") def find(node, tag): - return node.findtext(tag).strip() + return node.findtext(tag).strip() def do_asn(node): - asns.writerow((find(node, tag_orgHandle), - "%s-%s" % (find(node, tag_startAsNumber), - find(node, tag_endAsNumber)))) + asns.writerow((find(node, tag_orgHandle), + "%s-%s" % (find(node, tag_startAsNumber), + find(node, tag_endAsNumber)))) erx_table = { "AF" : "afrinic", @@ -71,19 +71,19 @@ erx_table = { "RX" : "ripe" } def do_net(node): - handle = find(node, tag_orgHandle) - for netblock in node.iter(tag_netBlock): - tag = find(netblock, tag_type) - startAddress = find(netblock, tag_startAddress) - endAddress = find(netblock, tag_endAddress) - if not startAddress.endswith(".000") and not startAddress.endswith(":0000"): - continue - if not endAddress.endswith(".255") and not endAddress.endswith(":FFFF"): - continue - if tag in ("DS", "DA", "IU"): - prefixes.writerow((handle, "%s-%s" % (startAddress, endAddress))) - elif tag in erx_table: - erx.writerow((erx_table[tag], "%s-%s" % (startAddress, endAddress))) + handle = find(node, tag_orgHandle) + for netblock in node.iter(tag_netBlock): + tag = find(netblock, tag_type) + startAddress = find(netblock, tag_startAddress) + endAddress = find(netblock, tag_endAddress) + if not startAddress.endswith(".000") and not startAddress.endswith(":0000"): + continue + if not endAddress.endswith(".255") and not endAddress.endswith(":FFFF"): + continue + if tag in ("DS", "DA", "IU"): + prefixes.writerow((handle, "%s-%s" % (startAddress, endAddress))) + elif tag in erx_table: + erx.writerow((erx_table[tag], "%s-%s" % (startAddress, endAddress))) dispatch = { tag_asn : do_asn, tag_net : do_net } @@ -95,19 +95,19 @@ root = None for event, node in lxml.etree.iterparse(sys.stdin): - if root is None: - root = node - while root.getparent() is not None: - root = root.getparent() + if root is None: + root = node + while root.getparent() is not None: + root = root.getparent() - if node.getparent() is root: + if node.getparent() is root: - if node.tag in dispatch: - dispatch[node.tag](node) + if node.tag in dispatch: + dispatch[node.tag](node) - node.clear() - while node.getprevious() is not None: - del node.getparent()[0] + node.clear() + while node.getprevious() is not None: + del node.getparent()[0] asns.close() prefixes.close() diff --git a/potpourri/cross_certify.py b/potpourri/cross_certify.py index fab7743b..4e6485b7 100644 --- a/potpourri/cross_certify.py +++ b/potpourri/cross_certify.py @@ -1,13 +1,13 @@ # $Id$ -# +# # Copyright (C) 2014 Dragon Research Labs ("DRL") # Portions copyright (C) 2009--2012 Internet Systems Consortium ("ISC") # Portions copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notices and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND DRL, ISC, AND ARIN DISCLAIM ALL # WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL, @@ -57,18 +57,18 @@ now = rpki.sundial.now() notAfter = now + args.lifetime try: - with open(args.serial, "r") as f: - serial = int(f.read().splitlines()[0], 16) + with open(args.serial, "r") as f: + serial = int(f.read().splitlines()[0], 16) except IOError: - serial = 1 + serial = 1 cert = args.ca.cross_certify(args.key, args.input, serial, notAfter, now) with open(args.serial, "w") as f: - f.write("%02x\n" % (serial + 1)) + f.write("%02x\n" % (serial + 1)) if args.out is None: - sys.stdout.write(cert.get_PEM()) + sys.stdout.write(cert.get_PEM()) else: - with open(args.out, "w") as f: - f.write(cert.get_PEM()) + with open(args.out, "w") as f: + f.write(cert.get_PEM()) diff --git a/potpourri/csvgrep.py b/potpourri/csvgrep.py index 68bdd259..3d558245 100644 --- a/potpourri/csvgrep.py +++ b/potpourri/csvgrep.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2010-2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -37,36 +37,36 @@ ipv4 = resource_set_ipv4() ipv6 = resource_set_ipv6() for datum in sys.argv[1:]: - if datum.replace("-", "").isdigit(): - t = asn - else: - t = ipv6 if ":" in datum else ipv4 - if "-" not in datum and "/" not in datum: - datum = datum + "-" + datum - try: - t.append(t.parse_str(datum)) - except: - print "Error attempting to parse", datum - raise + if datum.replace("-", "").isdigit(): + t = asn + else: + t = ipv6 if ":" in datum else ipv4 + if "-" not in datum and "/" not in datum: + datum = datum + "-" + datum + try: + t.append(t.parse_str(datum)) + except: + print "Error attempting to parse", datum + raise #print "Looking for: ASNs %s IPv4 %s IPv6 %s" % (asn, ipv4, ipv6) def matches(set1, datum): - set2 = set1.__class__(datum) - if set1.intersection(set2): - return set2 - else: - return False + set2 = set1.__class__(datum) + if set1.intersection(set2): + return set2 + else: + return False if asn: - for h, a in csv_reader("asns.csv", columns = 2): - m = matches(asn, a) - if m: - print h, m + for h, a in csv_reader("asns.csv", columns = 2): + m = matches(asn, a) + if m: + print h, m if ipv4 or ipv6: - for h, a in csv_reader("prefixes.csv", columns = 2): - t = ipv6 if ":" in a else ipv4 - m = t and matches(t, a) - if m: - print h, m + for h, a in csv_reader("prefixes.csv", columns = 2): + t = ipv6 if ":" in a else ipv4 + m = t and matches(t, a) + if m: + print h, m diff --git a/potpourri/expand-roa-prefixes.py b/potpourri/expand-roa-prefixes.py index ae34ea0a..c08f8abf 100644 --- a/potpourri/expand-roa-prefixes.py +++ b/potpourri/expand-roa-prefixes.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2011 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -27,53 +27,53 @@ import rpki.resource_set import rpki.ipaddrs class NotAPrefix(Exception): - """ - Address is not a proper prefix. - """ + """ + Address is not a proper prefix. + """ class address_range(object): - """ - Iterator for rpki.ipaddrs address objects. - """ + """ + Iterator for rpki.ipaddrs address objects. + """ - def __init__(self, start, stop, step): - self.addr = start - self.stop = stop - self.step = step - self.type = type(start) + def __init__(self, start, stop, step): + self.addr = start + self.stop = stop + self.step = step + self.type = type(start) - def __iter__(self): - while self.addr < self.stop: - yield self.addr - self.addr = self.type(self.addr + self.step) + def __iter__(self): + while self.addr < self.stop: + yield self.addr + self.addr = self.type(self.addr + self.step) def main(argv): - prefix_sets = [] - for arg in argv: - if ":" in arg: - prefix_sets.extend(rpki.resource_set.roa_prefix_set_ipv6(arg)) - else: - prefix_sets.extend(rpki.resource_set.roa_prefix_set_ipv4(arg)) + prefix_sets = [] + for arg in argv: + if ":" in arg: + prefix_sets.extend(rpki.resource_set.roa_prefix_set_ipv6(arg)) + else: + prefix_sets.extend(rpki.resource_set.roa_prefix_set_ipv4(arg)) - for prefix_set in prefix_sets: - sys.stdout.write("%s expands to:\n" % prefix_set) + for prefix_set in prefix_sets: + sys.stdout.write("%s expands to:\n" % prefix_set) - prefix_type = prefix_set.range_type.datum_type - prefix_min = prefix_set.prefix - prefix_max = prefix_set.prefix + (1L << (prefix_type.bits - prefix_set.prefixlen)) + prefix_type = prefix_set.range_type.datum_type + prefix_min = prefix_set.prefix + prefix_max = prefix_set.prefix + (1L << (prefix_type.bits - prefix_set.prefixlen)) - for prefixlen in xrange(prefix_set.prefixlen, prefix_set.max_prefixlen + 1): + for prefixlen in xrange(prefix_set.prefixlen, prefix_set.max_prefixlen + 1): - step = (1L << (prefix_type.bits - prefixlen)) - mask = step - 1 + step = (1L << (prefix_type.bits - prefixlen)) + mask = step - 1 - for addr in address_range(prefix_min, prefix_max, step): - if (addr & mask) != 0: - raise NotAPrefix, "%s is not a /%d prefix" % (addr, prefixlen) - sys.stdout.write(" %s/%d\n" % (addr, prefixlen)) + for addr in address_range(prefix_min, prefix_max, step): + if (addr & mask) != 0: + raise NotAPrefix, "%s is not a /%d prefix" % (addr, prefixlen) + sys.stdout.write(" %s/%d\n" % (addr, prefixlen)) - sys.stdout.write("\n") + sys.stdout.write("\n") if __name__ == "__main__": - main(sys.argv[1:] if len(sys.argv) > 1 else ["18.0.0.0/8-24"]) + main(sys.argv[1:] if len(sys.argv) > 1 else ["18.0.0.0/8-24"]) diff --git a/potpourri/extract-key.py b/potpourri/extract-key.py index b85c3d55..e185b255 100644 --- a/potpourri/extract-key.py +++ b/potpourri/extract-key.py @@ -2,11 +2,11 @@ # Copyright (C) 2014 Dragon Research Labs ("DRL") # Portions copyright (C) 2008 American Registry for Internet Numbers ("ARIN") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notices and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND AND ARIN DISCLAIM ALL # WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR @@ -49,16 +49,16 @@ args = parser.parse_args() cur = MySQLdb.connect(user = args.user, db = args.db, passwd = args.password).cursor() cur.execute( - """ - SELECT bsc.private_key_id, bsc.signing_cert - FROM bsc, self - WHERE self.self_handle = %s AND self.self_id = bsc.self_id AND bsc_handle = %s - """, - (args.self, args.bsc)) + """ + SELECT bsc.private_key_id, bsc.signing_cert + FROM bsc, self + WHERE self.self_handle = %s AND self.self_id = bsc.self_id AND bsc_handle = %s + """, + (args.self, args.bsc)) key, cer = cur.fetchone() print rpki.x509.RSA(DER = key).get_PEM() if cer: - print rpki.x509.X509(DER = cer).get_PEM() + print rpki.x509.X509(DER = cer).get_PEM() diff --git a/potpourri/fakerootd.py b/potpourri/fakerootd.py index 6275a2a9..22b1c117 100644 --- a/potpourri/fakerootd.py +++ b/potpourri/fakerootd.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2011 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -44,7 +44,6 @@ s6.listen(limit) print "Going to sleep at", datetime.datetime.utcnow() try: - signal.pause() + signal.pause() except KeyboardInterrupt: - sys.exit(0) - + sys.exit(0) diff --git a/potpourri/format-application-x-rpki.py b/potpourri/format-application-x-rpki.py index 184103f9..44428131 100644 --- a/potpourri/format-application-x-rpki.py +++ b/potpourri/format-application-x-rpki.py @@ -1,12 +1,12 @@ # $Id$ -# +# # Copyright (C) 2014 Dragon Research Labs ("DRL") # Portions copyright (C) 2010--2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notices and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL # WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR @@ -53,80 +53,80 @@ parser.add_argument("-u", "--unseen", action = "store_true", args = parser.parse_args() def pprint_cert(b64): - return rpki.POW.X509.derRead(base64.b64decode(b64)).pprint() - + return rpki.POW.X509.derRead(base64.b64decode(b64)).pprint() + def up_down(): - msg["X-RPKI-Up-Down-Type"] = xml.get("type") - msg["X-RPKI-Up-Down-Sender"] = xml.get("sender") - msg["X-RPKI-Up-Down-Recipient"] = xml.get("recipient") - msg["Subject"] = "Up-down %s %s => %s" % (xml.get("type"), xml.get("sender"), xml.get("recipient")) - for x in xml: - if x.tag.endswith("class"): - for y in x: - if y.tag.endswith("certificate") or y.tag.endswith("issuer"): - msg.attach(email.mime.text.MIMEText(pprint_cert(y.text))) + msg["X-RPKI-Up-Down-Type"] = xml.get("type") + msg["X-RPKI-Up-Down-Sender"] = xml.get("sender") + msg["X-RPKI-Up-Down-Recipient"] = xml.get("recipient") + msg["Subject"] = "Up-down %s %s => %s" % (xml.get("type"), xml.get("sender"), xml.get("recipient")) + for x in xml: + if x.tag.endswith("class"): + for y in x: + if y.tag.endswith("certificate") or y.tag.endswith("issuer"): + msg.attach(email.mime.text.MIMEText(pprint_cert(y.text))) def left_right(): - msg["X-RPKI-Left-Right-Type"] = xml.get("type") - msg["Subject"] = "Left-right %s" % xml.get("type") + msg["X-RPKI-Left-Right-Type"] = xml.get("type") + msg["Subject"] = "Left-right %s" % xml.get("type") def publication(): - msg["X-RPKI-Left-Right-Type"] = xml.get("type") - msg["Subject"] = "Publication %s" % xml.get("type") + msg["X-RPKI-Left-Right-Type"] = xml.get("type") + msg["Subject"] = "Publication %s" % xml.get("type") dispatch = { "{http://www.apnic.net/specs/rescerts/up-down/}message" : up_down, "{http://www.hactrn.net/uris/rpki/left-right-spec/}msg" : left_right, "{http://www.hactrn.net/uris/rpki/publication-spec/}msg" : publication } def fix_headers(): - if "X-RPKI-PID" in srcmsg or "X-RPKI-Object" in srcmsg: - msg["X-RPKI-PID"] = srcmsg["X-RPKI-PID"] - msg["X-RPKI-Object"] = srcmsg["X-RPKI-Object"] - else: - words = srcmsg["Subject"].split() - msg["X-RPKI-PID"] = words[1] - msg["X-RPKI-Object"] = " ".join(words[4:]) - + if "X-RPKI-PID" in srcmsg or "X-RPKI-Object" in srcmsg: + msg["X-RPKI-PID"] = srcmsg["X-RPKI-PID"] + msg["X-RPKI-Object"] = srcmsg["X-RPKI-Object"] + else: + words = srcmsg["Subject"].split() + msg["X-RPKI-PID"] = words[1] + msg["X-RPKI-Object"] = " ".join(words[4:]) + destination = None source = None try: - destination = mailbox.MH(args.output, factory = None, create = True) - source = mailbox.Maildir(args.input, factory = None) + destination = mailbox.MH(args.output, factory = None, create = True) + source = mailbox.Maildir(args.input, factory = None) - for srckey, srcmsg in source.iteritems(): - if args.unseen and "S" in srcmsg.get_flags(): - continue - assert not srcmsg.is_multipart() and srcmsg.get_content_type() == "application/x-rpki" - payload = srcmsg.get_payload(decode = True) - cms = rpki.POW.CMS.derRead(payload) - txt = cms.verify(rpki.POW.X509Store(), None, rpki.POW.CMS_NOCRL | rpki.POW.CMS_NO_SIGNER_CERT_VERIFY | rpki.POW.CMS_NO_ATTR_VERIFY | rpki.POW.CMS_NO_CONTENT_VERIFY) - xml = lxml.etree.fromstring(txt) - tag = xml.tag - if args.tag and tag != args.tag: - continue - msg = email.mime.multipart.MIMEMultipart("related") - msg["X-RPKI-Tag"] = tag - for i in ("Date", "Message-ID", "X-RPKI-Timestamp"): - msg[i] = srcmsg[i] - fix_headers() - if tag in dispatch: - dispatch[tag]() - if "Subject" not in msg: - msg["Subject"] = srcmsg["Subject"] - msg.attach(email.mime.text.MIMEText(txt)) - msg.attach(email.mime.application.MIMEApplication(payload, "x-rpki")) - msg.epilogue = "\n" # Force trailing newline - key = destination.add(msg) - print "Added", key - if args.kill: - del source[srckey] - elif args.mark: - srcmsg.set_subdir("cur") - srcmsg.add_flag("S") - source[srckey] = srcmsg + for srckey, srcmsg in source.iteritems(): + if args.unseen and "S" in srcmsg.get_flags(): + continue + assert not srcmsg.is_multipart() and srcmsg.get_content_type() == "application/x-rpki" + payload = srcmsg.get_payload(decode = True) + cms = rpki.POW.CMS.derRead(payload) + txt = cms.verify(rpki.POW.X509Store(), None, rpki.POW.CMS_NOCRL | rpki.POW.CMS_NO_SIGNER_CERT_VERIFY | rpki.POW.CMS_NO_ATTR_VERIFY | rpki.POW.CMS_NO_CONTENT_VERIFY) + xml = lxml.etree.fromstring(txt) + tag = xml.tag + if args.tag and tag != args.tag: + continue + msg = email.mime.multipart.MIMEMultipart("related") + msg["X-RPKI-Tag"] = tag + for i in ("Date", "Message-ID", "X-RPKI-Timestamp"): + msg[i] = srcmsg[i] + fix_headers() + if tag in dispatch: + dispatch[tag]() + if "Subject" not in msg: + msg["Subject"] = srcmsg["Subject"] + msg.attach(email.mime.text.MIMEText(txt)) + msg.attach(email.mime.application.MIMEApplication(payload, "x-rpki")) + msg.epilogue = "\n" # Force trailing newline + key = destination.add(msg) + print "Added", key + if args.kill: + del source[srckey] + elif args.mark: + srcmsg.set_subdir("cur") + srcmsg.add_flag("S") + source[srckey] = srcmsg finally: - if destination: - destination.close() - if source: - source.close() + if destination: + destination.close() + if source: + source.close() diff --git a/potpourri/gc_summary.py b/potpourri/gc_summary.py index 1f6987bf..61b21587 100644 --- a/potpourri/gc_summary.py +++ b/potpourri/gc_summary.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2010 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -20,93 +20,93 @@ import sys, os, time class datapoint(object): - outtype = os.getenv("OUTTYPE", "png") - outname = os.getenv("OUTNAME", "") - timefmt = os.getenv("TIMEFMT", "%T") - pretend = os.getenv("PRETEND_EVERYTHING_CHANGED", False) - threshold = int(os.getenv("THRESHOLD", "100")) + outtype = os.getenv("OUTTYPE", "png") + outname = os.getenv("OUTNAME", "") + timefmt = os.getenv("TIMEFMT", "%T") + pretend = os.getenv("PRETEND_EVERYTHING_CHANGED", False) + threshold = int(os.getenv("THRESHOLD", "100")) - raw = [] - filenames = [] + raw = [] + filenames = [] - def __init__(self, filename, timestamp, process, count, typesig, line): - self.filename = filename - self.timestamp = timestamp - self.process = process - self.count = count - self.typesig = typesig - self.line = line - self.key = "%s %s" % (filename, typesig) - self.raw.append(self) - if filename not in self.filenames: - self.filenames.append(filename) + def __init__(self, filename, timestamp, process, count, typesig, line): + self.filename = filename + self.timestamp = timestamp + self.process = process + self.count = count + self.typesig = typesig + self.line = line + self.key = "%s %s" % (filename, typesig) + self.raw.append(self) + if filename not in self.filenames: + self.filenames.append(filename) - def __cmp__(self, other): - c = cmp(self.key, other.key) - return c if c else cmp(self.timestamp, other.timestamp) + def __cmp__(self, other): + c = cmp(self.key, other.key) + return c if c else cmp(self.timestamp, other.timestamp) - @classmethod - def plot(cls): + @classmethod + def plot(cls): - print "# [%s] Looking for interesting records" % time.strftime("%T") - changed = {} - for i in cls.raw: - if i.key not in changed: - changed[i.key] = set() - changed[i.key].add(i.count) - if cls.pretend: - changed = set(changed.iterkeys()) - else: - changed = set(k for k, v in changed.iteritems() if max(v) - min(v) > cls.threshold) + print "# [%s] Looking for interesting records" % time.strftime("%T") + changed = {} + for i in cls.raw: + if i.key not in changed: + changed[i.key] = set() + changed[i.key].add(i.count) + if cls.pretend: + changed = set(changed.iterkeys()) + else: + changed = set(k for k, v in changed.iteritems() if max(v) - min(v) > cls.threshold) - if not changed: - print "# [%s] Apparently nothing worth reporting" % time.strftime("%T") - print "print 'Nothing to plot'" - return + if not changed: + print "# [%s] Apparently nothing worth reporting" % time.strftime("%T") + print "print 'Nothing to plot'" + return - print "# [%s] Header" % time.strftime("%T") - print "set xdata time" - print "set timefmt '%Y-%m-%dT%H:%M:%S'" - print "set format x '%s'" % cls.timefmt - print "set key right bottom" - if cls.outname: - print "set terminal", cls.outtype - print "set output '%s.%s'" % (cls.outname, cls.outtype) - print "set term png size 1024,1024" - print "plot", ", ".join("'-' using 1:2 with linespoints title '%s'" % i for i in changed) + print "# [%s] Header" % time.strftime("%T") + print "set xdata time" + print "set timefmt '%Y-%m-%dT%H:%M:%S'" + print "set format x '%s'" % cls.timefmt + print "set key right bottom" + if cls.outname: + print "set terminal", cls.outtype + print "set output '%s.%s'" % (cls.outname, cls.outtype) + print "set term png size 1024,1024" + print "plot", ", ".join("'-' using 1:2 with linespoints title '%s'" % i for i in changed) - print "# [%s] Sorting" % time.strftime("%T") - cls.raw.sort() + print "# [%s] Sorting" % time.strftime("%T") + cls.raw.sort() - key = None - proc = None - for i in cls.raw: - if i.key not in changed: - continue - if key is not None and i.key != key: + key = None + proc = None + for i in cls.raw: + if i.key not in changed: + continue + if key is not None and i.key != key: + print "e" + elif proc is not None and i.process != proc: + print "" + key = i.key + proc = i.process + print "#", i.key, i.line + print i.timestamp, i.count print "e" - elif proc is not None and i.process != proc: - print "" - key = i.key - proc = i.process - print "#", i.key, i.line - print i.timestamp, i.count - print "e" - if not cls.outname: - print "pause mouse any" + if not cls.outname: + print "pause mouse any" for filename in sys.argv[1:]: - print "# [%s] Reading %s" % (time.strftime("%T"), filename) - for line in open(filename): - if "gc_summary:" in line: - word = line.split(None, 6) - if word[4].isdigit() and word[5].startswith("(") and word[5].endswith(")"): - datapoint(filename = filename, - timestamp = word[0] + "T" + word[1], - process = word[2], - count = int(word[4]), - typesig = word[5], - line = line.strip()) - + print "# [%s] Reading %s" % (time.strftime("%T"), filename) + for line in open(filename): + if "gc_summary:" in line: + word = line.split(None, 6) + if word[4].isdigit() and word[5].startswith("(") and word[5].endswith(")"): + datapoint(filename = filename, + timestamp = word[0] + "T" + word[1], + process = word[2], + count = int(word[4]), + typesig = word[5], + line = line.strip()) + print "# [%s] Plotting" % time.strftime("%T") datapoint.plot() diff --git a/potpourri/generate-ripe-root-cert.py b/potpourri/generate-ripe-root-cert.py index 3407bc51..1b891dce 100644 --- a/potpourri/generate-ripe-root-cert.py +++ b/potpourri/generate-ripe-root-cert.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2010-2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -24,10 +24,10 @@ import lxml.etree from rpki.csv_utils import csv_writer def iterate_xml(filename, tag): - return lxml.etree.parse(filename).getroot().getiterator(tag) + return lxml.etree.parse(filename).getroot().getiterator(tag) def ns(tag): - return "{http://www.iana.org/assignments}" + tag + return "{http://www.iana.org/assignments}" + tag tag_description = ns("description") tag_designation = ns("designation") @@ -39,19 +39,19 @@ asns = csv_writer("asns.csv") prefixes = csv_writer("prefixes.csv") for record in iterate_xml("as-numbers.xml", tag_record): - if record.findtext(tag_description) == "Assigned by RIPE NCC": - asns.writerow(("RIPE", record.findtext(tag_number))) - + if record.findtext(tag_description) == "Assigned by RIPE NCC": + asns.writerow(("RIPE", record.findtext(tag_number))) + for record in iterate_xml("ipv4-address-space.xml", tag_record): - if record.findtext(tag_designation) in ("RIPE NCC", "Administered by RIPE NCC"): - prefix = record.findtext(tag_prefix) - p, l = prefix.split("/") - assert l == "8", "Violated /8 assumption: %r" % prefix - prefixes.writerow(("RIPE", "%d.0.0.0/8" % int(p))) - + if record.findtext(tag_designation) in ("RIPE NCC", "Administered by RIPE NCC"): + prefix = record.findtext(tag_prefix) + p, l = prefix.split("/") + assert l == "8", "Violated /8 assumption: %r" % prefix + prefixes.writerow(("RIPE", "%d.0.0.0/8" % int(p))) + for record in iterate_xml("ipv6-unicast-address-assignments.xml", tag_record): - if record.findtext(tag_description) == "RIPE NCC": - prefixes.writerow(("RIPE", record.findtext(tag_prefix))) + if record.findtext(tag_description) == "RIPE NCC": + prefixes.writerow(("RIPE", record.findtext(tag_prefix))) asns.close() prefixes.close() diff --git a/potpourri/gski.py b/potpourri/gski.py index 083a59c8..3faf22d6 100644 --- a/potpourri/gski.py +++ b/potpourri/gski.py @@ -17,5 +17,5 @@ import rpki.x509, sys for file in sys.argv[1:]: - cert = rpki.x509.X509(Auto_file = file) - print cert.gSKI(), cert.hSKI(), file + cert = rpki.x509.X509(Auto_file = file) + print cert.gSKI(), cert.hSKI(), file diff --git a/potpourri/guess-roas-from-routeviews.py b/potpourri/guess-roas-from-routeviews.py index d8fb9c4c..8e2ed81a 100644 --- a/potpourri/guess-roas-from-routeviews.py +++ b/potpourri/guess-roas-from-routeviews.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2009 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -39,25 +39,25 @@ from rpki.resource_set import roa_prefix_ipv4, resource_set_ipv4, resource_range roas = [] for filename in sys.argv[1:]: - resources = rpki.x509.X509(Auto_file = filename).get_3779resources().v4 + resources = rpki.x509.X509(Auto_file = filename).get_3779resources().v4 - while resources: - labels = str(resources[0].min).split(".") - labels.reverse() + while resources: + labels = str(resources[0].min).split(".") + labels.reverse() - try: - for answer in dns.resolver.query(".".join(labels) + ".asn.routeviews.org", "txt"): - asn, prefix, prefixlen = answer.strings - roa_prefix = roa_prefix_ipv4(v4addr(prefix), long(prefixlen)) - roa = "%s\t%s\t%s" % (roa_prefix, long(asn), filename) - if roa not in roas: - roas.append(roa) - resources = resources.difference(resource_set_ipv4([roa_prefix.to_resource_range()])) + try: + for answer in dns.resolver.query(".".join(labels) + ".asn.routeviews.org", "txt"): + asn, prefix, prefixlen = answer.strings + roa_prefix = roa_prefix_ipv4(v4addr(prefix), long(prefixlen)) + roa = "%s\t%s\t%s" % (roa_prefix, long(asn), filename) + if roa not in roas: + roas.append(roa) + resources = resources.difference(resource_set_ipv4([roa_prefix.to_resource_range()])) - except dns.resolver.NXDOMAIN: - resources = resources.difference(resource_set_ipv4([resource_range_ipv4(resources[0].min, v4addr(resources[0].min + 256))])) + except dns.resolver.NXDOMAIN: + resources = resources.difference(resource_set_ipv4([resource_range_ipv4(resources[0].min, v4addr(resources[0].min + 256))])) roas.sort() for roa in roas: - print roa + print roa diff --git a/potpourri/iana-to-csv.py b/potpourri/iana-to-csv.py index f803a21e..cf82c7e9 100644 --- a/potpourri/iana-to-csv.py +++ b/potpourri/iana-to-csv.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2010-2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -24,10 +24,10 @@ from rpki.csv_utils import csv_reader, csv_writer from rpki.resource_set import resource_bag def iterate_xml(filename, tag): - return lxml.etree.parse(filename).getroot().getiterator(tag) + return lxml.etree.parse(filename).getroot().getiterator(tag) def ns(tag): - return "{http://www.iana.org/assignments}" + tag + return "{http://www.iana.org/assignments}" + tag tag_description = ns("description") tag_designation = ns("designation") @@ -40,30 +40,30 @@ handles = {} rirs = { "legacy" : resource_bag() } for rir in ("AfriNIC", "APNIC", "ARIN", "LACNIC", "RIPE NCC"): - handle = rir.split()[0].lower() - handles[rir] = handles["Assigned by %s" % rir] = handles["Administered by %s" % rir] = handle - rirs[handle] = resource_bag() + handle = rir.split()[0].lower() + handles[rir] = handles["Assigned by %s" % rir] = handles["Administered by %s" % rir] = handle + rirs[handle] = resource_bag() asns = csv_writer("asns.csv") prefixes = csv_writer("prefixes.csv") for record in iterate_xml("as-numbers.xml", tag_record): - description = record.findtext(tag_description) - if description in handles: - asns.writerow((handles[description], record.findtext(tag_number))) - + description = record.findtext(tag_description) + if description in handles: + asns.writerow((handles[description], record.findtext(tag_number))) + for record in iterate_xml("ipv4-address-space.xml", tag_record): - designation = record.findtext(tag_designation) - if record.findtext(tag_status) != "RESERVED": - prefix, prefixlen = [int(i) for i in record.findtext(tag_prefix).split("/")] - if prefixlen != 8: - raise ValueError("%s violated /8 assumption" % record.findtext(tag_prefix)) - rirs[handles.get(designation, "legacy")] |= resource_bag.from_str("%d.0.0.0/8" % prefix) + designation = record.findtext(tag_designation) + if record.findtext(tag_status) != "RESERVED": + prefix, prefixlen = [int(i) for i in record.findtext(tag_prefix).split("/")] + if prefixlen != 8: + raise ValueError("%s violated /8 assumption" % record.findtext(tag_prefix)) + rirs[handles.get(designation, "legacy")] |= resource_bag.from_str("%d.0.0.0/8" % prefix) for record in iterate_xml("ipv6-unicast-address-assignments.xml", tag_record): - description = record.findtext(tag_description) - if record.findtext(tag_description) in handles: - rirs[handles[description]] |= resource_bag.from_str(record.findtext(tag_prefix)) + description = record.findtext(tag_description) + if record.findtext(tag_description) in handles: + rirs[handles[description]] |= resource_bag.from_str(record.findtext(tag_prefix)) erx = list(csv_reader("erx.csv")) assert all(r in rirs for r, p in erx) @@ -71,15 +71,15 @@ assert all(r in rirs for r, p in erx) erx_overrides = resource_bag.from_str(",".join(p for r, p in erx), allow_overlap = True) for rir in rirs: - if rir != "legacy": - rirs[rir] -= erx_overrides - rirs[rir] |= resource_bag.from_str(",".join(p for r, p in erx if r == rir), allow_overlap = True) + if rir != "legacy": + rirs[rir] -= erx_overrides + rirs[rir] |= resource_bag.from_str(",".join(p for r, p in erx if r == rir), allow_overlap = True) for rir, bag in rirs.iteritems(): - for p in bag.v4: - prefixes.writerow((rir, p)) - for p in bag.v6: - prefixes.writerow((rir, p)) + for p in bag.v4: + prefixes.writerow((rir, p)) + for p in bag.v6: + prefixes.writerow((rir, p)) asns.close() prefixes.close() diff --git a/potpourri/missing-oids.py b/potpourri/missing-oids.py index 16316eac..8557e841 100644 --- a/potpourri/missing-oids.py +++ b/potpourri/missing-oids.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2008 American Registry for Internet Numbers ("ARIN") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -23,16 +23,16 @@ import rpki.POW.pkix, rpki.oids need_header = True for oid, name in rpki.oids.oid2name.items(): - try: - rpki.POW.pkix.oid2obj(oid) - except: - o = rpki.POW.pkix.Oid() - o.set(oid) - if need_header: - print - print "# Local additions" - need_header = False - print - print "OID =", " ".join(("%02X" % ord(c)) for c in o.toString()) - print "Comment = RPKI project" - print "Description =", name, "(" + " ".join((str(i) for i in oid)) + ")" + try: + rpki.POW.pkix.oid2obj(oid) + except: + o = rpki.POW.pkix.Oid() + o.set(oid) + if need_header: + print + print "# Local additions" + need_header = False + print + print "OID =", " ".join(("%02X" % ord(c)) for c in o.toString()) + print "Comment = RPKI project" + print "Description =", name, "(" + " ".join((str(i) for i in oid)) + ")" diff --git a/potpourri/object-dates.py b/potpourri/object-dates.py index b99441d6..1be8677b 100644 --- a/potpourri/object-dates.py +++ b/potpourri/object-dates.py @@ -5,11 +5,11 @@ # RPKI objects. # Copyright (C) 2013--2014 Dragon Research Labs ("DRL") -# +# # Permission to use, copy, modify, and/or distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -28,21 +28,21 @@ extract_flags = (rpki.POW.CMS_NOCRL | rpki.POW.CMS_NO_CONTENT_VERIFY) def get_mft(fn): - cms = rpki.POW.Manifest.derReadFile(fn) - cms.verify(rpki.POW.X509Store(), None, extract_flags) - return cms, cms.certs()[0] + cms = rpki.POW.Manifest.derReadFile(fn) + cms.verify(rpki.POW.X509Store(), None, extract_flags) + return cms, cms.certs()[0] def get_roa(fn): - return None, rpki.POW.CMS.derReadFile(fn).certs()[0] + return None, rpki.POW.CMS.derReadFile(fn).certs()[0] def get_gbr(fn): - return None, rpki.POW.CMS.derReadFile(fn).certs()[0] + return None, rpki.POW.CMS.derReadFile(fn).certs()[0] def get_crl(fn): - return rpki.POW.CRL.derReadFile(fn), None + return rpki.POW.CRL.derReadFile(fn), None def get_cer(fn): - return None, rpki.POW.X509.derReadFile(fn) + return None, rpki.POW.X509.derReadFile(fn) dispatch = dict(mft = get_mft, roa = get_roa, @@ -51,13 +51,13 @@ dispatch = dict(mft = get_mft, cer = get_cer) for fn in sys.argv[1:]: - obj, cer = dispatch[os.path.splitext(fn)[1][1:]](fn) - print fn - if cer is not None: - print "notBefore: ", cer.getNotBefore() - if obj is not None: - print "thisUpdate:", obj.getThisUpdate() - print "nextUpdate:", obj.getNextUpdate() - if cer is not None: - print "notAfter: ", cer.getNotAfter() - print + obj, cer = dispatch[os.path.splitext(fn)[1][1:]](fn) + print fn + if cer is not None: + print "notBefore: ", cer.getNotBefore() + if obj is not None: + print "thisUpdate:", obj.getThisUpdate() + print "nextUpdate:", obj.getNextUpdate() + if cer is not None: + print "notAfter: ", cer.getNotAfter() + print diff --git a/potpourri/print-profile.py b/potpourri/print-profile.py index 081d2602..4012fa3f 100644 --- a/potpourri/print-profile.py +++ b/potpourri/print-profile.py @@ -17,4 +17,4 @@ import pstats, glob for f in glob.iglob("*.prof"): - pstats.Stats(f).sort_stats("time").print_stats(50) + pstats.Stats(f).sort_stats("time").print_stats(50) diff --git a/potpourri/rcynic-diff.py b/potpourri/rcynic-diff.py index 327a7b71..d5be51e0 100644 --- a/potpourri/rcynic-diff.py +++ b/potpourri/rcynic-diff.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2012 Internet Systems Consortium, Inc. ("ISC") -# +# # Permission to use, copy, modify, and/or distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -21,94 +21,94 @@ Diff a series of rcynic.xml files, sort of. import sys try: - from lxml.etree import ElementTree + from lxml.etree import ElementTree except ImportError: - from xml.etree.ElementTree import ElementTree + from xml.etree.ElementTree import ElementTree show_backup_generation = False show_rsync_transfer = False class Object(object): - def __init__(self, session, uri, generation): - self.session = session - self.uri = uri - self.generation = generation - self.labels = [] + def __init__(self, session, uri, generation): + self.session = session + self.uri = uri + self.generation = generation + self.labels = [] - def add(self, label): - self.labels.append(label) + def add(self, label): + self.labels.append(label) - def __cmp__(self, other): - return cmp(self.labels, other.labels) + def __cmp__(self, other): + return cmp(self.labels, other.labels) def show(old = None, new = None): - assert old is not None or new is not None - assert old is None or new is None or old.uri == new.uri - if old is None: - obj = new - labels = ["+" + label for label in new.labels] - elif new is None: - obj = old - labels = ["-" + label for label in old.labels] - else: - obj = new - labels = [] - for label in new.session.labels: - if label in new.labels and label in old.labels: - labels.append(label) - elif label in new.labels: - labels.append("+" + label) - elif label in old.labels: - labels.append("-" + label) - labels = " ".join(labels) - if show_backup_generation: - print " ", obj.uri, obj.generation, labels - else: - print " ", obj.uri, labels + assert old is not None or new is not None + assert old is None or new is None or old.uri == new.uri + if old is None: + obj = new + labels = ["+" + label for label in new.labels] + elif new is None: + obj = old + labels = ["-" + label for label in old.labels] + else: + obj = new + labels = [] + for label in new.session.labels: + if label in new.labels and label in old.labels: + labels.append(label) + elif label in new.labels: + labels.append("+" + label) + elif label in old.labels: + labels.append("-" + label) + labels = " ".join(labels) + if show_backup_generation: + print " ", obj.uri, obj.generation, labels + else: + print " ", obj.uri, labels class Session(dict): - def __init__(self, name): - self.name = name - tree = ElementTree(file = name) - self.labels = [elt.tag.strip() for elt in tree.find("labels")] - for elt in tree.findall("validation_status"): - generation = elt.get("generation") - status = elt.get("status") - uri = elt.text.strip() - if not show_rsync_transfer and status.startswith("rsync_transfer_"): - continue - if show_backup_generation: - key = (uri, generation) - elif generation == "backup": - continue - else: - key = uri - if key not in self: - self[key] = Object(self, uri, generation) - self[key].add(status) + def __init__(self, name): + self.name = name + tree = ElementTree(file = name) + self.labels = [elt.tag.strip() for elt in tree.find("labels")] + for elt in tree.findall("validation_status"): + generation = elt.get("generation") + status = elt.get("status") + uri = elt.text.strip() + if not show_rsync_transfer and status.startswith("rsync_transfer_"): + continue + if show_backup_generation: + key = (uri, generation) + elif generation == "backup": + continue + else: + key = uri + if key not in self: + self[key] = Object(self, uri, generation) + self[key].add(status) old_db = new_db = None for arg in sys.argv[1:]: - old_db = new_db - new_db = Session(arg) - - if old_db is None: - continue - - only_old = set(old_db) - set(new_db) - only_new = set(new_db) - set(old_db) - changed = set(key for key in (set(old_db) & set(new_db)) if old_db[key] != new_db[key]) - - if only_old or changed or only_new: - print "Comparing", old_db.name, "with", new_db.name - for key in sorted(only_old): - show(old = old_db[key]) - for key in sorted(changed): - show(old = old_db[key], new = new_db[key]) - for key in sorted(only_new): - show(new = new_db[key]) - print + old_db = new_db + new_db = Session(arg) + + if old_db is None: + continue + + only_old = set(old_db) - set(new_db) + only_new = set(new_db) - set(old_db) + changed = set(key for key in (set(old_db) & set(new_db)) if old_db[key] != new_db[key]) + + if only_old or changed or only_new: + print "Comparing", old_db.name, "with", new_db.name + for key in sorted(only_old): + show(old = old_db[key]) + for key in sorted(changed): + show(old = old_db[key], new = new_db[key]) + for key in sorted(only_new): + show(new = new_db[key]) + print diff --git a/potpourri/ripe-asns-to-csv.py b/potpourri/ripe-asns-to-csv.py index 50251ce8..0c85b901 100644 --- a/potpourri/ripe-asns-to-csv.py +++ b/potpourri/ripe-asns-to-csv.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2009-2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -28,81 +28,81 @@ from rpki.csv_utils import csv_writer class Handle(dict): - want_tags = () + want_tags = () - debug = False + debug = False - def set(self, tag, val): - if tag in self.want_tags: - self[tag] = "".join(val.split(" ")) + def set(self, tag, val): + if tag in self.want_tags: + self[tag] = "".join(val.split(" ")) - def check(self): - for tag in self.want_tags: - if not tag in self: - return False - if self.debug: - self.log() - return True + def check(self): + for tag in self.want_tags: + if not tag in self: + return False + if self.debug: + self.log() + return True - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, - " ".join("%s:%s" % (tag, self.get(tag, "?")) - for tag in self.want_tags)) + def __repr__(self): + return "<%s %s>" % (self.__class__.__name__, + " ".join("%s:%s" % (tag, self.get(tag, "?")) + for tag in self.want_tags)) - def log(self): - print repr(self) + def log(self): + print repr(self) - def finish(self, ctx): - self.check() + def finish(self, ctx): + self.check() class aut_num(Handle): - want_tags = ("aut-num", "mnt-by", "as-name") + want_tags = ("aut-num", "mnt-by", "as-name") - def set(self, tag, val): - if tag == "aut-num" and val.startswith("AS"): - val = val[2:] - Handle.set(self, tag, val) + def set(self, tag, val): + if tag == "aut-num" and val.startswith("AS"): + val = val[2:] + Handle.set(self, tag, val) - def finish(self, ctx): - if self.check(): - ctx.asns.writerow((self["mnt-by"], self["aut-num"])) + def finish(self, ctx): + if self.check(): + ctx.asns.writerow((self["mnt-by"], self["aut-num"])) class main(object): - types = dict((x.want_tags[0], x) for x in (aut_num,)) - - - def finish_statement(self, done): - if self.statement: - tag, sep, val = self.statement.partition(":") - assert sep, "Couldn't find separator in %r" % self.statement - tag = tag.strip().lower() - val = val.strip().upper() - if self.cur is None: - self.cur = self.types[tag]() if tag in self.types else False - if self.cur is not False: - self.cur.set(tag, val) - if done and self.cur: - self.cur.finish(self) - self.cur = None - - filenames = ("ripe.db.aut-num.gz",) - - def __init__(self): - self.asns = csv_writer("asns.csv") - for fn in self.filenames: - f = gzip.open(fn) - self.statement = "" - self.cur = None - for line in f: - line = line.expandtabs().partition("#")[0].rstrip("\n") - if line and not line[0].isalpha(): - self.statement += line[1:] if line[0] == "+" else line - else: - self.finish_statement(not line) - self.statement = line - self.finish_statement(True) - f.close() - self.asns.close() + types = dict((x.want_tags[0], x) for x in (aut_num,)) + + + def finish_statement(self, done): + if self.statement: + tag, sep, val = self.statement.partition(":") + assert sep, "Couldn't find separator in %r" % self.statement + tag = tag.strip().lower() + val = val.strip().upper() + if self.cur is None: + self.cur = self.types[tag]() if tag in self.types else False + if self.cur is not False: + self.cur.set(tag, val) + if done and self.cur: + self.cur.finish(self) + self.cur = None + + filenames = ("ripe.db.aut-num.gz",) + + def __init__(self): + self.asns = csv_writer("asns.csv") + for fn in self.filenames: + f = gzip.open(fn) + self.statement = "" + self.cur = None + for line in f: + line = line.expandtabs().partition("#")[0].rstrip("\n") + if line and not line[0].isalpha(): + self.statement += line[1:] if line[0] == "+" else line + else: + self.finish_statement(not line) + self.statement = line + self.finish_statement(True) + f.close() + self.asns.close() main() diff --git a/potpourri/ripe-to-csv.py b/potpourri/ripe-to-csv.py index b864345b..308917ce 100644 --- a/potpourri/ripe-to-csv.py +++ b/potpourri/ripe-to-csv.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2009-2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -41,98 +41,98 @@ from rpki.csv_utils import csv_writer class Handle(dict): - want_tags = () + want_tags = () - want_status = ("ASSIGNED", "ASSIGNEDPA", "ASSIGNEDPI") + want_status = ("ASSIGNED", "ASSIGNEDPA", "ASSIGNEDPI") - debug = False + debug = False - def set(self, tag, val): - if tag in self.want_tags: - self[tag] = "".join(val.split(" ")) + def set(self, tag, val): + if tag in self.want_tags: + self[tag] = "".join(val.split(" ")) - def check(self): - for tag in self.want_tags: - if not tag in self: - return False - if self.debug: - self.log() - return True + def check(self): + for tag in self.want_tags: + if not tag in self: + return False + if self.debug: + self.log() + return True - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, - " ".join("%s:%s" % (tag, self.get(tag, "?")) - for tag in self.want_tags)) + def __repr__(self): + return "<%s %s>" % (self.__class__.__name__, + " ".join("%s:%s" % (tag, self.get(tag, "?")) + for tag in self.want_tags)) - def log(self): - print repr(self) + def log(self): + print repr(self) - def finish(self, ctx): - self.check() + def finish(self, ctx): + self.check() class aut_num(Handle): - want_tags = ("aut-num", "mnt-by") # "as-name" + want_tags = ("aut-num", "mnt-by") # "as-name" - def set(self, tag, val): - if tag == "aut-num" and val.startswith("AS"): - val = val[2:] - Handle.set(self, tag, val) + def set(self, tag, val): + if tag == "aut-num" and val.startswith("AS"): + val = val[2:] + Handle.set(self, tag, val) - def finish(self, ctx): - if self.check(): - ctx.asns.writerow((self["mnt-by"], self["aut-num"])) + def finish(self, ctx): + if self.check(): + ctx.asns.writerow((self["mnt-by"], self["aut-num"])) class inetnum(Handle): - want_tags = ("inetnum", "netname", "status") # "mnt-by" - - def finish(self, ctx): - if self.check() and self["status"] in self.want_status: - ctx.prefixes.writerow((self["netname"], self["inetnum"])) + want_tags = ("inetnum", "netname", "status") # "mnt-by" + + def finish(self, ctx): + if self.check() and self["status"] in self.want_status: + ctx.prefixes.writerow((self["netname"], self["inetnum"])) class inet6num(Handle): - want_tags = ("inet6num", "netname", "status") # "mnt-by" + want_tags = ("inet6num", "netname", "status") # "mnt-by" - def finish(self, ctx): - if self.check() and self["status"] in self.want_status: - ctx.prefixes.writerow((self["netname"], self["inet6num"])) + def finish(self, ctx): + if self.check() and self["status"] in self.want_status: + ctx.prefixes.writerow((self["netname"], self["inet6num"])) class main(object): - types = dict((x.want_tags[0], x) for x in (aut_num, inetnum, inet6num)) - - def finish_statement(self, done): - if self.statement: - tag, sep, val = self.statement.partition(":") - assert sep, "Couldn't find separator in %r" % self.statement - tag = tag.strip().lower() - val = val.strip().upper() - if self.cur is None: - self.cur = self.types[tag]() if tag in self.types else False - if self.cur is not False: - self.cur.set(tag, val) - if done and self.cur: - self.cur.finish(self) - self.cur = None - - filenames = ("ripe.db.aut-num.gz", "ripe.db.inet6num.gz", "ripe.db.inetnum.gz") - - def __init__(self): - self.asns = csv_writer("asns.csv") - self.prefixes = csv_writer("prefixes.csv") - for fn in self.filenames: - f = gzip.open(fn) - self.statement = "" - self.cur = None - for line in f: - line = line.expandtabs().partition("#")[0].rstrip("\n") - if line and not line[0].isalpha(): - self.statement += line[1:] if line[0] == "+" else line - else: - self.finish_statement(not line) - self.statement = line - self.finish_statement(True) - f.close() - self.asns.close() - self.prefixes.close() + types = dict((x.want_tags[0], x) for x in (aut_num, inetnum, inet6num)) + + def finish_statement(self, done): + if self.statement: + tag, sep, val = self.statement.partition(":") + assert sep, "Couldn't find separator in %r" % self.statement + tag = tag.strip().lower() + val = val.strip().upper() + if self.cur is None: + self.cur = self.types[tag]() if tag in self.types else False + if self.cur is not False: + self.cur.set(tag, val) + if done and self.cur: + self.cur.finish(self) + self.cur = None + + filenames = ("ripe.db.aut-num.gz", "ripe.db.inet6num.gz", "ripe.db.inetnum.gz") + + def __init__(self): + self.asns = csv_writer("asns.csv") + self.prefixes = csv_writer("prefixes.csv") + for fn in self.filenames: + f = gzip.open(fn) + self.statement = "" + self.cur = None + for line in f: + line = line.expandtabs().partition("#")[0].rstrip("\n") + if line and not line[0].isalpha(): + self.statement += line[1:] if line[0] == "+" else line + else: + self.finish_statement(not line) + self.statement = line + self.finish_statement(True) + f.close() + self.asns.close() + self.prefixes.close() main() diff --git a/potpourri/roa-to-irr.py b/potpourri/roa-to-irr.py index 500596f8..748f37fa 100644 --- a/potpourri/roa-to-irr.py +++ b/potpourri/roa-to-irr.py @@ -1,12 +1,12 @@ # $Id$ -# +# # Copyright (C) 2014 Dragon Research Labs ("DRL") # Portions copyright (C) 2010--2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notices and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL # WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR @@ -34,128 +34,128 @@ from time import time, strftime, gmtime, asctime args = None class route(object): - """ - Interesting parts of a route object. - """ - - def __init__(self, label, uri, asnum, date, prefix, prefixlen, max_prefixlen): - self.label = label - self.uri = uri - self.asn = asnum - self.date = date - self.prefix = prefix - self.prefixlen = prefixlen - self.max_prefixlen = self.prefixlen if max_prefixlen is None else max_prefixlen - - def __cmp__(self, other): - result = cmp(self.asn, other.asn) - if result == 0: - result = cmp(self.prefix, other.prefix) - if result == 0: - result = cmp(self.prefixlen, other.prefixlen) - if result == 0: - result = cmp(self.max_prefixlen, other.max_prefixlen) - if result == 0: - result = cmp(self.date, other.date) - return result - - def __str__(self): - lines = "\n" if args.email else "" - lines += dedent('''\ - {self.label:<14s}{self.prefix}/{self.prefixlen} - descr: {self.prefix}/{self.prefixlen}-{self.max_prefixlen} - origin: AS{self.asn:d} - notify: {args.notify} - mnt-by: {args.mnt_by} - changed: {args.changed_by} {self.date} - source: {args.source} - ''').format(self = self, args = args) - if args.password is not None: - lines += "override: {}\n".format(args.password) - return lines - - def write(self, output_directory): - name = "{0.prefix}-{0.prefixlen}-{0.max_prefixlen}-AS{0.asn:d}-{0.date}".format(self) - with open(os.path.join(output_directory, name), "w") as f: - f.write(str(self)) + """ + Interesting parts of a route object. + """ + + def __init__(self, label, uri, asnum, date, prefix, prefixlen, max_prefixlen): + self.label = label + self.uri = uri + self.asn = asnum + self.date = date + self.prefix = prefix + self.prefixlen = prefixlen + self.max_prefixlen = self.prefixlen if max_prefixlen is None else max_prefixlen + + def __cmp__(self, other): + result = cmp(self.asn, other.asn) + if result == 0: + result = cmp(self.prefix, other.prefix) + if result == 0: + result = cmp(self.prefixlen, other.prefixlen) + if result == 0: + result = cmp(self.max_prefixlen, other.max_prefixlen) + if result == 0: + result = cmp(self.date, other.date) + return result + + def __str__(self): + lines = "\n" if args.email else "" + lines += dedent('''\ + {self.label:<14s}{self.prefix}/{self.prefixlen} + descr: {self.prefix}/{self.prefixlen}-{self.max_prefixlen} + origin: AS{self.asn:d} + notify: {args.notify} + mnt-by: {args.mnt_by} + changed: {args.changed_by} {self.date} + source: {args.source} + ''').format(self = self, args = args) + if args.password is not None: + lines += "override: {}\n".format(args.password) + return lines + + def write(self, output_directory): + name = "{0.prefix}-{0.prefixlen}-{0.max_prefixlen}-AS{0.asn:d}-{0.date}".format(self) + with open(os.path.join(output_directory, name), "w") as f: + f.write(str(self)) class route_list(list): - """ - A list of route objects. - """ - - def __init__(self, rcynic_dir): - for root, dirs, files in os.walk(rcynic_dir): - for f in files: - if f.endswith(".roa"): - path = os.path.join(root, f) - uri = "rsync://" + path[len(rcynic_dir):].lstrip("/") - roa = rpki.x509.ROA(DER_file = path) - roa.extract() - assert roa.get_POW().getVersion() == 0, "ROA version is {:d}, expected 0".format(roa.get_POW().getVersion()) - asnum = roa.get_POW().getASID() - notBefore = roa.get_POW().certs()[0].getNotBefore().strftime("%Y%m%d") - v4, v6 = roa.get_POW().getPrefixes() - if v4 is not None: - for prefix, prefixlen, max_prefixlen in v4: - self.append(route("route:", uri, asnum, notBefore, prefix, prefixlen, max_prefixlen)) - if v6 is not None: - for prefix, prefixlen, max_prefixlen in v6: - self.append(route("route6:", uri, asnum, notBefore, prefix, prefixlen, max_prefixlen)) - self.sort() - for i in xrange(len(self) - 2, -1, -1): - if self[i] == self[i + 1]: - del self[i + 1] + """ + A list of route objects. + """ + + def __init__(self, rcynic_dir): + for root, dirs, files in os.walk(rcynic_dir): + for f in files: + if f.endswith(".roa"): + path = os.path.join(root, f) + uri = "rsync://" + path[len(rcynic_dir):].lstrip("/") + roa = rpki.x509.ROA(DER_file = path) + roa.extract() + assert roa.get_POW().getVersion() == 0, "ROA version is {:d}, expected 0".format(roa.get_POW().getVersion()) + asnum = roa.get_POW().getASID() + notBefore = roa.get_POW().certs()[0].getNotBefore().strftime("%Y%m%d") + v4, v6 = roa.get_POW().getPrefixes() + if v4 is not None: + for prefix, prefixlen, max_prefixlen in v4: + self.append(route("route:", uri, asnum, notBefore, prefix, prefixlen, max_prefixlen)) + if v6 is not None: + for prefix, prefixlen, max_prefixlen in v6: + self.append(route("route6:", uri, asnum, notBefore, prefix, prefixlen, max_prefixlen)) + self.sort() + for i in xrange(len(self) - 2, -1, -1): + if self[i] == self[i + 1]: + del self[i + 1] def email_header(f): - if args.email: - now = time() - f.write(dedent('''\ - From {from_} {ctime} - Date: {date} - From: {from_} - Subject: Fake email header to make irr_rpsl_submit happy - Message-Id: <{pid}.{seconds}@{hostname}> - ''').format(from_ = args.from_, - ctime = asctime(gmtime(now)), - date = strftime("%d %b %Y %T %z", gmtime(now)), - pid = os.getpid(), - seconds = now, - hostname = gethostname())) + if args.email: + now = time() + f.write(dedent('''\ + From {from_} {ctime} + Date: {date} + From: {from_} + Subject: Fake email header to make irr_rpsl_submit happy + Message-Id: <{pid}.{seconds}@{hostname}> + ''').format(from_ = args.from_, + ctime = asctime(gmtime(now)), + date = strftime("%d %b %Y %T %z", gmtime(now)), + pid = os.getpid(), + seconds = now, + hostname = gethostname())) def main(): - global args - whoami = "{}@{}".format(os.getlogin(), gethostname()) - - parser = argparse.ArgumentParser(description = __doc__) - parser.add_argument("-c", "--changed_by", default = whoami, help = "override \"changed:\" value") - parser.add_argument("-f", "--from", dest="from_", default = whoami, help = "override \"from:\" header when using --email") - parser.add_argument("-m", "--mnt_by", default = "MAINT-RPKI", help = "override \"mnt-by:\" value") - parser.add_argument("-n", "--notify", default = whoami, help = "override \"notify:\" value") - parser.add_argument("-p", "--password", help = "specify \"override:\" password") - parser.add_argument("-s", "--source", default = "RPKI", help = "override \"source:\" value") - group = parser.add_mutually_exclusive_group() - group.add_argument("-e", "--email", action = "store_true", help = "generate fake RFC 822 header suitable for piping to irr_rpsl_submit") - group.add_argument("-d", "--output-directory", help = "write route and route6 objects to directory OUTPUT, one object per file") - parser.add_argument("authenticated_directory", help = "directory tree containing authenticated rcynic output") - args = parser.parse_args() - - if not os.path.isdir(args.authenticated_directory): - sys.exit('"{}" is not a directory'.format(args.authenticated_directory)) - - routes = route_list(args.authenticated_directory) - - if args.output_directory: - if not os.path.isdir(args.output_directory): - os.makedirs(args.output_directory) - for r in routes: - r.write(args.output_directory) - else: - email_header(sys.stdout) - for r in routes: - sys.stdout.write(str(r)) + global args + whoami = "{}@{}".format(os.getlogin(), gethostname()) + + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument("-c", "--changed_by", default = whoami, help = "override \"changed:\" value") + parser.add_argument("-f", "--from", dest="from_", default = whoami, help = "override \"from:\" header when using --email") + parser.add_argument("-m", "--mnt_by", default = "MAINT-RPKI", help = "override \"mnt-by:\" value") + parser.add_argument("-n", "--notify", default = whoami, help = "override \"notify:\" value") + parser.add_argument("-p", "--password", help = "specify \"override:\" password") + parser.add_argument("-s", "--source", default = "RPKI", help = "override \"source:\" value") + group = parser.add_mutually_exclusive_group() + group.add_argument("-e", "--email", action = "store_true", help = "generate fake RFC 822 header suitable for piping to irr_rpsl_submit") + group.add_argument("-d", "--output-directory", help = "write route and route6 objects to directory OUTPUT, one object per file") + parser.add_argument("authenticated_directory", help = "directory tree containing authenticated rcynic output") + args = parser.parse_args() + + if not os.path.isdir(args.authenticated_directory): + sys.exit('"{}" is not a directory'.format(args.authenticated_directory)) + + routes = route_list(args.authenticated_directory) + + if args.output_directory: + if not os.path.isdir(args.output_directory): + os.makedirs(args.output_directory) + for r in routes: + r.write(args.output_directory) + else: + email_header(sys.stdout) + for r in routes: + sys.stdout.write(str(r)) if __name__ == "__main__": - main() + main() diff --git a/potpourri/rrd-rcynic-history.py b/potpourri/rrd-rcynic-history.py index 8a0d50a8..45aec6c5 100644 --- a/potpourri/rrd-rcynic-history.py +++ b/potpourri/rrd-rcynic-history.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2011-2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -39,159 +39,159 @@ os.putenv("TZ", "UTC") time.tzset() def parse_utc(s): - return int(time.mktime(time.strptime(s, "%Y-%m-%dT%H:%M:%SZ"))) + return int(time.mktime(time.strptime(s, "%Y-%m-%dT%H:%M:%SZ"))) class Host(object): - """ - A host object represents all the data collected for one host for a given session. - """ - - def __init__(self, hostname, timestamp): - self.hostname = hostname - self.timestamp = timestamp - self.elapsed = 0 - self.connections = 0 - self.failures = 0 - self.uris = set() - - def add_connection(self, elt): - self.elapsed += parse_utc(elt.get("finished")) - parse_utc(elt.get("started")) - self.connections += 1 - if elt.get("error") is not None: - self.failures += 1 - - def add_object_uri(self, u): - self.uris.add(u) - - @property - def failed(self): - return 1 if self.failures > 0 else 0 - - @property - def objects(self): - return len(self.uris) - - field_table = (("timestamp", None, None, None), - ("connections", "GAUGE", "Connections", "FF0000"), - ("objects", "GAUGE", "Objects", "00FF00"), - ("elapsed", "GAUGE", "Fetch Time", "0000FF"), - ("failed", "ABSOLUTE", "Failed", "00FFFF")) - - @property - def field_values(self): - return tuple(str(getattr(self, field[0])) for field in self.field_table) - - @classmethod - def field_ds_specifiers(cls, heartbeat = 24 * 60 * 60, minimum = 0, maximum = "U"): - return ["DS:%s:%s:%s:%s:%s" % (field[0], field[1], heartbeat, minimum, maximum) - for field in cls.field_table if field[1] is not None] - - @classmethod - def field_graph_specifiers(cls, hostname): - result = [] - for field in cls.field_table: - if field[1] is not None: - result.append("DEF:%s=%s.rrd:%s:AVERAGE" % (field[0], hostname, field[0])) - result.append("'LINE1:%s#%s:%s'" % (field[0], field[3], field[2])) - return result - - def save(self, rrdtable): - rrdtable.add(self.hostname, self.field_values) + """ + A host object represents all the data collected for one host for a given session. + """ + + def __init__(self, hostname, timestamp): + self.hostname = hostname + self.timestamp = timestamp + self.elapsed = 0 + self.connections = 0 + self.failures = 0 + self.uris = set() + + def add_connection(self, elt): + self.elapsed += parse_utc(elt.get("finished")) - parse_utc(elt.get("started")) + self.connections += 1 + if elt.get("error") is not None: + self.failures += 1 + + def add_object_uri(self, u): + self.uris.add(u) + + @property + def failed(self): + return 1 if self.failures > 0 else 0 + + @property + def objects(self): + return len(self.uris) + + field_table = (("timestamp", None, None, None), + ("connections", "GAUGE", "Connections", "FF0000"), + ("objects", "GAUGE", "Objects", "00FF00"), + ("elapsed", "GAUGE", "Fetch Time", "0000FF"), + ("failed", "ABSOLUTE", "Failed", "00FFFF")) + + @property + def field_values(self): + return tuple(str(getattr(self, field[0])) for field in self.field_table) + + @classmethod + def field_ds_specifiers(cls, heartbeat = 24 * 60 * 60, minimum = 0, maximum = "U"): + return ["DS:%s:%s:%s:%s:%s" % (field[0], field[1], heartbeat, minimum, maximum) + for field in cls.field_table if field[1] is not None] + + @classmethod + def field_graph_specifiers(cls, hostname): + result = [] + for field in cls.field_table: + if field[1] is not None: + result.append("DEF:%s=%s.rrd:%s:AVERAGE" % (field[0], hostname, field[0])) + result.append("'LINE1:%s#%s:%s'" % (field[0], field[3], field[2])) + return result + + def save(self, rrdtable): + rrdtable.add(self.hostname, self.field_values) class Session(dict): - """ - A session corresponds to one XML file. This is a dictionary of Host - objects, keyed by hostname. - """ - - def __init__(self, timestamp): - dict.__init__(self) - self.timestamp = timestamp - - @property - def hostnames(self): - return set(self.iterkeys()) - - def add_connection(self, elt): - hostname = urlparse.urlparse(elt.text.strip()).hostname - if hostname not in self: - self[hostname] = Host(hostname, self.timestamp) - self[hostname].add_connection(elt) - - def add_object_uri(self, u): - h = urlparse.urlparse(u).hostname - if h and h in self: - self[h].add_object_uri(u) - - def save(self, rrdtable): - for h in self.itervalues(): - h.save(rrdtable) + """ + A session corresponds to one XML file. This is a dictionary of Host + objects, keyed by hostname. + """ + + def __init__(self, timestamp): + dict.__init__(self) + self.timestamp = timestamp + + @property + def hostnames(self): + return set(self.iterkeys()) + + def add_connection(self, elt): + hostname = urlparse.urlparse(elt.text.strip()).hostname + if hostname not in self: + self[hostname] = Host(hostname, self.timestamp) + self[hostname].add_connection(elt) + + def add_object_uri(self, u): + h = urlparse.urlparse(u).hostname + if h and h in self: + self[h].add_object_uri(u) + + def save(self, rrdtable): + for h in self.itervalues(): + h.save(rrdtable) class RRDTable(dict): - """ - Final data we're going to be sending to rrdtool. We need to buffer - it until we're done because we have to sort it. Might be easier - just to sort the maildir, then again it might be easier to get rid - of the maildir too once we're dealing with current data. We'll see. - """ - - def __init__(self, rrdtool = sys.stdout): - dict.__init__(self) - self.rrdtool = rrdtool - - def add(self, hostname, data): - if hostname not in self: - self[hostname] = [] - self[hostname].append(data) - - def sort(self): - for data in self.itervalues(): - data.sort() - - @property - def oldest(self): - return min(min(datum[0] for datum in data) for data in self.itervalues()) - - rras = tuple("RRA:AVERAGE:0.5:%s:9600" % steps for steps in (1, 4, 24)) - - def create(self): - start = self.oldest - ds_list = Host.field_ds_specifiers() - ds_list.extend(self.rras) - for hostname in self: - if not os.path.exists("%s.rrd" % hostname): - self.rrdtool("create %s.rrd --start %s --step 3600 %s\n" % (hostname, start, " ".join(ds_list))) - - def update(self): - for hostname, data in self.iteritems(): - for datum in data: - self.rrdtool("update %s.rrd %s\n" % (hostname, ":".join(str(d) for d in datum))) - - def graph(self): - for hostname in self: - self.rrdtool("graph %s.png --start -90d %s\n" % (hostname, " ".join(Host.field_graph_specifiers(hostname)))) + """ + Final data we're going to be sending to rrdtool. We need to buffer + it until we're done because we have to sort it. Might be easier + just to sort the maildir, then again it might be easier to get rid + of the maildir too once we're dealing with current data. We'll see. + """ + + def __init__(self, rrdtool = sys.stdout): + dict.__init__(self) + self.rrdtool = rrdtool + + def add(self, hostname, data): + if hostname not in self: + self[hostname] = [] + self[hostname].append(data) + + def sort(self): + for data in self.itervalues(): + data.sort() + + @property + def oldest(self): + return min(min(datum[0] for datum in data) for data in self.itervalues()) + + rras = tuple("RRA:AVERAGE:0.5:%s:9600" % steps for steps in (1, 4, 24)) + + def create(self): + start = self.oldest + ds_list = Host.field_ds_specifiers() + ds_list.extend(self.rras) + for hostname in self: + if not os.path.exists("%s.rrd" % hostname): + self.rrdtool("create %s.rrd --start %s --step 3600 %s\n" % (hostname, start, " ".join(ds_list))) + + def update(self): + for hostname, data in self.iteritems(): + for datum in data: + self.rrdtool("update %s.rrd %s\n" % (hostname, ":".join(str(d) for d in datum))) + + def graph(self): + for hostname in self: + self.rrdtool("graph %s.png --start -90d %s\n" % (hostname, " ".join(Host.field_graph_specifiers(hostname)))) mb = mailbox.Maildir("/u/sra/rpki/rcynic-xml", factory = None, create = False) rrdtable = RRDTable() for i, key in enumerate(mb.iterkeys(), 1): - sys.stderr.write("\r%s %d/%d..." % ("|\\-/"[i & 3], i, len(mb))) - - assert not mb[key].is_multipart() - input = ElementTreeFromString(mb[key].get_payload()) - date = input.get("date") - sys.stderr.write("%s..." % date) - session = Session(parse_utc(date)) - for elt in input.findall("rsync_history"): - session.add_connection(elt) - for elt in input.findall("validation_status"): - if elt.get("generation") == "current": - session.add_object_uri(elt.text.strip()) - session.save(rrdtable) - - # XXX - #if i > 4: break + sys.stderr.write("\r%s %d/%d..." % ("|\\-/"[i & 3], i, len(mb))) + + assert not mb[key].is_multipart() + input = ElementTreeFromString(mb[key].get_payload()) + date = input.get("date") + sys.stderr.write("%s..." % date) + session = Session(parse_utc(date)) + for elt in input.findall("rsync_history"): + session.add_connection(elt) + for elt in input.findall("validation_status"): + if elt.get("generation") == "current": + session.add_object_uri(elt.text.strip()) + session.save(rrdtable) + + # XXX + #if i > 4: break sys.stderr.write("\n") diff --git a/potpourri/rrdp-fetch.py b/potpourri/rrdp-fetch.py index 469c0c9f..b8d927ee 100755 --- a/potpourri/rrdp-fetch.py +++ b/potpourri/rrdp-fetch.py @@ -29,25 +29,25 @@ from urlparse import urlparse from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter class BadHash(Exception): - "Calculated hash value doesn't match expected hash value." + "Calculated hash value doesn't match expected hash value." def fetch(elt): - uri = elt.get("uri") - hash = elt.get("hash").lower() - print "Fetching", uri + uri = elt.get("uri") + hash = elt.get("hash").lower() + print "Fetching", uri - text = urlopen(uri).read() - h = sha256(text).encode("hex") - if h != hash: - raise BadHash("Bad hash for %s: expected %s got %s" % (uri, hash, h)) + text = urlopen(uri).read() + h = sha256(text).encode("hex") + if h != hash: + raise BadHash("Bad hash for %s: expected %s got %s" % (uri, hash, h)) - xml = XML(text) - rrdp.schema.assertValid(xml) + xml = XML(text) + rrdp.schema.assertValid(xml) - u = urlparse(uri) - fn = u.netloc + u.path + u = urlparse(uri) + fn = u.netloc + u.path - return elt, xml, fn + return elt, xml, fn parser = ArgumentParser(description = __doc__, formatter_class = ArgumentDefaultsHelpFormatter) parser.add_argument("uri", nargs = "?", @@ -65,4 +65,4 @@ deltas = [fetch(elt) for elt in updates.findall(rrdp.xmlns + "delta")] print updates print snapshot for delta in deltas: - print delta + print delta diff --git a/potpourri/show-key-identifiers.py b/potpourri/show-key-identifiers.py index fa2bae8b..4ba6219a 100644 --- a/potpourri/show-key-identifiers.py +++ b/potpourri/show-key-identifiers.py @@ -29,26 +29,26 @@ import rpki.oids def check_dir(s): - if os.path.isdir(s): - return os.path.abspath(s) - else: - raise argparse.ArgumentTypeError("%r is not a directory" % s) + if os.path.isdir(s): + return os.path.abspath(s) + else: + raise argparse.ArgumentTypeError("%r is not a directory" % s) def filename_to_uri(filename): - if not filename.startswith(args.rcynic_dir): - raise ValueError - return "rsync://" + filename[len(args.rcynic_dir):].lstrip("/") + if not filename.startswith(args.rcynic_dir): + raise ValueError + return "rsync://" + filename[len(args.rcynic_dir):].lstrip("/") def get_roa(fn): - return rpki.POW.CMS.derReadFile(fn).certs()[0] + return rpki.POW.CMS.derReadFile(fn).certs()[0] def get_crl(fn): - return rpki.POW.CRL.derReadFile(fn) + return rpki.POW.CRL.derReadFile(fn) def get_cer(fn): - return rpki.POW.X509.derReadFile(fn) + return rpki.POW.X509.derReadFile(fn) dispatch = dict(roa = get_roa, crl = get_crl, @@ -59,23 +59,23 @@ parser.add_argument("rcynic_dir", type = check_dir, help = "rcynic authenticated args = parser.parse_args() for root, dirs, files in os.walk(args.rcynic_dir): - for fn in files: - fn = os.path.join(root, fn) - fn2 = os.path.splitext(fn)[1][1:] - if fn2 not in dispatch: - continue - obj = dispatch[fn2](fn) - uri = filename_to_uri(fn) - try: - ski = obj.getSKI().encode("hex") - except: - ski = "" - try: - aki = obj.getAKI().encode("hex") - except: - aki = "" - try: - res = ",".join(",".join("%s-%s" % r2 for r2 in r1) for r1 in obj.getRFC3779() if r1 is not None) - except: - res = "" - print "\t".join((uri, ski, aki, res)) + for fn in files: + fn = os.path.join(root, fn) + fn2 = os.path.splitext(fn)[1][1:] + if fn2 not in dispatch: + continue + obj = dispatch[fn2](fn) + uri = filename_to_uri(fn) + try: + ski = obj.getSKI().encode("hex") + except: + ski = "" + try: + aki = obj.getAKI().encode("hex") + except: + aki = "" + try: + res = ",".join(",".join("%s-%s" % r2 for r2 in r1) for r1 in obj.getRFC3779() if r1 is not None) + except: + res = "" + print "\t".join((uri, ski, aki, res)) diff --git a/potpourri/show-tracking-data.py b/potpourri/show-tracking-data.py index 07e0a144..0fbb26c1 100644 --- a/potpourri/show-tracking-data.py +++ b/potpourri/show-tracking-data.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -31,9 +31,9 @@ import rpki.sundial rcynic_dir = sys.argv[1] for root, dirs, files in os.walk(rcynic_dir): - for f in files: - path = os.path.join(root, f) - date = rpki.sundial.datetime.utcfromtimestamp(os.stat(path).st_mtime) - uri = "rsync://" + path[len(rcynic_dir):].lstrip("/") - obj = rpki.x509.uri_dispatch(uri)(DER_file = path) - print date, obj.tracking_data(uri) + for f in files: + path = os.path.join(root, f) + date = rpki.sundial.datetime.utcfromtimestamp(os.stat(path).st_mtime) + uri = "rsync://" + path[len(rcynic_dir):].lstrip("/") + obj = rpki.x509.uri_dispatch(uri)(DER_file = path) + print date, obj.tracking_data(uri) diff --git a/potpourri/signed-object-dates.py b/potpourri/signed-object-dates.py index fefd9448..d5699252 100644 --- a/potpourri/signed-object-dates.py +++ b/potpourri/signed-object-dates.py @@ -5,11 +5,11 @@ # if the object is a manifest, also extract thisUpdate and nextUpdate. # Copyright (C) 2013 Dragon Research Labs ("DRL") -# +# # Permission to use, copy, modify, and/or distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -27,14 +27,14 @@ extract_flags = (rpki.POW.CMS_NOCRL | rpki.POW.CMS_NO_CONTENT_VERIFY) for fn in sys.argv[1:]: - cls = rpki.POW.Manifest if fn.endswith(".mft") else rpki.POW.CMS - cms = cls.derReadFile(fn) - cer = cms.certs()[0] - print fn - print " notBefore: ", cer.getNotBefore() - if fn.endswith(".mft"): - cms.verify(rpki.POW.X509Store(), None, extract_flags) - print " thisUpdate:", cms.getThisUpdate() - print " nextUpdate:", cms.getNextUpdate() - print " notAfter: ", cer.getNotAfter() - print + cls = rpki.POW.Manifest if fn.endswith(".mft") else rpki.POW.CMS + cms = cls.derReadFile(fn) + cer = cms.certs()[0] + print fn + print " notBefore: ", cer.getNotBefore() + if fn.endswith(".mft"): + cms.verify(rpki.POW.X509Store(), None, extract_flags) + print " thisUpdate:", cms.getThisUpdate() + print " nextUpdate:", cms.getNextUpdate() + print " notAfter: ", cer.getNotAfter() + print diff --git a/potpourri/testbed-rootcert.py b/potpourri/testbed-rootcert.py index 0716be2f..5e2e97c5 100644 --- a/potpourri/testbed-rootcert.py +++ b/potpourri/testbed-rootcert.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2009-2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -26,7 +26,7 @@ import sys from rpki.csv_utils import csv_reader if len(sys.argv) not in (2, 4): - sys.exit("Usage: %s holder [asns.csv prefixes.csv]" % sys.argv[0]) + sys.exit("Usage: %s holder [asns.csv prefixes.csv]" % sys.argv[0]) print '''\ [req] @@ -53,7 +53,7 @@ sbgp-ipAddrBlock = critical,@rfc3997_addrs "HOLDER" : sys.argv[1].upper() } for i, asn in enumerate(asn for handle, asn in csv_reader(sys.argv[2] if len(sys.argv) > 2 else "asns.csv", columns = 2)): - print "AS.%d = %s" % (i, asn) + print "AS.%d = %s" % (i, asn) print '''\ @@ -62,5 +62,5 @@ print '''\ ''' for i, prefix in enumerate(prefix for handle, prefix in csv_reader(sys.argv[3] if len(sys.argv) > 2 else "prefixes.csv", columns = 2)): - v = 6 if ":" in prefix else 4 - print "IPv%d.%d = %s" % (v, i, prefix) + v = 6 if ":" in prefix else 4 + print "IPv%d.%d = %s" % (v, i, prefix) diff --git a/potpourri/translate-handles.py b/potpourri/translate-handles.py index 49848277..124604e6 100644 --- a/potpourri/translate-handles.py +++ b/potpourri/translate-handles.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2010-2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -39,11 +39,11 @@ translations = dict((src, dst) for src, dst in csv_reader("translations.csv", co for filename in sys.argv[1:]: - f = csv_writer(filename) + f = csv_writer(filename) - for cols in csv_reader(filename): - if cols[0] in translations: - cols[0] = translations[cols[0]] - f.writerow(cols) + for cols in csv_reader(filename): + if cols[0] in translations: + cols[0] = translations[cols[0]] + f.writerow(cols) - f.close() + f.close() diff --git a/potpourri/upgrade-add-ghostbusters.py b/potpourri/upgrade-add-ghostbusters.py index 2548487c..2370e959 100644 --- a/potpourri/upgrade-add-ghostbusters.py +++ b/potpourri/upgrade-add-ghostbusters.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2009--2011 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -24,24 +24,24 @@ import getopt, sys, rpki.config, warnings from rpki.mysql_import import MySQLdb def fix(name, *statements): - db = MySQLdb.connect(db = cfg.get("sql-database", section = name), - user = cfg.get("sql-username", section = name), - passwd = cfg.get("sql-password", section = name)) - cur = db.cursor() - for statement in statements: - cur.execute(statement) - db.commit() - db.close() + db = MySQLdb.connect(db = cfg.get("sql-database", section = name), + user = cfg.get("sql-username", section = name), + passwd = cfg.get("sql-password", section = name)) + cur = db.cursor() + for statement in statements: + cur.execute(statement) + db.commit() + db.close() cfg_file = None opts, argv = getopt.getopt(sys.argv[1:], "c:h?", ["config=", "help"]) for o, a in opts: - if o in ("-h", "--help", "-?"): - print __doc__ - sys.exit(0) - if o in ("-c", "--config"): - cfg_file = a + if o in ("-h", "--help", "-?"): + print __doc__ + sys.exit(0) + if o in ("-c", "--config"): + cfg_file = a cfg = rpki.config.parser(filename = cfg_file, section = "myrpki") diff --git a/potpourri/validation-status-sql.py b/potpourri/validation-status-sql.py index fc52e64b..646d7d9b 100755 --- a/potpourri/validation-status-sql.py +++ b/potpourri/validation-status-sql.py @@ -34,186 +34,185 @@ import subprocess class Parser(object): - @staticmethod - def main(): - parser = argparse.ArgumentParser( - description = __doc__, - formatter_class = argparse.ArgumentDefaultsHelpFormatter) - group = parser.add_mutually_exclusive_group(required = True) - group.add_argument("--mailbox", "--mb", - help = "Maildir mailbox containing rcynic XML output") - group.add_argument("--tarballs", - help = "directory tree of tar files containing containing rcynic XML output") - parser.add_argument("--database", "--db", - default = "validation-status-sql.db", - help = "SQLite3 database") - parser.add_argument("--path-within-tarball", - default = "var/rcynic/data/rcynic.xml", - help = "rcynic.xml path name within tarball(s)") - parser.add_argument("--tar-extensions", nargs = "+", - default = ".tar .tar.gz .tgz .tar.bz2 .tbz .tar.xz .txz".split(), - help = "extensions to recognize as indicating tar files") - args = parser.parse_args() - if args.mailbox: - ParserMailbox(args) - else: - ParserTarball(args) - - def __init__(self, args): - self.args = args - self.init_sql() - self.init_hook() - self.index1() - self.parsed = 1 - for self.current, self.iterval in enumerate(self.iterator, 1): - self.parse_xml() - if self.parsed > 1: - sys.stderr.write("\n") - self.index2() - self.db.close() - - - def init_sql(self): - creating = not os.path.exists(self.args.database) - self.db = sqlite3.connect(self.args.database) - self.db.text_factory = str - self.db.executescript(''' - PRAGMA foreign_keys = off; - PRAGMA synchronous = off; - PRAGMA count_changes = off; - ''') - - if creating: - self.db.executescript(''' - CREATE TABLE sessions ( - session_id INTEGER PRIMARY KEY NOT NULL, - session DATETIME NOT NULL, - handle TEXT NOT NULL - ); - - CREATE TABLE uris ( - uri_id INTEGER PRIMARY KEY NOT NULL, - uri TEXT NOT NULL - ); - - CREATE TABLE codes ( - code_id INTEGER PRIMARY KEY NOT NULL, - code TEXT NOT NULL - ); - - CREATE TABLE generations ( - generation_id INTEGER PRIMARY KEY NOT NULL, - generation TEXT NOT NULL - ); - - CREATE TABLE events ( - id INTEGER PRIMARY KEY NOT NULL, - timestamp DATETIME NOT NULL, - session_id INTEGER NOT NULL REFERENCES sessions (session_id) ON DELETE RESTRICT ON UPDATE RESTRICT, - generation_id INTEGER NOT NULL REFERENCES generations (generation_id) ON DELETE RESTRICT ON UPDATE RESTRICT, - code_id INTEGER NOT NULL REFERENCES codes (code_id) ON DELETE RESTRICT ON UPDATE RESTRICT, - uri_id INTEGER NOT NULL REFERENCES uris (uri_id) ON DELETE RESTRICT ON UPDATE RESTRICT - ); - - CREATE VIEW status AS - SELECT id, handle, session, timestamp, generation, code, uri - FROM events - NATURAL JOIN sessions - NATURAL JOIN uris - NATURAL JOIN codes - NATURAL JOIN generations; - ''') - - - def index1(self): - self.db.executescript(''' - CREATE UNIQUE INDEX IF NOT EXISTS sessions_index ON sessions (session); - CREATE UNIQUE INDEX IF NOT EXISTS handles_index ON sessions (handle); - CREATE UNIQUE INDEX IF NOT EXISTS uris_index ON uris (uri); - CREATE UNIQUE INDEX IF NOT EXISTS codes_index ON codes (code); - CREATE UNIQUE INDEX IF NOT EXISTS generations_index ON generations (generation); - ''') - - - def index2(self): - self.db.executescript(''' - CREATE UNIQUE INDEX IF NOT EXISTS events_index ON events (uri_id, timestamp, code_id, generation_id); - ''') - - - def string_id(self, table, value): - field = table.rstrip("s") - try: - return self.db.execute("SELECT %s_id FROM %s WHERE %s = ?" % (field, table, field), (value,)).fetchone()[0] - except: - return self.db.execute("INSERT INTO %s (%s) VALUES (?)" % (table, field), (value,)).lastrowid - - - def parse_xml(self): - sys.stderr.write("\r%s %d/%d/%d...%s " % ("|\\-/"[self.current & 3], - self.current, self.parsed, self.total, self.handle)) - if self.db.execute("SELECT handle FROM sessions WHERE handle = ?", (self.handle,)).fetchone(): - return - xml = self.read_xml() - with self.db: - session_id = self.db.execute("INSERT INTO sessions (session, handle) VALUES (strftime('%s', ?), ?)", - (xml.get("date"), self.handle)).lastrowid - self.db.executemany("INSERT INTO events (session_id, timestamp, generation_id, code_id, uri_id) " - "VALUES (?, strftime('%s', ?), ?, ?, ?)", - ((session_id, - x.get("timestamp"), - self.string_id("generations", x.get("generation", "none")), - self.string_id("codes", x.get("status")), - self.string_id("uris", x.text.strip())) - for x in xml.findall("validation_status"))) - self.parsed += 1 + @staticmethod + def main(): + parser = argparse.ArgumentParser( + description = __doc__, + formatter_class = argparse.ArgumentDefaultsHelpFormatter) + group = parser.add_mutually_exclusive_group(required = True) + group.add_argument("--mailbox", "--mb", + help = "Maildir mailbox containing rcynic XML output") + group.add_argument("--tarballs", + help = "directory tree of tar files containing containing rcynic XML output") + parser.add_argument("--database", "--db", + default = "validation-status-sql.db", + help = "SQLite3 database") + parser.add_argument("--path-within-tarball", + default = "var/rcynic/data/rcynic.xml", + help = "rcynic.xml path name within tarball(s)") + parser.add_argument("--tar-extensions", nargs = "+", + default = ".tar .tar.gz .tgz .tar.bz2 .tbz .tar.xz .txz".split(), + help = "extensions to recognize as indicating tar files") + args = parser.parse_args() + if args.mailbox: + ParserMailbox(args) + else: + ParserTarball(args) + + def __init__(self, args): + self.args = args + self.init_sql() + self.init_hook() + self.index1() + self.parsed = 1 + for self.current, self.iterval in enumerate(self.iterator, 1): + self.parse_xml() + if self.parsed > 1: + sys.stderr.write("\n") + self.index2() + self.db.close() + + + def init_sql(self): + creating = not os.path.exists(self.args.database) + self.db = sqlite3.connect(self.args.database) + self.db.text_factory = str + self.db.executescript(''' + PRAGMA foreign_keys = off; + PRAGMA synchronous = off; + PRAGMA count_changes = off; + ''') + + if creating: + self.db.executescript(''' + CREATE TABLE sessions ( + session_id INTEGER PRIMARY KEY NOT NULL, + session DATETIME NOT NULL, + handle TEXT NOT NULL + ); + + CREATE TABLE uris ( + uri_id INTEGER PRIMARY KEY NOT NULL, + uri TEXT NOT NULL + ); + + CREATE TABLE codes ( + code_id INTEGER PRIMARY KEY NOT NULL, + code TEXT NOT NULL + ); + + CREATE TABLE generations ( + generation_id INTEGER PRIMARY KEY NOT NULL, + generation TEXT NOT NULL + ); + + CREATE TABLE events ( + id INTEGER PRIMARY KEY NOT NULL, + timestamp DATETIME NOT NULL, + session_id INTEGER NOT NULL REFERENCES sessions (session_id) ON DELETE RESTRICT ON UPDATE RESTRICT, + generation_id INTEGER NOT NULL REFERENCES generations (generation_id) ON DELETE RESTRICT ON UPDATE RESTRICT, + code_id INTEGER NOT NULL REFERENCES codes (code_id) ON DELETE RESTRICT ON UPDATE RESTRICT, + uri_id INTEGER NOT NULL REFERENCES uris (uri_id) ON DELETE RESTRICT ON UPDATE RESTRICT + ); + + CREATE VIEW status AS + SELECT id, handle, session, timestamp, generation, code, uri + FROM events + NATURAL JOIN sessions + NATURAL JOIN uris + NATURAL JOIN codes + NATURAL JOIN generations; + ''') + + + def index1(self): + self.db.executescript(''' + CREATE UNIQUE INDEX IF NOT EXISTS sessions_index ON sessions (session); + CREATE UNIQUE INDEX IF NOT EXISTS handles_index ON sessions (handle); + CREATE UNIQUE INDEX IF NOT EXISTS uris_index ON uris (uri); + CREATE UNIQUE INDEX IF NOT EXISTS codes_index ON codes (code); + CREATE UNIQUE INDEX IF NOT EXISTS generations_index ON generations (generation); + ''') + + + def index2(self): + self.db.executescript(''' + CREATE UNIQUE INDEX IF NOT EXISTS events_index ON events (uri_id, timestamp, code_id, generation_id); + ''') + + + def string_id(self, table, value): + field = table.rstrip("s") + try: + return self.db.execute("SELECT %s_id FROM %s WHERE %s = ?" % (field, table, field), (value,)).fetchone()[0] + except: + return self.db.execute("INSERT INTO %s (%s) VALUES (?)" % (table, field), (value,)).lastrowid + + + def parse_xml(self): + sys.stderr.write("\r%s %d/%d/%d...%s " % ("|\\-/"[self.current & 3], + self.current, self.parsed, self.total, self.handle)) + if self.db.execute("SELECT handle FROM sessions WHERE handle = ?", (self.handle,)).fetchone(): + return + xml = self.read_xml() + with self.db: + session_id = self.db.execute("INSERT INTO sessions (session, handle) VALUES (strftime('%s', ?), ?)", + (xml.get("date"), self.handle)).lastrowid + self.db.executemany("INSERT INTO events (session_id, timestamp, generation_id, code_id, uri_id) " + "VALUES (?, strftime('%s', ?), ?, ?, ?)", + ((session_id, + x.get("timestamp"), + self.string_id("generations", x.get("generation", "none")), + self.string_id("codes", x.get("status")), + self.string_id("uris", x.text.strip())) + for x in xml.findall("validation_status"))) + self.parsed += 1 class ParserTarball(Parser): - def init_hook(self): - self.total = 0 - for fn in self.iter_tarball_names(): - self.total += 1 - self.iterator = self.iter_tarball_names() + def init_hook(self): + self.total = 0 + for fn in self.iter_tarball_names(): + self.total += 1 + self.iterator = self.iter_tarball_names() - @property - def handle(self): - return self.iterval + @property + def handle(self): + return self.iterval - def read_xml(self): - return lxml.etree.ElementTree( - file = subprocess.Popen(("tar", "Oxf", self.iterval, self.args.path_within_tarball), - stdout = subprocess.PIPE).stdout).getroot() + def read_xml(self): + return lxml.etree.ElementTree( + file = subprocess.Popen(("tar", "Oxf", self.iterval, self.args.path_within_tarball), + stdout = subprocess.PIPE).stdout).getroot() - def iter_tarball_names(self): - if os.path.isdir(self.args.tarballs): - for root, dirs, files in os.walk(self.args.tarballs): - for fn in files: - if any(fn.endswith(ext) for ext in self.args.tar_extensions): - yield os.path.join(root, fn) - else: - yield self.args.tarballs + def iter_tarball_names(self): + if os.path.isdir(self.args.tarballs): + for root, dirs, files in os.walk(self.args.tarballs): + for fn in files: + if any(fn.endswith(ext) for ext in self.args.tar_extensions): + yield os.path.join(root, fn) + else: + yield self.args.tarballs class ParserMailbox(Parser): - def init_hook(self): - self.mb = mailbox.Maildir(self.args.mailbox, factory = None, create = False) - self.total = len(self.mb) - self.iterator = self.mb.iterkeys() + def init_hook(self): + self.mb = mailbox.Maildir(self.args.mailbox, factory = None, create = False) + self.total = len(self.mb) + self.iterator = self.mb.iterkeys() - @property - def handle(self): - return self.mb[self.iterval].get("Message-ID") + @property + def handle(self): + return self.mb[self.iterval].get("Message-ID") - def read_xml(self): - return lxml.etree.XML(self.mb[self.iterval].get_payload()) + def read_xml(self): + return lxml.etree.XML(self.mb[self.iterval].get_payload()) if __name__ == "__main__": - try: - Parser.main() - except KeyboardInterrupt: - pass - + try: + Parser.main() + except KeyboardInterrupt: + pass diff --git a/potpourri/whack-ripe-asns.py b/potpourri/whack-ripe-asns.py index 9c702271..ed4a6451 100644 --- a/potpourri/whack-ripe-asns.py +++ b/potpourri/whack-ripe-asns.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2010 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -44,14 +44,14 @@ sorter = subprocess.Popen(("sort", "-T.", "-n"), stdout = subprocess.PIPE) for line in sys.stdin: - handle, asn = line.split() + handle, asn = line.split() - if "-" in asn: - range_min, range_max = asn.split("-") - else: - range_min, range_max = asn, asn + if "-" in asn: + range_min, range_max = asn.split("-") + else: + range_min, range_max = asn, asn - sorter.stdin.write("%d %d\n" % (long(range_min), long(range_max))) + sorter.stdin.write("%d %d\n" % (long(range_min), long(range_max))) sorter.stdin.close() @@ -59,22 +59,22 @@ prev_min = None prev_max = None def show(): - if prev_min and prev_max: - sys.stdout.write("x\t%s-%s\n" % (prev_min, prev_max)) + if prev_min and prev_max: + sys.stdout.write("x\t%s-%s\n" % (prev_min, prev_max)) for line in sorter.stdout: - this_min, this_max = line.split() - this_min = long(this_min) - this_max = long(this_max) - - if prev_min and prev_max and prev_max + 1 >= this_min: - prev_min = min(prev_min, this_min) - prev_max = max(prev_max, this_max) - - else: - show() - prev_min = this_min - prev_max = this_max + this_min, this_max = line.split() + this_min = long(this_min) + this_max = long(this_max) + + if prev_min and prev_max and prev_max + 1 >= this_min: + prev_min = min(prev_min, this_min) + prev_max = max(prev_max, this_max) + + else: + show() + prev_min = this_min + prev_max = this_max show() diff --git a/potpourri/whack-ripe-prefixes.py b/potpourri/whack-ripe-prefixes.py index 52ea3f18..b3d9c39d 100644 --- a/potpourri/whack-ripe-prefixes.py +++ b/potpourri/whack-ripe-prefixes.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2010 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -49,21 +49,21 @@ sorter = subprocess.Popen(("sort", "-T.", "-n"), stdout = subprocess.PIPE) for line in sys.stdin: - handle, prefix = line.split() + handle, prefix = line.split() - if "-" in prefix: - range_min, range_max = prefix.split("-") - range_min = rpki.ipaddrs.parse(range_min) - range_max = rpki.ipaddrs.parse(range_max) + if "-" in prefix: + range_min, range_max = prefix.split("-") + range_min = rpki.ipaddrs.parse(range_min) + range_max = rpki.ipaddrs.parse(range_max) - else: - address, length = prefix.split("/") - address = rpki.ipaddrs.parse(address) - mask = (1L << (address.bits - int(length))) - 1 - range_min = address & ~mask - range_max = address | mask + else: + address, length = prefix.split("/") + address = rpki.ipaddrs.parse(address) + mask = (1L << (address.bits - int(length))) - 1 + range_min = address & ~mask + range_max = address | mask - sorter.stdin.write("%d %d\n" % (long(range_min), long(range_max))) + sorter.stdin.write("%d %d\n" % (long(range_min), long(range_max))) sorter.stdin.close() @@ -71,28 +71,28 @@ prev_min = None prev_max = None def address(number): - if number > 0xffffffff: - return rpki.ipaddrs.v6addr(number) - else: - return rpki.ipaddrs.v4addr(number) + if number > 0xffffffff: + return rpki.ipaddrs.v6addr(number) + else: + return rpki.ipaddrs.v4addr(number) def show(): - if prev_min and prev_max: - sys.stdout.write("x\t%s-%s\n" % (address(prev_min), address(prev_max))) + if prev_min and prev_max: + sys.stdout.write("x\t%s-%s\n" % (address(prev_min), address(prev_max))) for line in sorter.stdout: - this_min, this_max = line.split() - this_min = long(this_min) - this_max = long(this_max) - - if prev_min and prev_max and prev_max + 1 >= this_min: - prev_min = min(prev_min, this_min) - prev_max = max(prev_max, this_max) - - else: - show() - prev_min = this_min - prev_max = this_max + this_min, this_max = line.split() + this_min = long(this_min) + this_max = long(this_max) + + if prev_min and prev_max and prev_max + 1 >= this_min: + prev_min = min(prev_min, this_min) + prev_max = max(prev_max, this_max) + + else: + show() + prev_min = this_min + prev_max = this_max show() diff --git a/potpourri/x509-dot.py b/potpourri/x509-dot.py index 42e1543a..493199fd 100644 --- a/potpourri/x509-dot.py +++ b/potpourri/x509-dot.py @@ -36,123 +36,123 @@ import rpki.POW, sys, glob, os class x509(object): - ski = None - aki = None + ski = None + aki = None - show_file = False - show_ski = False - show_aki = False - show_issuer = True - show_subject = True + show_file = False + show_ski = False + show_aki = False + show_issuer = True + show_subject = True - cn_only = True + cn_only = True - subjects = {} + subjects = {} - def __init__(self, filename): + def __init__(self, filename): - while filename.startswith("./"): - filename = filename[2:] + while filename.startswith("./"): + filename = filename[2:] - self.filename = filename + self.filename = filename - f = open(filename, "rb") - text = f.read() - f.close() + f = open(filename, "rb") + text = f.read() + f.close() - if "-----BEGIN" in text: - self.pow = rpki.POW.X509.pemRead(text) - else: - self.pow = rpki.POW.X509.derRead(text) + if "-----BEGIN" in text: + self.pow = rpki.POW.X509.pemRead(text) + else: + self.pow = rpki.POW.X509.derRead(text) - try: - self.ski = ":".join(["%02X" % ord(i) for i in self.pow.getSKI()]) - except: - pass + try: + self.ski = ":".join(["%02X" % ord(i) for i in self.pow.getSKI()]) + except: + pass - try: - self.aki = ":".join(["%02X" % ord(i) for i in self.pow.getAKI()]) - except: - pass + try: + self.aki = ":".join(["%02X" % ord(i) for i in self.pow.getAKI()]) + except: + pass - self.subject = self.canonize(self.pow.getSubject()) - self.issuer = self.canonize(self.pow.getIssuer()) + self.subject = self.canonize(self.pow.getSubject()) + self.issuer = self.canonize(self.pow.getIssuer()) - if self.subject in self.subjects: - self.subjects[self.subject].append(self) - else: - self.subjects[self.subject] = [self] + if self.subject in self.subjects: + self.subjects[self.subject].append(self) + else: + self.subjects[self.subject] = [self] - def canonize(self, name): + def canonize(self, name): - # Probably should just use rpki.x509.X501DN class here. + # Probably should just use rpki.x509.X501DN class here. - try: - if self.cn_only and name[0][0][0] == "2.5.4.3": - return name[0][0][1] - except: - pass + try: + if self.cn_only and name[0][0][0] == "2.5.4.3": + return name[0][0][1] + except: + pass - return name + return name - def set_node(self, node): + def set_node(self, node): - self.node = node + self.node = node - def dot(self): + def dot(self): - label = [] + label = [] - if self.show_issuer: - label.append(("Issuer", self.issuer)) + if self.show_issuer: + label.append(("Issuer", self.issuer)) - if self.show_subject: - label.append(("Subject", self.subject)) + if self.show_subject: + label.append(("Subject", self.subject)) - if self.show_file: - label.append(("File", self.filename)) + if self.show_file: + label.append(("File", self.filename)) - if self.show_aki: - label.append(("AKI", self.aki)) + if self.show_aki: + label.append(("AKI", self.aki)) - if self.show_ski: - label.append(("SKI", self.ski)) + if self.show_ski: + label.append(("SKI", self.ski)) - print "#", repr(label) + print "#", repr(label) - if len(label) > 1: - print '%s [shape = record, label = "{%s}"];' % (self.node, "|".join("{%s|%s}" % (x, y) for x, y in label if y is not None)) - else: - print '%s [label = "%s"];' % (self.node, label[0][1]) + if len(label) > 1: + print '%s [shape = record, label = "{%s}"];' % (self.node, "|".join("{%s|%s}" % (x, y) for x, y in label if y is not None)) + else: + print '%s [label = "%s"];' % (self.node, label[0][1]) - for issuer in self.subjects.get(self.issuer, ()): + for issuer in self.subjects.get(self.issuer, ()): - if issuer is self: - print "# Issuer is self" - issuer = None + if issuer is self: + print "# Issuer is self" + issuer = None - if issuer is not None and self.aki is not None and self.ski is not None and self.aki == self.ski: - print "# Self-signed" - issuer = None + if issuer is not None and self.aki is not None and self.ski is not None and self.aki == self.ski: + print "# Self-signed" + issuer = None - if issuer is not None and self.aki is not None and issuer.ski is not None and self.aki != issuer.ski: - print "# AKI does not match issuer SKI" - issuer = None + if issuer is not None and self.aki is not None and issuer.ski is not None and self.aki != issuer.ski: + print "# AKI does not match issuer SKI" + issuer = None - if issuer is not None: - print "%s -> %s;" % (issuer.node, self.node) + if issuer is not None: + print "%s -> %s;" % (issuer.node, self.node) - print + print certs = [] for topdir in sys.argv[1:] or ["."]: - for dirpath, dirnames, filenames in os.walk(topdir): - certs += [x509(dirpath + "/" + filename) for filename in filenames if filename.endswith(".cer")] + for dirpath, dirnames, filenames in os.walk(topdir): + certs += [x509(dirpath + "/" + filename) for filename in filenames if filename.endswith(".cer")] for i, cert in enumerate(certs): - cert.set_node("cert_%d" % i) + cert.set_node("cert_%d" % i) print """\ digraph certificates { @@ -165,6 +165,6 @@ ratio = fill; """ for cert in certs: - cert.dot() + cert.dot() print "}" diff --git a/rp/rcynic/rcynic-cron b/rp/rcynic/rcynic-cron index 53bfea9f..3d38726c 100755 --- a/rp/rcynic/rcynic-cron +++ b/rp/rcynic/rcynic-cron @@ -35,34 +35,34 @@ import argparse import rpki.autoconf def run(*cmd, **kwargs): - chroot_this = kwargs.pop("chroot_this", False) - cwd = kwargs.pop("cwd", None) - pid = os.fork() - if pid == 0: - if chroot_this: - os.chdir(rpki.autoconf.RCYNIC_DIR) - elif cwd is not None: - os.chdir(cwd) - if we_are_root: - os.initgroups(pw.pw_name, pw.pw_gid) - if chroot_this: - os.chroot(rpki.autoconf.RCYNIC_DIR) - if we_are_root: - os.setgid(pw.pw_gid) - os.setuid(pw.pw_uid) - os.closerange(3, os.sysconf("SC_OPEN_MAX")) - os.execvp(cmd[0], cmd) - os._exit(1) - else: - status = os.waitpid(pid, 0)[1] - if status == 0: - return - elif os.WIFSIGNALED(status): - sys.exit("Process %s exited with signal %s" % (" ".join(cmd), os.WTERMSIG(status))) - elif os.WIFEXITED(status): - sys.exit("Program %s exited with status %s" % (" ".join(cmd), os.WEXITSTATUS(status))) + chroot_this = kwargs.pop("chroot_this", False) + cwd = kwargs.pop("cwd", None) + pid = os.fork() + if pid == 0: + if chroot_this: + os.chdir(rpki.autoconf.RCYNIC_DIR) + elif cwd is not None: + os.chdir(cwd) + if we_are_root: + os.initgroups(pw.pw_name, pw.pw_gid) + if chroot_this: + os.chroot(rpki.autoconf.RCYNIC_DIR) + if we_are_root: + os.setgid(pw.pw_gid) + os.setuid(pw.pw_uid) + os.closerange(3, os.sysconf("SC_OPEN_MAX")) + os.execvp(cmd[0], cmd) + os._exit(1) else: - sys.exit("Program %s exited for unknown reason %s" % (" ".join(cmd), status)) + status = os.waitpid(pid, 0)[1] + if status == 0: + return + elif os.WIFSIGNALED(status): + sys.exit("Process %s exited with signal %s" % (" ".join(cmd), os.WTERMSIG(status))) + elif os.WIFEXITED(status): + sys.exit("Program %s exited with status %s" % (" ".join(cmd), os.WEXITSTATUS(status))) + else: + sys.exit("Program %s exited for unknown reason %s" % (" ".join(cmd), status)) parser = argparse.ArgumentParser(description = __doc__) parser.add_argument("--chroot", action = "store_true", help = "run chrooted") @@ -71,29 +71,29 @@ args = parser.parse_args() we_are_root = os.getuid() == 0 if args.chroot and not we_are_root: - sys.exit("Only root can --chroot") + sys.exit("Only root can --chroot") try: - pw = pwd.getpwnam(rpki.autoconf.RCYNIC_USER) + pw = pwd.getpwnam(rpki.autoconf.RCYNIC_USER) except KeyError: - sys.exit("Could not find passwd entry for user %s" % rpki.autoconf.RCYNIC_USER) + sys.exit("Could not find passwd entry for user %s" % rpki.autoconf.RCYNIC_USER) try: - lock = os.open(os.path.join(rpki.autoconf.RCYNIC_DIR, "data/lock"), - os.O_RDONLY | os.O_CREAT | os.O_NONBLOCK, 0666) - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - if we_are_root: - os.fchown(lock, pw.pw_uid, pw.pw_gid) + lock = os.open(os.path.join(rpki.autoconf.RCYNIC_DIR, "data/lock"), + os.O_RDONLY | os.O_CREAT | os.O_NONBLOCK, 0666) + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + if we_are_root: + os.fchown(lock, pw.pw_uid, pw.pw_gid) except (IOError, OSError), e: - if e.errno == errno.EAGAIN: - sys.exit(0) # Another instance of this script is already running, exit silently - else: - sys.exit("Error %r opening lock %r" % (e.strerror, os.path.join(rpki.autoconf.RCYNIC_DIR, "data/lock"))) + if e.errno == errno.EAGAIN: + sys.exit(0) # Another instance of this script is already running, exit silently + else: + sys.exit("Error %r opening lock %r" % (e.strerror, os.path.join(rpki.autoconf.RCYNIC_DIR, "data/lock"))) if args.chroot: - run("/bin/rcynic", "-c", "/etc/rcynic.conf", chroot_this = True) + run("/bin/rcynic", "-c", "/etc/rcynic.conf", chroot_this = True) else: - run(os.path.join(rpki.autoconf.bindir, "rcynic"), "-c", os.path.join(rpki.autoconf.sysconfdir, "rcynic.conf")) + run(os.path.join(rpki.autoconf.bindir, "rcynic"), "-c", os.path.join(rpki.autoconf.sysconfdir, "rcynic.conf")) run(os.path.join(rpki.autoconf.bindir, "rpki-rtr"), "cronjob", @@ -102,9 +102,9 @@ run(os.path.join(rpki.autoconf.bindir, "rpki-rtr"), prog = os.path.join(rpki.autoconf.libexecdir, "rpkigui-rcynic") if os.path.exists(prog): - run(prog) + run(prog) if rpki.autoconf.RCYNIC_HTML_DIR and os.path.exists(os.path.dirname(rpki.autoconf.RCYNIC_HTML_DIR)): - run(os.path.join(rpki.autoconf.bindir, "rcynic-html"), - os.path.join(rpki.autoconf.RCYNIC_DIR, "data/rcynic.xml"), - rpki.autoconf.RCYNIC_HTML_DIR) + run(os.path.join(rpki.autoconf.bindir, "rcynic-html"), + os.path.join(rpki.autoconf.RCYNIC_DIR, "data/rcynic.xml"), + rpki.autoconf.RCYNIC_HTML_DIR) diff --git a/rp/rcynic/rcynic-html b/rp/rcynic/rcynic-html index ef566440..012bccad 100755 --- a/rp/rcynic/rcynic-html +++ b/rp/rcynic/rcynic-html @@ -32,361 +32,361 @@ import copy import rpki.autoconf try: - from lxml.etree import (ElementTree, Element, SubElement, Comment) + from lxml.etree import (ElementTree, Element, SubElement, Comment) except ImportError: - from xml.etree.ElementTree import (ElementTree, Element, SubElement, Comment) + from xml.etree.ElementTree import (ElementTree, Element, SubElement, Comment) session = None args = None def parse_options(): - global args - - parser = argparse.ArgumentParser(description = __doc__) - parser.add_argument("--refresh", type = int, default = 1800, - help = "refresh interval for generated HTML") - parser.add_argument("--hide-problems", action = "store_true", - help = "don't generate \"problems\" page") - parser.add_argument("--hide-graphs", action = "store_true", - help = "don't generate graphs") - parser.add_argument("--hide-object-counts", action = "store_true", - help = "don't display object counts") - parser.add_argument("--dont-update-rrds", action = "store_true", - help = "don't add new data to RRD databases") - parser.add_argument("--png-height", type = int, default = 190, - help = "height of PNG images") - parser.add_argument("--png-width", type = int, default = 1350, - help = "width of PNG images") - parser.add_argument("--svg-height", type = int, default = 600, - help = "height of SVG images") - parser.add_argument("--svg-width", type = int, default = 1200, - help = "width of SVG images") - parser.add_argument("--eps-height", type = int, default = 0, - help = "height of EPS images") - parser.add_argument("--eps-width", type = int, default = 0, - help = "width of EPS images") - parser.add_argument("--rrdtool-binary", default = rpki.autoconf.RRDTOOL, - help = "location of rrdtool binary") - parser.add_argument("input_file", type = argparse.FileType("r"), - help = "XML input file") - parser.add_argument("output_directory", - help = "output directory") - args = parser.parse_args() + global args + + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument("--refresh", type = int, default = 1800, + help = "refresh interval for generated HTML") + parser.add_argument("--hide-problems", action = "store_true", + help = "don't generate \"problems\" page") + parser.add_argument("--hide-graphs", action = "store_true", + help = "don't generate graphs") + parser.add_argument("--hide-object-counts", action = "store_true", + help = "don't display object counts") + parser.add_argument("--dont-update-rrds", action = "store_true", + help = "don't add new data to RRD databases") + parser.add_argument("--png-height", type = int, default = 190, + help = "height of PNG images") + parser.add_argument("--png-width", type = int, default = 1350, + help = "width of PNG images") + parser.add_argument("--svg-height", type = int, default = 600, + help = "height of SVG images") + parser.add_argument("--svg-width", type = int, default = 1200, + help = "width of SVG images") + parser.add_argument("--eps-height", type = int, default = 0, + help = "height of EPS images") + parser.add_argument("--eps-width", type = int, default = 0, + help = "width of EPS images") + parser.add_argument("--rrdtool-binary", default = rpki.autoconf.RRDTOOL, + help = "location of rrdtool binary") + parser.add_argument("input_file", type = argparse.FileType("r"), + help = "XML input file") + parser.add_argument("output_directory", + help = "output directory") + args = parser.parse_args() def parse_utc(s): - return int(time.mktime(time.strptime(s, "%Y-%m-%dT%H:%M:%SZ"))) + return int(time.mktime(time.strptime(s, "%Y-%m-%dT%H:%M:%SZ"))) class Label(object): - moods = ["bad", "warn", "good"] + moods = ["bad", "warn", "good"] - def __init__(self, elt): - self.code = elt.tag - self.mood = elt.get("kind") - self.text = elt.text.strip() - self.count = 0 + def __init__(self, elt): + self.code = elt.tag + self.mood = elt.get("kind") + self.text = elt.text.strip() + self.count = 0 - def get_count(self): - return self.count + def get_count(self): + return self.count - @property - def sort_key(self): - try: - return self.moods.index(self.mood) - except ValueError: - return len(self.moods) + @property + def sort_key(self): + try: + return self.moods.index(self.mood) + except ValueError: + return len(self.moods) class Validation_Status(object): - def __init__(self, elt, label_map): - self.uri = elt.text.strip() - self.timestamp = elt.get("timestamp") - self.generation = elt.get("generation") - self.hostname = urlparse.urlparse(self.uri).hostname or "[None]" - self.fn2 = os.path.splitext(self.uri)[1] or None if self.generation else None - self.label = label_map[elt.get("status")] + def __init__(self, elt, label_map): + self.uri = elt.text.strip() + self.timestamp = elt.get("timestamp") + self.generation = elt.get("generation") + self.hostname = urlparse.urlparse(self.uri).hostname or "[None]" + self.fn2 = os.path.splitext(self.uri)[1] or None if self.generation else None + self.label = label_map[elt.get("status")] - def sort_key(self): - return (self.label.sort_key, self.timestamp, self.hostname, self.fn2, self.generation) + def sort_key(self): + return (self.label.sort_key, self.timestamp, self.hostname, self.fn2, self.generation) - @property - def code(self): - return self.label.code + @property + def code(self): + return self.label.code - @property - def mood(self): - return self.label.mood + @property + def mood(self): + return self.label.mood - @property - def accepted(self): - return self.label.code == "object_accepted" + @property + def accepted(self): + return self.label.code == "object_accepted" - @property - def rejected(self): - return self.label.code == "object_rejected" + @property + def rejected(self): + return self.label.code == "object_rejected" - @property - def is_current(self): - return self.generation == "current" + @property + def is_current(self): + return self.generation == "current" - @property - def is_backup(self): - return self.generation == "backup" + @property + def is_backup(self): + return self.generation == "backup" - @property - def is_problem(self): - return self.label.mood != "good" + @property + def is_problem(self): + return self.label.mood != "good" - @property - def is_connection_problem(self): - return self.label.mood != "good" and self.label.code.startswith("rsync_transfer_") + @property + def is_connection_problem(self): + return self.label.mood != "good" and self.label.code.startswith("rsync_transfer_") - @property - def is_object_problem(self): - return self.label.mood != "good" and not self.label.code.startswith("rsync_transfer_") + @property + def is_object_problem(self): + return self.label.mood != "good" and not self.label.code.startswith("rsync_transfer_") - @property - def is_connection_detail(self): - return self.label.code.startswith("rsync_transfer_") + @property + def is_connection_detail(self): + return self.label.code.startswith("rsync_transfer_") - @property - def is_object_detail(self): - return not self.label.code.startswith("rsync_transfer_") + @property + def is_object_detail(self): + return not self.label.code.startswith("rsync_transfer_") class Problem_Mixin(object): - @property - def connection_problems(self): - result = [v for v in self.validation_status if v.is_connection_problem] - result.sort(key = Validation_Status.sort_key) - return result + @property + def connection_problems(self): + result = [v for v in self.validation_status if v.is_connection_problem] + result.sort(key = Validation_Status.sort_key) + return result - @property - def object_problems(self): - result = [v for v in self.validation_status if v.is_object_problem] - result.sort(key = Validation_Status.sort_key) - return result + @property + def object_problems(self): + result = [v for v in self.validation_status if v.is_object_problem] + result.sort(key = Validation_Status.sort_key) + return result class Host(Problem_Mixin): - def __init__(self, hostname, timestamp): - self.hostname = hostname - self.timestamp = timestamp - self.elapsed = 0 - self.connections = 0 - self.failures = 0 - self.uris = set() - self.graph = None - self.counters = {} - self.totals = {} - self.validation_status = [] - - def add_connection(self, elt): - self.elapsed += parse_utc(elt.get("finished")) - parse_utc(elt.get("started")) - self.connections += 1 - if elt.get("error") is not None: - self.failures += 1 - - def add_validation_status(self, v): - self.validation_status.append(v) - if v.generation == "current": - self.uris.add(v.uri) - self.counters[(v.fn2, v.generation, v.label)] = self.get_counter(v.fn2, v.generation, v.label) + 1 - self.totals[v.label] = self.get_total(v.label) + 1 - v.label.count += 1 - - def get_counter(self, fn2, generation, label): - return self.counters.get((fn2, generation, label), 0) - - def get_total(self, label): - return self.totals.get(label, 0) - - @property - def failed(self): - return 1 if self.failures > 0 else 0 - - @property - def objects(self): - return len(self.uris) - - field_table = (("connections", "GAUGE"), - ("objects", "GAUGE"), - ("elapsed", "GAUGE"), - ("failed", "ABSOLUTE")) - - rras = tuple("RRA:AVERAGE:0.5:%s:9600" % steps - for steps in (1, 4, 24)) - - @classmethod - def field_ds_specifiers(cls, heartbeat = 24 * 60 * 60, minimum = 0, maximum = "U"): - return ["DS:%s:%s:%s:%s:%s" % (field[0], field[1], heartbeat, minimum, maximum) - for field in cls.field_table] - - @property - def field_values(self): - return tuple(str(getattr(self, field[0])) for field in self.field_table) - - @classmethod - def field_defs(cls, filebase): - return ["DEF:%s=%s.rrd:%s:AVERAGE" % (field[0], filebase, field[0]) - for field in cls.field_table] - - graph_opts = ( - "--vertical-label", "Sync time (seconds)", - "--right-axis-label", "Objects (count)", - "--lower-limit", "0", - "--right-axis", "1:0", - "--full-size-mode" ) - - graph_cmds = ( - - # Split elapsed into separate data sets, so we can color - # differently to indicate how succesful transfer was. Intent is - # that exactly one of these be defined for every value in elapsed. - - r"CDEF:success=failed,UNKN,elapsed,IF", - r"CDEF:failure=connections,1,EQ,failed,*,elapsed,UNKN,IF", - r"CDEF:partial=connections,1,NE,failed,*,elapsed,UNKN,IF", - - # Show connection timing first, as color-coded semi-transparent - # areas with opaque borders. Intent is to make the colors stand - # out, since they're a major health indicator. Transparency is - # handled via an alpha channel (fourth octet of color code). We - # draw this stuff first so that later lines can overwrite it. - - r"AREA:success#00FF0080:Sync time (success)", - r"AREA:partial#FFA50080:Sync time (partial failure)", - r"AREA:failure#FF000080:Sync time (total failure)", - - r"LINE1:success#00FF00", # Green - r"LINE1:partial#FFA500", # Orange - r"LINE1:failure#FF0000", # Red - - # Now show object counts, as a simple black line. - - r"LINE1:objects#000000:Objects", # Black - - # Add averages over period to chart legend. - - r"VDEF:avg_elapsed=elapsed,AVERAGE", - r"VDEF:avg_connections=connections,AVERAGE", - r"VDEF:avg_objects=objects,AVERAGE", - r"COMMENT:\j", - r"GPRINT:avg_elapsed:Average sync time (seconds)\: %5.2lf", - r"GPRINT:avg_connections:Average connection count\: %5.2lf", - r"GPRINT:avg_objects:Average object count\: %5.2lf" ) - - graph_periods = (("week", "-1w"), - ("month", "-31d"), - ("year", "-1y")) - - def rrd_run(self, cmd): - try: - cmd = [str(i) for i in cmd] - cmd.insert(0, args.rrdtool_binary) - subprocess.check_call(cmd, stdout = open("/dev/null", "w")) - except OSError, e: - sys.exit("Problem running %s, perhaps you need to set --rrdtool-binary? (%s)" % (args.rrdtool_binary, e)) - except subprocess.CalledProcessError, e: - sys.exit("Failure running %s: %s" % (args.rrdtool_binary, e)) - - def rrd_update(self): - filename = os.path.join(args.output_directory, self.hostname) + ".rrd" - if not os.path.exists(filename): - cmd = ["create", filename, "--start", self.timestamp - 1, "--step", "3600"] - cmd.extend(self.field_ds_specifiers()) - cmd.extend(self.rras) - self.rrd_run(cmd) - self.rrd_run(["update", filename, - "%s:%s" % (self.timestamp, ":".join(str(v) for v in self.field_values))]) - - def rrd_graph(self, html): - # pylint: disable=W0622 - filebase = os.path.join(args.output_directory, self.hostname) - formats = [format for format in ("png", "svg", "eps") - if getattr(args, format + "_width") and getattr(args, format + "_height")] - for period, start in self.graph_periods: - for format in formats: - cmds = [ "graph", "%s_%s.%s" % (filebase, period, format), - "--title", "%s last %s" % (self.hostname, period), - "--start", start, - "--width", getattr(args, format + "_width"), - "--height", getattr(args, format + "_height"), - "--imgformat", format.upper() ] - cmds.extend(self.graph_opts) - cmds.extend(self.field_defs(filebase)) - cmds.extend(self.graph_cmds) - self.rrd_run(cmds) - img = Element("img", src = "%s_%s.png" % (self.hostname, period), - width = str(args.png_width), - height = str(args.png_height)) - if self.graph is None: - self.graph = copy.copy(img) - html.BodyElement("h2").text = "%s over last %s" % (self.hostname, period) - html.BodyElement("a", href = "%s_%s_svg.html" % (self.hostname, period)).append(img) - html.BodyElement("br") - svg_html = HTML("%s over last %s" % (self.hostname, period), - "%s_%s_svg" % (self.hostname, period)) - svg_html.BodyElement("img", src = "%s_%s.svg" % (self.hostname, period)) - svg_html.close() + def __init__(self, hostname, timestamp): + self.hostname = hostname + self.timestamp = timestamp + self.elapsed = 0 + self.connections = 0 + self.failures = 0 + self.uris = set() + self.graph = None + self.counters = {} + self.totals = {} + self.validation_status = [] + + def add_connection(self, elt): + self.elapsed += parse_utc(elt.get("finished")) - parse_utc(elt.get("started")) + self.connections += 1 + if elt.get("error") is not None: + self.failures += 1 + + def add_validation_status(self, v): + self.validation_status.append(v) + if v.generation == "current": + self.uris.add(v.uri) + self.counters[(v.fn2, v.generation, v.label)] = self.get_counter(v.fn2, v.generation, v.label) + 1 + self.totals[v.label] = self.get_total(v.label) + 1 + v.label.count += 1 + + def get_counter(self, fn2, generation, label): + return self.counters.get((fn2, generation, label), 0) + + def get_total(self, label): + return self.totals.get(label, 0) + + @property + def failed(self): + return 1 if self.failures > 0 else 0 + + @property + def objects(self): + return len(self.uris) + + field_table = (("connections", "GAUGE"), + ("objects", "GAUGE"), + ("elapsed", "GAUGE"), + ("failed", "ABSOLUTE")) + + rras = tuple("RRA:AVERAGE:0.5:%s:9600" % steps + for steps in (1, 4, 24)) + + @classmethod + def field_ds_specifiers(cls, heartbeat = 24 * 60 * 60, minimum = 0, maximum = "U"): + return ["DS:%s:%s:%s:%s:%s" % (field[0], field[1], heartbeat, minimum, maximum) + for field in cls.field_table] + + @property + def field_values(self): + return tuple(str(getattr(self, field[0])) for field in self.field_table) + + @classmethod + def field_defs(cls, filebase): + return ["DEF:%s=%s.rrd:%s:AVERAGE" % (field[0], filebase, field[0]) + for field in cls.field_table] + + graph_opts = ( + "--vertical-label", "Sync time (seconds)", + "--right-axis-label", "Objects (count)", + "--lower-limit", "0", + "--right-axis", "1:0", + "--full-size-mode" ) + + graph_cmds = ( + + # Split elapsed into separate data sets, so we can color + # differently to indicate how succesful transfer was. Intent is + # that exactly one of these be defined for every value in elapsed. + + r"CDEF:success=failed,UNKN,elapsed,IF", + r"CDEF:failure=connections,1,EQ,failed,*,elapsed,UNKN,IF", + r"CDEF:partial=connections,1,NE,failed,*,elapsed,UNKN,IF", + + # Show connection timing first, as color-coded semi-transparent + # areas with opaque borders. Intent is to make the colors stand + # out, since they're a major health indicator. Transparency is + # handled via an alpha channel (fourth octet of color code). We + # draw this stuff first so that later lines can overwrite it. + + r"AREA:success#00FF0080:Sync time (success)", + r"AREA:partial#FFA50080:Sync time (partial failure)", + r"AREA:failure#FF000080:Sync time (total failure)", + + r"LINE1:success#00FF00", # Green + r"LINE1:partial#FFA500", # Orange + r"LINE1:failure#FF0000", # Red + + # Now show object counts, as a simple black line. + + r"LINE1:objects#000000:Objects", # Black + + # Add averages over period to chart legend. + + r"VDEF:avg_elapsed=elapsed,AVERAGE", + r"VDEF:avg_connections=connections,AVERAGE", + r"VDEF:avg_objects=objects,AVERAGE", + r"COMMENT:\j", + r"GPRINT:avg_elapsed:Average sync time (seconds)\: %5.2lf", + r"GPRINT:avg_connections:Average connection count\: %5.2lf", + r"GPRINT:avg_objects:Average object count\: %5.2lf" ) + + graph_periods = (("week", "-1w"), + ("month", "-31d"), + ("year", "-1y")) + + def rrd_run(self, cmd): + try: + cmd = [str(i) for i in cmd] + cmd.insert(0, args.rrdtool_binary) + subprocess.check_call(cmd, stdout = open("/dev/null", "w")) + except OSError, e: + sys.exit("Problem running %s, perhaps you need to set --rrdtool-binary? (%s)" % (args.rrdtool_binary, e)) + except subprocess.CalledProcessError, e: + sys.exit("Failure running %s: %s" % (args.rrdtool_binary, e)) + + def rrd_update(self): + filename = os.path.join(args.output_directory, self.hostname) + ".rrd" + if not os.path.exists(filename): + cmd = ["create", filename, "--start", self.timestamp - 1, "--step", "3600"] + cmd.extend(self.field_ds_specifiers()) + cmd.extend(self.rras) + self.rrd_run(cmd) + self.rrd_run(["update", filename, + "%s:%s" % (self.timestamp, ":".join(str(v) for v in self.field_values))]) + + def rrd_graph(self, html): + # pylint: disable=W0622 + filebase = os.path.join(args.output_directory, self.hostname) + formats = [format for format in ("png", "svg", "eps") + if getattr(args, format + "_width") and getattr(args, format + "_height")] + for period, start in self.graph_periods: + for format in formats: + cmds = [ "graph", "%s_%s.%s" % (filebase, period, format), + "--title", "%s last %s" % (self.hostname, period), + "--start", start, + "--width", getattr(args, format + "_width"), + "--height", getattr(args, format + "_height"), + "--imgformat", format.upper() ] + cmds.extend(self.graph_opts) + cmds.extend(self.field_defs(filebase)) + cmds.extend(self.graph_cmds) + self.rrd_run(cmds) + img = Element("img", src = "%s_%s.png" % (self.hostname, period), + width = str(args.png_width), + height = str(args.png_height)) + if self.graph is None: + self.graph = copy.copy(img) + html.BodyElement("h2").text = "%s over last %s" % (self.hostname, period) + html.BodyElement("a", href = "%s_%s_svg.html" % (self.hostname, period)).append(img) + html.BodyElement("br") + svg_html = HTML("%s over last %s" % (self.hostname, period), + "%s_%s_svg" % (self.hostname, period)) + svg_html.BodyElement("img", src = "%s_%s.svg" % (self.hostname, period)) + svg_html.close() class Session(Problem_Mixin): - def __init__(self): - self.hosts = {} + def __init__(self): + self.hosts = {} - self.root = ElementTree(file = args.input_file).getroot() + self.root = ElementTree(file = args.input_file).getroot() - self.rcynic_version = self.root.get("rcynic-version") - self.rcynic_date = self.root.get("date") - self.timestamp = parse_utc(self.rcynic_date) + self.rcynic_version = self.root.get("rcynic-version") + self.rcynic_date = self.root.get("date") + self.timestamp = parse_utc(self.rcynic_date) - self.labels = [Label(elt) for elt in self.root.find("labels")] - self.load_validation_status() + self.labels = [Label(elt) for elt in self.root.find("labels")] + self.load_validation_status() - for elt in self.root.findall("rsync_history"): - self.get_host(urlparse.urlparse(elt.text.strip()).hostname).add_connection(elt) + for elt in self.root.findall("rsync_history"): + self.get_host(urlparse.urlparse(elt.text.strip()).hostname).add_connection(elt) - generations = set() - fn2s = set() + generations = set() + fn2s = set() - for v in self.validation_status: - self.get_host(v.hostname).add_validation_status(v) - generations.add(v.generation) - fn2s.add(v.fn2) + for v in self.validation_status: + self.get_host(v.hostname).add_validation_status(v) + generations.add(v.generation) + fn2s.add(v.fn2) - self.labels = [l for l in self.labels if l.count > 0] + self.labels = [l for l in self.labels if l.count > 0] - self.hostnames = sorted(self.hosts) - self.generations = sorted(generations) - self.fn2s = sorted(fn2s) + self.hostnames = sorted(self.hosts) + self.generations = sorted(generations) + self.fn2s = sorted(fn2s) - def load_validation_status(self): - label_map = dict((label.code, label) for label in self.labels) - full_validation_status = [Validation_Status(elt, label_map) - for elt in self.root.findall("validation_status")] - accepted_current = set(v.uri for v in full_validation_status - if v.is_current and v.accepted) - self.validation_status = [v for v in full_validation_status - if not v.is_backup - or v.uri not in accepted_current] + def load_validation_status(self): + label_map = dict((label.code, label) for label in self.labels) + full_validation_status = [Validation_Status(elt, label_map) + for elt in self.root.findall("validation_status")] + accepted_current = set(v.uri for v in full_validation_status + if v.is_current and v.accepted) + self.validation_status = [v for v in full_validation_status + if not v.is_backup + or v.uri not in accepted_current] - def get_host(self, hostname): - if hostname not in self.hosts: - self.hosts[hostname] = Host(hostname, self.timestamp) - return self.hosts[hostname] + def get_host(self, hostname): + if hostname not in self.hosts: + self.hosts[hostname] = Host(hostname, self.timestamp) + return self.hosts[hostname] - def get_sum(self, fn2, generation, label): - return sum(h.get_counter(fn2, generation, label) - for h in self.hosts.itervalues()) + def get_sum(self, fn2, generation, label): + return sum(h.get_counter(fn2, generation, label) + for h in self.hosts.itervalues()) - def rrd_update(self): - if not args.dont_update_rrds: - for h in self.hosts.itervalues(): - h.rrd_update() + def rrd_update(self): + if not args.dont_update_rrds: + for h in self.hosts.itervalues(): + h.rrd_update() css = ''' th, td { @@ -475,183 +475,183 @@ css = ''' class HTML(object): - def __init__(self, title, filebase): + def __init__(self, title, filebase): + + self.filename = os.path.join(args.output_directory, filebase + ".html") + + self.html = Element("html") + self.html.append(Comment(" Generators:\n" + + " " + session.rcynic_version + "\n" + + " $Id$\n")) + self.head = SubElement(self.html, "head") + self.body = SubElement(self.html, "body") + + title += " " + session.rcynic_date + SubElement(self.head, "title").text = title + SubElement(self.body, "h1").text = title + SubElement(self.head, "style", type = "text/css").text = css + + if args.refresh: + SubElement(self.head, "meta", { "http-equiv" : "Refresh", "content" : str(args.refresh) }) + + hostwidth = max(len(hostname) for hostname in session.hostnames) + + toc = SubElement(self.body, "ul", id = "nav") + SubElement(SubElement(toc, "li"), "a", href = "index.html").text = "Overview" + li = SubElement(toc, "li") + SubElement(li, "span").text = "Repositories" + ul = SubElement(li, "ul", style = "width: %sem" % hostwidth) + for hostname in session.hostnames: + SubElement(SubElement(ul, "li"), "a", href = "%s.html" % hostname).text = hostname + SubElement(SubElement(toc, "li"), "a", href = "problems.html").text = "Problems" + li = SubElement(toc, "li") + SubElement(li, "span").text = "All Details" + ul = SubElement(li, "ul", style = "width: 15em") + SubElement(SubElement(ul, "li"), "a", href = "connections.html").text = "All Connections" + SubElement(SubElement(ul, "li"), "a", href = "objects.html").text = "All Objects" + SubElement(self.body, "br") + + def close(self): + ElementTree(element = self.html).write(self.filename) + + def BodyElement(self, tag, **attrib): + return SubElement(self.body, tag, **attrib) + + def counter_table(self, data_func, total_func): + table = self.BodyElement("table", rules = "all", border = "1") + thead = SubElement(table, "thead") + tfoot = SubElement(table, "tfoot") + tbody = SubElement(table, "tbody") + tr = SubElement(thead, "tr") + SubElement(tr, "th") + for label in session.labels: + SubElement(tr, "th").text = label.text + for fn2 in session.fn2s: + for generation in session.generations: + counters = [data_func(fn2, generation, label) for label in session.labels] + if sum(counters) > 0: + tr = SubElement(tbody, "tr") + SubElement(tr, "td").text = ((generation or "") + " " + (fn2 or "")).strip() + for label, count in zip(session.labels, counters): + td = SubElement(tr, "td") + if count > 0: + td.set("class", label.mood) + td.text = str(count) + tr = SubElement(tfoot, "tr") + SubElement(tr, "td").text = "Total" + counters = [total_func(label) for label in session.labels] + for label, count in zip(session.labels, counters): + td = SubElement(tr, "td") + if count > 0: + td.set("class", label.mood) + td.text = str(count) + return table + + def object_count_table(self, session): # pylint: disable=W0621 + table = self.BodyElement("table", rules = "all", border = "1") + thead = SubElement(table, "thead") + tbody = SubElement(table, "tbody") + tfoot = SubElement(table, "tfoot") + fn2s = [fn2 for fn2 in session.fn2s if fn2 is not None] + total = dict((fn2, 0) for fn2 in fn2s) + for hostname in session.hostnames: + tr = SubElement(tbody, "tr") + SubElement(tr, "td").text = hostname + for fn2 in fn2s: + td = SubElement(tr, "td") + count = sum(uri.endswith(fn2) for uri in session.hosts[hostname].uris) + total[fn2] += count + if count > 0: + td.text = str(count) + trhead = SubElement(thead, "tr") + trfoot = SubElement(tfoot, "tr") + SubElement(trhead, "th").text = "Repository" + SubElement(trfoot, "td").text = "Total" + for fn2 in fn2s: + SubElement(trhead, "th").text = fn2 + SubElement(trfoot, "td").text = str(total[fn2]) + return table + + def detail_table(self, records): + if records: + table = self.BodyElement("table", rules = "all", border = "1") + thead = SubElement(table, "thead") + tbody = SubElement(table, "tbody") + tr = SubElement(thead, "tr") + SubElement(tr, "th").text = "Timestamp" + SubElement(tr, "th").text = "Generation" + SubElement(tr, "th").text = "Status" + SubElement(tr, "th").text = "URI" + for v in records: + tr = SubElement(tbody, "tr", { "class" : v.mood }) + SubElement(tr, "td").text = v.timestamp + SubElement(tr, "td").text = v.generation + SubElement(tr, "td").text = v.label.text + SubElement(tr, "td", { "class" : "uri"}).text = v.uri + return table + else: + self.BodyElement("p").text = "None found" + return None - self.filename = os.path.join(args.output_directory, filebase + ".html") +def main(): - self.html = Element("html") - self.html.append(Comment(" Generators:\n" + - " " + session.rcynic_version + "\n" + - " $Id$\n")) - self.head = SubElement(self.html, "head") - self.body = SubElement(self.html, "body") + global session - title += " " + session.rcynic_date - SubElement(self.head, "title").text = title - SubElement(self.body, "h1").text = title - SubElement(self.head, "style", type = "text/css").text = css + os.putenv("TZ", "UTC") + time.tzset() - if args.refresh: - SubElement(self.head, "meta", { "http-equiv" : "Refresh", "content" : str(args.refresh) }) + parse_options() - hostwidth = max(len(hostname) for hostname in session.hostnames) + session = Session() + session.rrd_update() - toc = SubElement(self.body, "ul", id = "nav") - SubElement(SubElement(toc, "li"), "a", href = "index.html").text = "Overview" - li = SubElement(toc, "li") - SubElement(li, "span").text = "Repositories" - ul = SubElement(li, "ul", style = "width: %sem" % hostwidth) for hostname in session.hostnames: - SubElement(SubElement(ul, "li"), "a", href = "%s.html" % hostname).text = hostname - SubElement(SubElement(toc, "li"), "a", href = "problems.html").text = "Problems" - li = SubElement(toc, "li") - SubElement(li, "span").text = "All Details" - ul = SubElement(li, "ul", style = "width: 15em") - SubElement(SubElement(ul, "li"), "a", href = "connections.html").text = "All Connections" - SubElement(SubElement(ul, "li"), "a", href = "objects.html").text = "All Objects" - SubElement(self.body, "br") - - def close(self): - ElementTree(element = self.html).write(self.filename) - - def BodyElement(self, tag, **attrib): - return SubElement(self.body, tag, **attrib) - - def counter_table(self, data_func, total_func): - table = self.BodyElement("table", rules = "all", border = "1") - thead = SubElement(table, "thead") - tfoot = SubElement(table, "tfoot") - tbody = SubElement(table, "tbody") - tr = SubElement(thead, "tr") - SubElement(tr, "th") - for label in session.labels: - SubElement(tr, "th").text = label.text - for fn2 in session.fn2s: - for generation in session.generations: - counters = [data_func(fn2, generation, label) for label in session.labels] - if sum(counters) > 0: - tr = SubElement(tbody, "tr") - SubElement(tr, "td").text = ((generation or "") + " " + (fn2 or "")).strip() - for label, count in zip(session.labels, counters): - td = SubElement(tr, "td") - if count > 0: - td.set("class", label.mood) - td.text = str(count) - tr = SubElement(tfoot, "tr") - SubElement(tr, "td").text = "Total" - counters = [total_func(label) for label in session.labels] - for label, count in zip(session.labels, counters): - td = SubElement(tr, "td") - if count > 0: - td.set("class", label.mood) - td.text = str(count) - return table - - def object_count_table(self, session): # pylint: disable=W0621 - table = self.BodyElement("table", rules = "all", border = "1") - thead = SubElement(table, "thead") - tbody = SubElement(table, "tbody") - tfoot = SubElement(table, "tfoot") - fn2s = [fn2 for fn2 in session.fn2s if fn2 is not None] - total = dict((fn2, 0) for fn2 in fn2s) + html = HTML("Repository details for %s" % hostname, hostname) + html.counter_table(session.hosts[hostname].get_counter, session.hosts[hostname].get_total) + if not args.hide_graphs: + session.hosts[hostname].rrd_graph(html) + if not args.hide_problems: + html.BodyElement("h2").text = "Connection Problems" + html.detail_table(session.hosts[hostname].connection_problems) + html.BodyElement("h2").text = "Object Problems" + html.detail_table(session.hosts[hostname].object_problems) + html.close() + + html = HTML("rcynic summary", "index") + html.BodyElement("h2").text = "Grand totals for all repositories" + html.counter_table(session.get_sum, Label.get_count) + if not args.hide_object_counts: + html.BodyElement("br") + html.BodyElement("hr") + html.BodyElement("br") + html.BodyElement("h2").text = "Current total object counts (distinct URIs)" + html.object_count_table(session) for hostname in session.hostnames: - tr = SubElement(tbody, "tr") - SubElement(tr, "td").text = hostname - for fn2 in fn2s: - td = SubElement(tr, "td") - count = sum(uri.endswith(fn2) for uri in session.hosts[hostname].uris) - total[fn2] += count - if count > 0: - td.text = str(count) - trhead = SubElement(thead, "tr") - trfoot = SubElement(tfoot, "tr") - SubElement(trhead, "th").text = "Repository" - SubElement(trfoot, "td").text = "Total" - for fn2 in fn2s: - SubElement(trhead, "th").text = fn2 - SubElement(trfoot, "td").text = str(total[fn2]) - return table - - def detail_table(self, records): - if records: - table = self.BodyElement("table", rules = "all", border = "1") - thead = SubElement(table, "thead") - tbody = SubElement(table, "tbody") - tr = SubElement(thead, "tr") - SubElement(tr, "th").text = "Timestamp" - SubElement(tr, "th").text = "Generation" - SubElement(tr, "th").text = "Status" - SubElement(tr, "th").text = "URI" - for v in records: - tr = SubElement(tbody, "tr", { "class" : v.mood }) - SubElement(tr, "td").text = v.timestamp - SubElement(tr, "td").text = v.generation - SubElement(tr, "td").text = v.label.text - SubElement(tr, "td", { "class" : "uri"}).text = v.uri - return table - else: - self.BodyElement("p").text = "None found" - return None - -def main(): - - global session - - os.putenv("TZ", "UTC") - time.tzset() - - parse_options() + html.BodyElement("br") + html.BodyElement("hr") + html.BodyElement("br") + html.BodyElement("h2").text = "Overview for repository %s" % hostname + html.counter_table(session.hosts[hostname].get_counter, session.hosts[hostname].get_total) + if not args.hide_graphs: + html.BodyElement("br") + html.BodyElement("a", href = "%s.html" % hostname).append(session.hosts[hostname].graph) + html.close() - session = Session() - session.rrd_update() + html = HTML("Problems", "problems") + html.BodyElement("h2").text = "Connection Problems" + html.detail_table(session.connection_problems) + html.BodyElement("h2").text = "Object Problems" + html.detail_table(session.object_problems) + html.close() - for hostname in session.hostnames: - html = HTML("Repository details for %s" % hostname, hostname) - html.counter_table(session.hosts[hostname].get_counter, session.hosts[hostname].get_total) - if not args.hide_graphs: - session.hosts[hostname].rrd_graph(html) - if not args.hide_problems: - html.BodyElement("h2").text = "Connection Problems" - html.detail_table(session.hosts[hostname].connection_problems) - html.BodyElement("h2").text = "Object Problems" - html.detail_table(session.hosts[hostname].object_problems) + html = HTML("All connections", "connections") + html.detail_table([v for v in session.validation_status if v.is_connection_detail]) html.close() - html = HTML("rcynic summary", "index") - html.BodyElement("h2").text = "Grand totals for all repositories" - html.counter_table(session.get_sum, Label.get_count) - if not args.hide_object_counts: - html.BodyElement("br") - html.BodyElement("hr") - html.BodyElement("br") - html.BodyElement("h2").text = "Current total object counts (distinct URIs)" - html.object_count_table(session) - for hostname in session.hostnames: - html.BodyElement("br") - html.BodyElement("hr") - html.BodyElement("br") - html.BodyElement("h2").text = "Overview for repository %s" % hostname - html.counter_table(session.hosts[hostname].get_counter, session.hosts[hostname].get_total) - if not args.hide_graphs: - html.BodyElement("br") - html.BodyElement("a", href = "%s.html" % hostname).append(session.hosts[hostname].graph) - html.close() - - html = HTML("Problems", "problems") - html.BodyElement("h2").text = "Connection Problems" - html.detail_table(session.connection_problems) - html.BodyElement("h2").text = "Object Problems" - html.detail_table(session.object_problems) - html.close() - - html = HTML("All connections", "connections") - html.detail_table([v for v in session.validation_status if v.is_connection_detail]) - html.close() - - html = HTML("All objects", "objects") - html.detail_table([v for v in session.validation_status if v.is_object_detail]) - html.close() + html = HTML("All objects", "objects") + html.detail_table([v for v in session.validation_status if v.is_object_detail]) + html.close() if __name__ == "__main__": - main() + main() diff --git a/rp/rcynic/rcynic-svn b/rp/rcynic/rcynic-svn index 28b24672..a9417d8d 100755 --- a/rp/rcynic/rcynic-svn +++ b/rp/rcynic/rcynic-svn @@ -27,50 +27,50 @@ import fcntl import os try: - from lxml.etree import ElementTree + from lxml.etree import ElementTree except ImportError: - from xml.etree.ElementTree import ElementTree + from xml.etree.ElementTree import ElementTree mime_types = ( - ("html", "application/xhtml+xml"), - ("cer", "application/pkix-cert"), - ("crl", "application/pkix-crl"), - ("mft", "application/rpki-manifest"), - ("mnf", "application/rpki-manifest"), - ("roa", "application/rpki-roa"), - ("gbr", "application/rpki-ghostbusters")) + ("html", "application/xhtml+xml"), + ("cer", "application/pkix-cert"), + ("crl", "application/pkix-crl"), + ("mft", "application/rpki-manifest"), + ("mnf", "application/rpki-manifest"), + ("roa", "application/rpki-roa"), + ("gbr", "application/rpki-ghostbusters")) def run(*argv, **kwargs): - """ - Run a program, displaying timing data when appropriate. - """ + """ + Run a program, displaying timing data when appropriate. + """ - _t0 = datetime.datetime.utcnow() - subprocess.check_call(argv, **kwargs) - if args.show_timing: - _t1 = datetime.datetime.utcnow() - print _t1, (_t1 - _t0), " ".join(argv) + _t0 = datetime.datetime.utcnow() + subprocess.check_call(argv, **kwargs) + if args.show_timing: + _t1 = datetime.datetime.utcnow() + print _t1, (_t1 - _t0), " ".join(argv) def runxml(*argv): - """ - - Run a program which produces XML output, displaying timing data when - appropriate and returning an ElementTree constructed from the - program's output. - """ - _t0 = datetime.datetime.utcnow() - p = subprocess.Popen(argv, stdout = subprocess.PIPE) - x = ElementTree(file = p.stdout) - s = p.wait() - if s: - raise subprocess.CalledProcessError(s, argv[0]) - if args.show_timing: - _t1 = datetime.datetime.utcnow() - print _t1, (_t1 - _t0), " ".join(argv) - return x + """ + + Run a program which produces XML output, displaying timing data when + appropriate and returning an ElementTree constructed from the + program's output. + """ + _t0 = datetime.datetime.utcnow() + p = subprocess.Popen(argv, stdout = subprocess.PIPE) + x = ElementTree(file = p.stdout) + s = p.wait() + if s: + raise subprocess.CalledProcessError(s, argv[0]) + if args.show_timing: + _t1 = datetime.datetime.utcnow() + print _t1, (_t1 - _t0), " ".join(argv) + return x # Main program. @@ -120,8 +120,8 @@ parser.add_argument("working_directory", help = \ args = parser.parse_args() if args.show_timing: - t0 = datetime.datetime.utcnow() - print t0, "Starting" + t0 = datetime.datetime.utcnow() + print t0, "Starting" # Lock out other instances of this program. We may want some more # sophsiticated approach when combining this with other programs, but @@ -141,18 +141,18 @@ run("svn", "update", "--quiet", args.working_directory) if args.files_to_archive: - if args.verbatim: - cmd = ["rsync", "--archive", "--quiet", "--delete"] - cmd.extend(args.files_to_archive) - cmd.append(args.working_directory) - run(*cmd) + if args.verbatim: + cmd = ["rsync", "--archive", "--quiet", "--delete"] + cmd.extend(args.files_to_archive) + cmd.append(args.working_directory) + run(*cmd) - else: - for src in args.files_to_archive: - cmd = ["rsync", "--archive", "--quiet", "--delete", "--copy-links"] - cmd.append(src.rstrip("/")) - cmd.append(args.working_directory.rstrip("/") + "/") - run(*cmd) + else: + for src in args.files_to_archive: + cmd = ["rsync", "--archive", "--quiet", "--delete", "--copy-links"] + cmd.append(src.rstrip("/")) + cmd.append(args.working_directory.rstrip("/") + "/") + run(*cmd) # Ask Subversion to add any new files, trying hard to get the MIME # types right. @@ -160,8 +160,8 @@ if args.files_to_archive: cmd = ["svn", "add", "--quiet", "--force", "--auto-props"] for fn2, mime_type in mime_types: - cmd.append("--config-option") - cmd.append("config:auto-props:*.%s=svn:mime-type=%s" % (fn2, mime_type)) + cmd.append("--config-option") + cmd.append("config:auto-props:*.%s=svn:mime-type=%s" % (fn2, mime_type)) cmd.append(".") @@ -177,9 +177,9 @@ missing = sorted(entry.get("path") deleted = [] for path in missing: - if not any(path.startswith(r) for r in deleted): - run("svn", "delete", "--quiet", path) - deleted.append(path + "/") + if not any(path.startswith(r) for r in deleted): + run("svn", "delete", "--quiet", path) + deleted.append(path + "/") # Commit our changes and update the working tree. @@ -187,5 +187,5 @@ run("svn", "commit", "--quiet", "--message", "Auto update.", args.working_direct run("svn", "update", "--quiet", args.working_directory) if args.show_timing: - t1 = datetime.datetime.utcnow() - print t1, t1 - t0, "total runtime" + t1 = datetime.datetime.utcnow() + print t1, t1 - t0, "total runtime" diff --git a/rp/rcynic/rcynic-text b/rp/rcynic/rcynic-text index db4126ce..d4a5b23e 100755 --- a/rp/rcynic/rcynic-text +++ b/rp/rcynic/rcynic-text @@ -25,96 +25,96 @@ import urlparse import textwrap try: - from lxml.etree import ElementTree + from lxml.etree import ElementTree except ImportError: - from xml.etree.ElementTree import ElementTree + from xml.etree.ElementTree import ElementTree class Label(object): - def __init__(self, elt): - self.tag = elt.tag - self.width = max(len(s) for s in elt.text.split()) - self.lines = textwrap.wrap(elt.text.strip(), width = self.width) - self.counter = 0 + def __init__(self, elt): + self.tag = elt.tag + self.width = max(len(s) for s in elt.text.split()) + self.lines = textwrap.wrap(elt.text.strip(), width = self.width) + self.counter = 0 - def line(self, n): - try: - return " " + self.lines[n].center(self.width) + " " - except IndexError: - return " " * (self.width + 2) + def line(self, n): + try: + return " " + self.lines[n].center(self.width) + " " + except IndexError: + return " " * (self.width + 2) - def add(self): - self.counter += 1 + def add(self): + self.counter += 1 - @property - def total(self): - return " " + str(self.counter).rjust(self.width) + " " + @property + def total(self): + return " " + str(self.counter).rjust(self.width) + " " - @property - def visible(self): - return self.counter > 0 + @property + def visible(self): + return self.counter > 0 class Host(object): - def __init__(self): - self.counters = {} + def __init__(self): + self.counters = {} - def add(self, label): - self.counters[label] = self.counters.get(label, 0) + 1 - label.add() + def add(self, label): + self.counters[label] = self.counters.get(label, 0) + 1 + label.add() - def total(self, label): - if label in self.counters: - return " " + str(self.counters[label]).rjust(label.width) + " " - else: - return " " * (label.width + 2) + def total(self, label): + if label in self.counters: + return " " + str(self.counters[label]).rjust(label.width) + " " + else: + return " " * (label.width + 2) class Session(object): - def __init__(self, labels): - self.hosts = {} - self.labels = labels - self.map = dict((label.tag, label) for label in labels) - - def add(self, elt): - label = self.map[elt.get("status")] - hostname = urlparse.urlparse(elt.text.strip()).hostname - if hostname not in self.hosts: - self.hosts[hostname] = Host() - self.hosts[hostname].add(label) - - def show(self): - visible = [label for label in self.labels if label.visible] - hostnames = sorted(hostname for hostname in self.hosts if hostname is not None) - hostwidth = max(len(hostname) for hostname in hostnames + ["Hostname"]) - separator = "+-%s-+-%s-+" % ( - "-" * hostwidth, - "-+-".join("-" * label.width for label in visible)) - print separator - for i in xrange(max(len(label.lines) for label in visible)): - print "| %s |%s|" % ( - ("Hostname" if i == 0 else "").ljust(hostwidth), - "|".join(label.line(i) for label in visible)) - print separator - for hostname in hostnames: - print "| %s |%s|" % ( - hostname.ljust(hostwidth), - "|".join(self.hosts[hostname].total(label) for label in visible)) - if hostnames: - print separator - print "| %s |%s|" % ( - "Total".ljust(hostwidth), - "|".join(label.total for label in visible)) - print separator + def __init__(self, labels): + self.hosts = {} + self.labels = labels + self.map = dict((label.tag, label) for label in labels) + + def add(self, elt): + label = self.map[elt.get("status")] + hostname = urlparse.urlparse(elt.text.strip()).hostname + if hostname not in self.hosts: + self.hosts[hostname] = Host() + self.hosts[hostname].add(label) + + def show(self): + visible = [label for label in self.labels if label.visible] + hostnames = sorted(hostname for hostname in self.hosts if hostname is not None) + hostwidth = max(len(hostname) for hostname in hostnames + ["Hostname"]) + separator = "+-%s-+-%s-+" % ( + "-" * hostwidth, + "-+-".join("-" * label.width for label in visible)) + print separator + for i in xrange(max(len(label.lines) for label in visible)): + print "| %s |%s|" % ( + ("Hostname" if i == 0 else "").ljust(hostwidth), + "|".join(label.line(i) for label in visible)) + print separator + for hostname in hostnames: + print "| %s |%s|" % ( + hostname.ljust(hostwidth), + "|".join(self.hosts[hostname].total(label) for label in visible)) + if hostnames: + print separator + print "| %s |%s|" % ( + "Total".ljust(hostwidth), + "|".join(label.total for label in visible)) + print separator def main(): - for filename in ([sys.stdin] if len(sys.argv) < 2 else sys.argv[1:]): - etree = ElementTree(file = filename) - session = Session([Label(elt) for elt in etree.find("labels")]) - for elt in etree.findall("validation_status"): - session.add(elt) - session.show() + for filename in ([sys.stdin] if len(sys.argv) < 2 else sys.argv[1:]): + etree = ElementTree(file = filename) + session = Session([Label(elt) for elt in etree.find("labels")]) + for elt in etree.findall("validation_status"): + session.add(elt) + session.show() if __name__ == "__main__": - main() + main() diff --git a/rp/rcynic/rpki-torrent.py b/rp/rcynic/rpki-torrent.py index 2c6aa64d..f9a3d620 100644 --- a/rp/rcynic/rpki-torrent.py +++ b/rp/rcynic/rpki-torrent.py @@ -46,688 +46,688 @@ import transmissionrpc tr_env_vars = ("TR_TORRENT_DIR", "TR_TORRENT_ID", "TR_TORRENT_NAME") class WrongServer(Exception): - "Hostname not in X.509v3 subjectAltName extension." + "Hostname not in X.509v3 subjectAltName extension." class UnexpectedRedirect(Exception): - "Unexpected HTTP redirect." + "Unexpected HTTP redirect." class WrongMode(Exception): - "Wrong operation for mode." + "Wrong operation for mode." class BadFormat(Exception): - "Zip file does not match our expectations." + "Zip file does not match our expectations." class InconsistentEnvironment(Exception): - "Environment variables received from Transmission aren't consistent." + "Environment variables received from Transmission aren't consistent." class TorrentNotReady(Exception): - "Torrent is not ready for checking." + "Torrent is not ready for checking." class TorrentDoesNotMatchManifest(Exception): - "Retrieved torrent does not match manifest." + "Retrieved torrent does not match manifest." class TorrentNameDoesNotMatchURL(Exception): - "Torrent name doesn't uniquely match a URL." + "Torrent name doesn't uniquely match a URL." class CouldNotFindTorrents(Exception): - "Could not find torrent(s) with given name(s)." + "Could not find torrent(s) with given name(s)." class UseTheSourceLuke(Exception): - "Use The Source, Luke." + "Use The Source, Luke." cfg = None def main(): - try: - syslog_flags = syslog.LOG_PID - if os.isatty(sys.stderr.fileno()): - syslog_flags |= syslog.LOG_PERROR - syslog.openlog("rpki-torrent", syslog_flags) - - # If I seriously expected this script to get a lot of further use, - # I might rewrite this using subparsers, but it'd be a bit tricky - # as argparse doesn't support making the subparser argument - # optional and transmission gives no sane way to provide arguments - # when running a completion script. So, for the moment, let's - # just fix the bugs accidently introduced while converting the - # universe to argparse without making any radical changes to the - # program structure here, even if the result looks kind of klunky. - - parser = argparse.ArgumentParser(description = __doc__) - parser.add_argument("-c", "--config", - help = "configuration file") - parser.add_argument("action", choices = ("poll", "generate", "mirror"), nargs = "?", - help = "action to take") - args = parser.parse_args() - - global cfg - cfg = MyConfigParser() - cfg.read(args.config or - [os.path.join(dn, fn) - for fn in ("rcynic.conf", "rpki.conf") - for dn in ("/var/rcynic/etc", "/usr/local/etc", "/etc")]) - - if cfg.act_as_generator: - if args.action == "generate": - generator_main() - elif args.action == "mirror": - mirror_main() - else: - raise UseTheSourceLuke - else: - if args.action is None and all(v in os.environ for v in tr_env_vars): - torrent_completion_main() - elif args.action == "poll": - poll_main() - else: - raise UseTheSourceLuke - - except: - for line in traceback.format_exc().splitlines(): - syslog.syslog(line) - sys.exit(1) + try: + syslog_flags = syslog.LOG_PID + if os.isatty(sys.stderr.fileno()): + syslog_flags |= syslog.LOG_PERROR + syslog.openlog("rpki-torrent", syslog_flags) + + # If I seriously expected this script to get a lot of further use, + # I might rewrite this using subparsers, but it'd be a bit tricky + # as argparse doesn't support making the subparser argument + # optional and transmission gives no sane way to provide arguments + # when running a completion script. So, for the moment, let's + # just fix the bugs accidently introduced while converting the + # universe to argparse without making any radical changes to the + # program structure here, even if the result looks kind of klunky. + + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument("-c", "--config", + help = "configuration file") + parser.add_argument("action", choices = ("poll", "generate", "mirror"), nargs = "?", + help = "action to take") + args = parser.parse_args() + + global cfg + cfg = MyConfigParser() + cfg.read(args.config or + [os.path.join(dn, fn) + for fn in ("rcynic.conf", "rpki.conf") + for dn in ("/var/rcynic/etc", "/usr/local/etc", "/etc")]) + + if cfg.act_as_generator: + if args.action == "generate": + generator_main() + elif args.action == "mirror": + mirror_main() + else: + raise UseTheSourceLuke + else: + if args.action is None and all(v in os.environ for v in tr_env_vars): + torrent_completion_main() + elif args.action == "poll": + poll_main() + else: + raise UseTheSourceLuke + + except: + for line in traceback.format_exc().splitlines(): + syslog.syslog(line) + sys.exit(1) def generator_main(): - import paramiko - - class SFTPClient(paramiko.SFTPClient): - def atomic_rename(self, oldpath, newpath): - oldpath = self._adjust_cwd(oldpath) - newpath = self._adjust_cwd(newpath) - self._log(paramiko.common.DEBUG, 'atomic_rename(%r, %r)' % (oldpath, newpath)) - self._request(paramiko.sftp.CMD_EXTENDED, "posix-rename@openssh.com", oldpath, newpath) - - z = ZipFile(url = cfg.generate_url, dn = cfg.zip_dir) - client = TransmissionClient() - - client.remove_torrents(z.torrent_name) - - download_dir = client.get_session().download_dir - torrent_dir = os.path.join(download_dir, z.torrent_name) - torrent_file = os.path.join(cfg.zip_dir, z.torrent_name + ".torrent") - - - syslog.syslog("Synchronizing local data from %s to %s" % (cfg.unauthenticated, torrent_dir)) - subprocess.check_call((cfg.rsync_prog, "--archive", "--delete", - os.path.normpath(cfg.unauthenticated) + "/", - os.path.normpath(torrent_dir) + "/")) - - syslog.syslog("Creating %s" % torrent_file) - try: - os.unlink(torrent_file) - except OSError, e: - if e.errno != errno.ENOENT: - raise - ignore_output_for_now = subprocess.check_output( # pylint: disable=W0612 - (cfg.mktorrent_prog, - "-a", cfg.tracker_url, - "-c", "RPKI unauthenticated data snapshot generated by rpki-torrent", - "-o", torrent_file, - torrent_dir)) - - syslog.syslog("Generating manifest") - manifest = create_manifest(download_dir, z.torrent_name) - - syslog.syslog("Loading %s with unlimited seeding" % torrent_file) - f = open(torrent_file, "rb") - client.add(base64.b64encode(f.read())) - f.close() - client.unlimited_seeding(z.torrent_name) - - syslog.syslog("Creating upload connection") - ssh = paramiko.Transport((cfg.sftp_host, cfg.sftp_port)) - try: - hostkeys = paramiko.util.load_host_keys(cfg.sftp_hostkey_file)[cfg.sftp_host]["ssh-rsa"] - except ConfigParser.Error: - hostkeys = None - ssh.connect( - username = cfg.sftp_user, - hostkey = hostkeys, - pkey = paramiko.RSAKey.from_private_key_file(cfg.sftp_private_key_file)) - sftp = SFTPClient.from_transport(ssh) - - zip_filename = os.path.join("data", os.path.basename(z.filename)) - zip_tempname = zip_filename + ".new" - - syslog.syslog("Creating %s" % zip_tempname) - f = sftp.open(zip_tempname, "wb") - z.set_output_stream(f) - - syslog.syslog("Writing %s to zip" % torrent_file) - z.write( - torrent_file, - arcname = os.path.basename(torrent_file), - compress_type = zipfile.ZIP_DEFLATED) - - manifest_name = z.torrent_name + ".manifest" - - syslog.syslog("Writing %s to zip" % manifest_name) - zi = zipfile.ZipInfo(manifest_name, time.gmtime()[:6]) - zi.external_attr = (stat.S_IFREG | 0644) << 16 - zi.internal_attr = 1 # Text, not binary - z.writestr(zi, - "".join("%s %s\n" % (v, k) for k, v in manifest.iteritems()), - zipfile.ZIP_DEFLATED) - - syslog.syslog("Closing %s and renaming to %s" % (zip_tempname, zip_filename)) - z.close() - f.close() - sftp.atomic_rename(zip_tempname, zip_filename) - - syslog.syslog("Closing upload connection") - ssh.close() - -def mirror_main(): - client = TransmissionClient() - torrent_names = [] - - for zip_url in cfg.zip_urls: - if zip_url != cfg.generate_url: - z = ZipFile(url = zip_url, dn = cfg.zip_dir, ta = cfg.zip_ta) - if z.fetch(): - client.remove_torrents(z.torrent_name) - syslog.syslog("Mirroring torrent %s" % z.torrent_name) - client.add(z.get_torrent()) - torrent_names.append(z.torrent_name) - - if torrent_names: - client.unlimited_seeding(*torrent_names) + import paramiko + class SFTPClient(paramiko.SFTPClient): + def atomic_rename(self, oldpath, newpath): + oldpath = self._adjust_cwd(oldpath) + newpath = self._adjust_cwd(newpath) + self._log(paramiko.common.DEBUG, 'atomic_rename(%r, %r)' % (oldpath, newpath)) + self._request(paramiko.sftp.CMD_EXTENDED, "posix-rename@openssh.com", oldpath, newpath) -def poll_main(): - for zip_url in cfg.zip_urls: - - z = ZipFile(url = zip_url, dn = cfg.zip_dir, ta = cfg.zip_ta) + z = ZipFile(url = cfg.generate_url, dn = cfg.zip_dir) client = TransmissionClient() - if z.fetch(): - client.remove_torrents(z.torrent_name) - syslog.syslog("Adding torrent %s" % z.torrent_name) - client.add(z.get_torrent()) - - elif cfg.run_rcynic_anyway: - run_rcynic(client, z) - - -def torrent_completion_main(): - torrent_name = os.getenv("TR_TORRENT_NAME") - torrent_id = int(os.getenv("TR_TORRENT_ID")) - - z = ZipFile(url = cfg.find_url(torrent_name), dn = cfg.zip_dir, ta = cfg.zip_ta) - client = TransmissionClient() - torrent = client.info([torrent_id]).popitem()[1] + client.remove_torrents(z.torrent_name) - if torrent.name != torrent_name: - raise InconsistentEnvironment("Torrent name %s does not match ID %d" % (torrent_name, torrent_id)) + download_dir = client.get_session().download_dir + torrent_dir = os.path.join(download_dir, z.torrent_name) + torrent_file = os.path.join(cfg.zip_dir, z.torrent_name + ".torrent") - if z.torrent_name != torrent_name: - raise InconsistentEnvironment("Torrent name %s does not match torrent name in zip file %s" % (torrent_name, z.torrent_name)) - if torrent is None or torrent.progress != 100: - raise TorrentNotReady("Torrent %s not ready for checking, how did I get here?" % torrent_name) + syslog.syslog("Synchronizing local data from %s to %s" % (cfg.unauthenticated, torrent_dir)) + subprocess.check_call((cfg.rsync_prog, "--archive", "--delete", + os.path.normpath(cfg.unauthenticated) + "/", + os.path.normpath(torrent_dir) + "/")) - log_email("Download complete %s" % z.url) - - run_rcynic(client, z) - - -def run_rcynic(client, z): - """ - Run rcynic and any post-processing we might want. - """ - - if cfg.lockfile is not None: - syslog.syslog("Acquiring lock %s" % cfg.lockfile) - lock = os.open(cfg.lockfile, os.O_WRONLY | os.O_CREAT, 0600) - fcntl.flock(lock, fcntl.LOCK_EX) - else: - lock = None - - syslog.syslog("Checking manifest against disk") - - download_dir = client.get_session().download_dir - - manifest_from_disk = create_manifest(download_dir, z.torrent_name) - manifest_from_zip = z.get_manifest() + syslog.syslog("Creating %s" % torrent_file) + try: + os.unlink(torrent_file) + except OSError, e: + if e.errno != errno.ENOENT: + raise + ignore_output_for_now = subprocess.check_output( # pylint: disable=W0612 + (cfg.mktorrent_prog, + "-a", cfg.tracker_url, + "-c", "RPKI unauthenticated data snapshot generated by rpki-torrent", + "-o", torrent_file, + torrent_dir)) + + syslog.syslog("Generating manifest") + manifest = create_manifest(download_dir, z.torrent_name) + + syslog.syslog("Loading %s with unlimited seeding" % torrent_file) + f = open(torrent_file, "rb") + client.add(base64.b64encode(f.read())) + f.close() + client.unlimited_seeding(z.torrent_name) - excess_files = set(manifest_from_disk) - set(manifest_from_zip) - for fn in excess_files: - del manifest_from_disk[fn] + syslog.syslog("Creating upload connection") + ssh = paramiko.Transport((cfg.sftp_host, cfg.sftp_port)) + try: + hostkeys = paramiko.util.load_host_keys(cfg.sftp_hostkey_file)[cfg.sftp_host]["ssh-rsa"] + except ConfigParser.Error: + hostkeys = None + ssh.connect( + username = cfg.sftp_user, + hostkey = hostkeys, + pkey = paramiko.RSAKey.from_private_key_file(cfg.sftp_private_key_file)) + sftp = SFTPClient.from_transport(ssh) + + zip_filename = os.path.join("data", os.path.basename(z.filename)) + zip_tempname = zip_filename + ".new" + + syslog.syslog("Creating %s" % zip_tempname) + f = sftp.open(zip_tempname, "wb") + z.set_output_stream(f) + + syslog.syslog("Writing %s to zip" % torrent_file) + z.write( + torrent_file, + arcname = os.path.basename(torrent_file), + compress_type = zipfile.ZIP_DEFLATED) + + manifest_name = z.torrent_name + ".manifest" + + syslog.syslog("Writing %s to zip" % manifest_name) + zi = zipfile.ZipInfo(manifest_name, time.gmtime()[:6]) + zi.external_attr = (stat.S_IFREG | 0644) << 16 + zi.internal_attr = 1 # Text, not binary + z.writestr(zi, + "".join("%s %s\n" % (v, k) for k, v in manifest.iteritems()), + zipfile.ZIP_DEFLATED) + + syslog.syslog("Closing %s and renaming to %s" % (zip_tempname, zip_filename)) + z.close() + f.close() + sftp.atomic_rename(zip_tempname, zip_filename) - if manifest_from_disk != manifest_from_zip: - raise TorrentDoesNotMatchManifest("Manifest for torrent %s does not match what we got" % - z.torrent_name) + syslog.syslog("Closing upload connection") + ssh.close() - if excess_files: - syslog.syslog("Cleaning up excess files") - for fn in excess_files: - os.unlink(os.path.join(download_dir, fn)) +def mirror_main(): + client = TransmissionClient() + torrent_names = [] - syslog.syslog("Running rcynic") - log_email("Starting rcynic %s" % z.url) - subprocess.check_call((cfg.rcynic_prog, - "-c", cfg.rcynic_conf, - "-u", os.path.join(client.get_session().download_dir, z.torrent_name))) - log_email("Completed rcynic %s" % z.url) + for zip_url in cfg.zip_urls: + if zip_url != cfg.generate_url: + z = ZipFile(url = zip_url, dn = cfg.zip_dir, ta = cfg.zip_ta) + if z.fetch(): + client.remove_torrents(z.torrent_name) + syslog.syslog("Mirroring torrent %s" % z.torrent_name) + client.add(z.get_torrent()) + torrent_names.append(z.torrent_name) - for cmd in cfg.post_rcynic_commands: - syslog.syslog("Running post-rcynic command: %s" % cmd) - subprocess.check_call(cmd, shell = True) + if torrent_names: + client.unlimited_seeding(*torrent_names) - if lock is not None: - syslog.syslog("Releasing lock %s" % cfg.lockfile) - os.close(lock) -# See http://www.minstrel.org.uk/papers/sftp/ for details on how to -# set up safe upload-only SFTP directories on the server. In -# particular http://www.minstrel.org.uk/papers/sftp/builtin/ is likely -# to be the right path. +def poll_main(): + for zip_url in cfg.zip_urls: + z = ZipFile(url = zip_url, dn = cfg.zip_dir, ta = cfg.zip_ta) + client = TransmissionClient() -class ZipFile(object): - """ - Augmented version of standard python zipfile.ZipFile class, with - some extra methods and specialized capabilities. - - All methods of the standard zipfile.ZipFile class are supported, but - the constructor arguments are different, and opening the zip file - itself is deferred until a call which requires this, since the file - may first need to be fetched via HTTPS. - """ - - def __init__(self, url, dn, ta = None, verbose = True): - self.url = url - self.dir = dn - self.ta = ta - self.verbose = verbose - self.filename = os.path.join(dn, os.path.basename(url)) - self.changed = False - self.zf = None - self.peercert = None - self.torrent_name, zip_ext = os.path.splitext(os.path.basename(url)) - if zip_ext != ".zip": - raise BadFormat - - - def __getattr__(self, name): - if self.zf is None: - self.zf = zipfile.ZipFile(self.filename) - return getattr(self.zf, name) - - - def build_opener(self): - """ - Voodoo to create a urllib2.OpenerDirector object with TLS - certificate checking enabled and a hook to set self.peercert so - our caller can check the subjectAltName field. + if z.fetch(): + client.remove_torrents(z.torrent_name) + syslog.syslog("Adding torrent %s" % z.torrent_name) + client.add(z.get_torrent()) - You probably don't want to look at this if you can avoid it. - """ + elif cfg.run_rcynic_anyway: + run_rcynic(client, z) - assert self.ta is not None - # Yes, we're constructing one-off classes. Look away, look away. +def torrent_completion_main(): + torrent_name = os.getenv("TR_TORRENT_NAME") + torrent_id = int(os.getenv("TR_TORRENT_ID")) - class HTTPSConnection(httplib.HTTPSConnection): - zip = self - def connect(self): - sock = socket.create_connection((self.host, self.port), self.timeout) - if getattr(self, "_tunnel_host", None): - self.sock = sock - self._tunnel() - self.sock = ssl.wrap_socket(sock, - keyfile = self.key_file, - certfile = self.cert_file, - cert_reqs = ssl.CERT_REQUIRED, - ssl_version = ssl.PROTOCOL_TLSv1, - ca_certs = self.zip.ta) - self.zip.peercert = self.sock.getpeercert() + z = ZipFile(url = cfg.find_url(torrent_name), dn = cfg.zip_dir, ta = cfg.zip_ta) + client = TransmissionClient() + torrent = client.info([torrent_id]).popitem()[1] - class HTTPSHandler(urllib2.HTTPSHandler): - def https_open(self, req): - return self.do_open(HTTPSConnection, req) + if torrent.name != torrent_name: + raise InconsistentEnvironment("Torrent name %s does not match ID %d" % (torrent_name, torrent_id)) - return urllib2.build_opener(HTTPSHandler) + if z.torrent_name != torrent_name: + raise InconsistentEnvironment("Torrent name %s does not match torrent name in zip file %s" % (torrent_name, z.torrent_name)) + if torrent is None or torrent.progress != 100: + raise TorrentNotReady("Torrent %s not ready for checking, how did I get here?" % torrent_name) - def check_subjectAltNames(self): - """ - Check self.peercert against URL to make sure we were talking to - the right HTTPS server. - """ + log_email("Download complete %s" % z.url) - hostname = urlparse.urlparse(self.url).hostname - subjectAltNames = set(i[1] - for i in self.peercert.get("subjectAltName", ()) - if i[0] == "DNS") - if hostname not in subjectAltNames: - raise WrongServer + run_rcynic(client, z) - def download_file(self, r, bufsize = 4096): +def run_rcynic(client, z): """ - Downloaded file to disk. + Run rcynic and any post-processing we might want. """ - tempname = self.filename + ".new" - f = open(tempname, "wb") - n = int(r.info()["Content-Length"]) - for i in xrange(0, n - bufsize, bufsize): # pylint: disable=W0612 - f.write(r.read(bufsize)) - f.write(r.read()) - f.close() - mtime = email.utils.mktime_tz(email.utils.parsedate_tz(r.info()["Last-Modified"])) - os.utime(tempname, (mtime, mtime)) - os.rename(tempname, self.filename) + if cfg.lockfile is not None: + syslog.syslog("Acquiring lock %s" % cfg.lockfile) + lock = os.open(cfg.lockfile, os.O_WRONLY | os.O_CREAT, 0600) + fcntl.flock(lock, fcntl.LOCK_EX) + else: + lock = None + syslog.syslog("Checking manifest against disk") - def set_output_stream(self, stream): - """ - Set up this zip file for writing to a network stream. - """ + download_dir = client.get_session().download_dir - assert self.zf is None - self.zf = zipfile.ZipFile(stream, "w") + manifest_from_disk = create_manifest(download_dir, z.torrent_name) + manifest_from_zip = z.get_manifest() + excess_files = set(manifest_from_disk) - set(manifest_from_zip) + for fn in excess_files: + del manifest_from_disk[fn] - def fetch(self): - """ - Fetch zip file from URL given to constructor. - """ + if manifest_from_disk != manifest_from_zip: + raise TorrentDoesNotMatchManifest("Manifest for torrent %s does not match what we got" % + z.torrent_name) - headers = { "User-Agent" : "rpki-torrent" } - try: - headers["If-Modified-Since"] = email.utils.formatdate( - os.path.getmtime(self.filename), False, True) - except OSError: - pass + if excess_files: + syslog.syslog("Cleaning up excess files") + for fn in excess_files: + os.unlink(os.path.join(download_dir, fn)) - syslog.syslog("Checking %s..." % self.url) - try: - r = self.build_opener().open(urllib2.Request(self.url, None, headers)) - syslog.syslog("%s has changed, starting download" % self.url) - self.changed = True - log_email("Downloading %s" % self.url) - except urllib2.HTTPError, e: - if e.code == 304: - syslog.syslog("%s has not changed" % self.url) - elif e.code == 404: - syslog.syslog("%s does not exist" % self.url) - else: - raise - r = None - - self.check_subjectAltNames() + syslog.syslog("Running rcynic") + log_email("Starting rcynic %s" % z.url) + subprocess.check_call((cfg.rcynic_prog, + "-c", cfg.rcynic_conf, + "-u", os.path.join(client.get_session().download_dir, z.torrent_name))) + log_email("Completed rcynic %s" % z.url) - if r is not None and r.geturl() != self.url: - raise UnexpectedRedirect + for cmd in cfg.post_rcynic_commands: + syslog.syslog("Running post-rcynic command: %s" % cmd) + subprocess.check_call(cmd, shell = True) - if r is not None: - self.download_file(r) - r.close() + if lock is not None: + syslog.syslog("Releasing lock %s" % cfg.lockfile) + os.close(lock) - return self.changed +# See http://www.minstrel.org.uk/papers/sftp/ for details on how to +# set up safe upload-only SFTP directories on the server. In +# particular http://www.minstrel.org.uk/papers/sftp/builtin/ is likely +# to be the right path. - def check_format(self): - """ - Make sure that format of zip file matches our preconceptions: it - should contain two files, one of which is the .torrent file, the - other is the manifest, with names derived from the torrent name - inferred from the URL. +class ZipFile(object): """ + Augmented version of standard python zipfile.ZipFile class, with + some extra methods and specialized capabilities. - if set(self.namelist()) != set((self.torrent_name + ".torrent", self.torrent_name + ".manifest")): - raise BadFormat - - - def get_torrent(self): - """ - Extract torrent file from zip file, encoded in Base64 because - that's what the transmisionrpc library says it wants. + All methods of the standard zipfile.ZipFile class are supported, but + the constructor arguments are different, and opening the zip file + itself is deferred until a call which requires this, since the file + may first need to be fetched via HTTPS. """ - self.check_format() - return base64.b64encode(self.read(self.torrent_name + ".torrent")) + def __init__(self, url, dn, ta = None, verbose = True): + self.url = url + self.dir = dn + self.ta = ta + self.verbose = verbose + self.filename = os.path.join(dn, os.path.basename(url)) + self.changed = False + self.zf = None + self.peercert = None + self.torrent_name, zip_ext = os.path.splitext(os.path.basename(url)) + if zip_ext != ".zip": + raise BadFormat + + + def __getattr__(self, name): + if self.zf is None: + self.zf = zipfile.ZipFile(self.filename) + return getattr(self.zf, name) + + + def build_opener(self): + """ + Voodoo to create a urllib2.OpenerDirector object with TLS + certificate checking enabled and a hook to set self.peercert so + our caller can check the subjectAltName field. + + You probably don't want to look at this if you can avoid it. + """ + + assert self.ta is not None + + # Yes, we're constructing one-off classes. Look away, look away. + + class HTTPSConnection(httplib.HTTPSConnection): + zip = self + def connect(self): + sock = socket.create_connection((self.host, self.port), self.timeout) + if getattr(self, "_tunnel_host", None): + self.sock = sock + self._tunnel() + self.sock = ssl.wrap_socket(sock, + keyfile = self.key_file, + certfile = self.cert_file, + cert_reqs = ssl.CERT_REQUIRED, + ssl_version = ssl.PROTOCOL_TLSv1, + ca_certs = self.zip.ta) + self.zip.peercert = self.sock.getpeercert() + + class HTTPSHandler(urllib2.HTTPSHandler): + def https_open(self, req): + return self.do_open(HTTPSConnection, req) + + return urllib2.build_opener(HTTPSHandler) + + + def check_subjectAltNames(self): + """ + Check self.peercert against URL to make sure we were talking to + the right HTTPS server. + """ + + hostname = urlparse.urlparse(self.url).hostname + subjectAltNames = set(i[1] + for i in self.peercert.get("subjectAltName", ()) + if i[0] == "DNS") + if hostname not in subjectAltNames: + raise WrongServer + + + def download_file(self, r, bufsize = 4096): + """ + Downloaded file to disk. + """ + + tempname = self.filename + ".new" + f = open(tempname, "wb") + n = int(r.info()["Content-Length"]) + for i in xrange(0, n - bufsize, bufsize): # pylint: disable=W0612 + f.write(r.read(bufsize)) + f.write(r.read()) + f.close() + mtime = email.utils.mktime_tz(email.utils.parsedate_tz(r.info()["Last-Modified"])) + os.utime(tempname, (mtime, mtime)) + os.rename(tempname, self.filename) + + + def set_output_stream(self, stream): + """ + Set up this zip file for writing to a network stream. + """ + + assert self.zf is None + self.zf = zipfile.ZipFile(stream, "w") + + + def fetch(self): + """ + Fetch zip file from URL given to constructor. + """ + + headers = { "User-Agent" : "rpki-torrent" } + try: + headers["If-Modified-Since"] = email.utils.formatdate( + os.path.getmtime(self.filename), False, True) + except OSError: + pass + + syslog.syslog("Checking %s..." % self.url) + try: + r = self.build_opener().open(urllib2.Request(self.url, None, headers)) + syslog.syslog("%s has changed, starting download" % self.url) + self.changed = True + log_email("Downloading %s" % self.url) + except urllib2.HTTPError, e: + if e.code == 304: + syslog.syslog("%s has not changed" % self.url) + elif e.code == 404: + syslog.syslog("%s does not exist" % self.url) + else: + raise + r = None + + self.check_subjectAltNames() + + if r is not None and r.geturl() != self.url: + raise UnexpectedRedirect + + if r is not None: + self.download_file(r) + r.close() + + return self.changed + + + def check_format(self): + """ + Make sure that format of zip file matches our preconceptions: it + should contain two files, one of which is the .torrent file, the + other is the manifest, with names derived from the torrent name + inferred from the URL. + """ + + if set(self.namelist()) != set((self.torrent_name + ".torrent", self.torrent_name + ".manifest")): + raise BadFormat + + + def get_torrent(self): + """ + Extract torrent file from zip file, encoded in Base64 because + that's what the transmisionrpc library says it wants. + """ + + self.check_format() + return base64.b64encode(self.read(self.torrent_name + ".torrent")) + + + def get_manifest(self): + """ + Extract manifest from zip file, as a dictionary. + + For the moment we're fixing up the internal file names from the + format that the existing shell-script prototype uses, but this + should go away once this program both generates and checks the + manifests. + """ + + self.check_format() + result = {} + for line in self.open(self.torrent_name + ".manifest"): + h, fn = line.split() + # + # Fixup for earlier manifest format, this should go away + if not fn.startswith(self.torrent_name): + fn = os.path.normpath(os.path.join(self.torrent_name, fn)) + # + result[fn] = h + return result - def get_manifest(self): +def create_manifest(topdir, torrent_name): """ - Extract manifest from zip file, as a dictionary. - - For the moment we're fixing up the internal file names from the - format that the existing shell-script prototype uses, but this - should go away once this program both generates and checks the - manifests. + Generate a manifest, expressed as a dictionary. """ - self.check_format() result = {} - for line in self.open(self.torrent_name + ".manifest"): - h, fn = line.split() - # - # Fixup for earlier manifest format, this should go away - if not fn.startswith(self.torrent_name): - fn = os.path.normpath(os.path.join(self.torrent_name, fn)) - # - result[fn] = h + topdir = os.path.abspath(topdir) + for dirpath, dirnames, filenames in os.walk(os.path.join(topdir, torrent_name)): # pylint: disable=W0612 + for filename in filenames: + filename = os.path.join(dirpath, filename) + f = open(filename, "rb") + result[os.path.relpath(filename, topdir)] = hashlib.sha256(f.read()).hexdigest() + f.close() return result -def create_manifest(topdir, torrent_name): - """ - Generate a manifest, expressed as a dictionary. - """ - - result = {} - topdir = os.path.abspath(topdir) - for dirpath, dirnames, filenames in os.walk(os.path.join(topdir, torrent_name)): # pylint: disable=W0612 - for filename in filenames: - filename = os.path.join(dirpath, filename) - f = open(filename, "rb") - result[os.path.relpath(filename, topdir)] = hashlib.sha256(f.read()).hexdigest() - f.close() - return result - - def log_email(msg, subj = None): - try: - if not msg.endswith("\n"): - msg += "\n" - if subj is None: - subj = msg.partition("\n")[0] - m = email.mime.text.MIMEText(msg) - m["Date"] = time.strftime("%d %b %Y %H:%M:%S +0000", time.gmtime()) - m["From"] = cfg.log_email - m["To"] = cfg.log_email - m["Subject"] = subj - s = smtplib.SMTP("localhost") - s.sendmail(cfg.log_email, [cfg.log_email], m.as_string()) - s.quit() - except ConfigParser.Error: - pass + try: + if not msg.endswith("\n"): + msg += "\n" + if subj is None: + subj = msg.partition("\n")[0] + m = email.mime.text.MIMEText(msg) + m["Date"] = time.strftime("%d %b %Y %H:%M:%S +0000", time.gmtime()) + m["From"] = cfg.log_email + m["To"] = cfg.log_email + m["Subject"] = subj + s = smtplib.SMTP("localhost") + s.sendmail(cfg.log_email, [cfg.log_email], m.as_string()) + s.quit() + except ConfigParser.Error: + pass class TransmissionClient(transmissionrpc.client.Client): - """ - Extension of transmissionrpc.client.Client. - """ - - def __init__(self, **kwargs): - kwargs.setdefault("address", "127.0.0.1") - kwargs.setdefault("user", cfg.transmission_username) - kwargs.setdefault("password", cfg.transmission_password) - transmissionrpc.client.Client.__init__(self, **kwargs) - - - def find_torrents(self, *names): - """ - Find torrents with given name(s), return id(s). - """ - - result = [i for i, t in self.list().iteritems() if t.name in names] - if not result: - raise CouldNotFindTorrents - return result - - - def remove_torrents(self, *names): """ - Remove any torrents with the given name(s). + Extension of transmissionrpc.client.Client. """ - try: - ids = self.find_torrents(*names) - except CouldNotFindTorrents: - pass - else: - syslog.syslog("Removing torrent%s %s (%s)" % ( - "" if len(ids) == 1 else "s", - ", ".join(names), - ", ".join("#%s" % i for i in ids))) - self.remove(ids) + def __init__(self, **kwargs): + kwargs.setdefault("address", "127.0.0.1") + kwargs.setdefault("user", cfg.transmission_username) + kwargs.setdefault("password", cfg.transmission_password) + transmissionrpc.client.Client.__init__(self, **kwargs) - def unlimited_seeding(self, *names): - """ - Set unlimited seeding for specified torrents. - """ - # Apparently seedRatioMode = 2 means "no limit" - try: - self.change(self.find_torrents(*names), seedRatioMode = 2) - except CouldNotFindTorrents: - syslog.syslog("Couldn't tweak seedRatioMode, blundering onwards") + def find_torrents(self, *names): + """ + Find torrents with given name(s), return id(s). + """ + result = [i for i, t in self.list().iteritems() if t.name in names] + if not result: + raise CouldNotFindTorrents + return result -class MyConfigParser(ConfigParser.RawConfigParser): - rpki_torrent_section = "rpki-torrent" + def remove_torrents(self, *names): + """ + Remove any torrents with the given name(s). + """ - @property - def zip_dir(self): - return self.get(self.rpki_torrent_section, "zip_dir") + try: + ids = self.find_torrents(*names) + except CouldNotFindTorrents: + pass + else: + syslog.syslog("Removing torrent%s %s (%s)" % ( + "" if len(ids) == 1 else "s", + ", ".join(names), + ", ".join("#%s" % i for i in ids))) + self.remove(ids) - @property - def zip_ta(self): - return self.get(self.rpki_torrent_section, "zip_ta") + def unlimited_seeding(self, *names): + """ + Set unlimited seeding for specified torrents. + """ - @property - def rcynic_prog(self): - return self.get(self.rpki_torrent_section, "rcynic_prog") + # Apparently seedRatioMode = 2 means "no limit" + try: + self.change(self.find_torrents(*names), seedRatioMode = 2) + except CouldNotFindTorrents: + syslog.syslog("Couldn't tweak seedRatioMode, blundering onwards") - @property - def rcynic_conf(self): - return self.get(self.rpki_torrent_section, "rcynic_conf") - @property - def run_rcynic_anyway(self): - return self.getboolean(self.rpki_torrent_section, "run_rcynic_anyway") - - @property - def generate_url(self): - return self.get(self.rpki_torrent_section, "generate_url") - - @property - def act_as_generator(self): - try: - return self.get(self.rpki_torrent_section, "generate_url") != "" - except ConfigParser.Error: - return False - - @property - def rsync_prog(self): - return self.get(self.rpki_torrent_section, "rsync_prog") - - @property - def mktorrent_prog(self): - return self.get(self.rpki_torrent_section, "mktorrent_prog") - - @property - def tracker_url(self): - return self.get(self.rpki_torrent_section, "tracker_url") - - @property - def sftp_host(self): - return self.get(self.rpki_torrent_section, "sftp_host") - - @property - def sftp_port(self): - try: - return self.getint(self.rpki_torrent_section, "sftp_port") - except ConfigParser.Error: - return 22 - - @property - def sftp_user(self): - return self.get(self.rpki_torrent_section, "sftp_user") - - @property - def sftp_hostkey_file(self): - return self.get(self.rpki_torrent_section, "sftp_hostkey_file") - - @property - def sftp_private_key_file(self): - return self.get(self.rpki_torrent_section, "sftp_private_key_file") - - @property - def lockfile(self): - try: - return self.get(self.rpki_torrent_section, "lockfile") - except ConfigParser.Error: - return None - - @property - def unauthenticated(self): - try: - return self.get(self.rpki_torrent_section, "unauthenticated") - except ConfigParser.Error: - return self.get("rcynic", "unauthenticated") - - @property - def log_email(self): - return self.get(self.rpki_torrent_section, "log_email") - - @property - def transmission_username(self): - try: - return self.get(self.rpki_torrent_section, "transmission_username") - except ConfigParser.Error: - return None +class MyConfigParser(ConfigParser.RawConfigParser): - @property - def transmission_password(self): - try: - return self.get(self.rpki_torrent_section, "transmission_password") - except ConfigParser.Error: - return None - - def multioption_iter(self, name, getter = None): - if getter is None: - getter = self.get - if self.has_option(self.rpki_torrent_section, name): - yield getter(self.rpki_torrent_section, name) - name += "." - names = [i for i in self.options(self.rpki_torrent_section) if i.startswith(name) and i[len(name):].isdigit()] - names.sort(key = lambda s: int(s[len(name):])) # pylint: disable=W0631 - for name in names: - yield getter(self.rpki_torrent_section, name) - - @property - def zip_urls(self): - return self.multioption_iter("zip_url") - - @property - def post_rcynic_commands(self): - return self.multioption_iter("post_rcynic_command") - - def find_url(self, torrent_name): - urls = [u for u in self.zip_urls - if os.path.splitext(os.path.basename(u))[0] == torrent_name] - if len(urls) != 1: - raise TorrentNameDoesNotMatchURL("Can't find URL matching torrent name %s" % torrent_name) - return urls[0] + rpki_torrent_section = "rpki-torrent" + + @property + def zip_dir(self): + return self.get(self.rpki_torrent_section, "zip_dir") + + @property + def zip_ta(self): + return self.get(self.rpki_torrent_section, "zip_ta") + + @property + def rcynic_prog(self): + return self.get(self.rpki_torrent_section, "rcynic_prog") + + @property + def rcynic_conf(self): + return self.get(self.rpki_torrent_section, "rcynic_conf") + + @property + def run_rcynic_anyway(self): + return self.getboolean(self.rpki_torrent_section, "run_rcynic_anyway") + + @property + def generate_url(self): + return self.get(self.rpki_torrent_section, "generate_url") + + @property + def act_as_generator(self): + try: + return self.get(self.rpki_torrent_section, "generate_url") != "" + except ConfigParser.Error: + return False + + @property + def rsync_prog(self): + return self.get(self.rpki_torrent_section, "rsync_prog") + + @property + def mktorrent_prog(self): + return self.get(self.rpki_torrent_section, "mktorrent_prog") + + @property + def tracker_url(self): + return self.get(self.rpki_torrent_section, "tracker_url") + + @property + def sftp_host(self): + return self.get(self.rpki_torrent_section, "sftp_host") + + @property + def sftp_port(self): + try: + return self.getint(self.rpki_torrent_section, "sftp_port") + except ConfigParser.Error: + return 22 + + @property + def sftp_user(self): + return self.get(self.rpki_torrent_section, "sftp_user") + + @property + def sftp_hostkey_file(self): + return self.get(self.rpki_torrent_section, "sftp_hostkey_file") + + @property + def sftp_private_key_file(self): + return self.get(self.rpki_torrent_section, "sftp_private_key_file") + + @property + def lockfile(self): + try: + return self.get(self.rpki_torrent_section, "lockfile") + except ConfigParser.Error: + return None + + @property + def unauthenticated(self): + try: + return self.get(self.rpki_torrent_section, "unauthenticated") + except ConfigParser.Error: + return self.get("rcynic", "unauthenticated") + + @property + def log_email(self): + return self.get(self.rpki_torrent_section, "log_email") + + @property + def transmission_username(self): + try: + return self.get(self.rpki_torrent_section, "transmission_username") + except ConfigParser.Error: + return None + + @property + def transmission_password(self): + try: + return self.get(self.rpki_torrent_section, "transmission_password") + except ConfigParser.Error: + return None + + def multioption_iter(self, name, getter = None): + if getter is None: + getter = self.get + if self.has_option(self.rpki_torrent_section, name): + yield getter(self.rpki_torrent_section, name) + name += "." + names = [i for i in self.options(self.rpki_torrent_section) if i.startswith(name) and i[len(name):].isdigit()] + names.sort(key = lambda s: int(s[len(name):])) # pylint: disable=W0631 + for name in names: + yield getter(self.rpki_torrent_section, name) + + @property + def zip_urls(self): + return self.multioption_iter("zip_url") + + @property + def post_rcynic_commands(self): + return self.multioption_iter("post_rcynic_command") + + def find_url(self, torrent_name): + urls = [u for u in self.zip_urls + if os.path.splitext(os.path.basename(u))[0] == torrent_name] + if len(urls) != 1: + raise TorrentNameDoesNotMatchURL("Can't find URL matching torrent name %s" % torrent_name) + return urls[0] if __name__ == "__main__": - main() + main() diff --git a/rp/rcynic/validation_status b/rp/rcynic/validation_status index a3ee36f1..f961f473 100755 --- a/rp/rcynic/validation_status +++ b/rp/rcynic/validation_status @@ -23,14 +23,14 @@ Flat text listing of elements from rcynic.xml. import sys try: - from lxml.etree import ElementTree + from lxml.etree import ElementTree except ImportError: - from xml.etree.ElementTree import ElementTree + from xml.etree.ElementTree import ElementTree for filename in ([sys.stdin] if len(sys.argv) < 2 else sys.argv[1:]): - for elt in ElementTree(file = filename).findall("validation_status"): - print "%s %8s %-40s %s" % ( - elt.get("timestamp"), - elt.get("generation"), - elt.get("status"), - elt.text.strip()) + for elt in ElementTree(file = filename).findall("validation_status"): + print "%s %8s %-40s %s" % ( + elt.get("timestamp"), + elt.get("generation"), + elt.get("status"), + elt.text.strip()) diff --git a/rp/rpki-rtr/rpki-rtr b/rp/rpki-rtr/rpki-rtr index 5ad4cf26..7f3e6b4f 100755 --- a/rp/rpki-rtr/rpki-rtr +++ b/rp/rpki-rtr/rpki-rtr @@ -19,5 +19,5 @@ # PERFORMANCE OF THIS SOFTWARE. if __name__ == "__main__": - from rpki.rtr.main import main - main() + from rpki.rtr.main import main + main() diff --git a/rp/utils/find_roa b/rp/utils/find_roa index 2b537bf4..15a2f25f 100755 --- a/rp/utils/find_roa +++ b/rp/utils/find_roa @@ -28,109 +28,109 @@ import rpki.oids def check_dir(s): - if os.path.isdir(s): - return os.path.abspath(s) - else: - raise argparse.ArgumentTypeError("%r is not a directory" % s) + if os.path.isdir(s): + return os.path.abspath(s) + else: + raise argparse.ArgumentTypeError("%r is not a directory" % s) def filename_to_uri(filename): - if not filename.startswith(args.rcynic_dir): - raise ValueError - return "rsync://" + filename[len(args.rcynic_dir):].lstrip("/") + if not filename.startswith(args.rcynic_dir): + raise ValueError + return "rsync://" + filename[len(args.rcynic_dir):].lstrip("/") def uri_to_filename(uri): - if not uri.startswith("rsync://"): - raise ValueError - return os.path.join(args.rcynic_dir, uri[len("rsync://"):]) + if not uri.startswith("rsync://"): + raise ValueError + return os.path.join(args.rcynic_dir, uri[len("rsync://"):]) class Prefix(object): - """ - One prefix parsed from the command line. - """ - - def __init__(self, val): - addr, length = val.split("/") - length, sep, maxlength = length.partition("-") # pylint: disable=W0612 - self.prefix = rpki.POW.IPAddress(addr) - self.length = int(length) - self.maxlength = int(maxlength) if maxlength else self.length - if self.maxlength < self.length or self.length < 0 or self.length > self.prefix.bits: - raise ValueError - if self.prefix & ((1 << (self.prefix.bits - self.length)) - 1) != 0: - raise ValueError - - def matches(self, roa): # pylint: disable=W0621 - return any(self.prefix == prefix and - self.length == length and - (not args.match_maxlength or - self.maxlength == maxlength or - (maxlength is None and - self.length == self.maxlength)) - for prefix, length, maxlength in roa.prefixes) + """ + One prefix parsed from the command line. + """ + + def __init__(self, val): + addr, length = val.split("/") + length, sep, maxlength = length.partition("-") # pylint: disable=W0612 + self.prefix = rpki.POW.IPAddress(addr) + self.length = int(length) + self.maxlength = int(maxlength) if maxlength else self.length + if self.maxlength < self.length or self.length < 0 or self.length > self.prefix.bits: + raise ValueError + if self.prefix & ((1 << (self.prefix.bits - self.length)) - 1) != 0: + raise ValueError + + def matches(self, roa): # pylint: disable=W0621 + return any(self.prefix == prefix and + self.length == length and + (not args.match_maxlength or + self.maxlength == maxlength or + (maxlength is None and + self.length == self.maxlength)) + for prefix, length, maxlength in roa.prefixes) class ROA(rpki.POW.ROA): # pylint: disable=W0232 - """ - Aspects of a ROA that we care about. - """ - - @classmethod - def parse(cls, fn): # pylint: disable=W0621 - assert fn.startswith(args.rcynic_dir) - self = cls.derReadFile(fn) - self.fn = fn - self.extractWithoutVerifying() - v4, v6 = self.getPrefixes() - self.prefixes = (v4 or ()) + (v6 or ()) - return self - - @property - def uri(self): - return filename_to_uri(self.fn) - - @property - def formatted_prefixes(self): - for prefix in self.prefixes: - if prefix[2] is None or prefix[1] == prefix[2]: - yield "%s/%d" % (prefix[0], prefix[1]) - else: - yield "%s/%d-%d" % (prefix[0], prefix[1], prefix[2]) - - def __str__(self): - prefixes = " ".join(self.formatted_prefixes) - plural = "es" if " " in prefixes else "" - if args.show_inception: - return "signingTime %s ASN %s prefix%s %s" % (self.signingTime(), self.getASID(), plural, prefixes) - else: - return "ASN %s prefix%s %s" % (self.getASID(), plural, prefixes) - - def show(self): - print "%s %s" % (self, self.fn if args.show_filenames else self.uri) - - def show_expiration(self): - print self - x = self.certs()[0] - fn = self.fn # pylint: disable=W0621 - uri = self.uri - while uri is not None: - name = fn if args.show_filenames else uri - if args.show_inception: - print "notBefore", x.getNotBefore(), "notAfter", x.getNotAfter(), name - else: - print x.getNotAfter(), name - for uri in x.getAIA() or (): - if uri.startswith("rsync://"): - break - else: - break - fn = uri_to_filename(uri) - if not os.path.exists(fn): - print "***** MISSING ******", uri - break - x = rpki.POW.X509.derReadFile(fn) - print + """ + Aspects of a ROA that we care about. + """ + + @classmethod + def parse(cls, fn): # pylint: disable=W0621 + assert fn.startswith(args.rcynic_dir) + self = cls.derReadFile(fn) + self.fn = fn + self.extractWithoutVerifying() + v4, v6 = self.getPrefixes() + self.prefixes = (v4 or ()) + (v6 or ()) + return self + + @property + def uri(self): + return filename_to_uri(self.fn) + + @property + def formatted_prefixes(self): + for prefix in self.prefixes: + if prefix[2] is None or prefix[1] == prefix[2]: + yield "%s/%d" % (prefix[0], prefix[1]) + else: + yield "%s/%d-%d" % (prefix[0], prefix[1], prefix[2]) + + def __str__(self): + prefixes = " ".join(self.formatted_prefixes) + plural = "es" if " " in prefixes else "" + if args.show_inception: + return "signingTime %s ASN %s prefix%s %s" % (self.signingTime(), self.getASID(), plural, prefixes) + else: + return "ASN %s prefix%s %s" % (self.getASID(), plural, prefixes) + + def show(self): + print "%s %s" % (self, self.fn if args.show_filenames else self.uri) + + def show_expiration(self): + print self + x = self.certs()[0] + fn = self.fn # pylint: disable=W0621 + uri = self.uri + while uri is not None: + name = fn if args.show_filenames else uri + if args.show_inception: + print "notBefore", x.getNotBefore(), "notAfter", x.getNotAfter(), name + else: + print x.getNotAfter(), name + for uri in x.getAIA() or (): + if uri.startswith("rsync://"): + break + else: + break + fn = uri_to_filename(uri) + if not os.path.exists(fn): + print "***** MISSING ******", uri + break + x = rpki.POW.X509.derReadFile(fn) + print parser = argparse.ArgumentParser(description = __doc__) @@ -145,14 +145,14 @@ args = parser.parse_args() # If there's some way to automate this in the parser, I don't know what it is, so just catch it here. if args.all != (not args.prefixes): - parser.error("--all and prefix list are mutually exclusive") + parser.error("--all and prefix list are mutually exclusive") for root, dirs, files in os.walk(args.rcynic_dir): - for fn in files: - if fn.endswith(".roa"): - roa = ROA.parse(os.path.join(root, fn)) - if args.all or any(prefix.matches(roa) for prefix in args.prefixes): - if args.show_expiration: - roa.show_expiration() - else: - roa.show() + for fn in files: + if fn.endswith(".roa"): + roa = ROA.parse(os.path.join(root, fn)) + if args.all or any(prefix.matches(roa) for prefix in args.prefixes): + if args.show_expiration: + roa.show_expiration() + else: + roa.show() diff --git a/rp/utils/hashdir b/rp/utils/hashdir index d3fe393c..c7c18350 100755 --- a/rp/utils/hashdir +++ b/rp/utils/hashdir @@ -30,10 +30,10 @@ import argparse import rpki.POW def check_dir(s): - if os.path.isdir(s): - return os.path.abspath(s) - else: - raise argparse.ArgumentTypeError("%r is not a directory" % s) + if os.path.isdir(s): + return os.path.abspath(s) + else: + raise argparse.ArgumentTypeError("%r is not a directory" % s) parser = argparse.ArgumentParser(description = __doc__) parser.add_argument("-v", "--verbose", action = "store_true", help = "whistle while you work") @@ -42,26 +42,26 @@ parser.add_argument("output_dir", help = "name of output directory to create") args = parser.parse_args() if not os.path.isdir(args.output_dir): - os.makedirs(args.output_dir) + os.makedirs(args.output_dir) for root, dirs, files in os.walk(args.rcynic_dir): - for ifn in files: - ifn = os.path.join(root, ifn) - if ifn.endswith(".cer"): - obj = rpki.POW.X509.derReadFile(ifn) - fmt = "%08x.%%d" % obj.getSubjectHash() - elif ifn.endswith(".crl"): - obj = rpki.POW.CRL.derReadFile(ifn) - fmt = "%08x.r%%d" % obj.getIssuerHash() - else: - continue - for i in xrange(1000000): - ofn = os.path.join(args.output_dir, fmt % i) - if not os.path.exists(ofn): - with open(ofn, "w") as f: - f.write(obj.pemWrite()) - if args.verbose: - print ofn, "<=", ifn - break - else: - sys.exit("No path name available for %s (%s)" % (ifn, ofn)) + for ifn in files: + ifn = os.path.join(root, ifn) + if ifn.endswith(".cer"): + obj = rpki.POW.X509.derReadFile(ifn) + fmt = "%08x.%%d" % obj.getSubjectHash() + elif ifn.endswith(".crl"): + obj = rpki.POW.CRL.derReadFile(ifn) + fmt = "%08x.r%%d" % obj.getIssuerHash() + else: + continue + for i in xrange(1000000): + ofn = os.path.join(args.output_dir, fmt % i) + if not os.path.exists(ofn): + with open(ofn, "w") as f: + f.write(obj.pemWrite()) + if args.verbose: + print ofn, "<=", ifn + break + else: + sys.exit("No path name available for %s (%s)" % (ifn, ofn)) diff --git a/rp/utils/print_roa b/rp/utils/print_roa index c96a7c66..78ae244f 100755 --- a/rp/utils/print_roa +++ b/rp/utils/print_roa @@ -26,18 +26,18 @@ import rpki.POW class ROA(rpki.POW.ROA): # pylint: disable=W0232 - @staticmethod - def _format_prefix(p): - if p[2] in (None, p[1]): - return "%s/%d" % (p[0], p[1]) - else: - return "%s/%d-%d" % (p[0], p[1], p[2]) + @staticmethod + def _format_prefix(p): + if p[2] in (None, p[1]): + return "%s/%d" % (p[0], p[1]) + else: + return "%s/%d-%d" % (p[0], p[1], p[2]) - def parse(self): - self.extractWithoutVerifying() - v4, v6 = self.getPrefixes() - self.v4_prefixes = [self._format_prefix(p) for p in (v4 or ())] - self.v6_prefixes = [self._format_prefix(p) for p in (v6 or ())] + def parse(self): + self.extractWithoutVerifying() + v4, v6 = self.getPrefixes() + self.v4_prefixes = [self._format_prefix(p) for p in (v4 or ())] + self.v6_prefixes = [self._format_prefix(p) for p in (v6 or ())] parser = argparse.ArgumentParser(description = __doc__) parser.add_argument("-b", "--brief", action = "store_true", help = "show only ASN and prefix(es)") @@ -47,27 +47,27 @@ parser.add_argument("roas", nargs = "+", type = ROA.derReadFile, help = "ROA(s) args = parser.parse_args() for roa in args.roas: - roa.parse() - if args.brief: - if args.signing_time: - print roa.signingTime(), - print roa.getASID(), " ".join(roa.v4_prefixes + roa.v6_prefixes) - else: - print "ROA Version: ", roa.getVersion() - print "SigningTime: ", roa.signingTime() - print "asID: ", roa.getASID() - if roa.v4_prefixes: - print " addressFamily:", 1 - for prefix in roa.v4_prefixes: - print " IPAddress:", prefix - if roa.v6_prefixes: - print " addressFamily:", 2 - for prefix in roa.v6_prefixes: - print " IPAddress:", prefix - if args.cms: - print roa.pprint() - for cer in roa.certs(): - print cer.pprint() - for crl in roa.crls(): - print crl.pprint() - print + roa.parse() + if args.brief: + if args.signing_time: + print roa.signingTime(), + print roa.getASID(), " ".join(roa.v4_prefixes + roa.v6_prefixes) + else: + print "ROA Version: ", roa.getVersion() + print "SigningTime: ", roa.signingTime() + print "asID: ", roa.getASID() + if roa.v4_prefixes: + print " addressFamily:", 1 + for prefix in roa.v4_prefixes: + print " IPAddress:", prefix + if roa.v6_prefixes: + print " addressFamily:", 2 + for prefix in roa.v6_prefixes: + print " IPAddress:", prefix + if args.cms: + print roa.pprint() + for cer in roa.certs(): + print cer.pprint() + for crl in roa.crls(): + print crl.pprint() + print diff --git a/rp/utils/print_rpki_manifest b/rp/utils/print_rpki_manifest index ce9b25ea..83e5c16e 100755 --- a/rp/utils/print_rpki_manifest +++ b/rp/utils/print_rpki_manifest @@ -31,20 +31,20 @@ parser.add_argument("manifests", nargs = "+", type = rpki.POW.Manifest.derReadFi args = parser.parse_args() for mft in args.manifests: - mft.extractWithoutVerifying() - print "Manifest Version:", mft.getVersion() - print "SigningTime: ", mft.signingTime() - print "Number: ", mft.getManifestNumber() - print "thisUpdate: ", mft.getThisUpdate() - print "nextUpdate: ", mft.getNextUpdate() - print "fileHashAlg: ", rpki.oids.oid2name(mft.getAlgorithm()) - for i, fah in enumerate(mft.getFiles()): - name, obj_hash = fah - print "fileList[%3d]: %s %s" % (i, ":".join(("%02X" % ord(h) for h in obj_hash)), name) - if args.cms: - print mft.pprint() - for cer in mft.certs(): - print cer.pprint() - for crl in mft.crls(): - print crl.pprint() - print + mft.extractWithoutVerifying() + print "Manifest Version:", mft.getVersion() + print "SigningTime: ", mft.signingTime() + print "Number: ", mft.getManifestNumber() + print "thisUpdate: ", mft.getThisUpdate() + print "nextUpdate: ", mft.getNextUpdate() + print "fileHashAlg: ", rpki.oids.oid2name(mft.getAlgorithm()) + for i, fah in enumerate(mft.getFiles()): + name, obj_hash = fah + print "fileList[%3d]: %s %s" % (i, ":".join(("%02X" % ord(h) for h in obj_hash)), name) + if args.cms: + print mft.pprint() + for cer in mft.certs(): + print cer.pprint() + for crl in mft.crls(): + print crl.pprint() + print diff --git a/rp/utils/scan_roas b/rp/utils/scan_roas index 4f3dc7f0..f4489f32 100755 --- a/rp/utils/scan_roas +++ b/rp/utils/scan_roas @@ -27,29 +27,29 @@ import argparse import rpki.POW def check_dir(d): - if not os.path.isdir(d): - raise argparse.ArgumentTypeError("%r is not a directory" % d) - return d + if not os.path.isdir(d): + raise argparse.ArgumentTypeError("%r is not a directory" % d) + return d class ROA(rpki.POW.ROA): # pylint: disable=W0232 - @classmethod - def parse(cls, fn): # pylint: disable=W0621 - self = cls.derReadFile(fn) - self.extractWithoutVerifying() - return self + @classmethod + def parse(cls, fn): # pylint: disable=W0621 + self = cls.derReadFile(fn) + self.extractWithoutVerifying() + return self - @property - def prefixes(self): - v4, v6 = self.getPrefixes() - for prefix, length, maxlength in (v4 or ()) + (v6 or ()): - if maxlength is None or length == maxlength: - yield "%s/%d" % (prefix, length) - else: - yield "%s/%d-%d" % (prefix, length, maxlength) + @property + def prefixes(self): + v4, v6 = self.getPrefixes() + for prefix, length, maxlength in (v4 or ()) + (v6 or ()): + if maxlength is None or length == maxlength: + yield "%s/%d" % (prefix, length) + else: + yield "%s/%d-%d" % (prefix, length, maxlength) - def __str__(self): - return "%s %s %s" % (self.signingTime(), self.getASID(), " ".join(self.prefixes)) + def __str__(self): + return "%s %s %s" % (self.signingTime(), self.getASID(), " ".join(self.prefixes)) parser = argparse.ArgumentParser(description = __doc__) parser.add_argument("rcynic_dir", nargs = "+", type = check_dir, @@ -57,7 +57,7 @@ parser.add_argument("rcynic_dir", nargs = "+", type = check_dir, args = parser.parse_args() for rcynic_dir in args.rcynic_dir: - for root, dirs, files in os.walk(rcynic_dir): - for fn in files: - if fn.endswith(".roa"): - print ROA.parse(os.path.join(root, fn)) + for root, dirs, files in os.walk(rcynic_dir): + for fn in files: + if fn.endswith(".roa"): + print ROA.parse(os.path.join(root, fn)) diff --git a/rp/utils/scan_routercerts b/rp/utils/scan_routercerts index 081a6293..9a13d7a9 100755 --- a/rp/utils/scan_routercerts +++ b/rp/utils/scan_routercerts @@ -28,9 +28,9 @@ import rpki.POW import rpki.oids def check_dir(s): - if not os.path.isdir(s): - raise argparse.ArgumentTypeError("%r is not a directory" % s) - return s + if not os.path.isdir(s): + raise argparse.ArgumentTypeError("%r is not a directory" % s) + return s parser = argparse.ArgumentParser(description = __doc__) parser.add_argument("rcynic_dir", type = check_dir, help = "rcynic authenticated output directory") @@ -38,20 +38,20 @@ args = parser.parse_args() for root, dirs, files in os.walk(args.rcynic_dir): - for fn in files: + for fn in files: - if not fn.endswith(".cer"): - continue + if not fn.endswith(".cer"): + continue - x = rpki.POW.X509.derReadFile(os.path.join(root, fn)) + x = rpki.POW.X509.derReadFile(os.path.join(root, fn)) - if rpki.oids.id_kp_bgpsec_router not in (x.getEKU() or ()): - continue + if rpki.oids.id_kp_bgpsec_router not in (x.getEKU() or ()): + continue - sys.stdout.write(base64.urlsafe_b64encode(x.getSKI()).rstrip("=")) + sys.stdout.write(base64.urlsafe_b64encode(x.getSKI()).rstrip("=")) - for min_asn, max_asn in x.getRFC3779()[0]: - for asn in xrange(min_asn, max_asn + 1): - sys.stdout.write(" %s" % asn) + for min_asn, max_asn in x.getRFC3779()[0]: + for asn in xrange(min_asn, max_asn + 1): + sys.stdout.write(" %s" % asn) - sys.stdout.write(" %s\n" % base64.b64encode(x.getPublicKey().derWritePublic())) + sys.stdout.write(" %s\n" % base64.b64encode(x.getPublicKey().derWritePublic())) diff --git a/rp/utils/uri b/rp/utils/uri index df6e710b..4fecf73a 100755 --- a/rp/utils/uri +++ b/rp/utils/uri @@ -29,44 +29,44 @@ import rpki.POW class Certificate(object): - @staticmethod - def first_whatever(uris, prefix): - if uris is not None: - for uri in uris: - if uri.startswith(prefix): - return uri - return None + @staticmethod + def first_whatever(uris, prefix): + if uris is not None: + for uri in uris: + if uri.startswith(prefix): + return uri + return None - def first_rsync(self, uris): - return self.first_whatever(uris, "rsync://") + def first_rsync(self, uris): + return self.first_whatever(uris, "rsync://") - def first_http(self, uris): - return self.first_whatever(uris, "http://") + def first_http(self, uris): + return self.first_whatever(uris, "http://") - def __init__(self, fn): - try: - x = rpki.POW.X509.derReadFile(fn) - except: # pylint: disable=W0702 - try: - cms = rpki.POW.CMS.derReadFile(fn) - cms.extractWithoutVerifying() - x = cms.certs()[0] - except: - raise ValueError - sia = x.getSIA() or (None, None, None, None) - self.fn = fn - self.uris = ( - ("AIA:caIssuers", self.first_rsync(x.getAIA())), - ("SIA:caRepository", self.first_rsync(sia[0])), - ("SIA:rpkiManifest", self.first_rsync(sia[1])), - ("SIA:signedObject", self.first_rsync(sia[2])), - ("SIA:rpkiNotify", self.first_http(sia[3])), - ("CRLDP", self.first_rsync(x.getCRLDP()))) + def __init__(self, fn): + try: + x = rpki.POW.X509.derReadFile(fn) + except: # pylint: disable=W0702 + try: + cms = rpki.POW.CMS.derReadFile(fn) + cms.extractWithoutVerifying() + x = cms.certs()[0] + except: + raise ValueError + sia = x.getSIA() or (None, None, None, None) + self.fn = fn + self.uris = ( + ("AIA:caIssuers", self.first_rsync(x.getAIA())), + ("SIA:caRepository", self.first_rsync(sia[0])), + ("SIA:rpkiManifest", self.first_rsync(sia[1])), + ("SIA:signedObject", self.first_rsync(sia[2])), + ("SIA:rpkiNotify", self.first_http(sia[3])), + ("CRLDP", self.first_rsync(x.getCRLDP()))) - def __str__(self): - words = [self.fn] if args.single_line else ["File: " + self.fn] - words.extend(" %s: %s" % (tag, uri) for tag, uri in self.uris if uri is not None) - return ("" if args.single_line else "\n").join(words) + def __str__(self): + words = [self.fn] if args.single_line else ["File: " + self.fn] + words.extend(" %s: %s" % (tag, uri) for tag, uri in self.uris if uri is not None) + return ("" if args.single_line else "\n").join(words) parser = argparse.ArgumentParser(description = __doc__) parser.add_argument("-s", "--single-line", action = "store_true", help = "single output line per object") @@ -74,4 +74,4 @@ parser.add_argument("certs", nargs = "+", type = Certificate, help = "RPKI objec args = parser.parse_args() for cert in args.certs: - print cert + print cert diff --git a/rpki/adns.py b/rpki/adns.py index c5af3549..b0f235e7 100644 --- a/rpki/adns.py +++ b/rpki/adns.py @@ -31,14 +31,14 @@ import rpki.sundial import rpki.log try: - import dns.resolver, dns.rdatatype, dns.rdataclass, dns.name, dns.message - import dns.inet, dns.exception, dns.query, dns.rcode, dns.ipv4, dns.ipv6 + import dns.resolver, dns.rdatatype, dns.rdataclass, dns.name, dns.message + import dns.inet, dns.exception, dns.query, dns.rcode, dns.ipv4, dns.ipv6 except ImportError: - if __name__ == "__main__": - sys.stderr.write("DNSPython not available, skipping rpki.adns unit test\n") - sys.exit(0) - else: - raise + if __name__ == "__main__": + sys.stderr.write("DNSPython not available, skipping rpki.adns unit test\n") + sys.exit(0) + else: + raise logger = logging.getLogger(__name__) @@ -47,7 +47,7 @@ logger = logging.getLogger(__name__) resolver = dns.resolver.Resolver() if resolver.cache is None: - resolver.cache = dns.resolver.Cache() + resolver.cache = dns.resolver.Cache() ## @var nameservers # Nameservers from resolver.nameservers converted to (af, address) @@ -58,327 +58,327 @@ if resolver.cache is None: nameservers = [] for ns in resolver.nameservers: - try: - nameservers.append((socket.AF_INET, dns.ipv4.inet_aton(ns))) - continue - except: - pass - try: - nameservers.append((socket.AF_INET6, dns.ipv6.inet_aton(ns))) - continue - except: - pass - logger.error("Couldn't parse nameserver address %r", ns) + try: + nameservers.append((socket.AF_INET, dns.ipv4.inet_aton(ns))) + continue + except: + pass + try: + nameservers.append((socket.AF_INET6, dns.ipv6.inet_aton(ns))) + continue + except: + pass + logger.error("Couldn't parse nameserver address %r", ns) class dispatcher(asyncore.dispatcher): - """ - Basic UDP socket reader for use with asyncore. - """ - - def __init__(self, cb, eb, af, bufsize = 65535): - asyncore.dispatcher.__init__(self) - self.cb = cb - self.eb = eb - self.af = af - self.bufsize = bufsize - self.create_socket(af, socket.SOCK_DGRAM) - - def handle_read(self): - """ - Receive a packet, hand it off to query class callback. - """ - - wire, from_address = self.recvfrom(self.bufsize) - self.cb(self.af, from_address[0], from_address[1], wire) - - def handle_error(self): - """ - Pass errors to query class errback. - """ - - self.eb(sys.exc_info()[1]) - - def handle_connect(self): - """ - Quietly ignore UDP "connection" events. - """ - - pass - - def writable(self): """ - We don't need to hear about UDP socket becoming writable. + Basic UDP socket reader for use with asyncore. """ - return False + def __init__(self, cb, eb, af, bufsize = 65535): + asyncore.dispatcher.__init__(self) + self.cb = cb + self.eb = eb + self.af = af + self.bufsize = bufsize + self.create_socket(af, socket.SOCK_DGRAM) + def handle_read(self): + """ + Receive a packet, hand it off to query class callback. + """ -class query(object): - """ - Simplified (no search paths) asynchronous adaptation of - dns.resolver.Resolver.query() (q.v.). - """ - - def __init__(self, cb, eb, qname, qtype = dns.rdatatype.A, qclass = dns.rdataclass.IN): - if isinstance(qname, (str, unicode)): - qname = dns.name.from_text(qname) - if isinstance(qtype, str): - qtype = dns.rdatatype.from_text(qtype) - if isinstance(qclass, str): - qclass = dns.rdataclass.from_text(qclass) - assert qname.is_absolute() - self.cb = cb - self.eb = eb - self.qname = qname - self.qtype = qtype - self.qclass = qclass - self.start = time.time() - rpki.async.event_defer(self.go) - - def go(self): - """ - Start running the query. Check our cache before doing network - query; if we find an answer there, just return it. Otherwise - start the network query. - """ - - if resolver.cache: - answer = resolver.cache.get((self.qname, self.qtype, self.qclass)) - else: - answer = None - if answer: - self.cb(self, answer) - else: - self.timer = rpki.async.timer() - self.sockets = {} - self.request = dns.message.make_query(self.qname, self.qtype, self.qclass) - if resolver.keyname is not None: - self.request.use_tsig(resolver.keyring, resolver.keyname, resolver.keyalgorithm) - self.request.use_edns(resolver.edns, resolver.ednsflags, resolver.payload) - self.response = None - self.backoff = 0.10 - self.nameservers = nameservers[:] - self.loop1() - - def loop1(self): - """ - Outer loop. If we haven't got a response yet and still have - nameservers to check, start inner loop. Otherwise, we're done. - """ + wire, from_address = self.recvfrom(self.bufsize) + self.cb(self.af, from_address[0], from_address[1], wire) - self.timer.cancel() - if self.response is None and self.nameservers: - self.iterator = rpki.async.iterator(self.nameservers[:], self.loop2, self.done2) - else: - self.done1() + def handle_error(self): + """ + Pass errors to query class errback. + """ - def loop2(self, iterator, nameserver): - """ - Inner loop. Send query to next nameserver in our list, unless - we've hit the overall timeout for this query. - """ + self.eb(sys.exc_info()[1]) - self.timer.cancel() - try: - timeout = resolver._compute_timeout(self.start) - except dns.resolver.Timeout, e: - self.lose(e) - else: - af, addr = nameserver - if af not in self.sockets: - self.sockets[af] = dispatcher(self.socket_cb, self.socket_eb, af) - self.sockets[af].sendto(self.request.to_wire(), - (dns.inet.inet_ntop(af, addr), resolver.port)) - self.timer.set_handler(self.socket_timeout) - self.timer.set_errback(self.socket_eb) - self.timer.set(rpki.sundial.timedelta(seconds = timeout)) - - def socket_timeout(self): - """ - No answer from nameserver, move on to next one (inner loop). - """ + def handle_connect(self): + """ + Quietly ignore UDP "connection" events. + """ - self.response = None - self.iterator() + pass - def socket_eb(self, e): - """ - UDP socket signaled error. If it really is some kind of socket - error, handle as if we've timed out on this nameserver; otherwise, - pass error back to caller. - """ + def writable(self): + """ + We don't need to hear about UDP socket becoming writable. + """ - self.timer.cancel() - if isinstance(e, socket.error): - self.response = None - self.iterator() - else: - self.lose(e) + return False - def socket_cb(self, af, from_host, from_port, wire): - """ - Received a packet that might be a DNS message. If it doesn't look - like it came from one of our nameservers, just drop it and leave - the timer running. Otherwise, try parsing it: if it's an answer, - we're done, otherwise handle error appropriately and move on to - next nameserver. - """ - sender = (af, dns.inet.inet_pton(af, from_host)) - if from_port != resolver.port or sender not in self.nameservers: - return - self.timer.cancel() - try: - self.response = dns.message.from_wire(wire, keyring = self.request.keyring, request_mac = self.request.mac, one_rr_per_rrset = False) - except dns.exception.FormError: - self.nameservers.remove(sender) - else: - rcode = self.response.rcode() - if rcode in (dns.rcode.NOERROR, dns.rcode.NXDOMAIN): - self.done1() - return - if rcode != dns.rcode.SERVFAIL: - self.nameservers.remove(sender) - self.response = None - self.iterator() - - def done2(self): +class query(object): """ - Done with inner loop. If we still haven't got an answer and - haven't (yet?) eliminated all of our nameservers, wait a little - while before starting the cycle again, unless we've hit the - timeout threshold for the whole query. + Simplified (no search paths) asynchronous adaptation of + dns.resolver.Resolver.query() (q.v.). """ - if self.response is None and self.nameservers: - try: - delay = rpki.sundial.timedelta(seconds = min(resolver._compute_timeout(self.start), self.backoff)) - self.backoff *= 2 - self.timer.set_handler(self.loop1) - self.timer.set_errback(self.lose) - self.timer.set(delay) - except dns.resolver.Timeout, e: - self.lose(e) - else: - self.loop1() - - def cleanup(self): - """ - Shut down our timer and sockets. - """ + def __init__(self, cb, eb, qname, qtype = dns.rdatatype.A, qclass = dns.rdataclass.IN): + if isinstance(qname, (str, unicode)): + qname = dns.name.from_text(qname) + if isinstance(qtype, str): + qtype = dns.rdatatype.from_text(qtype) + if isinstance(qclass, str): + qclass = dns.rdataclass.from_text(qclass) + assert qname.is_absolute() + self.cb = cb + self.eb = eb + self.qname = qname + self.qtype = qtype + self.qclass = qclass + self.start = time.time() + rpki.async.event_defer(self.go) + + def go(self): + """ + Start running the query. Check our cache before doing network + query; if we find an answer there, just return it. Otherwise + start the network query. + """ + + if resolver.cache: + answer = resolver.cache.get((self.qname, self.qtype, self.qclass)) + else: + answer = None + if answer: + self.cb(self, answer) + else: + self.timer = rpki.async.timer() + self.sockets = {} + self.request = dns.message.make_query(self.qname, self.qtype, self.qclass) + if resolver.keyname is not None: + self.request.use_tsig(resolver.keyring, resolver.keyname, resolver.keyalgorithm) + self.request.use_edns(resolver.edns, resolver.ednsflags, resolver.payload) + self.response = None + self.backoff = 0.10 + self.nameservers = nameservers[:] + self.loop1() + + def loop1(self): + """ + Outer loop. If we haven't got a response yet and still have + nameservers to check, start inner loop. Otherwise, we're done. + """ + + self.timer.cancel() + if self.response is None and self.nameservers: + self.iterator = rpki.async.iterator(self.nameservers[:], self.loop2, self.done2) + else: + self.done1() + + def loop2(self, iterator, nameserver): + """ + Inner loop. Send query to next nameserver in our list, unless + we've hit the overall timeout for this query. + """ + + self.timer.cancel() + try: + timeout = resolver._compute_timeout(self.start) + except dns.resolver.Timeout, e: + self.lose(e) + else: + af, addr = nameserver + if af not in self.sockets: + self.sockets[af] = dispatcher(self.socket_cb, self.socket_eb, af) + self.sockets[af].sendto(self.request.to_wire(), + (dns.inet.inet_ntop(af, addr), resolver.port)) + self.timer.set_handler(self.socket_timeout) + self.timer.set_errback(self.socket_eb) + self.timer.set(rpki.sundial.timedelta(seconds = timeout)) + + def socket_timeout(self): + """ + No answer from nameserver, move on to next one (inner loop). + """ + + self.response = None + self.iterator() + + def socket_eb(self, e): + """ + UDP socket signaled error. If it really is some kind of socket + error, handle as if we've timed out on this nameserver; otherwise, + pass error back to caller. + """ + + self.timer.cancel() + if isinstance(e, socket.error): + self.response = None + self.iterator() + else: + self.lose(e) + + def socket_cb(self, af, from_host, from_port, wire): + """ + Received a packet that might be a DNS message. If it doesn't look + like it came from one of our nameservers, just drop it and leave + the timer running. Otherwise, try parsing it: if it's an answer, + we're done, otherwise handle error appropriately and move on to + next nameserver. + """ + + sender = (af, dns.inet.inet_pton(af, from_host)) + if from_port != resolver.port or sender not in self.nameservers: + return + self.timer.cancel() + try: + self.response = dns.message.from_wire(wire, keyring = self.request.keyring, request_mac = self.request.mac, one_rr_per_rrset = False) + except dns.exception.FormError: + self.nameservers.remove(sender) + else: + rcode = self.response.rcode() + if rcode in (dns.rcode.NOERROR, dns.rcode.NXDOMAIN): + self.done1() + return + if rcode != dns.rcode.SERVFAIL: + self.nameservers.remove(sender) + self.response = None + self.iterator() + + def done2(self): + """ + Done with inner loop. If we still haven't got an answer and + haven't (yet?) eliminated all of our nameservers, wait a little + while before starting the cycle again, unless we've hit the + timeout threshold for the whole query. + """ + + if self.response is None and self.nameservers: + try: + delay = rpki.sundial.timedelta(seconds = min(resolver._compute_timeout(self.start), self.backoff)) + self.backoff *= 2 + self.timer.set_handler(self.loop1) + self.timer.set_errback(self.lose) + self.timer.set(delay) + except dns.resolver.Timeout, e: + self.lose(e) + else: + self.loop1() + + def cleanup(self): + """ + Shut down our timer and sockets. + """ + + self.timer.cancel() + for s in self.sockets.itervalues(): + s.close() - self.timer.cancel() - for s in self.sockets.itervalues(): - s.close() + def lose(self, e): + """ + Something bad happened. Clean up, then pass error back to caller. + """ + + self.cleanup() + self.eb(self, e) + + def done1(self): + """ + Done with outer loop. If we got a useful answer, cache it, then + pass it back to caller; if we got an error, pass the appropriate + exception back to caller. + """ + + self.cleanup() + try: + if not self.nameservers: + raise dns.resolver.NoNameservers + if self.response.rcode() == dns.rcode.NXDOMAIN: + raise dns.resolver.NXDOMAIN + answer = dns.resolver.Answer(self.qname, self.qtype, self.qclass, self.response) + if resolver.cache: + resolver.cache.put((self.qname, self.qtype, self.qclass), answer) + self.cb(self, answer) + except (rpki.async.ExitNow, SystemExit): + raise + except Exception, e: + self.lose(e) - def lose(self, e): - """ - Something bad happened. Clean up, then pass error back to caller. - """ +class getaddrinfo(object): - self.cleanup() - self.eb(self, e) + typemap = { dns.rdatatype.A : socket.AF_INET, + dns.rdatatype.AAAA : socket.AF_INET6 } + + def __init__(self, cb, eb, host, address_families = typemap.values()): + self.cb = cb + self.eb = eb + self.host = host + self.result = [] + self.queries = [query(self.done, self.lose, host, qtype) + for qtype in self.typemap + if self.typemap[qtype] in address_families] + + def done(self, q, answer): + if answer is not None: + for a in answer: + self.result.append((self.typemap[a.rdtype], a.address)) + self.queries.remove(q) + if not self.queries: + self.cb(self.result) - def done1(self): - """ - Done with outer loop. If we got a useful answer, cache it, then - pass it back to caller; if we got an error, pass the appropriate - exception back to caller. - """ + def lose(self, q, e): + if isinstance(e, dns.resolver.NoAnswer): + self.done(q, None) + else: + for q in self.queries: + q.cleanup() + self.eb(e) - self.cleanup() - try: - if not self.nameservers: - raise dns.resolver.NoNameservers - if self.response.rcode() == dns.rcode.NXDOMAIN: - raise dns.resolver.NXDOMAIN - answer = dns.resolver.Answer(self.qname, self.qtype, self.qclass, self.response) - if resolver.cache: - resolver.cache.put((self.qname, self.qtype, self.qclass), answer) - self.cb(self, answer) - except (rpki.async.ExitNow, SystemExit): - raise - except Exception, e: - self.lose(e) +if __name__ == "__main__": -class getaddrinfo(object): + rpki.log.init("test-adns") + print "Some adns tests may take a minute or two, please be patient" - typemap = { dns.rdatatype.A : socket.AF_INET, - dns.rdatatype.AAAA : socket.AF_INET6 } - - def __init__(self, cb, eb, host, address_families = typemap.values()): - self.cb = cb - self.eb = eb - self.host = host - self.result = [] - self.queries = [query(self.done, self.lose, host, qtype) - for qtype in self.typemap - if self.typemap[qtype] in address_families] - - def done(self, q, answer): - if answer is not None: - for a in answer: - self.result.append((self.typemap[a.rdtype], a.address)) - self.queries.remove(q) - if not self.queries: - self.cb(self.result) - - def lose(self, q, e): - if isinstance(e, dns.resolver.NoAnswer): - self.done(q, None) - else: - for q in self.queries: - q.cleanup() - self.eb(e) + class test_getaddrinfo(object): -if __name__ == "__main__": + def __init__(self, qname): + self.qname = qname + getaddrinfo(self.done, self.lose, qname) - rpki.log.init("test-adns") - print "Some adns tests may take a minute or two, please be patient" + def done(self, result): + print "getaddrinfo(%s) returned: %s" % ( + self.qname, + ", ".join(str(r) for r in result)) - class test_getaddrinfo(object): + def lose(self, e): + print "getaddrinfo(%s) failed: %r" % (self.qname, e) - def __init__(self, qname): - self.qname = qname - getaddrinfo(self.done, self.lose, qname) + class test_query(object): - def done(self, result): - print "getaddrinfo(%s) returned: %s" % ( - self.qname, - ", ".join(str(r) for r in result)) + def __init__(self, qname, qtype = dns.rdatatype.A, qclass = dns.rdataclass.IN): + self.qname = qname + self.qtype = qtype + self.qclass = qclass + query(self.done, self.lose, qname, qtype = qtype, qclass = qclass) - def lose(self, e): - print "getaddrinfo(%s) failed: %r" % (self.qname, e) + def done(self, q, result): + print "query(%s, %s, %s) returned: %s" % ( + self.qname, + dns.rdatatype.to_text(self.qtype), + dns.rdataclass.to_text(self.qclass), + ", ".join(str(r) for r in result)) - class test_query(object): + def lose(self, q, e): + print "getaddrinfo(%s, %s, %s) failed: %r" % ( + self.qname, + dns.rdatatype.to_text(self.qtype), + dns.rdataclass.to_text(self.qclass), + e) - def __init__(self, qname, qtype = dns.rdatatype.A, qclass = dns.rdataclass.IN): - self.qname = qname - self.qtype = qtype - self.qclass = qclass - query(self.done, self.lose, qname, qtype = qtype, qclass = qclass) + if True: + for t in (dns.rdatatype.A, dns.rdatatype.AAAA, dns.rdatatype.HINFO): + test_query("subvert-rpki.hactrn.net", t) + test_query("nonexistant.rpki.net") + test_query("subvert-rpki.hactrn.net", qclass = dns.rdataclass.CH) - def done(self, q, result): - print "query(%s, %s, %s) returned: %s" % ( - self.qname, - dns.rdatatype.to_text(self.qtype), - dns.rdataclass.to_text(self.qclass), - ", ".join(str(r) for r in result)) + for h in ("subvert-rpki.hactrn.net", "nonexistant.rpki.net"): + test_getaddrinfo(h) - def lose(self, q, e): - print "getaddrinfo(%s, %s, %s) failed: %r" % ( - self.qname, - dns.rdatatype.to_text(self.qtype), - dns.rdataclass.to_text(self.qclass), - e) - - if True: - for t in (dns.rdatatype.A, dns.rdatatype.AAAA, dns.rdatatype.HINFO): - test_query("subvert-rpki.hactrn.net", t) - test_query("nonexistant.rpki.net") - test_query("subvert-rpki.hactrn.net", qclass = dns.rdataclass.CH) - - for h in ("subvert-rpki.hactrn.net", "nonexistant.rpki.net"): - test_getaddrinfo(h) - - rpki.async.event_loop() + rpki.async.event_loop() diff --git a/rpki/cli.py b/rpki/cli.py index 35999cb0..51ac0367 100644 --- a/rpki/cli.py +++ b/rpki/cli.py @@ -28,244 +28,244 @@ import argparse import traceback try: - import readline - have_readline = True + import readline + have_readline = True except ImportError: - have_readline = False + have_readline = False class BadCommandSyntax(Exception): - "Bad command line syntax." + "Bad command line syntax." class ExitArgparse(Exception): - "Exit method from ArgumentParser." + "Exit method from ArgumentParser." - def __init__(self, message = None, status = 0): - super(ExitArgparse, self).__init__() - self.message = message - self.status = status + def __init__(self, message = None, status = 0): + super(ExitArgparse, self).__init__() + self.message = message + self.status = status class Cmd(cmd.Cmd): - """ - Customized subclass of Python cmd module. - """ + """ + Customized subclass of Python cmd module. + """ - emptyline_repeats_last_command = False + emptyline_repeats_last_command = False - EOF_exits_command_loop = True + EOF_exits_command_loop = True - identchars = cmd.IDENTCHARS + "/-." + identchars = cmd.IDENTCHARS + "/-." - histfile = None + histfile = None - last_command_failed = False + last_command_failed = False - def onecmd(self, line): - """ - Wrap error handling around cmd.Cmd.onecmd(). Might want to do - something kinder than showing a traceback, eventually. - """ + def onecmd(self, line): + """ + Wrap error handling around cmd.Cmd.onecmd(). Might want to do + something kinder than showing a traceback, eventually. + """ - self.last_command_failed = False - try: - return cmd.Cmd.onecmd(self, line) - except SystemExit: - raise - except ExitArgparse, e: - if e.message is not None: - print e.message - self.last_command_failed = e.status != 0 - return False - except BadCommandSyntax, e: - print e - except: - traceback.print_exc() - self.last_command_failed = True - return False - - def do_EOF(self, arg): - if self.EOF_exits_command_loop and self.prompt: - print - return self.EOF_exits_command_loop - - def do_exit(self, arg): - """ - Exit program. - """ + self.last_command_failed = False + try: + return cmd.Cmd.onecmd(self, line) + except SystemExit: + raise + except ExitArgparse, e: + if e.message is not None: + print e.message + self.last_command_failed = e.status != 0 + return False + except BadCommandSyntax, e: + print e + except: + traceback.print_exc() + self.last_command_failed = True + return False + + def do_EOF(self, arg): + if self.EOF_exits_command_loop and self.prompt: + print + return self.EOF_exits_command_loop + + def do_exit(self, arg): + """ + Exit program. + """ + + return True + + do_quit = do_exit + + def emptyline(self): + """ + Handle an empty line. cmd module default is to repeat the last + command, which I find to be violation of the principal of least + astonishment, so my preference is that an empty line does nothing. + """ + + if self.emptyline_repeats_last_command: + cmd.Cmd.emptyline(self) + + def filename_complete(self, text, line, begidx, endidx): + """ + Filename completion handler, with hack to restore what I consider + the normal (bash-like) behavior when one hits the completion key + and there's only one match. + """ + + result = glob.glob(text + "*") + if len(result) == 1: + path = result.pop() + if os.path.isdir(path) or (os.path.islink(path) and os.path.isdir(os.path.join(path, "."))): + result.append(path + os.path.sep) + else: + result.append(path + " ") + return result + + def completenames(self, text, *ignored): + """ + Command name completion handler, with hack to restore what I + consider the normal (bash-like) behavior when one hits the + completion key and there's only one match. + """ + + result = cmd.Cmd.completenames(self, text, *ignored) + if len(result) == 1: + result[0] += " " + return result + + def help_help(self): + """ + Type "help [topic]" for help on a command, + or just "help" for a list of commands. + """ + + self.stdout.write(self.help_help.__doc__ + "\n") + + def complete_help(self, *args): + """ + Better completion function for help command arguments. + """ + + text = args[0] + names = self.get_names() + result = [] + for prefix in ("do_", "help_"): + result.extend(s[len(prefix):] for s in names if s.startswith(prefix + text) and s != "do_EOF") + return result + + if have_readline: + + def cmdloop_with_history(self): + """ + Better command loop, with history file and tweaked readline + completion delimiters. + """ + + old_completer_delims = readline.get_completer_delims() + if self.histfile is not None: + try: + readline.read_history_file(self.histfile) + except IOError: + pass + try: + readline.set_completer_delims("".join(set(old_completer_delims) - set(self.identchars))) + self.cmdloop() + finally: + if self.histfile is not None and readline.get_current_history_length(): + readline.write_history_file(self.histfile) + readline.set_completer_delims(old_completer_delims) + + else: + + cmdloop_with_history = cmd.Cmd.cmdloop - return True - do_quit = do_exit - def emptyline(self): +def yes_or_no(prompt, default = None, require_full_word = False): """ - Handle an empty line. cmd module default is to repeat the last - command, which I find to be violation of the principal of least - astonishment, so my preference is that an empty line does nothing. + Ask a yes-or-no question. """ - if self.emptyline_repeats_last_command: - cmd.Cmd.emptyline(self) + prompt = prompt.rstrip() + _yes_or_no_prompts[default] + while True: + answer = raw_input(prompt).strip().lower() + if not answer and default is not None: + return default + if answer == "yes" or (not require_full_word and answer.startswith("y")): + return True + if answer == "no" or (not require_full_word and answer.startswith("n")): + return False + print 'Please answer "yes" or "no"' - def filename_complete(self, text, line, begidx, endidx): - """ - Filename completion handler, with hack to restore what I consider - the normal (bash-like) behavior when one hits the completion key - and there's only one match. - """ +_yes_or_no_prompts = { + True : ' ("yes" or "no" ["yes"]) ', + False : ' ("yes" or "no" ["no"]) ', + None : ' ("yes" or "no") ' } - result = glob.glob(text + "*") - if len(result) == 1: - path = result.pop() - if os.path.isdir(path) or (os.path.islink(path) and os.path.isdir(os.path.join(path, "."))): - result.append(path + os.path.sep) - else: - result.append(path + " ") - return result - def completenames(self, text, *ignored): +class NonExitingArgumentParser(argparse.ArgumentParser): """ - Command name completion handler, with hack to restore what I - consider the normal (bash-like) behavior when one hits the - completion key and there's only one match. + ArgumentParser tweaked to throw ExitArgparse exception + rather than using sys.exit(), for use with command loop. """ - result = cmd.Cmd.completenames(self, text, *ignored) - if len(result) == 1: - result[0] += " " - return result + def exit(self, status = 0, message = None): + raise ExitArgparse(status = status, message = message) - def help_help(self): - """ - Type "help [topic]" for help on a command, - or just "help" for a list of commands. - """ - - self.stdout.write(self.help_help.__doc__ + "\n") - def complete_help(self, *args): - """ - Better completion function for help command arguments. +def parsecmd(subparsers, *arg_clauses): """ + Decorator to combine the argparse and cmd modules. - text = args[0] - names = self.get_names() - result = [] - for prefix in ("do_", "help_"): - result.extend(s[len(prefix):] for s in names if s.startswith(prefix + text) and s != "do_EOF") - return result + subparsers is an instance of argparse.ArgumentParser (or subclass) which was + returned by calling the .add_subparsers() method on an ArgumentParser instance + intended to handle parsing for the entire program on the command line. - if have_readline: - - def cmdloop_with_history(self): - """ - Better command loop, with history file and tweaked readline - completion delimiters. - """ - - old_completer_delims = readline.get_completer_delims() - if self.histfile is not None: - try: - readline.read_history_file(self.histfile) - except IOError: - pass - try: - readline.set_completer_delims("".join(set(old_completer_delims) - set(self.identchars))) - self.cmdloop() - finally: - if self.histfile is not None and readline.get_current_history_length(): - readline.write_history_file(self.histfile) - readline.set_completer_delims(old_completer_delims) - - else: - - cmdloop_with_history = cmd.Cmd.cmdloop + arg_clauses is a series of defarg() invocations defining arguments to be parsed + by the argparse code. + The decorator will use arg_clauses to construct two separate argparse parser + instances: one will be attached to the global parser as a subparser, the + other will be used to parse arguments for this command when invoked by cmd. + The decorator will replace the original do_whatever method with a wrapped version + which uses the local argparse instance to parse the single string supplied by + the cmd module. -def yes_or_no(prompt, default = None, require_full_word = False): - """ - Ask a yes-or-no question. - """ - - prompt = prompt.rstrip() + _yes_or_no_prompts[default] - while True: - answer = raw_input(prompt).strip().lower() - if not answer and default is not None: - return default - if answer == "yes" or (not require_full_word and answer.startswith("y")): - return True - if answer == "no" or (not require_full_word and answer.startswith("n")): - return False - print 'Please answer "yes" or "no"' - -_yes_or_no_prompts = { - True : ' ("yes" or "no" ["yes"]) ', - False : ' ("yes" or "no" ["no"]) ', - None : ' ("yes" or "no") ' } - - -class NonExitingArgumentParser(argparse.ArgumentParser): - """ - ArgumentParser tweaked to throw ExitArgparse exception - rather than using sys.exit(), for use with command loop. - """ - - def exit(self, status = 0, message = None): - raise ExitArgparse(status = status, message = message) + The intent is that, from the command's point of view, all of this should work + pretty much the same way regardless of whether the command was invoked from + the global command line or from within the cmd command loop. Either way, + the command method should get an argparse.Namespace object. + In theory, we could generate a completion handler from the argparse definitions, + much as the separate argcomplete package does. In practice this is a lot of + work and I'm not ready to get into that just yet. + """ -def parsecmd(subparsers, *arg_clauses): - """ - Decorator to combine the argparse and cmd modules. - - subparsers is an instance of argparse.ArgumentParser (or subclass) which was - returned by calling the .add_subparsers() method on an ArgumentParser instance - intended to handle parsing for the entire program on the command line. - - arg_clauses is a series of defarg() invocations defining arguments to be parsed - by the argparse code. - - The decorator will use arg_clauses to construct two separate argparse parser - instances: one will be attached to the global parser as a subparser, the - other will be used to parse arguments for this command when invoked by cmd. - - The decorator will replace the original do_whatever method with a wrapped version - which uses the local argparse instance to parse the single string supplied by - the cmd module. - - The intent is that, from the command's point of view, all of this should work - pretty much the same way regardless of whether the command was invoked from - the global command line or from within the cmd command loop. Either way, - the command method should get an argparse.Namespace object. - - In theory, we could generate a completion handler from the argparse definitions, - much as the separate argcomplete package does. In practice this is a lot of - work and I'm not ready to get into that just yet. - """ - - def decorate(func): - assert func.__name__.startswith("do_") - parser = NonExitingArgumentParser(description = func.__doc__, - prog = func.__name__[3:], - add_help = False) - subparser = subparsers.add_parser(func.__name__[3:], - description = func.__doc__, - help = func.__doc__.lstrip().partition("\n")[0]) - for positional, keywords in arg_clauses: - parser.add_argument(*positional, **keywords) - subparser.add_argument(*positional, **keywords) - subparser.set_defaults(func = func) - def wrapped(self, arg): - return func(self, parser.parse_args(shlex.split(arg))) - wrapped.argparser = parser - wrapped.__doc__ = func.__doc__ - return wrapped - return decorate + def decorate(func): + assert func.__name__.startswith("do_") + parser = NonExitingArgumentParser(description = func.__doc__, + prog = func.__name__[3:], + add_help = False) + subparser = subparsers.add_parser(func.__name__[3:], + description = func.__doc__, + help = func.__doc__.lstrip().partition("\n")[0]) + for positional, keywords in arg_clauses: + parser.add_argument(*positional, **keywords) + subparser.add_argument(*positional, **keywords) + subparser.set_defaults(func = func) + def wrapped(self, arg): + return func(self, parser.parse_args(shlex.split(arg))) + wrapped.argparser = parser + wrapped.__doc__ = func.__doc__ + return wrapped + return decorate def cmdarg(*positional, **keywords): - """ - Syntactic sugar to let us use keyword arguments normally when constructing - arguments for deferred calls to argparse.ArgumentParser.add_argument(). - """ + """ + Syntactic sugar to let us use keyword arguments normally when constructing + arguments for deferred calls to argparse.ArgumentParser.add_argument(). + """ - return positional, keywords + return positional, keywords diff --git a/rpki/config.py b/rpki/config.py index 99041259..5dd03a6d 100644 --- a/rpki/config.py +++ b/rpki/config.py @@ -33,10 +33,10 @@ logger = logging.getLogger(__name__) # Default name of config file if caller doesn't specify one explictly. try: - import rpki.autoconf - default_filename = os.path.join(rpki.autoconf.sysconfdir, "rpki.conf") + import rpki.autoconf + default_filename = os.path.join(rpki.autoconf.sysconfdir, "rpki.conf") except ImportError: - default_filename = None + default_filename = None ## @var rpki_conf_envname # Name of environment variable containing config file name. @@ -44,230 +44,230 @@ except ImportError: rpki_conf_envname = "RPKI_CONF" class parser(object): - """ - Extensions to stock Python ConfigParser: - - Read config file and set default section while initializing parser object. - - Support for OpenSSL-style subscripted options and a limited form of - OpenSSL-style indirect variable references (${section::option}). - - get-methods with default values and default section name. - - If no filename is given to the constructor (filename and - set_filename both None), we check for an environment variable naming - the config file, then finally we check for a global config file if - autoconf provided a directory name to check. - - NB: Programs which accept a configuration filename on the command - lines should pass that filename using set_filename so that we can - set the magic environment variable. Constraints from some external - libraries (principally Django) sometimes require library code to - look things up in the configuration file without the knowledge of - the controlling program, but setting the environment variable - insures that everybody's reading from the same script, as it were. - """ - - # Odd keyword-only calling sequence is a defense against old code - # that thinks it knows how __init__() handles positional arguments. - - def __init__(self, **kwargs): - section = kwargs.pop("section", None) - allow_missing = kwargs.pop("allow_missing", False) - set_filename = kwargs.pop("set_filename", None) - filename = kwargs.pop("filename", set_filename) - - assert not kwargs, "Unexpected keyword arguments: " + ", ".join("%s = %r" % kv for kv in kwargs.iteritems()) - - if set_filename is not None: - os.environ[rpki_conf_envname] = set_filename - - self.cfg = ConfigParser.RawConfigParser() - self.default_section = section - - self.filename = filename or os.getenv(rpki_conf_envname) or default_filename - - try: - with open(self.filename, "r") as f: - self.cfg.readfp(f) - except IOError: - if allow_missing: - self.filename = None - else: - raise - - - def has_section(self, section): """ - Test whether a section exists. - """ - - return self.cfg.has_section(section) + Extensions to stock Python ConfigParser: + Read config file and set default section while initializing parser object. - def has_option(self, option, section = None): - """ - Test whether an option exists. - """ + Support for OpenSSL-style subscripted options and a limited form of + OpenSSL-style indirect variable references (${section::option}). - if section is None: - section = self.default_section - return self.cfg.has_option(section, option) + get-methods with default values and default section name. + If no filename is given to the constructor (filename and + set_filename both None), we check for an environment variable naming + the config file, then finally we check for a global config file if + autoconf provided a directory name to check. - def multiget(self, option, section = None): + NB: Programs which accept a configuration filename on the command + lines should pass that filename using set_filename so that we can + set the magic environment variable. Constraints from some external + libraries (principally Django) sometimes require library code to + look things up in the configuration file without the knowledge of + the controlling program, but setting the environment variable + insures that everybody's reading from the same script, as it were. """ - Parse OpenSSL-style foo.0, foo.1, ... subscripted options. - Returns iteration of values matching the specified option name. - """ + # Odd keyword-only calling sequence is a defense against old code + # that thinks it knows how __init__() handles positional arguments. - matches = [] - if section is None: - section = self.default_section - if self.cfg.has_option(section, option): - yield self.cfg.get(section, option) - option += "." - matches = [o for o in self.cfg.options(section) if o.startswith(option) and o[len(option):].isdigit()] - matches.sort() - for option in matches: - yield self.cfg.get(section, option) + def __init__(self, **kwargs): + section = kwargs.pop("section", None) + allow_missing = kwargs.pop("allow_missing", False) + set_filename = kwargs.pop("set_filename", None) + filename = kwargs.pop("filename", set_filename) + assert not kwargs, "Unexpected keyword arguments: " + ", ".join("%s = %r" % kv for kv in kwargs.iteritems()) - _regexp = re.compile("\\${(.*?)::(.*?)}") + if set_filename is not None: + os.environ[rpki_conf_envname] = set_filename - def _repl(self, m): - """ - Replacement function for indirect variable substitution. - This is intended for use with re.subn(). - """ + self.cfg = ConfigParser.RawConfigParser() + self.default_section = section - section, option = m.group(1, 2) - if section == "ENV": - return os.getenv(option, "") - else: - return self.cfg.get(section, option) + self.filename = filename or os.getenv(rpki_conf_envname) or default_filename + try: + with open(self.filename, "r") as f: + self.cfg.readfp(f) + except IOError: + if allow_missing: + self.filename = None + else: + raise - def get(self, option, default = None, section = None): - """ - Get an option, perhaps with a default value. - """ - if section is None: - section = self.default_section - if default is not None and not self.cfg.has_option(section, option): - return default - val = self.cfg.get(section, option) - while True: - val, modified = self._regexp.subn(self._repl, val, 1) - if not modified: - return val + def has_section(self, section): + """ + Test whether a section exists. + """ + return self.cfg.has_section(section) - def getboolean(self, option, default = None, section = None): - """ - Get a boolean option, perhaps with a default value. - """ - v = self.get(option, default, section) - if isinstance(v, str): - v = v.lower() - if v not in self.cfg._boolean_states: - raise ValueError("Not a boolean: %s" % v) - v = self.cfg._boolean_states[v] - return v + def has_option(self, option, section = None): + """ + Test whether an option exists. + """ + if section is None: + section = self.default_section + return self.cfg.has_option(section, option) - def getint(self, option, default = None, section = None): - """ - Get an integer option, perhaps with a default value. - """ - return int(self.get(option, default, section)) + def multiget(self, option, section = None): + """ + Parse OpenSSL-style foo.0, foo.1, ... subscripted options. + Returns iteration of values matching the specified option name. + """ - def getlong(self, option, default = None, section = None): - """ - Get a long integer option, perhaps with a default value. - """ - - return long(self.get(option, default, section)) - - - def set_global_flags(self): - """ - Consolidated control for all the little global control flags - scattered through the libraries. This isn't a particularly good - place for this function to live, but it has to live somewhere and - making it a method of the config parser from which it gets all of - its data is less silly than the available alternatives. - """ - - # pylint: disable=W0621 - import rpki.x509 - import rpki.log - import rpki.daemonize - - for line in self.multiget("configure_logger"): - try: - name, level = line.split() - logging.getLogger(name).setLevel(getattr(logging, level.upper())) - except Exception, e: - logger.warning("Could not process configure_logger line %r: %s", line, e) - - try: - rpki.x509.CMS_object.debug_cms_certs = self.getboolean("debug_cms_certs") - except ConfigParser.NoOptionError: - pass - - try: - rpki.x509.XML_CMS_object.dump_outbound_cms = rpki.x509.DeadDrop(self.get("dump_outbound_cms")) - except OSError, e: - logger.warning("Couldn't initialize mailbox %s: %s", self.get("dump_outbound_cms"), e) - except ConfigParser.NoOptionError: - pass - - try: - rpki.x509.XML_CMS_object.dump_inbound_cms = rpki.x509.DeadDrop(self.get("dump_inbound_cms")) - except OSError, e: - logger.warning("Couldn't initialize mailbox %s: %s", self.get("dump_inbound_cms"), e) - except ConfigParser.NoOptionError: - pass - - try: - rpki.x509.XML_CMS_object.check_inbound_schema = self.getboolean("check_inbound_schema") - except ConfigParser.NoOptionError: - pass - - try: - rpki.x509.XML_CMS_object.check_outbound_schema = self.getboolean("check_outbound_schema") - except ConfigParser.NoOptionError: - pass - - try: - rpki.log.enable_tracebacks = self.getboolean("enable_tracebacks") - except ConfigParser.NoOptionError: - pass - - try: - rpki.daemonize.default_pid_directory = self.get("pid_directory") - except ConfigParser.NoOptionError: - pass - - try: - rpki.daemonize.pid_filename = self.get("pid_filename") - except ConfigParser.NoOptionError: - pass - - try: - rpki.x509.generate_insecure_debug_only_rsa_key = rpki.x509.insecure_debug_only_rsa_key_generator(*self.get("insecure-debug-only-rsa-key-db").split()) - except ConfigParser.NoOptionError: - pass - except: # pylint: disable=W0702 - logger.warning("insecure-debug-only-rsa-key-db configured but initialization failed, check for corrupted database file") - - try: - rpki.up_down.content_type = self.get("up_down_content_type") - except ConfigParser.NoOptionError: - pass + matches = [] + if section is None: + section = self.default_section + if self.cfg.has_option(section, option): + yield self.cfg.get(section, option) + option += "." + matches = [o for o in self.cfg.options(section) if o.startswith(option) and o[len(option):].isdigit()] + matches.sort() + for option in matches: + yield self.cfg.get(section, option) + + + _regexp = re.compile("\\${(.*?)::(.*?)}") + + def _repl(self, m): + """ + Replacement function for indirect variable substitution. + This is intended for use with re.subn(). + """ + + section, option = m.group(1, 2) + if section == "ENV": + return os.getenv(option, "") + else: + return self.cfg.get(section, option) + + + def get(self, option, default = None, section = None): + """ + Get an option, perhaps with a default value. + """ + + if section is None: + section = self.default_section + if default is not None and not self.cfg.has_option(section, option): + return default + val = self.cfg.get(section, option) + while True: + val, modified = self._regexp.subn(self._repl, val, 1) + if not modified: + return val + + + def getboolean(self, option, default = None, section = None): + """ + Get a boolean option, perhaps with a default value. + """ + + v = self.get(option, default, section) + if isinstance(v, str): + v = v.lower() + if v not in self.cfg._boolean_states: + raise ValueError("Not a boolean: %s" % v) + v = self.cfg._boolean_states[v] + return v + + + def getint(self, option, default = None, section = None): + """ + Get an integer option, perhaps with a default value. + """ + + return int(self.get(option, default, section)) + + + def getlong(self, option, default = None, section = None): + """ + Get a long integer option, perhaps with a default value. + """ + + return long(self.get(option, default, section)) + + + def set_global_flags(self): + """ + Consolidated control for all the little global control flags + scattered through the libraries. This isn't a particularly good + place for this function to live, but it has to live somewhere and + making it a method of the config parser from which it gets all of + its data is less silly than the available alternatives. + """ + + # pylint: disable=W0621 + import rpki.x509 + import rpki.log + import rpki.daemonize + + for line in self.multiget("configure_logger"): + try: + name, level = line.split() + logging.getLogger(name).setLevel(getattr(logging, level.upper())) + except Exception, e: + logger.warning("Could not process configure_logger line %r: %s", line, e) + + try: + rpki.x509.CMS_object.debug_cms_certs = self.getboolean("debug_cms_certs") + except ConfigParser.NoOptionError: + pass + + try: + rpki.x509.XML_CMS_object.dump_outbound_cms = rpki.x509.DeadDrop(self.get("dump_outbound_cms")) + except OSError, e: + logger.warning("Couldn't initialize mailbox %s: %s", self.get("dump_outbound_cms"), e) + except ConfigParser.NoOptionError: + pass + + try: + rpki.x509.XML_CMS_object.dump_inbound_cms = rpki.x509.DeadDrop(self.get("dump_inbound_cms")) + except OSError, e: + logger.warning("Couldn't initialize mailbox %s: %s", self.get("dump_inbound_cms"), e) + except ConfigParser.NoOptionError: + pass + + try: + rpki.x509.XML_CMS_object.check_inbound_schema = self.getboolean("check_inbound_schema") + except ConfigParser.NoOptionError: + pass + + try: + rpki.x509.XML_CMS_object.check_outbound_schema = self.getboolean("check_outbound_schema") + except ConfigParser.NoOptionError: + pass + + try: + rpki.log.enable_tracebacks = self.getboolean("enable_tracebacks") + except ConfigParser.NoOptionError: + pass + + try: + rpki.daemonize.default_pid_directory = self.get("pid_directory") + except ConfigParser.NoOptionError: + pass + + try: + rpki.daemonize.pid_filename = self.get("pid_filename") + except ConfigParser.NoOptionError: + pass + + try: + rpki.x509.generate_insecure_debug_only_rsa_key = rpki.x509.insecure_debug_only_rsa_key_generator(*self.get("insecure-debug-only-rsa-key-db").split()) + except ConfigParser.NoOptionError: + pass + except: # pylint: disable=W0702 + logger.warning("insecure-debug-only-rsa-key-db configured but initialization failed, check for corrupted database file") + + try: + rpki.up_down.content_type = self.get("up_down_content_type") + except ConfigParser.NoOptionError: + pass diff --git a/rpki/csv_utils.py b/rpki/csv_utils.py index 9034e96b..2864693c 100644 --- a/rpki/csv_utils.py +++ b/rpki/csv_utils.py @@ -22,93 +22,93 @@ import csv import os class BadCSVSyntax(Exception): - """ - Bad CSV syntax. - """ + """ + Bad CSV syntax. + """ class csv_reader(object): - """ - Reader for tab-delimited text that's (slightly) friendlier than the - stock Python csv module (which isn't intended for direct use by - humans anyway, and neither was this package originally, but that - seems to be the way that it has evolved...). - - Columns parameter specifies how many columns users of the reader - expect to see; lines with fewer columns will be padded with None - values. - - Original API design for this class courtesy of Warren Kumari, but - don't blame him if you don't like what I did with his ideas. - """ - - def __init__(self, filename, columns = None, min_columns = None, comment_characters = "#;"): - assert columns is None or isinstance(columns, int) - assert min_columns is None or isinstance(min_columns, int) - if columns is not None and min_columns is None: - min_columns = columns - self.filename = filename - self.columns = columns - self.min_columns = min_columns - self.comment_characters = comment_characters - self.file = open(filename, "r") - - def __iter__(self): - line_number = 0 - for line in self.file: - line_number += 1 - line = line.strip() - if not line or line[0] in self.comment_characters: - continue - fields = line.split() - if self.min_columns is not None and len(fields) < self.min_columns: - raise BadCSVSyntax("%s:%d: Not enough columns in line %r" % (self.filename, line_number, line)) - if self.columns is not None and len(fields) > self.columns: - raise BadCSVSyntax("%s:%d: Too many columns in line %r" % (self.filename, line_number, line)) - if self.columns is not None and len(fields) < self.columns: - fields += tuple(None for i in xrange(self.columns - len(fields))) - yield fields - - def __enter__(self): - return self - - def __exit__(self, _type, value, traceback): - self.file.close() + """ + Reader for tab-delimited text that's (slightly) friendlier than the + stock Python csv module (which isn't intended for direct use by + humans anyway, and neither was this package originally, but that + seems to be the way that it has evolved...). + + Columns parameter specifies how many columns users of the reader + expect to see; lines with fewer columns will be padded with None + values. + + Original API design for this class courtesy of Warren Kumari, but + don't blame him if you don't like what I did with his ideas. + """ + + def __init__(self, filename, columns = None, min_columns = None, comment_characters = "#;"): + assert columns is None or isinstance(columns, int) + assert min_columns is None or isinstance(min_columns, int) + if columns is not None and min_columns is None: + min_columns = columns + self.filename = filename + self.columns = columns + self.min_columns = min_columns + self.comment_characters = comment_characters + self.file = open(filename, "r") + + def __iter__(self): + line_number = 0 + for line in self.file: + line_number += 1 + line = line.strip() + if not line or line[0] in self.comment_characters: + continue + fields = line.split() + if self.min_columns is not None and len(fields) < self.min_columns: + raise BadCSVSyntax("%s:%d: Not enough columns in line %r" % (self.filename, line_number, line)) + if self.columns is not None and len(fields) > self.columns: + raise BadCSVSyntax("%s:%d: Too many columns in line %r" % (self.filename, line_number, line)) + if self.columns is not None and len(fields) < self.columns: + fields += tuple(None for i in xrange(self.columns - len(fields))) + yield fields + + def __enter__(self): + return self + + def __exit__(self, _type, value, traceback): + self.file.close() class csv_writer(object): - """ - Writer object for tab delimited text. We just use the stock CSV - module in excel-tab mode for this. + """ + Writer object for tab delimited text. We just use the stock CSV + module in excel-tab mode for this. - If "renmwo" is set (default), the file will be written to - a temporary name and renamed to the real filename after closing. - """ + If "renmwo" is set (default), the file will be written to + a temporary name and renamed to the real filename after closing. + """ - def __init__(self, filename, renmwo = True): - self.filename = filename - self.renmwo = "%s.~renmwo%d~" % (filename, os.getpid()) if renmwo else filename - self.file = open(self.renmwo, "w") - self.writer = csv.writer(self.file, dialect = csv.get_dialect("excel-tab")) + def __init__(self, filename, renmwo = True): + self.filename = filename + self.renmwo = "%s.~renmwo%d~" % (filename, os.getpid()) if renmwo else filename + self.file = open(self.renmwo, "w") + self.writer = csv.writer(self.file, dialect = csv.get_dialect("excel-tab")) - def __enter__(self): - return self + def __enter__(self): + return self - def __exit__(self, _type, value, traceback): - self.close() + def __exit__(self, _type, value, traceback): + self.close() - def close(self): - """ - Close this writer. - """ + def close(self): + """ + Close this writer. + """ - if self.file is not None: - self.file.close() - self.file = None - if self.filename != self.renmwo: - os.rename(self.renmwo, self.filename) + if self.file is not None: + self.file.close() + self.file = None + if self.filename != self.renmwo: + os.rename(self.renmwo, self.filename) - def __getattr__(self, attr): - """ - Fake inheritance from whatever object csv.writer deigns to give us. - """ + def __getattr__(self, attr): + """ + Fake inheritance from whatever object csv.writer deigns to give us. + """ - return getattr(self.writer, attr) + return getattr(self.writer, attr) diff --git a/rpki/daemonize.py b/rpki/daemonize.py index 6a825566..bd59fca0 100644 --- a/rpki/daemonize.py +++ b/rpki/daemonize.py @@ -80,56 +80,56 @@ default_pid_directory = "/var/run/rpki" pid_filename = None def daemon(nochdir = False, noclose = False, pidfile = None): - """ - Make this program become a daemon, like 4.4BSD daemon(3), and - write its pid out to a file with cleanup on exit. - """ - - if pidfile is None: - if pid_filename is None: - prog = os.path.splitext(os.path.basename(sys.argv[0]))[0] - pidfile = os.path.join(default_pid_directory, "%s.pid" % prog) + """ + Make this program become a daemon, like 4.4BSD daemon(3), and + write its pid out to a file with cleanup on exit. + """ + + if pidfile is None: + if pid_filename is None: + prog = os.path.splitext(os.path.basename(sys.argv[0]))[0] + pidfile = os.path.join(default_pid_directory, "%s.pid" % prog) + else: + pidfile = pid_filename + + old_sighup_action = signal.signal(signal.SIGHUP, signal.SIG_IGN) + + try: + pid = os.fork() + except OSError, e: + sys.exit("fork() failed: %d (%s)" % (e.errno, e.strerror)) else: - pidfile = pid_filename + if pid > 0: + os._exit(0) - old_sighup_action = signal.signal(signal.SIGHUP, signal.SIG_IGN) + if not nochdir: + os.chdir("/") - try: - pid = os.fork() - except OSError, e: - sys.exit("fork() failed: %d (%s)" % (e.errno, e.strerror)) - else: - if pid > 0: - os._exit(0) + os.setsid() - if not nochdir: - os.chdir("/") + if not noclose: + sys.stdout.flush() + sys.stderr.flush() + fd = os.open(os.devnull, os.O_RDWR) + os.dup2(fd, 0) + os.dup2(fd, 1) + os.dup2(fd, 2) + if fd > 2: + os.close(fd) - os.setsid() + signal.signal(signal.SIGHUP, old_sighup_action) - if not noclose: - sys.stdout.flush() - sys.stderr.flush() - fd = os.open(os.devnull, os.O_RDWR) - os.dup2(fd, 0) - os.dup2(fd, 1) - os.dup2(fd, 2) - if fd > 2: - os.close(fd) + def delete_pid_file(): + try: + os.unlink(pidfile) + except OSError: + pass - signal.signal(signal.SIGHUP, old_sighup_action) + atexit.register(delete_pid_file) - def delete_pid_file(): try: - os.unlink(pidfile) - except OSError: - pass - - atexit.register(delete_pid_file) - - try: - f = open(pidfile, "w") - f.write("%d\n" % os.getpid()) - f.close() - except IOError, e: - logger.warning("Couldn't write PID file %s: %s", pidfile, e.strerror) + f = open(pidfile, "w") + f.write("%d\n" % os.getpid()) + f.close() + except IOError, e: + logger.warning("Couldn't write PID file %s: %s", pidfile, e.strerror) diff --git a/rpki/django_settings/common.py b/rpki/django_settings/common.py index 2f676660..3860d40b 100644 --- a/rpki/django_settings/common.py +++ b/rpki/django_settings/common.py @@ -58,7 +58,7 @@ class DatabaseConfigurator(object): default_sql_engine = "mysql" - def configure(self, cfg, section): + def configure(self, cfg, section): # pylint: disable=W0621 self.cfg = cfg self.section = section engine = cfg.get("sql-engine", section = section, diff --git a/rpki/exceptions.py b/rpki/exceptions.py index f456dfc5..cbdb9f83 100644 --- a/rpki/exceptions.py +++ b/rpki/exceptions.py @@ -22,222 +22,222 @@ Exception definitions for RPKI modules. """ class RPKI_Exception(Exception): - "Base class for RPKI exceptions." + "Base class for RPKI exceptions." class NotInDatabase(RPKI_Exception): - "Lookup failed for an object expected to be in the database." + "Lookup failed for an object expected to be in the database." class BadURISyntax(RPKI_Exception): - "Illegal syntax for a URI." + "Illegal syntax for a URI." class BadStatusCode(RPKI_Exception): - "Unrecognized protocol status code." + "Unrecognized protocol status code." class BadQuery(RPKI_Exception): - "Unexpected protocol query." + "Unexpected protocol query." class DBConsistancyError(RPKI_Exception): - "Found multiple matches for a database query that shouldn't ever return that." + "Found multiple matches for a database query that shouldn't ever return that." class CMSVerificationFailed(RPKI_Exception): - "Verification of a CMS message failed." + "Verification of a CMS message failed." class HTTPRequestFailed(RPKI_Exception): - "HTTP request failed." + "HTTP request failed." class DERObjectConversionError(RPKI_Exception): - "Error trying to convert a DER-based object from one representation to another." + "Error trying to convert a DER-based object from one representation to another." class NotACertificateChain(RPKI_Exception): - "Certificates don't form a proper chain." + "Certificates don't form a proper chain." class BadContactURL(RPKI_Exception): - "Error trying to parse contact URL." + "Error trying to parse contact URL." class BadClassNameSyntax(RPKI_Exception): - "Illegal syntax for a class_name." + "Illegal syntax for a class_name." class BadIssueResponse(RPKI_Exception): - "issue_response PDU with wrong number of classes or certificates." + "issue_response PDU with wrong number of classes or certificates." class NotImplementedYet(RPKI_Exception): - "Internal error -- not implemented yet." + "Internal error -- not implemented yet." class BadPKCS10(RPKI_Exception): - "Bad PKCS #10 object." + "Bad PKCS #10 object." class UpstreamError(RPKI_Exception): - "Received an error from upstream." + "Received an error from upstream." class ChildNotFound(RPKI_Exception): - "Could not find specified child in database." + "Could not find specified child in database." class BSCNotFound(RPKI_Exception): - "Could not find specified BSC in database." + "Could not find specified BSC in database." class BadSender(RPKI_Exception): - "Unexpected XML sender value." + "Unexpected XML sender value." class ClassNameMismatch(RPKI_Exception): - "class_name does not match child context." + "class_name does not match child context." class ClassNameUnknown(RPKI_Exception): - "Unknown class_name." + "Unknown class_name." class SKIMismatch(RPKI_Exception): - "SKI value in response does not match request." + "SKI value in response does not match request." class SubprocessError(RPKI_Exception): - "Subprocess returned unexpected error." + "Subprocess returned unexpected error." class BadIRDBReply(RPKI_Exception): - "Unexpected reply to IRDB query." + "Unexpected reply to IRDB query." class NotFound(RPKI_Exception): - "Object not found in database." + "Object not found in database." class MustBePrefix(RPKI_Exception): - "Resource range cannot be expressed as a prefix." + "Resource range cannot be expressed as a prefix." class TLSValidationError(RPKI_Exception): - "TLS certificate validation error." + "TLS certificate validation error." class MultipleTLSEECert(TLSValidationError): - "Received more than one TLS EE certificate." + "Received more than one TLS EE certificate." class ReceivedTLSCACert(TLSValidationError): - "Received CA certificate via TLS." + "Received CA certificate via TLS." class WrongEContentType(RPKI_Exception): - "Received wrong CMS eContentType." + "Received wrong CMS eContentType." class EmptyPEM(RPKI_Exception): - "Couldn't find PEM block to convert." + "Couldn't find PEM block to convert." class UnexpectedCMSCerts(RPKI_Exception): - "Received CMS certs when not expecting any." + "Received CMS certs when not expecting any." class UnexpectedCMSCRLs(RPKI_Exception): - "Received CMS CRLs when not expecting any." + "Received CMS CRLs when not expecting any." class MissingCMSEEcert(RPKI_Exception): - "Didn't receive CMS EE cert when expecting one." + "Didn't receive CMS EE cert when expecting one." class MissingCMSCRL(RPKI_Exception): - "Didn't receive CMS CRL when expecting one." + "Didn't receive CMS CRL when expecting one." class UnparsableCMSDER(RPKI_Exception): - "Alleged CMS DER wasn't parsable." + "Alleged CMS DER wasn't parsable." class CMSCRLNotSet(RPKI_Exception): - "CMS CRL has not been configured." + "CMS CRL has not been configured." class ServerShuttingDown(RPKI_Exception): - "Server is shutting down." + "Server is shutting down." class NoActiveCA(RPKI_Exception): - "No active ca_detail for specified class." + "No active ca_detail for specified class." class BadClientURL(RPKI_Exception): - "URL given to HTTP client does not match profile." + "URL given to HTTP client does not match profile." class ClientNotFound(RPKI_Exception): - "Could not find specified client in database." + "Could not find specified client in database." class BadExtension(RPKI_Exception): - "Forbidden X.509 extension." + "Forbidden X.509 extension." class ForbiddenURI(RPKI_Exception): - "Forbidden URI, does not start with correct base URI." + "Forbidden URI, does not start with correct base URI." class HTTPClientAborted(RPKI_Exception): - "HTTP client connection closed while in request-sent state." + "HTTP client connection closed while in request-sent state." class BadPublicationReply(RPKI_Exception): - "Unexpected reply to publication query." + "Unexpected reply to publication query." class DuplicateObject(RPKI_Exception): - "Attempt to create an object that already exists." + "Attempt to create an object that already exists." class EmptyROAPrefixList(RPKI_Exception): - "Can't create ROA with an empty prefix list." + "Can't create ROA with an empty prefix list." class NoCoveringCertForROA(RPKI_Exception): - "Couldn't find a covering certificate to generate ROA." + "Couldn't find a covering certificate to generate ROA." class BSCNotReady(RPKI_Exception): - "BSC not yet in a usable state, signing_cert not set." + "BSC not yet in a usable state, signing_cert not set." class HTTPUnexpectedState(RPKI_Exception): - "HTTP event occurred in an unexpected state." + "HTTP event occurred in an unexpected state." class HTTPBadVersion(RPKI_Exception): - "HTTP couldn't parse HTTP version." + "HTTP couldn't parse HTTP version." class HandleTranslationError(RPKI_Exception): - "Internal error translating protocol handle -> SQL id." + "Internal error translating protocol handle -> SQL id." class NoObjectAtURI(RPKI_Exception): - "No object published at specified URI." + "No object published at specified URI." class ExistingObjectAtURI(RPKI_Exception): - "An object has already been published at specified URI." + "An object has already been published at specified URI." class DifferentObjectAtURI(RPKI_Exception): - "An object with a different hash exists at specified URI." + "An object with a different hash exists at specified URI." class CMSContentNotSet(RPKI_Exception): - """ - Inner content of a CMS_object has not been set. If object is known - to be valid, the .extract() method should be able to set the - content; otherwise, only the .verify() method (which checks - signatures) is safe. - """ + """ + Inner content of a CMS_object has not been set. If object is known + to be valid, the .extract() method should be able to set the + content; otherwise, only the .verify() method (which checks + signatures) is safe. + """ class HTTPTimeout(RPKI_Exception): - "HTTP connection timed out." + "HTTP connection timed out." class BadIPResource(RPKI_Exception): - "Parse failure for alleged IP resource string." + "Parse failure for alleged IP resource string." class BadROAPrefix(RPKI_Exception): - "Parse failure for alleged ROA prefix string." + "Parse failure for alleged ROA prefix string." class CommandParseFailure(RPKI_Exception): - "Failed to parse command line." + "Failed to parse command line." class CMSCertHasExpired(RPKI_Exception): - "CMS certificate has expired." + "CMS certificate has expired." class TrustedCMSCertHasExpired(RPKI_Exception): - "Trusted CMS certificate has expired." + "Trusted CMS certificate has expired." class MultipleCMSEECert(RPKI_Exception): - "Can't have more than one CMS EE certificate in validation chain." + "Can't have more than one CMS EE certificate in validation chain." class ResourceOverlap(RPKI_Exception): - "Overlapping resources in resource_set." + "Overlapping resources in resource_set." class CMSReplay(RPKI_Exception): - "Possible CMS replay attack detected." + "Possible CMS replay attack detected." class PastNotAfter(RPKI_Exception): - "Requested notAfter value is already in the past." + "Requested notAfter value is already in the past." class NullValidityInterval(RPKI_Exception): - "Requested validity interval is null." + "Requested validity interval is null." class BadX510DN(RPKI_Exception): - "X.510 distinguished name does not match profile." + "X.510 distinguished name does not match profile." class BadAutonomousSystemNumber(RPKI_Exception): - "Bad AutonomousSystem number." + "Bad AutonomousSystem number." class WrongEKU(RPKI_Exception): - "Extended Key Usage extension does not match profile." + "Extended Key Usage extension does not match profile." class UnexpectedUpDownResponse(RPKI_Exception): - "Up-down message is not of the expected type." + "Up-down message is not of the expected type." class BadContentType(RPKI_Exception): - "Bad HTTP Content-Type." + "Bad HTTP Content-Type." diff --git a/rpki/fields.py b/rpki/fields.py index a470e272..1390d4ac 100644 --- a/rpki/fields.py +++ b/rpki/fields.py @@ -35,78 +35,78 @@ logger = logging.getLogger(__name__) class EnumField(models.PositiveSmallIntegerField): - """ - An enumeration type that uses strings in Python and small integers - in SQL. - """ + """ + An enumeration type that uses strings in Python and small integers + in SQL. + """ - description = "An enumeration type" + description = "An enumeration type" - __metaclass__ = models.SubfieldBase + __metaclass__ = models.SubfieldBase - def __init__(self, *args, **kwargs): - if isinstance(kwargs.get("choices"), (tuple, list)) and isinstance(kwargs["choices"][0], (str, unicode)): - kwargs["choices"] = tuple(enumerate(kwargs["choices"], 1)) - # Might need something here to handle string-valued default parameter - models.PositiveSmallIntegerField.__init__(self, *args, **kwargs) - self.enum_i2s = dict(self.flatchoices) - self.enum_s2i = dict((v, k) for k, v in self.flatchoices) + def __init__(self, *args, **kwargs): + if isinstance(kwargs.get("choices"), (tuple, list)) and isinstance(kwargs["choices"][0], (str, unicode)): + kwargs["choices"] = tuple(enumerate(kwargs["choices"], 1)) + # Might need something here to handle string-valued default parameter + models.PositiveSmallIntegerField.__init__(self, *args, **kwargs) + self.enum_i2s = dict(self.flatchoices) + self.enum_s2i = dict((v, k) for k, v in self.flatchoices) - def to_python(self, value): - return self.enum_i2s.get(value, value) + def to_python(self, value): + return self.enum_i2s.get(value, value) - def get_prep_value(self, value): - return self.enum_s2i.get(value, value) + def get_prep_value(self, value): + return self.enum_s2i.get(value, value) class SundialField(models.DateTimeField): - """ - A field type for our customized datetime objects. - """ - __metaclass__ = models.SubfieldBase + """ + A field type for our customized datetime objects. + """ + __metaclass__ = models.SubfieldBase - description = "A datetime type using our customized datetime objects" + description = "A datetime type using our customized datetime objects" - def to_python(self, value): - if isinstance(value, rpki.sundial.pydatetime.datetime): - return rpki.sundial.datetime.from_datetime( - models.DateTimeField.to_python(self, value)) - else: - return value + def to_python(self, value): + if isinstance(value, rpki.sundial.pydatetime.datetime): + return rpki.sundial.datetime.from_datetime( + models.DateTimeField.to_python(self, value)) + else: + return value - def get_prep_value(self, value): - if isinstance(value, rpki.sundial.datetime): - return value.to_datetime() - else: - return value + def get_prep_value(self, value): + if isinstance(value, rpki.sundial.datetime): + return value.to_datetime() + else: + return value class BlobField(models.Field): - """ - Old BLOB field type, predating Django's BinaryField type. + """ + Old BLOB field type, predating Django's BinaryField type. - Do not use, this is only here for backwards compatabilty during migrations. - """ + Do not use, this is only here for backwards compatabilty during migrations. + """ - __metaclass__ = models.SubfieldBase - description = "Raw BLOB type without ASN.1 encoding/decoding" + __metaclass__ = models.SubfieldBase + description = "Raw BLOB type without ASN.1 encoding/decoding" - def __init__(self, *args, **kwargs): - self.blob_type = kwargs.pop("blob_type", None) - kwargs["serialize"] = False - kwargs["blank"] = True - kwargs["default"] = None - models.Field.__init__(self, *args, **kwargs) + def __init__(self, *args, **kwargs): + self.blob_type = kwargs.pop("blob_type", None) + kwargs["serialize"] = False + kwargs["blank"] = True + kwargs["default"] = None + models.Field.__init__(self, *args, **kwargs) - def db_type(self, connection): - if self.blob_type is not None: - return self.blob_type - elif connection.settings_dict['ENGINE'] == "django.db.backends.mysql": - return "LONGBLOB" - elif connection.settings_dict['ENGINE'] == "django.db.backends.posgresql": - return "bytea" - else: - return "BLOB" + def db_type(self, connection): + if self.blob_type is not None: + return self.blob_type + elif connection.settings_dict['ENGINE'] == "django.db.backends.mysql": + return "LONGBLOB" + elif connection.settings_dict['ENGINE'] == "django.db.backends.posgresql": + return "bytea" + else: + return "BLOB" # For reasons which now escape me, I had a few fields in the old @@ -124,70 +124,70 @@ class BlobField(models.Field): # backwards compatability during migrations, class DERField(models.BinaryField): - """ - Field class for DER objects, with automatic translation between - ASN.1 and Python types. This is an abstract class, concrete field - classes are derived from it. - """ - - def __init__(self, *args, **kwargs): - kwargs["blank"] = True - kwargs["default"] = None - super(DERField, self).__init__(*args, **kwargs) - - def deconstruct(self): - name, path, args, kwargs = super(DERField, self).deconstruct() - del kwargs["blank"] - del kwargs["default"] - return name, path, args, kwargs - - def from_db_value(self, value, expression, connection, context): - if value is not None: - value = self.rpki_type(DER = str(value)) - return value - - def to_python(self, value): - value = super(DERField, self).to_python(value) - if value is not None and not isinstance(value, self.rpki_type): - value = self.rpki_type(DER = str(value)) - return value - - def get_prep_value(self, value): - if value is not None: - value = value.get_DER() - return super(DERField, self).get_prep_value(value) + """ + Field class for DER objects, with automatic translation between + ASN.1 and Python types. This is an abstract class, concrete field + classes are derived from it. + """ + + def __init__(self, *args, **kwargs): + kwargs["blank"] = True + kwargs["default"] = None + super(DERField, self).__init__(*args, **kwargs) + + def deconstruct(self): + name, path, args, kwargs = super(DERField, self).deconstruct() + del kwargs["blank"] + del kwargs["default"] + return name, path, args, kwargs + + def from_db_value(self, value, expression, connection, context): + if value is not None: + value = self.rpki_type(DER = str(value)) + return value + + def to_python(self, value): + value = super(DERField, self).to_python(value) + if value is not None and not isinstance(value, self.rpki_type): + value = self.rpki_type(DER = str(value)) + return value + + def get_prep_value(self, value): + if value is not None: + value = value.get_DER() + return super(DERField, self).get_prep_value(value) class CertificateField(DERField): - description = "X.509 certificate" - rpki_type = rpki.x509.X509 + description = "X.509 certificate" + rpki_type = rpki.x509.X509 class RSAPrivateKeyField(DERField): - description = "RSA keypair" - rpki_type = rpki.x509.RSA + description = "RSA keypair" + rpki_type = rpki.x509.RSA KeyField = RSAPrivateKeyField class PublicKeyField(DERField): - description = "RSA keypair" - rpki_type = rpki.x509.PublicKey + description = "RSA keypair" + rpki_type = rpki.x509.PublicKey class CRLField(DERField): - description = "Certificate Revocation List" - rpki_type = rpki.x509.CRL + description = "Certificate Revocation List" + rpki_type = rpki.x509.CRL class PKCS10Field(DERField): - description = "PKCS #10 certificate request" - rpki_type = rpki.x509.PKCS10 + description = "PKCS #10 certificate request" + rpki_type = rpki.x509.PKCS10 class ManifestField(DERField): - description = "RPKI Manifest" - rpki_type = rpki.x509.SignedManifest + description = "RPKI Manifest" + rpki_type = rpki.x509.SignedManifest class ROAField(DERField): - description = "ROA" - rpki_type = rpki.x509.ROA + description = "ROA" + rpki_type = rpki.x509.ROA class GhostbusterField(DERField): - description = "Ghostbuster Record" - rpki_type = rpki.x509.Ghostbuster + description = "Ghostbuster Record" + rpki_type = rpki.x509.Ghostbuster diff --git a/rpki/gui/app/forms.py b/rpki/gui/app/forms.py index 306b8dce..4a95c8da 100644 --- a/rpki/gui/app/forms.py +++ b/rpki/gui/app/forms.py @@ -170,105 +170,105 @@ def ROARequestFormFactory(conf): """ class Cls(forms.Form): - """Form for entering a ROA request. - - Handles both IPv4 and IPv6.""" - - prefix = forms.CharField( - widget=forms.TextInput(attrs={ - 'autofocus': 'true', 'placeholder': 'Prefix', - 'class': 'span4' - }) - ) - max_prefixlen = forms.CharField( - required=False, - widget=forms.TextInput(attrs={ - 'placeholder': 'Max len', - 'class': 'span1' - }) - ) - asn = forms.IntegerField( - widget=forms.TextInput(attrs={ - 'placeholder': 'ASN', - 'class': 'span1' - }) - ) - protect_children = forms.BooleanField(required=False) - - def __init__(self, *args, **kwargs): - kwargs['auto_id'] = False - super(Cls, self).__init__(*args, **kwargs) - self.conf = conf # conf is the arg to ROARequestFormFactory - self.inline = True - self.use_table = False - - def _as_resource_range(self): - """Convert the prefix in the form to a - rpki.resource_set.resource_range_ip object. - - If there is no mask provided, assume the closest classful mask. - - """ - prefix = self.cleaned_data.get('prefix') - if '/' not in prefix: - p = IPAddress(prefix) - - # determine the first nonzero bit starting from the lsb and - # subtract from the address size to find the closest classful - # mask that contains this single address - prefixlen = 0 - while (p != 0) and (p & 1) == 0: - prefixlen = prefixlen + 1 - p = p >> 1 - mask = p.bits - (8 * (prefixlen / 8)) - prefix = prefix + '/' + str(mask) - - return resource_range_ip.parse_str(prefix) - - def clean_asn(self): - value = self.cleaned_data.get('asn') - if value < 0: - raise forms.ValidationError('AS must be a positive value or 0') - return value - - def clean_prefix(self): - try: - r = self._as_resource_range() - except: - raise forms.ValidationError('invalid prefix') - - manager = models.ResourceRangeAddressV4 if r.version == 4 else models.ResourceRangeAddressV6 - if not manager.objects.filter(cert__conf=self.conf, - prefix_min__lte=r.min, - prefix_max__gte=r.max).exists(): - raise forms.ValidationError('prefix is not allocated to you') - return str(r) - - def clean_max_prefixlen(self): - v = self.cleaned_data.get('max_prefixlen') - if v: - if v[0] == '/': - v = v[1:] # allow user to specify /24 - try: - if int(v) < 0: - raise forms.ValidationError('max prefix length must be positive or 0') - except ValueError: - raise forms.ValidationError('invalid integer value') - return v - - def clean(self): - if 'prefix' in self.cleaned_data: - r = self._as_resource_range() - max_prefixlen = self.cleaned_data.get('max_prefixlen') - max_prefixlen = int(max_prefixlen) if max_prefixlen else r.prefixlen() - if max_prefixlen < r.prefixlen(): - raise forms.ValidationError( - 'max prefix length must be greater than or equal to the prefix length') - if max_prefixlen > r.min.bits: - raise forms.ValidationError( - 'max prefix length (%d) is out of range for IP version (%d)' % (max_prefixlen, r.min.bits)) - self.cleaned_data['max_prefixlen'] = str(max_prefixlen) - return self.cleaned_data + """Form for entering a ROA request. + + Handles both IPv4 and IPv6.""" + + prefix = forms.CharField( + widget=forms.TextInput(attrs={ + 'autofocus': 'true', 'placeholder': 'Prefix', + 'class': 'span4' + }) + ) + max_prefixlen = forms.CharField( + required=False, + widget=forms.TextInput(attrs={ + 'placeholder': 'Max len', + 'class': 'span1' + }) + ) + asn = forms.IntegerField( + widget=forms.TextInput(attrs={ + 'placeholder': 'ASN', + 'class': 'span1' + }) + ) + protect_children = forms.BooleanField(required=False) + + def __init__(self, *args, **kwargs): + kwargs['auto_id'] = False + super(Cls, self).__init__(*args, **kwargs) + self.conf = conf # conf is the arg to ROARequestFormFactory + self.inline = True + self.use_table = False + + def _as_resource_range(self): + """Convert the prefix in the form to a + rpki.resource_set.resource_range_ip object. + + If there is no mask provided, assume the closest classful mask. + + """ + prefix = self.cleaned_data.get('prefix') + if '/' not in prefix: + p = IPAddress(prefix) + + # determine the first nonzero bit starting from the lsb and + # subtract from the address size to find the closest classful + # mask that contains this single address + prefixlen = 0 + while (p != 0) and (p & 1) == 0: + prefixlen = prefixlen + 1 + p = p >> 1 + mask = p.bits - (8 * (prefixlen / 8)) + prefix = prefix + '/' + str(mask) + + return resource_range_ip.parse_str(prefix) + + def clean_asn(self): + value = self.cleaned_data.get('asn') + if value < 0: + raise forms.ValidationError('AS must be a positive value or 0') + return value + + def clean_prefix(self): + try: + r = self._as_resource_range() + except: + raise forms.ValidationError('invalid prefix') + + manager = models.ResourceRangeAddressV4 if r.version == 4 else models.ResourceRangeAddressV6 + if not manager.objects.filter(cert__conf=self.conf, + prefix_min__lte=r.min, + prefix_max__gte=r.max).exists(): + raise forms.ValidationError('prefix is not allocated to you') + return str(r) + + def clean_max_prefixlen(self): + v = self.cleaned_data.get('max_prefixlen') + if v: + if v[0] == '/': + v = v[1:] # allow user to specify /24 + try: + if int(v) < 0: + raise forms.ValidationError('max prefix length must be positive or 0') + except ValueError: + raise forms.ValidationError('invalid integer value') + return v + + def clean(self): + if 'prefix' in self.cleaned_data: + r = self._as_resource_range() + max_prefixlen = self.cleaned_data.get('max_prefixlen') + max_prefixlen = int(max_prefixlen) if max_prefixlen else r.prefixlen() + if max_prefixlen < r.prefixlen(): + raise forms.ValidationError( + 'max prefix length must be greater than or equal to the prefix length') + if max_prefixlen > r.min.bits: + raise forms.ValidationError( + 'max prefix length (%d) is out of range for IP version (%d)' % (max_prefixlen, r.min.bits)) + self.cleaned_data['max_prefixlen'] = str(max_prefixlen) + return self.cleaned_data return Cls diff --git a/rpki/gui/app/views.py b/rpki/gui/app/views.py index 28b8a498..1d468a07 100644 --- a/rpki/gui/app/views.py +++ b/rpki/gui/app/views.py @@ -148,27 +148,27 @@ def generic_import(request, queryset, configure, form_class=None, if handle == '': handle = None try: - # configure_repository returns None, so can't use tuple expansion - # here. Unpack the tuple below if post_import_redirect is None. - r = configure(z, tmpf.name, handle) + # configure_repository returns None, so can't use tuple expansion + # here. Unpack the tuple below if post_import_redirect is None. + r = configure(z, tmpf.name, handle) except lxml.etree.XMLSyntaxError as e: - logger.exception('caught XMLSyntaxError while parsing uploaded file') + logger.exception('caught XMLSyntaxError while parsing uploaded file') messages.error( request, 'The uploaded file has an invalid XML syntax' ) else: - # force rpkid run now - z.synchronize_ca(poke=True) - if post_import_redirect: - url = post_import_redirect - else: - _, handle = r - url = queryset.get(issuer=conf, - handle=handle).get_absolute_url() - return http.HttpResponseRedirect(url) + # force rpkid run now + z.synchronize_ca(poke=True) + if post_import_redirect: + url = post_import_redirect + else: + _, handle = r + url = queryset.get(issuer=conf, + handle=handle).get_absolute_url() + return http.HttpResponseRedirect(url) finally: - os.remove(tmpf.name) + os.remove(tmpf.name) else: form = form_class() @@ -474,10 +474,10 @@ def child_add_prefix(request, pk): child.address_ranges.create(start_ip=str(r.min), end_ip=str(r.max), version=version) Zookeeper( - handle=conf.handle, - logstream=logstream, - disable_signal_handlers=True - ).run_rpkid_now() + handle=conf.handle, + logstream=logstream, + disable_signal_handlers=True + ).run_rpkid_now() return http.HttpResponseRedirect(child.get_absolute_url()) else: form = forms.AddNetForm(child=child) @@ -497,10 +497,10 @@ def child_add_asn(request, pk): r = resource_range_as.parse_str(asns) child.asns.create(start_as=r.min, end_as=r.max) Zookeeper( - handle=conf.handle, - logstream=logstream, - disable_signal_handlers=True - ).run_rpkid_now() + handle=conf.handle, + logstream=logstream, + disable_signal_handlers=True + ).run_rpkid_now() return http.HttpResponseRedirect(child.get_absolute_url()) else: form = forms.AddASNForm(child=child) @@ -531,10 +531,10 @@ def child_edit(request, pk): models.ChildASN.objects.filter(child=child).exclude(pk__in=form.cleaned_data.get('as_ranges')).delete() models.ChildNet.objects.filter(child=child).exclude(pk__in=form.cleaned_data.get('address_ranges')).delete() Zookeeper( - handle=conf.handle, - logstream=logstream, - disable_signal_handlers=True - ).run_rpkid_now() + handle=conf.handle, + logstream=logstream, + disable_signal_handlers=True + ).run_rpkid_now() return http.HttpResponseRedirect(child.get_absolute_url()) else: form = form_class(initial={ @@ -713,27 +713,27 @@ def roa_create_multi(request): v = [] rng.chop_into_prefixes(v) init.extend([{'asn': asn, 'prefix': str(p)} for p in v]) - extra = 0 if init else 1 + extra = 0 if init else 1 formset = formset_factory(forms.ROARequestFormFactory(conf), extra=extra)(initial=init) elif request.method == 'POST': formset = formset_factory(forms.ROARequestFormFactory(conf), extra=0)(request.POST, request.FILES) - # We need to check .has_changed() because .is_valid() will return true - # if the user clicks the Preview button without filling in the blanks - # in the ROA form, leaving the form invalid from this view's POV. + # We need to check .has_changed() because .is_valid() will return true + # if the user clicks the Preview button without filling in the blanks + # in the ROA form, leaving the form invalid from this view's POV. if formset.has_changed() and formset.is_valid(): routes = [] v = [] query = Q() # for matching routes roas = [] for form in formset: - asn = form.cleaned_data['asn'] - rng = resource_range_ip.parse_str(form.cleaned_data['prefix']) - max_prefixlen = int(form.cleaned_data['max_prefixlen']) + asn = form.cleaned_data['asn'] + rng = resource_range_ip.parse_str(form.cleaned_data['prefix']) + max_prefixlen = int(form.cleaned_data['max_prefixlen']) protect_children = form.cleaned_data['protect_children'] roas.append((rng, max_prefixlen, asn, protect_children)) - v.append({'prefix': str(rng), 'max_prefixlen': max_prefixlen, - 'asn': asn}) + v.append({'prefix': str(rng), 'max_prefixlen': max_prefixlen, + 'asn': asn}) query |= Q(prefix_min__gte=rng.min, prefix_max__lte=rng.max) @@ -1451,10 +1451,10 @@ class RouterImportView(FormView): def form_valid(self, form): conf = get_conf(self.request.user, self.request.session['handle']) - tmpf = NamedTemporaryFile(prefix='import', suffix='.xml', - delete=False) - tmpf.write(form.cleaned_data['xml'].read()) - tmpf.close() + tmpf = NamedTemporaryFile(prefix='import', suffix='.xml', + delete=False) + tmpf.write(form.cleaned_data['xml'].read()) + tmpf.close() z = Zookeeper(handle=conf.handle, disable_signal_handlers=True) z.add_router_certificate_request(tmpf.name) z.run_rpkid_now() diff --git a/rpki/gui/cacheview/tests.py b/rpki/gui/cacheview/tests.py index daca07bf..c2958c72 100644 --- a/rpki/gui/cacheview/tests.py +++ b/rpki/gui/cacheview/tests.py @@ -21,4 +21,3 @@ Another way to test that 1 + 1 is equal to 2. >>> 1 + 1 == 2 True """} - diff --git a/rpki/http_simple.py b/rpki/http_simple.py index ee9cac35..6f73def5 100644 --- a/rpki/http_simple.py +++ b/rpki/http_simple.py @@ -31,106 +31,106 @@ default_content_type = "application/x-rpki" class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): - """ - HTTP request handler simple RPKI servers. - """ - - def do_POST(self): - try: - content_type = self.headers.get("Content-Type") - content_length = self.headers.get("Content-Length") - for handler_path, handler, handler_content_type in self.rpki_handlers: - if self.path.startswith(handler_path) and content_type in handler_content_type: - return handler(self, - self.rfile.read() - if content_length is None else - self.rfile.read(int(content_length))) - self.send_error(404, "No handler for path %s" % self.path) - except Exception, e: - logger.exception("Unhandled exception") - self.send_error(501, "Unhandled exception %s" % e) - - def send_cms_response(self, der): - self.send_response(200) - self.send_header("Content-Type", default_content_type) - self.send_header("Content-Length", str(len(der))) - self.end_headers() - self.wfile.write(der) - - def log_message(self, *args): - logger.info(*args, extra = dict(context = "%s:%s" % self.client_address)) - - def send_error(self, code, message = None): - # BaseHTTPRequestHandler.send_error() generates HTML error messages, - # which we don't want, so we override the method to suppress this. - self.send_response(code, message) - self.send_header("Content-Type", default_content_type) - self.send_header("Connection", "close") - self.end_headers() + """ + HTTP request handler simple RPKI servers. + """ + + def do_POST(self): + try: + content_type = self.headers.get("Content-Type") + content_length = self.headers.get("Content-Length") + for handler_path, handler, handler_content_type in self.rpki_handlers: + if self.path.startswith(handler_path) and content_type in handler_content_type: + return handler(self, + self.rfile.read() + if content_length is None else + self.rfile.read(int(content_length))) + self.send_error(404, "No handler for path %s" % self.path) + except Exception, e: + logger.exception("Unhandled exception") + self.send_error(501, "Unhandled exception %s" % e) + + def send_cms_response(self, der): + self.send_response(200) + self.send_header("Content-Type", default_content_type) + self.send_header("Content-Length", str(len(der))) + self.end_headers() + self.wfile.write(der) + + def log_message(self, *args): + logger.info(*args, extra = dict(context = "%s:%s" % self.client_address)) + + def send_error(self, code, message = None): + # BaseHTTPRequestHandler.send_error() generates HTML error messages, + # which we don't want, so we override the method to suppress this. + self.send_response(code, message) + self.send_header("Content-Type", default_content_type) + self.send_header("Connection", "close") + self.end_headers() def server(handlers, port, host = ""): - """ - Run an HTTP server and wait (forever) for connections. - """ + """ + Run an HTTP server and wait (forever) for connections. + """ - if isinstance(handlers, (tuple, list)): - handlers = tuple(h[:3] if len(h) > 2 else (h[0], h[1], default_content_type) - for h in handlers) - else: - handlers = (("/", handlers, default_content_type),) + if isinstance(handlers, (tuple, list)): + handlers = tuple(h[:3] if len(h) > 2 else (h[0], h[1], default_content_type) + for h in handlers) + else: + handlers = (("/", handlers, default_content_type),) - class RequestHandler(HTTPRequestHandler): - rpki_handlers = handlers + class RequestHandler(HTTPRequestHandler): + rpki_handlers = handlers - BaseHTTPServer.HTTPServer((host, port), RequestHandler).serve_forever() + BaseHTTPServer.HTTPServer((host, port), RequestHandler).serve_forever() class BadURL(Exception): - "Bad contact URL" + "Bad contact URL" class RequestFailed(Exception): - "HTTP returned failure" + "HTTP returned failure" class BadContentType(Exception): - "Wrong HTTP Content-Type" + "Wrong HTTP Content-Type" def client(proto_cms_msg, client_key, client_cert, server_ta, server_cert, url, q_msg, debug = False, replay_track = None, client_crl = None, content_type = default_content_type): - """ - Issue single a query and return the response, handling all the CMS and XML goo. - """ + """ + Issue single a query and return the response, handling all the CMS and XML goo. + """ - u = urlparse.urlparse(url) + u = urlparse.urlparse(url) - if u.scheme not in ("", "http") or u.username or u.password or u.params or u.query or u.fragment: - raise BadURL("Unusable URL %s", url) + if u.scheme not in ("", "http") or u.username or u.password or u.params or u.query or u.fragment: + raise BadURL("Unusable URL %s", url) - q_cms = proto_cms_msg() - q_der = q_cms.wrap(q_msg, client_key, client_cert, client_crl) + q_cms = proto_cms_msg() + q_der = q_cms.wrap(q_msg, client_key, client_cert, client_crl) - if debug: - debug.write("\n" + q_cms.pretty_print_content() + "\n") + if debug: + debug.write("\n" + q_cms.pretty_print_content() + "\n") - http = httplib.HTTPConnection(u.hostname, u.port or httplib.HTTP_PORT) - http.request("POST", u.path, q_der, {"Content-Type" : content_type}) - r = http.getresponse() + http = httplib.HTTPConnection(u.hostname, u.port or httplib.HTTP_PORT) + http.request("POST", u.path, q_der, {"Content-Type" : content_type}) + r = http.getresponse() - if r.status != 200: - raise RequestFailed("HTTP request failed with status %r reason %r" % (r.status, r.reason)) + if r.status != 200: + raise RequestFailed("HTTP request failed with status %r reason %r" % (r.status, r.reason)) - if r.getheader("Content-Type") != content_type: - raise BadContentType("HTTP Content-Type %r, expected %r" % (r.getheader("Content-Type"), content_type)) + if r.getheader("Content-Type") != content_type: + raise BadContentType("HTTP Content-Type %r, expected %r" % (r.getheader("Content-Type"), content_type)) - r_der = r.read() - r_cms = proto_cms_msg(DER = r_der) - r_msg = r_cms.unwrap((server_ta, server_cert)) + r_der = r.read() + r_cms = proto_cms_msg(DER = r_der) + r_msg = r_cms.unwrap((server_ta, server_cert)) - if replay_track is not None: - replay_track.cms_timestamp = r_cms.check_replay(replay_track.cms_timestamp, url) + if replay_track is not None: + replay_track.cms_timestamp = r_cms.check_replay(replay_track.cms_timestamp, url) - if debug: - debug.write("\n" + r_cms.pretty_print_content() + "\n") + if debug: + debug.write("\n" + r_cms.pretty_print_content() + "\n") - return r_msg + return r_msg diff --git a/rpki/ipaddrs.py b/rpki/ipaddrs.py index 25eefd0d..5117585c 100644 --- a/rpki/ipaddrs.py +++ b/rpki/ipaddrs.py @@ -48,99 +48,99 @@ once, here, thus avoiding a lot of duplicate code elsewhere. import socket, struct class v4addr(long): - """ - IPv4 address. + """ + IPv4 address. - Derived from long, but supports IPv4 print syntax. - """ + Derived from long, but supports IPv4 print syntax. + """ - bits = 32 - ipversion = 4 + bits = 32 + ipversion = 4 - def __new__(cls, x): - """ - Construct a v4addr object. - """ + def __new__(cls, x): + """ + Construct a v4addr object. + """ - if isinstance(x, unicode): - x = x.encode("ascii") - if isinstance(x, str): - return cls.from_bytes(socket.inet_pton(socket.AF_INET, ".".join(str(int(i)) for i in x.split(".")))) - else: - return long.__new__(cls, x) + if isinstance(x, unicode): + x = x.encode("ascii") + if isinstance(x, str): + return cls.from_bytes(socket.inet_pton(socket.AF_INET, ".".join(str(int(i)) for i in x.split(".")))) + else: + return long.__new__(cls, x) - def to_bytes(self): - """ - Convert a v4addr object to a raw byte string. - """ + def to_bytes(self): + """ + Convert a v4addr object to a raw byte string. + """ - return struct.pack("!I", long(self)) + return struct.pack("!I", long(self)) - @classmethod - def from_bytes(cls, x): - """ - Convert from a raw byte string to a v4addr object. - """ + @classmethod + def from_bytes(cls, x): + """ + Convert from a raw byte string to a v4addr object. + """ - return cls(struct.unpack("!I", x)[0]) + return cls(struct.unpack("!I", x)[0]) - def __str__(self): - """ - Convert a v4addr object to string format. - """ + def __str__(self): + """ + Convert a v4addr object to string format. + """ - return socket.inet_ntop(socket.AF_INET, self.to_bytes()) + return socket.inet_ntop(socket.AF_INET, self.to_bytes()) class v6addr(long): - """ - IPv6 address. + """ + IPv6 address. - Derived from long, but supports IPv6 print syntax. - """ + Derived from long, but supports IPv6 print syntax. + """ - bits = 128 - ipversion = 6 + bits = 128 + ipversion = 6 - def __new__(cls, x): - """ - Construct a v6addr object. - """ + def __new__(cls, x): + """ + Construct a v6addr object. + """ - if isinstance(x, unicode): - x = x.encode("ascii") - if isinstance(x, str): - return cls.from_bytes(socket.inet_pton(socket.AF_INET6, x)) - else: - return long.__new__(cls, x) + if isinstance(x, unicode): + x = x.encode("ascii") + if isinstance(x, str): + return cls.from_bytes(socket.inet_pton(socket.AF_INET6, x)) + else: + return long.__new__(cls, x) - def to_bytes(self): - """ - Convert a v6addr object to a raw byte string. - """ + def to_bytes(self): + """ + Convert a v6addr object to a raw byte string. + """ - return struct.pack("!QQ", long(self) >> 64, long(self) & 0xFFFFFFFFFFFFFFFF) + return struct.pack("!QQ", long(self) >> 64, long(self) & 0xFFFFFFFFFFFFFFFF) - @classmethod - def from_bytes(cls, x): - """ - Convert from a raw byte string to a v6addr object. - """ + @classmethod + def from_bytes(cls, x): + """ + Convert from a raw byte string to a v6addr object. + """ - x = struct.unpack("!QQ", x) - return cls((x[0] << 64) | x[1]) + x = struct.unpack("!QQ", x) + return cls((x[0] << 64) | x[1]) - def __str__(self): - """ - Convert a v6addr object to string format. - """ + def __str__(self): + """ + Convert a v6addr object to string format. + """ - return socket.inet_ntop(socket.AF_INET6, self.to_bytes()) + return socket.inet_ntop(socket.AF_INET6, self.to_bytes()) def parse(s): - """ - Parse a string as either an IPv4 or IPv6 address, and return object of appropriate class. - """ + """ + Parse a string as either an IPv4 or IPv6 address, and return object of appropriate class. + """ - if isinstance(s, unicode): - s = s.encode("ascii") - return v6addr(s) if ":" in s else v4addr(s) + if isinstance(s, unicode): + s = s.encode("ascii") + return v6addr(s) if ":" in s else v4addr(s) diff --git a/rpki/irdb/models.py b/rpki/irdb/models.py index d2d6256b..4ff5734a 100644 --- a/rpki/irdb/models.py +++ b/rpki/irdb/models.py @@ -65,480 +65,480 @@ ee_certificate_lifetime = rpki.sundial.timedelta(days = 60) # Field classes class HandleField(django.db.models.CharField): - """ - A handle field class. Replace this with SlugField? - """ + """ + A handle field class. Replace this with SlugField? + """ - description = 'A "handle" in one of the RPKI protocols' + description = 'A "handle" in one of the RPKI protocols' - def __init__(self, *args, **kwargs): - kwargs["max_length"] = 120 - django.db.models.CharField.__init__(self, *args, **kwargs) + def __init__(self, *args, **kwargs): + kwargs["max_length"] = 120 + django.db.models.CharField.__init__(self, *args, **kwargs) class SignedReferralField(DERField): - description = "CMS signed object containing XML" - rpki_type = rpki.x509.SignedReferral + description = "CMS signed object containing XML" + rpki_type = rpki.x509.SignedReferral # Custom managers class CertificateManager(django.db.models.Manager): - def get_or_certify(self, **kwargs): - """ - Sort of like .get_or_create(), but for models containing - certificates which need to be generated based on other fields. - - Takes keyword arguments like .get(), checks for existing object. - If none, creates a new one; if found an existing object but some - of the non-key fields don't match, updates the existing object. - Runs certification method for new or updated objects. Returns a - tuple consisting of the object and a boolean indicating whether - anything has changed. - """ + def get_or_certify(self, **kwargs): + """ + Sort of like .get_or_create(), but for models containing + certificates which need to be generated based on other fields. + + Takes keyword arguments like .get(), checks for existing object. + If none, creates a new one; if found an existing object but some + of the non-key fields don't match, updates the existing object. + Runs certification method for new or updated objects. Returns a + tuple consisting of the object and a boolean indicating whether + anything has changed. + """ - changed = False + changed = False - try: - obj = self.get(**self._get_or_certify_keys(kwargs)) + try: + obj = self.get(**self._get_or_certify_keys(kwargs)) - except self.model.DoesNotExist: - obj = self.model(**kwargs) - changed = True + except self.model.DoesNotExist: + obj = self.model(**kwargs) + changed = True - else: - for k in kwargs: - if getattr(obj, k) != kwargs[k]: - setattr(obj, k, kwargs[k]) - changed = True + else: + for k in kwargs: + if getattr(obj, k) != kwargs[k]: + setattr(obj, k, kwargs[k]) + changed = True - if changed: - obj.avow() - obj.save() + if changed: + obj.avow() + obj.save() - return obj, changed + return obj, changed - def _get_or_certify_keys(self, kwargs): - assert len(self.model._meta.unique_together) == 1 - return dict((k, kwargs[k]) for k in self.model._meta.unique_together[0]) + def _get_or_certify_keys(self, kwargs): + assert len(self.model._meta.unique_together) == 1 + return dict((k, kwargs[k]) for k in self.model._meta.unique_together[0]) class ResourceHolderCAManager(CertificateManager): - def _get_or_certify_keys(self, kwargs): - return { "handle" : kwargs["handle"] } + def _get_or_certify_keys(self, kwargs): + return { "handle" : kwargs["handle"] } class ServerCAManager(CertificateManager): - def _get_or_certify_keys(self, kwargs): - return { "pk" : 1 } + def _get_or_certify_keys(self, kwargs): + return { "pk" : 1 } class ResourceHolderEEManager(CertificateManager): - def _get_or_certify_keys(self, kwargs): - return { "issuer" : kwargs["issuer"] } + def _get_or_certify_keys(self, kwargs): + return { "issuer" : kwargs["issuer"] } ### class CA(django.db.models.Model): - certificate = CertificateField() - private_key = RSAPrivateKeyField() - latest_crl = CRLField() - - # Might want to bring these into line with what rpkid does. Current - # variables here were chosen to map easily to what OpenSSL command - # line tool was keeping on disk. - - next_serial = django.db.models.BigIntegerField(default = 1) - next_crl_number = django.db.models.BigIntegerField(default = 1) - last_crl_update = SundialField() - next_crl_update = SundialField() - - class Meta: - abstract = True - - def avow(self): - if self.private_key is None: - self.private_key = rpki.x509.RSA.generate(quiet = True) - now = rpki.sundial.now() - notAfter = now + ca_certificate_lifetime - self.certificate = rpki.x509.X509.bpki_self_certify( - keypair = self.private_key, - subject_name = self.subject_name, - serial = self.next_serial, - now = now, - notAfter = notAfter) - self.next_serial += 1 - self.generate_crl() - return self.certificate - - def certify(self, subject_name, subject_key, validity_interval, is_ca, pathLenConstraint = None): - now = rpki.sundial.now() - notAfter = now + validity_interval - result = self.certificate.bpki_certify( - keypair = self.private_key, - subject_name = subject_name, - subject_key = subject_key, - serial = self.next_serial, - now = now, - notAfter = notAfter, - is_ca = is_ca, - pathLenConstraint = pathLenConstraint) - self.next_serial += 1 - return result - - def revoke(self, cert): - Revocation.objects.create( - issuer = self, - revoked = rpki.sundial.now(), - serial = cert.certificate.getSerial(), - expires = cert.certificate.getNotAfter() + crl_interval) - cert.delete() - self.generate_crl() - - def generate_crl(self): - now = rpki.sundial.now() - self.revocations.filter(expires__lt = now).delete() - revoked = [(r.serial, r.revoked) for r in self.revocations.all()] - self.latest_crl = rpki.x509.CRL.generate( - keypair = self.private_key, - issuer = self.certificate, - serial = self.next_crl_number, - thisUpdate = now, - nextUpdate = now + crl_interval, - revokedCertificates = revoked) - self.last_crl_update = now - self.next_crl_update = now + crl_interval - self.next_crl_number += 1 + certificate = CertificateField() + private_key = RSAPrivateKeyField() + latest_crl = CRLField() + + # Might want to bring these into line with what rpkid does. Current + # variables here were chosen to map easily to what OpenSSL command + # line tool was keeping on disk. + + next_serial = django.db.models.BigIntegerField(default = 1) + next_crl_number = django.db.models.BigIntegerField(default = 1) + last_crl_update = SundialField() + next_crl_update = SundialField() + + class Meta: + abstract = True + + def avow(self): + if self.private_key is None: + self.private_key = rpki.x509.RSA.generate(quiet = True) + now = rpki.sundial.now() + notAfter = now + ca_certificate_lifetime + self.certificate = rpki.x509.X509.bpki_self_certify( + keypair = self.private_key, + subject_name = self.subject_name, + serial = self.next_serial, + now = now, + notAfter = notAfter) + self.next_serial += 1 + self.generate_crl() + return self.certificate + + def certify(self, subject_name, subject_key, validity_interval, is_ca, pathLenConstraint = None): + now = rpki.sundial.now() + notAfter = now + validity_interval + result = self.certificate.bpki_certify( + keypair = self.private_key, + subject_name = subject_name, + subject_key = subject_key, + serial = self.next_serial, + now = now, + notAfter = notAfter, + is_ca = is_ca, + pathLenConstraint = pathLenConstraint) + self.next_serial += 1 + return result + + def revoke(self, cert): + Revocation.objects.create( + issuer = self, + revoked = rpki.sundial.now(), + serial = cert.certificate.getSerial(), + expires = cert.certificate.getNotAfter() + crl_interval) + cert.delete() + self.generate_crl() + + def generate_crl(self): + now = rpki.sundial.now() + self.revocations.filter(expires__lt = now).delete() + revoked = [(r.serial, r.revoked) for r in self.revocations.all()] + self.latest_crl = rpki.x509.CRL.generate( + keypair = self.private_key, + issuer = self.certificate, + serial = self.next_crl_number, + thisUpdate = now, + nextUpdate = now + crl_interval, + revokedCertificates = revoked) + self.last_crl_update = now + self.next_crl_update = now + crl_interval + self.next_crl_number += 1 class ServerCA(CA): - objects = ServerCAManager() + objects = ServerCAManager() - def __unicode__(self): - return "" + def __unicode__(self): + return "" - @property - def subject_name(self): - if self.certificate is not None: - return self.certificate.getSubject() - else: - return rpki.x509.X501DN.from_cn("%s BPKI server CA" % socket.gethostname()) + @property + def subject_name(self): + if self.certificate is not None: + return self.certificate.getSubject() + else: + return rpki.x509.X501DN.from_cn("%s BPKI server CA" % socket.gethostname()) class ResourceHolderCA(CA): - handle = HandleField(unique = True) - objects = ResourceHolderCAManager() + handle = HandleField(unique = True) + objects = ResourceHolderCAManager() - def __unicode__(self): - return self.handle + def __unicode__(self): + return self.handle - @property - def subject_name(self): - if self.certificate is not None: - return self.certificate.getSubject() - else: - return rpki.x509.X501DN.from_cn("%s BPKI resource CA" % self.handle) + @property + def subject_name(self): + if self.certificate is not None: + return self.certificate.getSubject() + else: + return rpki.x509.X501DN.from_cn("%s BPKI resource CA" % self.handle) class Certificate(django.db.models.Model): - certificate = CertificateField() - objects = CertificateManager() + certificate = CertificateField() + objects = CertificateManager() - class Meta: - abstract = True - unique_together = ("issuer", "handle") + class Meta: + abstract = True + unique_together = ("issuer", "handle") - def revoke(self): - self.issuer.revoke(self) + def revoke(self): + self.issuer.revoke(self) class CrossCertification(Certificate): - handle = HandleField() - ta = CertificateField() + handle = HandleField() + ta = CertificateField() - class Meta: - abstract = True + class Meta: + abstract = True - def avow(self): - self.certificate = self.issuer.certify( - subject_name = self.ta.getSubject(), - subject_key = self.ta.getPublicKey(), - validity_interval = ee_certificate_lifetime, - is_ca = True, - pathLenConstraint = 0) + def avow(self): + self.certificate = self.issuer.certify( + subject_name = self.ta.getSubject(), + subject_key = self.ta.getPublicKey(), + validity_interval = ee_certificate_lifetime, + is_ca = True, + pathLenConstraint = 0) - def __unicode__(self): - return self.handle + def __unicode__(self): + return self.handle class HostedCA(Certificate): - issuer = django.db.models.ForeignKey(ServerCA) - hosted = django.db.models.OneToOneField(ResourceHolderCA, related_name = "hosted_by") + issuer = django.db.models.ForeignKey(ServerCA) + hosted = django.db.models.OneToOneField(ResourceHolderCA, related_name = "hosted_by") - def avow(self): - self.certificate = self.issuer.certify( - subject_name = self.hosted.certificate.getSubject(), - subject_key = self.hosted.certificate.getPublicKey(), - validity_interval = ee_certificate_lifetime, - is_ca = True, - pathLenConstraint = 1) + def avow(self): + self.certificate = self.issuer.certify( + subject_name = self.hosted.certificate.getSubject(), + subject_key = self.hosted.certificate.getPublicKey(), + validity_interval = ee_certificate_lifetime, + is_ca = True, + pathLenConstraint = 1) - class Meta: - unique_together = ("issuer", "hosted") + class Meta: + unique_together = ("issuer", "hosted") - def __unicode__(self): - return self.hosted.handle + def __unicode__(self): + return self.hosted.handle class Revocation(django.db.models.Model): - serial = django.db.models.BigIntegerField() - revoked = SundialField() - expires = SundialField() + serial = django.db.models.BigIntegerField() + revoked = SundialField() + expires = SundialField() - class Meta: - abstract = True - unique_together = ("issuer", "serial") + class Meta: + abstract = True + unique_together = ("issuer", "serial") class ServerRevocation(Revocation): - issuer = django.db.models.ForeignKey(ServerCA, related_name = "revocations") + issuer = django.db.models.ForeignKey(ServerCA, related_name = "revocations") class ResourceHolderRevocation(Revocation): - issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "revocations") + issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "revocations") class EECertificate(Certificate): - private_key = RSAPrivateKeyField() + private_key = RSAPrivateKeyField() - class Meta: - abstract = True + class Meta: + abstract = True - def avow(self): - if self.private_key is None: - self.private_key = rpki.x509.RSA.generate(quiet = True) - self.certificate = self.issuer.certify( - subject_name = self.subject_name, - subject_key = self.private_key.get_public(), - validity_interval = ee_certificate_lifetime, - is_ca = False) + def avow(self): + if self.private_key is None: + self.private_key = rpki.x509.RSA.generate(quiet = True) + self.certificate = self.issuer.certify( + subject_name = self.subject_name, + subject_key = self.private_key.get_public(), + validity_interval = ee_certificate_lifetime, + is_ca = False) class ServerEE(EECertificate): - issuer = django.db.models.ForeignKey(ServerCA, related_name = "ee_certificates") - purpose = EnumField(choices = ("rpkid", "pubd", "irdbd", "irbe")) + issuer = django.db.models.ForeignKey(ServerCA, related_name = "ee_certificates") + purpose = EnumField(choices = ("rpkid", "pubd", "irdbd", "irbe")) - class Meta: - unique_together = ("issuer", "purpose") + class Meta: + unique_together = ("issuer", "purpose") - @property - def subject_name(self): - return rpki.x509.X501DN.from_cn("%s BPKI %s EE" % (socket.gethostname(), - self.get_purpose_display())) + @property + def subject_name(self): + return rpki.x509.X501DN.from_cn("%s BPKI %s EE" % (socket.gethostname(), + self.get_purpose_display())) class Referral(EECertificate): - issuer = django.db.models.OneToOneField(ResourceHolderCA, related_name = "referral_certificate") - objects = ResourceHolderEEManager() + issuer = django.db.models.OneToOneField(ResourceHolderCA, related_name = "referral_certificate") + objects = ResourceHolderEEManager() - @property - def subject_name(self): - return rpki.x509.X501DN.from_cn("%s BPKI Referral EE" % self.issuer.handle) + @property + def subject_name(self): + return rpki.x509.X501DN.from_cn("%s BPKI Referral EE" % self.issuer.handle) class Turtle(django.db.models.Model): - service_uri = django.db.models.CharField(max_length = 255) + service_uri = django.db.models.CharField(max_length = 255) class Rootd(EECertificate, Turtle): - issuer = django.db.models.OneToOneField(ResourceHolderCA, related_name = "rootd") - objects = ResourceHolderEEManager() + issuer = django.db.models.OneToOneField(ResourceHolderCA, related_name = "rootd") + objects = ResourceHolderEEManager() - @property - def subject_name(self): - return rpki.x509.X501DN.from_cn("%s BPKI rootd EE" % self.issuer.handle) + @property + def subject_name(self): + return rpki.x509.X501DN.from_cn("%s BPKI rootd EE" % self.issuer.handle) class BSC(Certificate): - issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "bscs") - handle = HandleField() - pkcs10 = PKCS10Field() + issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "bscs") + handle = HandleField() + pkcs10 = PKCS10Field() - def avow(self): - self.certificate = self.issuer.certify( - subject_name = self.pkcs10.getSubject(), - subject_key = self.pkcs10.getPublicKey(), - validity_interval = ee_certificate_lifetime, - is_ca = False) + def avow(self): + self.certificate = self.issuer.certify( + subject_name = self.pkcs10.getSubject(), + subject_key = self.pkcs10.getPublicKey(), + validity_interval = ee_certificate_lifetime, + is_ca = False) - def __unicode__(self): - return self.handle + def __unicode__(self): + return self.handle class ResourceSet(django.db.models.Model): - valid_until = SundialField() + valid_until = SundialField() - class Meta: - abstract = True + class Meta: + abstract = True - @property - def resource_bag(self): - raw_asn, raw_net = self._select_resource_bag() - asns = rpki.resource_set.resource_set_as.from_django( - (a.start_as, a.end_as) for a in raw_asn) - ipv4 = rpki.resource_set.resource_set_ipv4.from_django( - (a.start_ip, a.end_ip) for a in raw_net if a.version == "IPv4") - ipv6 = rpki.resource_set.resource_set_ipv6.from_django( - (a.start_ip, a.end_ip) for a in raw_net if a.version == "IPv6") - return rpki.resource_set.resource_bag( - valid_until = self.valid_until, asn = asns, v4 = ipv4, v6 = ipv6) + @property + def resource_bag(self): + raw_asn, raw_net = self._select_resource_bag() + asns = rpki.resource_set.resource_set_as.from_django( + (a.start_as, a.end_as) for a in raw_asn) + ipv4 = rpki.resource_set.resource_set_ipv4.from_django( + (a.start_ip, a.end_ip) for a in raw_net if a.version == "IPv4") + ipv6 = rpki.resource_set.resource_set_ipv6.from_django( + (a.start_ip, a.end_ip) for a in raw_net if a.version == "IPv6") + return rpki.resource_set.resource_bag( + valid_until = self.valid_until, asn = asns, v4 = ipv4, v6 = ipv6) - # Writing of .setter method deferred until something needs it. + # Writing of .setter method deferred until something needs it. class ResourceSetASN(django.db.models.Model): - start_as = django.db.models.BigIntegerField() - end_as = django.db.models.BigIntegerField() + start_as = django.db.models.BigIntegerField() + end_as = django.db.models.BigIntegerField() - class Meta: - abstract = True + class Meta: + abstract = True - def as_resource_range(self): - return rpki.resource_set.resource_range_as(self.start_as, self.end_as) + def as_resource_range(self): + return rpki.resource_set.resource_range_as(self.start_as, self.end_as) class ResourceSetNet(django.db.models.Model): - start_ip = django.db.models.CharField(max_length = 40) - end_ip = django.db.models.CharField(max_length = 40) - version = EnumField(choices = ip_version_choices) + start_ip = django.db.models.CharField(max_length = 40) + end_ip = django.db.models.CharField(max_length = 40) + version = EnumField(choices = ip_version_choices) - class Meta: - abstract = True + class Meta: + abstract = True - def as_resource_range(self): - return rpki.resource_set.resource_range_ip.from_strings(self.start_ip, self.end_ip) + def as_resource_range(self): + return rpki.resource_set.resource_range_ip.from_strings(self.start_ip, self.end_ip) class Child(CrossCertification, ResourceSet): - issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "children") - name = django.db.models.TextField(null = True, blank = True) - - def _select_resource_bag(self): - child_asn = rpki.irdb.models.ChildASN.objects.raw(""" - SELECT * - FROM irdb_childasn - WHERE child_id = %s - """, [self.id]) - child_net = list(rpki.irdb.models.ChildNet.objects.raw(""" - SELECT * - FROM irdb_childnet - WHERE child_id = %s - """, [self.id])) - return child_asn, child_net - - class Meta: - unique_together = ("issuer", "handle") + issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "children") + name = django.db.models.TextField(null = True, blank = True) + + def _select_resource_bag(self): + child_asn = rpki.irdb.models.ChildASN.objects.raw(""" + SELECT * + FROM irdb_childasn + WHERE child_id = %s + """, [self.id]) + child_net = list(rpki.irdb.models.ChildNet.objects.raw(""" + SELECT * + FROM irdb_childnet + WHERE child_id = %s + """, [self.id])) + return child_asn, child_net + + class Meta: + unique_together = ("issuer", "handle") class ChildASN(ResourceSetASN): - child = django.db.models.ForeignKey(Child, related_name = "asns") + child = django.db.models.ForeignKey(Child, related_name = "asns") - class Meta: - unique_together = ("child", "start_as", "end_as") + class Meta: + unique_together = ("child", "start_as", "end_as") class ChildNet(ResourceSetNet): - child = django.db.models.ForeignKey(Child, related_name = "address_ranges") + child = django.db.models.ForeignKey(Child, related_name = "address_ranges") - class Meta: - unique_together = ("child", "start_ip", "end_ip", "version") + class Meta: + unique_together = ("child", "start_ip", "end_ip", "version") class Parent(CrossCertification, Turtle): - issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "parents") - parent_handle = HandleField() - child_handle = HandleField() - repository_type = EnumField(choices = ("none", "offer", "referral")) - referrer = HandleField(null = True, blank = True) - referral_authorization = SignedReferralField(null = True, blank = True) + issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "parents") + parent_handle = HandleField() + child_handle = HandleField() + repository_type = EnumField(choices = ("none", "offer", "referral")) + referrer = HandleField(null = True, blank = True) + referral_authorization = SignedReferralField(null = True, blank = True) - # This shouldn't be necessary - class Meta: - unique_together = ("issuer", "handle") + # This shouldn't be necessary + class Meta: + unique_together = ("issuer", "handle") class ROARequest(django.db.models.Model): - issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "roa_requests") - asn = django.db.models.BigIntegerField() - - @property - def roa_prefix_bag(self): - prefixes = list(rpki.irdb.models.ROARequestPrefix.objects.raw(""" - SELECT * - FROM irdb_roarequestprefix - WHERE roa_request_id = %s - """, [self.id])) - v4 = rpki.resource_set.roa_prefix_set_ipv4.from_django( - (p.prefix, p.prefixlen, p.max_prefixlen) for p in prefixes if p.version == "IPv4") - v6 = rpki.resource_set.roa_prefix_set_ipv6.from_django( - (p.prefix, p.prefixlen, p.max_prefixlen) for p in prefixes if p.version == "IPv6") - return rpki.resource_set.roa_prefix_bag(v4 = v4, v6 = v6) - - # Writing of .setter method deferred until something needs it. + issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "roa_requests") + asn = django.db.models.BigIntegerField() + + @property + def roa_prefix_bag(self): + prefixes = list(rpki.irdb.models.ROARequestPrefix.objects.raw(""" + SELECT * + FROM irdb_roarequestprefix + WHERE roa_request_id = %s + """, [self.id])) + v4 = rpki.resource_set.roa_prefix_set_ipv4.from_django( + (p.prefix, p.prefixlen, p.max_prefixlen) for p in prefixes if p.version == "IPv4") + v6 = rpki.resource_set.roa_prefix_set_ipv6.from_django( + (p.prefix, p.prefixlen, p.max_prefixlen) for p in prefixes if p.version == "IPv6") + return rpki.resource_set.roa_prefix_bag(v4 = v4, v6 = v6) + + # Writing of .setter method deferred until something needs it. class ROARequestPrefix(django.db.models.Model): - roa_request = django.db.models.ForeignKey(ROARequest, related_name = "prefixes") - version = EnumField(choices = ip_version_choices) - prefix = django.db.models.CharField(max_length = 40) - prefixlen = django.db.models.PositiveSmallIntegerField() - max_prefixlen = django.db.models.PositiveSmallIntegerField() + roa_request = django.db.models.ForeignKey(ROARequest, related_name = "prefixes") + version = EnumField(choices = ip_version_choices) + prefix = django.db.models.CharField(max_length = 40) + prefixlen = django.db.models.PositiveSmallIntegerField() + max_prefixlen = django.db.models.PositiveSmallIntegerField() - def as_roa_prefix(self): - if self.version == 'IPv4': - return rpki.resource_set.roa_prefix_ipv4(rpki.POW.IPAddress(self.prefix), self.prefixlen, self.max_prefixlen) - else: - return rpki.resource_set.roa_prefix_ipv6(rpki.POW.IPAddress(self.prefix), self.prefixlen, self.max_prefixlen) + def as_roa_prefix(self): + if self.version == 'IPv4': + return rpki.resource_set.roa_prefix_ipv4(rpki.POW.IPAddress(self.prefix), self.prefixlen, self.max_prefixlen) + else: + return rpki.resource_set.roa_prefix_ipv6(rpki.POW.IPAddress(self.prefix), self.prefixlen, self.max_prefixlen) - def as_resource_range(self): - return self.as_roa_prefix().to_resource_range() + def as_resource_range(self): + return self.as_roa_prefix().to_resource_range() - class Meta: - unique_together = ("roa_request", "version", "prefix", "prefixlen", "max_prefixlen") + class Meta: + unique_together = ("roa_request", "version", "prefix", "prefixlen", "max_prefixlen") class GhostbusterRequest(django.db.models.Model): - issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "ghostbuster_requests") - parent = django.db.models.ForeignKey(Parent, related_name = "ghostbuster_requests", null = True) - vcard = django.db.models.TextField() + issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "ghostbuster_requests") + parent = django.db.models.ForeignKey(Parent, related_name = "ghostbuster_requests", null = True) + vcard = django.db.models.TextField() class EECertificateRequest(ResourceSet): - issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "ee_certificate_requests") - pkcs10 = PKCS10Field() - gski = django.db.models.CharField(max_length = 27) - cn = django.db.models.CharField(max_length = 64) - sn = django.db.models.CharField(max_length = 64) - eku = django.db.models.TextField(null = True) - - def _select_resource_bag(self): - ee_asn = rpki.irdb.models.EECertificateRequestASN.objects.raw(""" - SELECT * - FROM irdb_eecertificaterequestasn - WHERE ee_certificate_request_id = %s - """, [self.id]) - ee_net = rpki.irdb.models.EECertificateRequestNet.objects.raw(""" - SELECT * - FROM irdb_eecertificaterequestnet - WHERE ee_certificate_request_id = %s - """, [self.id]) - return ee_asn, ee_net - - class Meta: - unique_together = ("issuer", "gski") + issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "ee_certificate_requests") + pkcs10 = PKCS10Field() + gski = django.db.models.CharField(max_length = 27) + cn = django.db.models.CharField(max_length = 64) + sn = django.db.models.CharField(max_length = 64) + eku = django.db.models.TextField(null = True) + + def _select_resource_bag(self): + ee_asn = rpki.irdb.models.EECertificateRequestASN.objects.raw(""" + SELECT * + FROM irdb_eecertificaterequestasn + WHERE ee_certificate_request_id = %s + """, [self.id]) + ee_net = rpki.irdb.models.EECertificateRequestNet.objects.raw(""" + SELECT * + FROM irdb_eecertificaterequestnet + WHERE ee_certificate_request_id = %s + """, [self.id]) + return ee_asn, ee_net + + class Meta: + unique_together = ("issuer", "gski") class EECertificateRequestASN(ResourceSetASN): - ee_certificate_request = django.db.models.ForeignKey(EECertificateRequest, related_name = "asns") + ee_certificate_request = django.db.models.ForeignKey(EECertificateRequest, related_name = "asns") - class Meta: - unique_together = ("ee_certificate_request", "start_as", "end_as") + class Meta: + unique_together = ("ee_certificate_request", "start_as", "end_as") class EECertificateRequestNet(ResourceSetNet): - ee_certificate_request = django.db.models.ForeignKey(EECertificateRequest, related_name = "address_ranges") + ee_certificate_request = django.db.models.ForeignKey(EECertificateRequest, related_name = "address_ranges") - class Meta: - unique_together = ("ee_certificate_request", "start_ip", "end_ip", "version") + class Meta: + unique_together = ("ee_certificate_request", "start_ip", "end_ip", "version") class Repository(CrossCertification): - issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "repositories") - client_handle = HandleField() - service_uri = django.db.models.CharField(max_length = 255) - sia_base = django.db.models.TextField() - rrdp_notification_uri = django.db.models.TextField(null = True) - turtle = django.db.models.OneToOneField(Turtle, related_name = "repository") + issuer = django.db.models.ForeignKey(ResourceHolderCA, related_name = "repositories") + client_handle = HandleField() + service_uri = django.db.models.CharField(max_length = 255) + sia_base = django.db.models.TextField() + rrdp_notification_uri = django.db.models.TextField(null = True) + turtle = django.db.models.OneToOneField(Turtle, related_name = "repository") - # This shouldn't be necessary - class Meta: - unique_together = ("issuer", "handle") + # This shouldn't be necessary + class Meta: + unique_together = ("issuer", "handle") class Client(CrossCertification): - issuer = django.db.models.ForeignKey(ServerCA, related_name = "clients") - sia_base = django.db.models.TextField() + issuer = django.db.models.ForeignKey(ServerCA, related_name = "clients") + sia_base = django.db.models.TextField() - # This shouldn't be necessary - class Meta: - unique_together = ("issuer", "handle") + # This shouldn't be necessary + class Meta: + unique_together = ("issuer", "handle") diff --git a/rpki/irdb/router.py b/rpki/irdb/router.py index 0aaf53ce..3cbd52f9 100644 --- a/rpki/irdb/router.py +++ b/rpki/irdb/router.py @@ -27,69 +27,69 @@ accomplishes this. """ class DBContextRouter(object): - """ - A Django database router for use with multiple IRDBs. - - This router is designed to work in conjunction with the - rpki.irdb.database context handler (q.v.). - """ - - _app = "irdb" - - _database = None - - def db_for_read(self, model, **hints): - if model._meta.app_label == self._app: - return self._database - else: - return None - - def db_for_write(self, model, **hints): - if model._meta.app_label == self._app: - return self._database - else: - return None - - def allow_relation(self, obj1, obj2, **hints): - if self._database is None: - return None - elif obj1._meta.app_label == self._app and obj2._meta.app_label == self._app: - return True - else: - return None - - def allow_migrate(self, db, model): - if db == self._database and model._meta.app_label == self._app: - return True - else: - return None + """ + A Django database router for use with multiple IRDBs. + + This router is designed to work in conjunction with the + rpki.irdb.database context handler (q.v.). + """ + + _app = "irdb" + + _database = None + + def db_for_read(self, model, **hints): + if model._meta.app_label == self._app: + return self._database + else: + return None + + def db_for_write(self, model, **hints): + if model._meta.app_label == self._app: + return self._database + else: + return None + + def allow_relation(self, obj1, obj2, **hints): + if self._database is None: + return None + elif obj1._meta.app_label == self._app and obj2._meta.app_label == self._app: + return True + else: + return None + + def allow_migrate(self, db, model): + if db == self._database and model._meta.app_label == self._app: + return True + else: + return None class database(object): - """ - Context manager for use with DBContextRouter. Use thusly: - - with rpki.irdb.database("blarg"): - do_stuff() - - This binds IRDB operations to database blarg for the duration of - the call to do_stuff(), then restores the prior state. - """ - - def __init__(self, name, on_entry = None, on_exit = None): - if not isinstance(name, str): - raise ValueError("database name must be a string, not %r" % name) - self.name = name - self.on_entry = on_entry - self.on_exit = on_exit - - def __enter__(self): - if self.on_entry is not None: - self.on_entry() - self.former = DBContextRouter._database - DBContextRouter._database = self.name - - def __exit__(self, _type, value, traceback): - assert DBContextRouter._database is self.name - DBContextRouter._database = self.former - if self.on_exit is not None: - self.on_exit() + """ + Context manager for use with DBContextRouter. Use thusly: + + with rpki.irdb.database("blarg"): + do_stuff() + + This binds IRDB operations to database blarg for the duration of + the call to do_stuff(), then restores the prior state. + """ + + def __init__(self, name, on_entry = None, on_exit = None): + if not isinstance(name, str): + raise ValueError("database name must be a string, not %r" % name) + self.name = name + self.on_entry = on_entry + self.on_exit = on_exit + + def __enter__(self): + if self.on_entry is not None: + self.on_entry() + self.former = DBContextRouter._database + DBContextRouter._database = self.name + + def __exit__(self, _type, value, traceback): + assert DBContextRouter._database is self.name + DBContextRouter._database = self.former + if self.on_exit is not None: + self.on_exit() diff --git a/rpki/irdb/zookeeper.py b/rpki/irdb/zookeeper.py index 7202f421..a65f1f5f 100644 --- a/rpki/irdb/zookeeper.py +++ b/rpki/irdb/zookeeper.py @@ -96,1651 +96,1654 @@ class CouldntFindRepoParent(Exception): "Couldn't find repository's parent." def B64Element(e, tag, obj, **kwargs): - """ - Create an XML element containing Base64 encoded data taken from a - DER object. - """ - - if e is None: - se = Element(tag, **kwargs) - else: - se = SubElement(e, tag, **kwargs) - if e is not None and e.text is None: - e.text = "\n" - se.text = "\n" + obj.get_Base64() - se.tail = "\n" - return se - -class PEM_writer(object): - """ - Write PEM files to disk, keeping track of which ones we've already - written and setting the file mode appropriately. - - Comparing the old file with what we're about to write serves no real - purpose except to calm users who find repeated messages about - writing the same file confusing. - """ - - def __init__(self, logstream = None): - self.wrote = set() - self.logstream = logstream - - def __call__(self, filename, obj, compare = True): - filename = os.path.realpath(filename) - if filename in self.wrote: - return - tempname = filename - pem = obj.get_PEM() - if not filename.startswith("/dev/"): - try: - if compare and pem == open(filename, "r").read(): - return - except: # pylint: disable=W0702 - pass - tempname += ".%s.tmp" % os.getpid() - mode = 0400 if filename.endswith(".key") else 0444 - if self.logstream is not None: - self.logstream.write("Writing %s\n" % filename) - f = os.fdopen(os.open(tempname, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode), "w") - f.write(pem) - f.close() - if tempname != filename: - os.rename(tempname, filename) - self.wrote.add(filename) - - -def etree_read(filename_or_etree_wrapper, schema = rpki.relaxng.oob_setup): - """ - Read an etree from a file, verifying then stripping XML namespace - cruft. As a convenience, we also accept an etree_wrapper object in - place of a filename, in which case we deepcopy the etree directly - from the etree_wrapper and there's no need for a file. - """ - - if isinstance(filename_or_etree_wrapper, etree_wrapper): - e = copy.deepcopy(filename_or_etree_wrapper.etree) - else: - e = ElementTree(file = filename_or_etree_wrapper).getroot() - schema.assertValid(e) - return e - - -class etree_wrapper(object): - """ - Wrapper for ETree objects so we can return them as function results - without requiring the caller to understand much about them. - """ - - def __init__(self, e, msg = None, debug = False, schema = rpki.relaxng.oob_setup): - self.msg = msg - e = copy.deepcopy(e) - if debug: - print ElementToString(e) - schema.assertValid(e) - self.etree = e - - def __str__(self): - return ElementToString(self.etree) - - def save(self, filename, logstream = None): - filename = os.path.realpath(filename) - tempname = filename - if not filename.startswith("/dev/"): - tempname += ".%s.tmp" % os.getpid() - ElementTree(self.etree).write(tempname) - if tempname != filename: - os.rename(tempname, filename) - if logstream is not None: - logstream.write("Wrote %s\n" % filename) - if self.msg is not None: - logstream.write(self.msg + "\n") - - @property - def file(self): - from cStringIO import StringIO - return StringIO(ElementToString(self.etree)) - - -class Zookeeper(object): - - ## @var show_xml - # If not None, a file-like object to which to prettyprint XML, for debugging. - - show_xml = None - - def __init__(self, cfg = None, handle = None, logstream = None, disable_signal_handlers = False): - - if cfg is None: - cfg = rpki.config.parser() - - if handle is None: - handle = cfg.get("handle", section = myrpki_section) - - self.cfg = cfg - - self.logstream = logstream - self.disable_signal_handlers = disable_signal_handlers - - self.run_rpkid = cfg.getboolean("run_rpkid", section = myrpki_section) - self.run_pubd = cfg.getboolean("run_pubd", section = myrpki_section) - self.run_rootd = cfg.getboolean("run_rootd", section = myrpki_section) - - if self.run_rootd and (not self.run_pubd or not self.run_rpkid): - raise CantRunRootd("Can't run rootd unless also running rpkid and pubd") - - self.default_repository = cfg.get("default_repository", "", section = myrpki_section) - self.pubd_contact_info = cfg.get("pubd_contact_info", "", section = myrpki_section) - - self.rsync_module = cfg.get("publication_rsync_module", section = myrpki_section) - self.rsync_server = cfg.get("publication_rsync_server", section = myrpki_section) - - self.reset_identity(handle) - - - def reset_identity(self, handle): - """ - Select handle of current resource holding entity. - """ - - if handle is None: - raise MissingHandle - self.handle = handle - - - def set_logstream(self, logstream): - """ - Set log stream for this Zookeeper. The log stream is a file-like - object, or None to suppress all logging. - """ - - self.logstream = logstream - - - def log(self, msg): - """ - Send some text to this Zookeeper's log stream, if one is set. - """ - - if self.logstream is not None: - self.logstream.write(msg) - self.logstream.write("\n") - - - @property - def resource_ca(self): - """ - Get ResourceHolderCA object associated with current handle. - """ - - if self.handle is None: - raise HandleNotSet - return rpki.irdb.models.ResourceHolderCA.objects.get(handle = self.handle) - - - @property - def server_ca(self): - """ - Get ServerCA object. - """ - - return rpki.irdb.models.ServerCA.objects.get() - - - @django.db.transaction.atomic - def initialize_server_bpki(self): """ - Initialize server BPKI portion of an RPKI installation. Reads the - configuration file and generates the initial BPKI server - certificates needed to start daemons. + Create an XML element containing Base64 encoded data taken from a + DER object. """ - if self.run_rpkid or self.run_pubd: - server_ca, created = rpki.irdb.models.ServerCA.objects.get_or_certify() - rpki.irdb.models.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "irbe") - - if self.run_rpkid: - rpki.irdb.models.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "rpkid") - rpki.irdb.models.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "irdbd") - - if self.run_pubd: - rpki.irdb.models.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "pubd") - - - @django.db.transaction.atomic - def initialize_resource_bpki(self): - """ - Initialize the resource-holding BPKI for an RPKI installation. - Returns XML describing the resource holder. - - This method is present primarily for backwards compatibility with - the old combined initialize() method which initialized both the - server BPKI and the default resource-holding BPKI in a single - method call. In the long run we want to replace this with - something that takes a handle as argument and creates the - resource-holding BPKI idenity if needed. - """ - - resource_ca, created = rpki.irdb.models.ResourceHolderCA.objects.get_or_certify(handle = self.handle) - return self.generate_identity() - - - def initialize(self): - """ - Backwards compatibility wrapper: calls initialize_server_bpki() - and initialize_resource_bpki(), returns latter's result. - """ - - self.initialize_server_bpki() - return self.initialize_resource_bpki() - - - def generate_identity(self): - """ - Generate identity XML. Broken out of .initialize() because it's - easier for the GUI this way. - """ - - e = Element(tag_oob_child_request, nsmap = oob_nsmap, version = oob_version, - child_handle = self.handle) - B64Element(e, tag_oob_child_bpki_ta, self.resource_ca.certificate) - return etree_wrapper(e, msg = 'This is the "identity" file you will need to send to your parent') - - - @django.db.transaction.atomic - def delete_tenant(self): - """ - Delete the ResourceHolderCA object corresponding to the current handle. - This corresponds to deleting an rpkid object. - - This code assumes the normal Django cascade-on-delete behavior, - that is, we assume that deleting the ResourceHolderCA object - deletes all the subordinate objects that refer to it via foreign - key relationships. - """ - - resource_ca = self.resource_ca - if resource_ca is not None: - resource_ca.delete() + if e is None: + se = Element(tag, **kwargs) else: - self.log("No such ResourceHolderCA \"%s\"" % self.handle) - - - @django.db.transaction.atomic - def configure_rootd(self): - - assert self.run_rpkid and self.run_pubd and self.run_rootd - - rpki.irdb.models.Rootd.objects.get_or_certify( - issuer = self.resource_ca, - service_uri = "http://localhost:%s/" % self.cfg.get("rootd_server_port", - section = myrpki_section)) - - return self.generate_rootd_repository_offer() - - - def generate_rootd_repository_offer(self): - """ - Generate repository offer for rootd. Split out of - configure_rootd() because that's easier for the GUI. - """ - - try: - self.resource_ca.repositories.get(handle = self.handle) - return None - - except rpki.irdb.models.Repository.DoesNotExist: - e = Element(tag_oob_publisher_request, nsmap = oob_nsmap, version = oob_version, - publisher_handle = self.handle) - B64Element(e, tag_oob_publisher_bpki_ta, self.resource_ca.certificate) - return etree_wrapper(e, msg = 'This is the "repository offer" file for you to use if you want to publish in your own repository') - - - def write_bpki_files(self): - """ - Write out BPKI certificate, key, and CRL files for daemons that - need them. - """ + se = SubElement(e, tag, **kwargs) + if e is not None and e.text is None: + e.text = "\n" + se.text = "\n" + obj.get_Base64() + se.tail = "\n" + return se - writer = PEM_writer(self.logstream) - - if self.run_rpkid: - rpkid = self.server_ca.ee_certificates.get(purpose = "rpkid") - writer(self.cfg.get("bpki-ta", section = rpkid_section), self.server_ca.certificate) - writer(self.cfg.get("rpkid-key", section = rpkid_section), rpkid.private_key) - writer(self.cfg.get("rpkid-cert", section = rpkid_section), rpkid.certificate) - writer(self.cfg.get("irdb-cert", section = rpkid_section), - self.server_ca.ee_certificates.get(purpose = "irdbd").certificate) - writer(self.cfg.get("irbe-cert", section = rpkid_section), - self.server_ca.ee_certificates.get(purpose = "irbe").certificate) - - if self.run_pubd: - pubd = self.server_ca.ee_certificates.get(purpose = "pubd") - writer(self.cfg.get("bpki-ta", section = pubd_section), self.server_ca.certificate) - writer(self.cfg.get("pubd-key", section = pubd_section), pubd.private_key) - writer(self.cfg.get("pubd-cert", section = pubd_section), pubd.certificate) - writer(self.cfg.get("irbe-cert", section = pubd_section), - self.server_ca.ee_certificates.get(purpose = "irbe").certificate) - - if self.run_rootd: - try: - rootd = rpki.irdb.models.ResourceHolderCA.objects.get(handle = self.handle).rootd - writer(self.cfg.get("bpki-ta", section = rootd_section), self.server_ca.certificate) - writer(self.cfg.get("rootd-bpki-crl", section = rootd_section), self.server_ca.latest_crl) - writer(self.cfg.get("rootd-bpki-key", section = rootd_section), rootd.private_key) - writer(self.cfg.get("rootd-bpki-cert", section = rootd_section), rootd.certificate) - writer(self.cfg.get("child-bpki-cert", section = rootd_section), rootd.issuer.certificate) - except rpki.irdb.models.ResourceHolderCA.DoesNotExist: - self.log("rootd enabled but resource holding entity not yet configured, skipping rootd setup") - except rpki.irdb.models.Rootd.DoesNotExist: - self.log("rootd enabled but not yet configured, skipping rootd setup") - - - @django.db.transaction.atomic - def update_bpki(self): +class PEM_writer(object): """ - Update BPKI certificates. Assumes an existing RPKI installation. - - Basic plan here is to reissue all BPKI certificates we can, right - now. In the long run we might want to be more clever about only - touching ones that need maintenance, but this will do for a start. + Write PEM files to disk, keeping track of which ones we've already + written and setting the file mode appropriately. + + Comparing the old file with what we're about to write serves no real + purpose except to calm users who find repeated messages about + writing the same file confusing. + """ + + def __init__(self, logstream = None): + self.wrote = set() + self.logstream = logstream + + def __call__(self, filename, obj, compare = True): + filename = os.path.realpath(filename) + if filename in self.wrote: + return + tempname = filename + pem = obj.get_PEM() + if not filename.startswith("/dev/"): + try: + if compare and pem == open(filename, "r").read(): + return + except: # pylint: disable=W0702 + pass + tempname += ".%s.tmp" % os.getpid() + mode = 0400 if filename.endswith(".key") else 0444 + if self.logstream is not None: + self.logstream.write("Writing %s\n" % filename) + f = os.fdopen(os.open(tempname, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode), "w") + f.write(pem) + f.close() + if tempname != filename: + os.rename(tempname, filename) + self.wrote.add(filename) - We also reissue CRLs for all CAs. - - Most likely this should be run under cron. - """ - for model in (rpki.irdb.models.ServerCA, - rpki.irdb.models.ResourceHolderCA, - rpki.irdb.models.ServerEE, - rpki.irdb.models.Referral, - rpki.irdb.models.Rootd, - rpki.irdb.models.HostedCA, - rpki.irdb.models.BSC, - rpki.irdb.models.Child, - rpki.irdb.models.Parent, - rpki.irdb.models.Client, - rpki.irdb.models.Repository): - for obj in model.objects.all(): - self.log("Regenerating BPKI certificate %s" % obj.certificate.getSubject()) - obj.avow() - obj.save() - - self.log("Regenerating Server BPKI CRL") - self.server_ca.generate_crl() - self.server_ca.save() - - for ca in rpki.irdb.models.ResourceHolderCA.objects.all(): - self.log("Regenerating BPKI CRL for Resource Holder %s" % ca.handle) - ca.generate_crl() - ca.save() - - - @staticmethod - def _compose_left_right_query(): +def etree_read(filename_or_etree_wrapper, schema = rpki.relaxng.oob_setup): """ - Compose top level element of a left-right query. + Read an etree from a file, verifying then stripping XML namespace + cruft. As a convenience, we also accept an etree_wrapper object in + place of a filename, in which case we deepcopy the etree directly + from the etree_wrapper and there's no need for a file. """ - return Element(rpki.left_right.tag_msg, nsmap = rpki.left_right.nsmap, - type = "query", version = rpki.left_right.version) + if isinstance(filename_or_etree_wrapper, etree_wrapper): + e = copy.deepcopy(filename_or_etree_wrapper.etree) + else: + e = ElementTree(file = filename_or_etree_wrapper).getroot() + schema.assertValid(e) + return e - @staticmethod - def _compose_publication_control_query(): +class etree_wrapper(object): """ - Compose top level element of a publication-control query. + Wrapper for ETree objects so we can return them as function results + without requiring the caller to understand much about them. """ - return Element(rpki.publication_control.tag_msg, nsmap = rpki.publication_control.nsmap, - type = "query", version = rpki.publication_control.version) + def __init__(self, e, msg = None, debug = False, schema = rpki.relaxng.oob_setup): + self.msg = msg + e = copy.deepcopy(e) + if debug: + print ElementToString(e) + schema.assertValid(e) + self.etree = e + def __str__(self): + return ElementToString(self.etree) - @django.db.transaction.atomic - def synchronize_bpki(self): - """ - Synchronize BPKI updates. This is separate from .update_bpki() - because this requires rpkid to be running and none of the other - BPKI update stuff does; there may be circumstances under which it - makes sense to do the rest of the BPKI update and allow this to - fail with a warning. - """ + def save(self, filename, logstream = None): + filename = os.path.realpath(filename) + tempname = filename + if not filename.startswith("/dev/"): + tempname += ".%s.tmp" % os.getpid() + ElementTree(self.etree).write(tempname) + if tempname != filename: + os.rename(tempname, filename) + if logstream is not None: + logstream.write("Wrote %s\n" % filename) + if self.msg is not None: + logstream.write(self.msg + "\n") - if self.run_rpkid: - q_msg = self._compose_left_right_query() - - for ca in rpki.irdb.models.ResourceHolderCA.objects.all(): - q_pdu = SubElement(q_msg, rpki.left_right.tag_tenant, - action = "set", - tag = "%s__tenant" % ca.handle, - tenant_handle = ca.handle) - SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = ca.certificate.get_Base64() - - for bsc in rpki.irdb.models.BSC.objects.all(): - q_pdu = SubElement(q_msg, rpki.left_right.tag_bsc, - action = "set", - tag = "%s__bsc__%s" % (bsc.issuer.handle, bsc.handle), - tenant_handle = bsc.issuer.handle, - bsc_handle = bsc.handle) - SubElement(q_pdu, rpki.left_right.tag_signing_cert).text = bsc.certificate.get_Base64() - SubElement(q_pdu, rpki.left_right.tag_signing_cert_crl).text = bsc.issuer.latest_crl.get_Base64() - - for repository in rpki.irdb.models.Repository.objects.all(): - q_pdu = SubElement(q_msg, rpki.left_right.tag_repository, - action = "set", - tag = "%s__repository__%s" % (repository.issuer.handle, repository.handle), - tenant_handle = repository.issuer.handle, - repository_handle = repository.handle) - SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = repository.certificate.get_Base64() - - for parent in rpki.irdb.models.Parent.objects.all(): - q_pdu = SubElement(q_msg, rpki.left_right.tag_parent, - action = "set", - tag = "%s__parent__%s" % (parent.issuer.handle, parent.handle), - tenant_handle = parent.issuer.handle, - parent_handle = parent.handle) - SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = parent.certificate.get_Base64() - - for rootd in rpki.irdb.models.Rootd.objects.all(): - q_pdu = SubElement(q_msg, rpki.left_right.tag_parent, - action = "set", - tag = "%s__rootd" % rootd.issuer.handle, - tenant_handle = rootd.issuer.handle, - parent_handle = rootd.issuer.handle) - SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = rootd.certificate.get_Base64() - - for child in rpki.irdb.models.Child.objects.all(): - q_pdu = SubElement(q_msg, rpki.left_right.tag_child, - action = "set", - tag = "%s__child__%s" % (child.issuer.handle, child.handle), - tenant_handle = child.issuer.handle, - child_handle = child.handle) - SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = child.certificate.get_Base64() - - if len(q_msg) > 0: - self.call_rpkid(q_msg) + @property + def file(self): + from cStringIO import StringIO + return StringIO(ElementToString(self.etree)) - if self.run_pubd: - q_msg = self._compose_publication_control_query() - for client in self.server_ca.clients.all(): - q_pdu = SubElement(q_msg, rpki.publication_control.tag_client, action = "set", client_handle = client.handle) - SubElement(q_pdu, rpki.publication_control.tag_bpki_cert).text = client.certificate.get_Base64() +class Zookeeper(object): - if len(q_msg) > 0: - self.call_pubd(q_msg) + ## @var show_xml + # If not None, a file-like object to which to prettyprint XML, for debugging. + show_xml = None - @django.db.transaction.atomic - def configure_child(self, filename, child_handle = None, valid_until = None): - """ - Configure a new child of this RPKI entity, given the child's XML - identity file as an input. Extracts the child's data from the - XML, cross-certifies the child's resource-holding BPKI - certificate, and generates an XML file describing the relationship - between the child and this parent, including this parent's BPKI - data and up-down protocol service URI. - """ + def __init__(self, cfg = None, handle = None, logstream = None, disable_signal_handlers = False): - x = etree_read(filename) + if cfg is None: + cfg = rpki.config.parser() - if x.tag != tag_oob_child_request: - raise BadXMLMessage("Expected %s, got %s", tag_oob_child_request, x.tag) + if handle is None: + handle = cfg.get("handle", section = myrpki_section) - if child_handle is None: - child_handle = x.get("child_handle") + self.cfg = cfg - if valid_until is None: - valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 365) - else: - valid_until = rpki.sundial.datetime.fromXMLtime(valid_until) - if valid_until < rpki.sundial.now(): - raise PastExpiration("Specified new expiration time %s has passed" % valid_until) + self.logstream = logstream + self.disable_signal_handlers = disable_signal_handlers - self.log("Child calls itself %r, we call it %r" % (x.get("child_handle"), child_handle)) + self.run_rpkid = cfg.getboolean("run_rpkid", section = myrpki_section) + self.run_pubd = cfg.getboolean("run_pubd", section = myrpki_section) + self.run_rootd = cfg.getboolean("run_rootd", section = myrpki_section) - child, created = rpki.irdb.models.Child.objects.get_or_certify( - issuer = self.resource_ca, - handle = child_handle, - ta = rpki.x509.X509(Base64 = x.findtext(tag_oob_child_bpki_ta)), - valid_until = valid_until) + if self.run_rootd and (not self.run_pubd or not self.run_rpkid): + raise CantRunRootd("Can't run rootd unless also running rpkid and pubd") - return self.generate_parental_response(child), child_handle + self.default_repository = cfg.get("default_repository", "", section = myrpki_section) + self.pubd_contact_info = cfg.get("pubd_contact_info", "", section = myrpki_section) + self.rsync_module = cfg.get("publication_rsync_module", section = myrpki_section) + self.rsync_server = cfg.get("publication_rsync_server", section = myrpki_section) - @django.db.transaction.atomic - def generate_parental_response(self, child): - """ - Generate parental response XML. Broken out of .configure_child() - for GUI. - """ + self.reset_identity(handle) - service_uri = "http://%s:%s/up-down/%s/%s" % ( - self.cfg.get("rpkid_server_host", section = myrpki_section), - self.cfg.get("rpkid_server_port", section = myrpki_section), - self.handle, child.handle) - e = Element(tag_oob_parent_response, nsmap = oob_nsmap, version = oob_version, - service_uri = service_uri, - child_handle = child.handle, - parent_handle = self.handle) - B64Element(e, tag_oob_parent_bpki_ta, self.resource_ca.certificate) + def reset_identity(self, handle): + """ + Select handle of current resource holding entity. + """ - try: - if self.default_repository: - repo = self.resource_ca.repositories.get(handle = self.default_repository) - else: - repo = self.resource_ca.repositories.get() - except rpki.irdb.models.Repository.DoesNotExist: - repo = None + if handle is None: + raise MissingHandle + self.handle = handle - if repo is None: - self.log("Couldn't find any usable repositories, not giving referral") - elif repo.handle == self.handle: - SubElement(e, tag_oob_offer) + def set_logstream(self, logstream): + """ + Set log stream for this Zookeeper. The log stream is a file-like + object, or None to suppress all logging. + """ - else: - proposed_sia_base = repo.sia_base + child.handle + "/" - referral_cert, created = rpki.irdb.models.Referral.objects.get_or_certify(issuer = self.resource_ca) - auth = rpki.x509.SignedReferral() - auth.set_content(B64Element(None, tag_oob_authorization, child.ta, - nsmap = oob_nsmap, version = oob_version, - authorized_sia_base = proposed_sia_base)) - auth.schema_check() - auth.sign(referral_cert.private_key, referral_cert.certificate, self.resource_ca.latest_crl) - B64Element(e, tag_oob_referral, auth, referrer = repo.client_handle) + self.logstream = logstream - return etree_wrapper(e, msg = "Send this file back to the child you just configured") + def log(self, msg): + """ + Send some text to this Zookeeper's log stream, if one is set. + """ - @django.db.transaction.atomic - def delete_child(self, child_handle): - """ - Delete a child of this RPKI entity. - """ - - self.resource_ca.children.get(handle = child_handle).delete() + if self.logstream is not None: + self.logstream.write(msg) + self.logstream.write("\n") - @django.db.transaction.atomic - def configure_parent(self, filename, parent_handle = None): - """ - Configure a new parent of this RPKI entity, given the output of - the parent's configure_child command as input. Reads the parent's - response XML, extracts the parent's BPKI and service URI - information, cross-certifies the parent's BPKI data into this - entity's BPKI, and checks for offers or referrals of publication - service. If a publication offer or referral is present, we - generate a request-for-service message to that repository, in case - the user wants to avail herself of the referral or offer. - """ + @property + def resource_ca(self): + """ + Get ResourceHolderCA object associated with current handle. + """ - x = etree_read(filename) + if self.handle is None: + raise HandleNotSet + return rpki.irdb.models.ResourceHolderCA.objects.get(handle = self.handle) - if x.tag != tag_oob_parent_response: - raise BadXMLMessage("Expected %s, got %s", tag_oob_parent_response, x.tag) - if parent_handle is None: - parent_handle = x.get("parent_handle") + @property + def server_ca(self): + """ + Get ServerCA object. + """ - offer = x.find(tag_oob_offer) - referral = x.find(tag_oob_referral) + return rpki.irdb.models.ServerCA.objects.get() - if offer is not None: - repository_type = "offer" - referrer = None - referral_authorization = None - elif referral is not None: - repository_type = "referral" - referrer = referral.get("referrer") - referral_authorization = rpki.x509.SignedReferral(Base64 = referral.text) + @django.db.transaction.atomic + def initialize_server_bpki(self): + """ + Initialize server BPKI portion of an RPKI installation. Reads the + configuration file and generates the initial BPKI server + certificates needed to start daemons. + """ - else: - repository_type = "none" - referrer = None - referral_authorization = None + if self.run_rpkid or self.run_pubd: + server_ca, created = rpki.irdb.models.ServerCA.objects.get_or_certify() + rpki.irdb.models.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "irbe") - self.log("Parent calls itself %r, we call it %r" % (x.get("parent_handle"), parent_handle)) - self.log("Parent calls us %r" % x.get("child_handle")) + if self.run_rpkid: + rpki.irdb.models.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "rpkid") + rpki.irdb.models.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "irdbd") - parent, created = rpki.irdb.models.Parent.objects.get_or_certify( - issuer = self.resource_ca, - handle = parent_handle, - child_handle = x.get("child_handle"), - parent_handle = x.get("parent_handle"), - service_uri = x.get("service_uri"), - ta = rpki.x509.X509(Base64 = x.findtext(tag_oob_parent_bpki_ta)), - repository_type = repository_type, - referrer = referrer, - referral_authorization = referral_authorization) + if self.run_pubd: + rpki.irdb.models.ServerEE.objects.get_or_certify(issuer = server_ca, purpose = "pubd") - return self.generate_repository_request(parent), parent_handle + @django.db.transaction.atomic + def initialize_resource_bpki(self): + """ + Initialize the resource-holding BPKI for an RPKI installation. + Returns XML describing the resource holder. - def generate_repository_request(self, parent): - """ - Generate repository request for a given parent. - """ + This method is present primarily for backwards compatibility with + the old combined initialize() method which initialized both the + server BPKI and the default resource-holding BPKI in a single + method call. In the long run we want to replace this with + something that takes a handle as argument and creates the + resource-holding BPKI idenity if needed. + """ - e = Element(tag_oob_publisher_request, nsmap = oob_nsmap, version = oob_version, - publisher_handle = self.handle) - B64Element(e, tag_oob_publisher_bpki_ta, self.resource_ca.certificate) - if parent.repository_type == "referral": - B64Element(e, tag_oob_referral, parent.referral_authorization, - referrer = parent.referrer) + resource_ca, created = rpki.irdb.models.ResourceHolderCA.objects.get_or_certify(handle = self.handle) + return self.generate_identity() - return etree_wrapper(e, msg = "This is the file to send to the repository operator") + def initialize(self): + """ + Backwards compatibility wrapper: calls initialize_server_bpki() + and initialize_resource_bpki(), returns latter's result. + """ - @django.db.transaction.atomic - def delete_parent(self, parent_handle): - """ - Delete a parent of this RPKI entity. - """ + self.initialize_server_bpki() + return self.initialize_resource_bpki() - self.resource_ca.parents.get(handle = parent_handle).delete() + def generate_identity(self): + """ + Generate identity XML. Broken out of .initialize() because it's + easier for the GUI this way. + """ - @django.db.transaction.atomic - def delete_rootd(self): - """ - Delete rootd associated with this RPKI entity. - """ + e = Element(tag_oob_child_request, nsmap = oob_nsmap, version = oob_version, + child_handle = self.handle) + B64Element(e, tag_oob_child_bpki_ta, self.resource_ca.certificate) + return etree_wrapper(e, msg = 'This is the "identity" file you will need to send to your parent') - self.resource_ca.rootd.delete() + @django.db.transaction.atomic + def delete_tenant(self): + """ + Delete the ResourceHolderCA object corresponding to the current handle. + This corresponds to deleting an rpkid object. - @django.db.transaction.atomic - def configure_publication_client(self, filename, sia_base = None, flat = False): - """ - Configure publication server to know about a new client, given the - client's request-for-service message as input. Reads the client's - request for service, cross-certifies the client's BPKI data, and - generates a response message containing the repository's BPKI data - and service URI. - """ + This code assumes the normal Django cascade-on-delete behavior, + that is, we assume that deleting the ResourceHolderCA object + deletes all the subordinate objects that refer to it via foreign + key relationships. + """ - x = etree_read(filename) - - if x.tag != tag_oob_publisher_request: - raise BadXMLMessage("Expected %s, got %s", tag_oob_publisher_request, x.tag) - - client_ta = rpki.x509.X509(Base64 = x.findtext(tag_oob_publisher_bpki_ta)) - - referral = x.find(tag_oob_referral) - - default_sia_base = "rsync://{self.rsync_server}/{self.rsync_module}/{handle}/".format( - self = self, handle = x.get("publisher_handle")) - - if sia_base is None and flat: - self.log("Flat publication structure forced, homing client at top-level") - sia_base = default_sia_base - - if sia_base is None and referral is not None: - self.log("This looks like a referral, checking") - try: - referrer = referral.get("referrer") - referrer = self.server_ca.clients.get(handle = referrer) - referral = rpki.x509.SignedReferral(Base64 = referral.text) - referral = referral.unwrap(ta = (referrer.certificate, self.server_ca.certificate)) - if rpki.x509.X509(Base64 = referral.text) != client_ta: - raise BadXMLMessage("Referral trust anchor does not match") - sia_base = referral.get("authorized_sia_base") - except rpki.irdb.models.Client.DoesNotExist: - self.log("We have no record of the client ({}) alleged to have made this referral".format(referrer)) - - if sia_base is None and referral is None: - self.log("This might be an offer, checking") - try: - parent = rpki.irdb.models.ResourceHolderCA.objects.get(children__ta = client_ta) - if "/" in parent.repositories.get(ta = self.server_ca.certificate).client_handle: - self.log("Client's parent is not top-level, this is not a valid offer") + resource_ca = self.resource_ca + if resource_ca is not None: + resource_ca.delete() else: - self.log("Found client and its parent, nesting") - sia_base = "rsync://{self.rsync_server}/{self.rsync_module}/{parent_handle}/{client_handle}/".format( - self = self, parent_handle = parent.handle, client_handle = x.get("publisher_handle")) - except rpki.irdb.models.Repository.DoesNotExist: - self.log("Found client's parent, but repository isn't set, this shouldn't happen!") - except rpki.irdb.models.ResourceHolderCA.DoesNotExist: - try: - rpki.irdb.models.Rootd.objects.get(issuer__certificate = client_ta) - self.log("This client's parent is rootd") - sia_base = default_sia_base - except rpki.irdb.models.Rootd.DoesNotExist: - self.log("We don't host this client's parent, so we didn't make an offer") - - if sia_base is None: - self.log("Don't know where else to nest this client, so defaulting to top-level") - sia_base = default_sia_base + self.log("No such ResourceHolderCA \"%s\"" % self.handle) - if not sia_base.startswith("rsync://"): - raise BadXMLMessage("Malformed sia_base parameter %r, should start with 'rsync://'" % sia_base) - client_handle = "/".join(sia_base.rstrip("/").split("/")[4:]) + @django.db.transaction.atomic + def configure_rootd(self): - self.log("Client calls itself %r, we call it %r" % ( - x.get("publisher_handle"), client_handle)) + assert self.run_rpkid and self.run_pubd and self.run_rootd - client, created = rpki.irdb.models.Client.objects.get_or_certify( - issuer = self.server_ca, - handle = client_handle, - ta = client_ta, - sia_base = sia_base) + rpki.irdb.models.Rootd.objects.get_or_certify( + issuer = self.resource_ca, + service_uri = "http://localhost:%s/" % self.cfg.get("rootd_server_port", + section = myrpki_section)) - return self.generate_repository_response(client), client_handle + return self.generate_rootd_repository_offer() - def generate_repository_response(self, client): - """ - Generate repository response XML to a given client. - """ - - service_uri = "http://{host}:{port}/client/{handle}".format( - host = self.cfg.get("pubd_server_host", section = myrpki_section), - port = self.cfg.get("pubd_server_port", section = myrpki_section), - handle = client.handle) - - rrdp_uri = self.cfg.get("publication_rrdp_notification_uri", section = myrpki_section, - default = "") or None - - e = Element(tag_oob_repository_response, nsmap = oob_nsmap, version = oob_version, - service_uri = service_uri, - publisher_handle = client.handle, - sia_base = client.sia_base) - - if rrdp_uri is not None: - e.set("rrdp_notification_uri", rrdp_uri) - - B64Element(e, tag_oob_repository_bpki_ta, self.server_ca.certificate) - return etree_wrapper(e, msg = "Send this file back to the publication client you just configured") + def generate_rootd_repository_offer(self): + """ + Generate repository offer for rootd. Split out of + configure_rootd() because that's easier for the GUI. + """ + try: + self.resource_ca.repositories.get(handle = self.handle) + return None - @django.db.transaction.atomic - def delete_publication_client(self, client_handle): - """ - Delete a publication client of this RPKI entity. - """ + except rpki.irdb.models.Repository.DoesNotExist: + e = Element(tag_oob_publisher_request, nsmap = oob_nsmap, version = oob_version, + publisher_handle = self.handle) + B64Element(e, tag_oob_publisher_bpki_ta, self.resource_ca.certificate) + return etree_wrapper(e, msg = 'This is the "repository offer" file for you to use if you want to publish in your own repository') + + + def write_bpki_files(self): + """ + Write out BPKI certificate, key, and CRL files for daemons that + need them. + """ + + writer = PEM_writer(self.logstream) + + if self.run_rpkid: + rpkid = self.server_ca.ee_certificates.get(purpose = "rpkid") + writer(self.cfg.get("bpki-ta", section = rpkid_section), self.server_ca.certificate) + writer(self.cfg.get("rpkid-key", section = rpkid_section), rpkid.private_key) + writer(self.cfg.get("rpkid-cert", section = rpkid_section), rpkid.certificate) + writer(self.cfg.get("irdb-cert", section = rpkid_section), + self.server_ca.ee_certificates.get(purpose = "irdbd").certificate) + writer(self.cfg.get("irbe-cert", section = rpkid_section), + self.server_ca.ee_certificates.get(purpose = "irbe").certificate) + + if self.run_pubd: + pubd = self.server_ca.ee_certificates.get(purpose = "pubd") + writer(self.cfg.get("bpki-ta", section = pubd_section), self.server_ca.certificate) + writer(self.cfg.get("pubd-key", section = pubd_section), pubd.private_key) + writer(self.cfg.get("pubd-cert", section = pubd_section), pubd.certificate) + writer(self.cfg.get("irbe-cert", section = pubd_section), + self.server_ca.ee_certificates.get(purpose = "irbe").certificate) + + if self.run_rootd: + try: + rootd = rpki.irdb.models.ResourceHolderCA.objects.get(handle = self.handle).rootd + writer(self.cfg.get("bpki-ta", section = rootd_section), self.server_ca.certificate) + writer(self.cfg.get("rootd-bpki-crl", section = rootd_section), self.server_ca.latest_crl) + writer(self.cfg.get("rootd-bpki-key", section = rootd_section), rootd.private_key) + writer(self.cfg.get("rootd-bpki-cert", section = rootd_section), rootd.certificate) + writer(self.cfg.get("child-bpki-cert", section = rootd_section), rootd.issuer.certificate) + except rpki.irdb.models.ResourceHolderCA.DoesNotExist: + self.log("rootd enabled but resource holding entity not yet configured, skipping rootd setup") + except rpki.irdb.models.Rootd.DoesNotExist: + self.log("rootd enabled but not yet configured, skipping rootd setup") + + + @django.db.transaction.atomic + def update_bpki(self): + """ + Update BPKI certificates. Assumes an existing RPKI installation. + + Basic plan here is to reissue all BPKI certificates we can, right + now. In the long run we might want to be more clever about only + touching ones that need maintenance, but this will do for a start. + + We also reissue CRLs for all CAs. + + Most likely this should be run under cron. + """ + + for model in (rpki.irdb.models.ServerCA, + rpki.irdb.models.ResourceHolderCA, + rpki.irdb.models.ServerEE, + rpki.irdb.models.Referral, + rpki.irdb.models.Rootd, + rpki.irdb.models.HostedCA, + rpki.irdb.models.BSC, + rpki.irdb.models.Child, + rpki.irdb.models.Parent, + rpki.irdb.models.Client, + rpki.irdb.models.Repository): + for obj in model.objects.all(): + self.log("Regenerating BPKI certificate %s" % obj.certificate.getSubject()) + obj.avow() + obj.save() + + self.log("Regenerating Server BPKI CRL") + self.server_ca.generate_crl() + self.server_ca.save() + + for ca in rpki.irdb.models.ResourceHolderCA.objects.all(): + self.log("Regenerating BPKI CRL for Resource Holder %s" % ca.handle) + ca.generate_crl() + ca.save() + + + @staticmethod + def _compose_left_right_query(): + """ + Compose top level element of a left-right query. + """ + + return Element(rpki.left_right.tag_msg, nsmap = rpki.left_right.nsmap, + type = "query", version = rpki.left_right.version) + + + @staticmethod + def _compose_publication_control_query(): + """ + Compose top level element of a publication-control query. + """ + + return Element(rpki.publication_control.tag_msg, nsmap = rpki.publication_control.nsmap, + type = "query", version = rpki.publication_control.version) + + + @django.db.transaction.atomic + def synchronize_bpki(self): + """ + Synchronize BPKI updates. This is separate from .update_bpki() + because this requires rpkid to be running and none of the other + BPKI update stuff does; there may be circumstances under which it + makes sense to do the rest of the BPKI update and allow this to + fail with a warning. + """ + + if self.run_rpkid: + q_msg = self._compose_left_right_query() + + for ca in rpki.irdb.models.ResourceHolderCA.objects.all(): + q_pdu = SubElement(q_msg, rpki.left_right.tag_tenant, + action = "set", + tag = "%s__tenant" % ca.handle, + tenant_handle = ca.handle) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = ca.certificate.get_Base64() + + for bsc in rpki.irdb.models.BSC.objects.all(): + q_pdu = SubElement(q_msg, rpki.left_right.tag_bsc, + action = "set", + tag = "%s__bsc__%s" % (bsc.issuer.handle, bsc.handle), + tenant_handle = bsc.issuer.handle, + bsc_handle = bsc.handle) + SubElement(q_pdu, rpki.left_right.tag_signing_cert).text = bsc.certificate.get_Base64() + SubElement(q_pdu, rpki.left_right.tag_signing_cert_crl).text = bsc.issuer.latest_crl.get_Base64() + + for repository in rpki.irdb.models.Repository.objects.all(): + q_pdu = SubElement(q_msg, rpki.left_right.tag_repository, + action = "set", + tag = "%s__repository__%s" % (repository.issuer.handle, repository.handle), + tenant_handle = repository.issuer.handle, + repository_handle = repository.handle) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = repository.certificate.get_Base64() + + for parent in rpki.irdb.models.Parent.objects.all(): + q_pdu = SubElement(q_msg, rpki.left_right.tag_parent, + action = "set", + tag = "%s__parent__%s" % (parent.issuer.handle, parent.handle), + tenant_handle = parent.issuer.handle, + parent_handle = parent.handle) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = parent.certificate.get_Base64() + + for rootd in rpki.irdb.models.Rootd.objects.all(): + q_pdu = SubElement(q_msg, rpki.left_right.tag_parent, + action = "set", + tag = "%s__rootd" % rootd.issuer.handle, + tenant_handle = rootd.issuer.handle, + parent_handle = rootd.issuer.handle) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = rootd.certificate.get_Base64() + + for child in rpki.irdb.models.Child.objects.all(): + q_pdu = SubElement(q_msg, rpki.left_right.tag_child, + action = "set", + tag = "%s__child__%s" % (child.issuer.handle, child.handle), + tenant_handle = child.issuer.handle, + child_handle = child.handle) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = child.certificate.get_Base64() + + if len(q_msg) > 0: + self.call_rpkid(q_msg) + + if self.run_pubd: + q_msg = self._compose_publication_control_query() + + for client in self.server_ca.clients.all(): + q_pdu = SubElement(q_msg, rpki.publication_control.tag_client, action = "set", client_handle = client.handle) + SubElement(q_pdu, rpki.publication_control.tag_bpki_cert).text = client.certificate.get_Base64() + + if len(q_msg) > 0: + self.call_pubd(q_msg) + + + @django.db.transaction.atomic + def configure_child(self, filename, child_handle = None, valid_until = None): + """ + Configure a new child of this RPKI entity, given the child's XML + identity file as an input. Extracts the child's data from the + XML, cross-certifies the child's resource-holding BPKI + certificate, and generates an XML file describing the relationship + between the child and this parent, including this parent's BPKI + data and up-down protocol service URI. + """ + + x = etree_read(filename) + + if x.tag != tag_oob_child_request: + raise BadXMLMessage("Expected %s, got %s", tag_oob_child_request, x.tag) + + if child_handle is None: + child_handle = x.get("child_handle") + + if valid_until is None: + valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 365) + else: + valid_until = rpki.sundial.datetime.fromXMLtime(valid_until) + if valid_until < rpki.sundial.now(): + raise PastExpiration("Specified new expiration time %s has passed" % valid_until) - self.server_ca.clients.get(handle = client_handle).delete() + self.log("Child calls itself %r, we call it %r" % (x.get("child_handle"), child_handle)) + child, created = rpki.irdb.models.Child.objects.get_or_certify( + issuer = self.resource_ca, + handle = child_handle, + ta = rpki.x509.X509(Base64 = x.findtext(tag_oob_child_bpki_ta)), + valid_until = valid_until) - @django.db.transaction.atomic - def configure_repository(self, filename, parent_handle = None): - """ - Configure a publication repository for this RPKI entity, given the - repository's response to our request-for-service message as input. - Reads the repository's response, extracts and cross-certifies the - BPKI data and service URI, and links the repository data with the - corresponding parent data in our local database. - """ + return self.generate_parental_response(child), child_handle - x = etree_read(filename) - if x.tag != tag_oob_repository_response: - raise BadXMLMessage("Expected %s, got %s", tag_oob_repository_response, x.tag) + @django.db.transaction.atomic + def generate_parental_response(self, child): + """ + Generate parental response XML. Broken out of .configure_child() + for GUI. + """ - self.log("Repository calls us %r" % (x.get("publisher_handle"))) + service_uri = "http://%s:%s/up-down/%s/%s" % ( + self.cfg.get("rpkid_server_host", section = myrpki_section), + self.cfg.get("rpkid_server_port", section = myrpki_section), + self.handle, child.handle) - if parent_handle is not None: - self.log("Explicit parent_handle given") - try: - if parent_handle == self.handle: - turtle = self.resource_ca.rootd - else: - turtle = self.resource_ca.parents.get(handle = parent_handle) - except (rpki.irdb.models.Parent.DoesNotExist, rpki.irdb.models.Rootd.DoesNotExist): - self.log("Could not find parent %r in our database" % parent_handle) - raise CouldntFindRepoParent + e = Element(tag_oob_parent_response, nsmap = oob_nsmap, version = oob_version, + service_uri = service_uri, + child_handle = child.handle, + parent_handle = self.handle) + B64Element(e, tag_oob_parent_bpki_ta, self.resource_ca.certificate) - else: - turtles = [] - for parent in self.resource_ca.parents.all(): try: - _ = parent.repository + if self.default_repository: + repo = self.resource_ca.repositories.get(handle = self.default_repository) + else: + repo = self.resource_ca.repositories.get() except rpki.irdb.models.Repository.DoesNotExist: - turtles.append(parent) - try: - _ = self.resource_ca.rootd.repository - except rpki.irdb.models.Repository.DoesNotExist: - turtles.append(self.resource_ca.rootd) - except rpki.irdb.models.Rootd.DoesNotExist: - pass - if len(turtles) != 1: - self.log("No explicit parent_handle given and unable to guess") - raise CouldntFindRepoParent - turtle = turtles[0] - if isinstance(turtle, rpki.irdb.models.Rootd): - parent_handle = self.handle - else: - parent_handle = turtle.handle - self.log("No explicit parent_handle given, guessing parent {}".format(parent_handle)) - - rpki.irdb.models.Repository.objects.get_or_certify( - issuer = self.resource_ca, - handle = parent_handle, - client_handle = x.get("publisher_handle"), - service_uri = x.get("service_uri"), - sia_base = x.get("sia_base"), - rrdp_notification_uri = x.get("rrdp_notification_uri"), - ta = rpki.x509.X509(Base64 = x.findtext(tag_oob_repository_bpki_ta)), - turtle = turtle) - - - @django.db.transaction.atomic - def delete_repository(self, repository_handle): - """ - Delete a repository of this RPKI entity. - """ - - self.resource_ca.repositories.get(handle = repository_handle).delete() - - - @django.db.transaction.atomic - def renew_children(self, child_handle, valid_until = None): - """ - Update validity period for one child entity or, if child_handle is - None, for all child entities. - """ - - if child_handle is None: - children = self.resource_ca.children.all() - else: - children = self.resource_ca.children.filter(handle = child_handle) - - if valid_until is None: - valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 365) - else: - valid_until = rpki.sundial.datetime.fromXMLtime(valid_until) - if valid_until < rpki.sundial.now(): - raise PastExpiration("Specified new expiration time %s has passed" % valid_until) - - self.log("New validity date %s" % valid_until) + repo = None - for child in children: - child.valid_until = valid_until - child.save() - - - @django.db.transaction.atomic - def load_prefixes(self, filename, ignore_missing_children = False): - """ - Whack IRDB to match prefixes.csv. - """ + if repo is None: + self.log("Couldn't find any usable repositories, not giving referral") - grouped4 = {} - grouped6 = {} + elif repo.handle == self.handle: + SubElement(e, tag_oob_offer) - for handle, prefix in csv_reader(filename, columns = 2): - grouped = grouped6 if ":" in prefix else grouped4 - if handle not in grouped: - grouped[handle] = [] - grouped[handle].append(prefix) - - primary_keys = [] - - for version, grouped, rset in ((4, grouped4, rpki.resource_set.resource_set_ipv4), - (6, grouped6, rpki.resource_set.resource_set_ipv6)): - for handle, prefixes in grouped.iteritems(): - try: - child = self.resource_ca.children.get(handle = handle) - except rpki.irdb.models.Child.DoesNotExist: - if not ignore_missing_children: - raise else: - for prefix in rset(",".join(prefixes)): - obj, created = rpki.irdb.models.ChildNet.objects.get_or_create( - child = child, - start_ip = str(prefix.min), - end_ip = str(prefix.max), - version = version) - primary_keys.append(obj.pk) - - q = rpki.irdb.models.ChildNet.objects - q = q.filter(child__issuer = self.resource_ca) - q = q.exclude(pk__in = primary_keys) - q.delete() - - - @django.db.transaction.atomic - def load_asns(self, filename, ignore_missing_children = False): - """ - Whack IRDB to match asns.csv. - """ - - grouped = {} - - for handle, asn in csv_reader(filename, columns = 2): - if handle not in grouped: - grouped[handle] = [] - grouped[handle].append(asn) - - primary_keys = [] - - for handle, asns in grouped.iteritems(): - try: - child = self.resource_ca.children.get(handle = handle) - except rpki.irdb.models.Child.DoesNotExist: - if not ignore_missing_children: - raise - else: - for asn in rpki.resource_set.resource_set_as(",".join(asns)): - obj, created = rpki.irdb.models.ChildASN.objects.get_or_create( - child = child, - start_as = str(asn.min), - end_as = str(asn.max)) - primary_keys.append(obj.pk) - - q = rpki.irdb.models.ChildASN.objects - q = q.filter(child__issuer = self.resource_ca) - q = q.exclude(pk__in = primary_keys) - q.delete() - - - @django.db.transaction.atomic - def load_roa_requests(self, filename): - """ - Whack IRDB to match roa.csv. - """ - - grouped = {} - - # format: p/n-m asn group - for pnm, asn, group in csv_reader(filename, columns = 3): - key = (asn, group) - if key not in grouped: - grouped[key] = [] - grouped[key].append(pnm) - - # Deleting and recreating all the ROA requests is inefficient, - # but rpkid's current representation of ROA requests is wrong - # (see #32), so it's not worth a lot of effort here as we're - # just going to have to rewrite this soon anyway. + proposed_sia_base = repo.sia_base + child.handle + "/" + referral_cert, created = rpki.irdb.models.Referral.objects.get_or_certify(issuer = self.resource_ca) + auth = rpki.x509.SignedReferral() + auth.set_content(B64Element(None, tag_oob_authorization, child.ta, + nsmap = oob_nsmap, version = oob_version, + authorized_sia_base = proposed_sia_base)) + auth.schema_check() + auth.sign(referral_cert.private_key, referral_cert.certificate, self.resource_ca.latest_crl) + B64Element(e, tag_oob_referral, auth, referrer = repo.client_handle) + + return etree_wrapper(e, msg = "Send this file back to the child you just configured") + + + @django.db.transaction.atomic + def delete_child(self, child_handle): + """ + Delete a child of this RPKI entity. + """ + + self.resource_ca.children.get(handle = child_handle).delete() + + + @django.db.transaction.atomic + def configure_parent(self, filename, parent_handle = None): + """ + Configure a new parent of this RPKI entity, given the output of + the parent's configure_child command as input. Reads the parent's + response XML, extracts the parent's BPKI and service URI + information, cross-certifies the parent's BPKI data into this + entity's BPKI, and checks for offers or referrals of publication + service. If a publication offer or referral is present, we + generate a request-for-service message to that repository, in case + the user wants to avail herself of the referral or offer. + """ + + x = etree_read(filename) + + if x.tag != tag_oob_parent_response: + raise BadXMLMessage("Expected %s, got %s", tag_oob_parent_response, x.tag) + + if parent_handle is None: + parent_handle = x.get("parent_handle") + + offer = x.find(tag_oob_offer) + referral = x.find(tag_oob_referral) + + if offer is not None: + repository_type = "offer" + referrer = None + referral_authorization = None + + elif referral is not None: + repository_type = "referral" + referrer = referral.get("referrer") + referral_authorization = rpki.x509.SignedReferral(Base64 = referral.text) - self.resource_ca.roa_requests.all().delete() - - for key, pnms in grouped.iteritems(): - asn, group = key - - roa_request = self.resource_ca.roa_requests.create(asn = asn) - - for pnm in pnms: - if ":" in pnm: - p = rpki.resource_set.roa_prefix_ipv6.parse_str(pnm) - v = 6 else: - p = rpki.resource_set.roa_prefix_ipv4.parse_str(pnm) - v = 4 - roa_request.prefixes.create( - version = v, - prefix = str(p.prefix), - prefixlen = int(p.prefixlen), - max_prefixlen = int(p.max_prefixlen)) - + repository_type = "none" + referrer = None + referral_authorization = None + + self.log("Parent calls itself %r, we call it %r" % (x.get("parent_handle"), parent_handle)) + self.log("Parent calls us %r" % x.get("child_handle")) + + parent, created = rpki.irdb.models.Parent.objects.get_or_certify( + issuer = self.resource_ca, + handle = parent_handle, + child_handle = x.get("child_handle"), + parent_handle = x.get("parent_handle"), + service_uri = x.get("service_uri"), + ta = rpki.x509.X509(Base64 = x.findtext(tag_oob_parent_bpki_ta)), + repository_type = repository_type, + referrer = referrer, + referral_authorization = referral_authorization) + + return self.generate_repository_request(parent), parent_handle + + + def generate_repository_request(self, parent): + """ + Generate repository request for a given parent. + """ + + e = Element(tag_oob_publisher_request, nsmap = oob_nsmap, version = oob_version, + publisher_handle = self.handle) + B64Element(e, tag_oob_publisher_bpki_ta, self.resource_ca.certificate) + if parent.repository_type == "referral": + B64Element(e, tag_oob_referral, parent.referral_authorization, + referrer = parent.referrer) + + return etree_wrapper(e, msg = "This is the file to send to the repository operator") + + + @django.db.transaction.atomic + def delete_parent(self, parent_handle): + """ + Delete a parent of this RPKI entity. + """ + + self.resource_ca.parents.get(handle = parent_handle).delete() + + + @django.db.transaction.atomic + def delete_rootd(self): + """ + Delete rootd associated with this RPKI entity. + """ + + self.resource_ca.rootd.delete() + + + @django.db.transaction.atomic + def configure_publication_client(self, filename, sia_base = None, flat = False): + """ + Configure publication server to know about a new client, given the + client's request-for-service message as input. Reads the client's + request for service, cross-certifies the client's BPKI data, and + generates a response message containing the repository's BPKI data + and service URI. + """ + + x = etree_read(filename) + + if x.tag != tag_oob_publisher_request: + raise BadXMLMessage("Expected %s, got %s", tag_oob_publisher_request, x.tag) + + client_ta = rpki.x509.X509(Base64 = x.findtext(tag_oob_publisher_bpki_ta)) + + referral = x.find(tag_oob_referral) + + default_sia_base = "rsync://{self.rsync_server}/{self.rsync_module}/{handle}/".format( + self = self, + handle = x.get("publisher_handle")) + + if sia_base is None and flat: + self.log("Flat publication structure forced, homing client at top-level") + sia_base = default_sia_base + + if sia_base is None and referral is not None: + self.log("This looks like a referral, checking") + try: + referrer = referral.get("referrer") + referrer = self.server_ca.clients.get(handle = referrer) + referral = rpki.x509.SignedReferral(Base64 = referral.text) + referral = referral.unwrap(ta = (referrer.certificate, self.server_ca.certificate)) + if rpki.x509.X509(Base64 = referral.text) != client_ta: + raise BadXMLMessage("Referral trust anchor does not match") + sia_base = referral.get("authorized_sia_base") + except rpki.irdb.models.Client.DoesNotExist: + self.log("We have no record of the client ({}) alleged to have made this referral".format(referrer)) + + if sia_base is None and referral is None: + self.log("This might be an offer, checking") + try: + parent = rpki.irdb.models.ResourceHolderCA.objects.get(children__ta = client_ta) + if "/" in parent.repositories.get(ta = self.server_ca.certificate).client_handle: + self.log("Client's parent is not top-level, this is not a valid offer") + else: + self.log("Found client and its parent, nesting") + sia_base = "rsync://{self.rsync_server}/{self.rsync_module}/{parent_handle}/{client_handle}/".format( + self = self, + parent_handle = parent.handle, + client_handle = x.get("publisher_handle")) + except rpki.irdb.models.Repository.DoesNotExist: + self.log("Found client's parent, but repository isn't set, this shouldn't happen!") + except rpki.irdb.models.ResourceHolderCA.DoesNotExist: + try: + rpki.irdb.models.Rootd.objects.get(issuer__certificate = client_ta) + self.log("This client's parent is rootd") + sia_base = default_sia_base + except rpki.irdb.models.Rootd.DoesNotExist: + self.log("We don't host this client's parent, so we didn't make an offer") + + if sia_base is None: + self.log("Don't know where else to nest this client, so defaulting to top-level") + sia_base = default_sia_base + + if not sia_base.startswith("rsync://"): + raise BadXMLMessage("Malformed sia_base parameter %r, should start with 'rsync://'" % sia_base) + + client_handle = "/".join(sia_base.rstrip("/").split("/")[4:]) + + self.log("Client calls itself %r, we call it %r" % ( + x.get("publisher_handle"), client_handle)) + + client, created = rpki.irdb.models.Client.objects.get_or_certify( + issuer = self.server_ca, + handle = client_handle, + ta = client_ta, + sia_base = sia_base) + + return self.generate_repository_response(client), client_handle + + + def generate_repository_response(self, client): + """ + Generate repository response XML to a given client. + """ + + service_uri = "http://{host}:{port}/client/{handle}".format( + host = self.cfg.get("pubd_server_host", section = myrpki_section), + port = self.cfg.get("pubd_server_port", section = myrpki_section), + handle = client.handle) + + rrdp_uri = self.cfg.get("publication_rrdp_notification_uri", section = myrpki_section, + default = "") or None + + e = Element(tag_oob_repository_response, nsmap = oob_nsmap, version = oob_version, + service_uri = service_uri, + publisher_handle = client.handle, + sia_base = client.sia_base) + + if rrdp_uri is not None: + e.set("rrdp_notification_uri", rrdp_uri) + + B64Element(e, tag_oob_repository_bpki_ta, self.server_ca.certificate) + return etree_wrapper(e, msg = "Send this file back to the publication client you just configured") + + + @django.db.transaction.atomic + def delete_publication_client(self, client_handle): + """ + Delete a publication client of this RPKI entity. + """ + + self.server_ca.clients.get(handle = client_handle).delete() + + + @django.db.transaction.atomic + def configure_repository(self, filename, parent_handle = None): + """ + Configure a publication repository for this RPKI entity, given the + repository's response to our request-for-service message as input. + Reads the repository's response, extracts and cross-certifies the + BPKI data and service URI, and links the repository data with the + corresponding parent data in our local database. + """ + + x = etree_read(filename) + + if x.tag != tag_oob_repository_response: + raise BadXMLMessage("Expected %s, got %s", tag_oob_repository_response, x.tag) + + self.log("Repository calls us %r" % (x.get("publisher_handle"))) + + if parent_handle is not None: + self.log("Explicit parent_handle given") + try: + if parent_handle == self.handle: + turtle = self.resource_ca.rootd + else: + turtle = self.resource_ca.parents.get(handle = parent_handle) + except (rpki.irdb.models.Parent.DoesNotExist, rpki.irdb.models.Rootd.DoesNotExist): + self.log("Could not find parent %r in our database" % parent_handle) + raise CouldntFindRepoParent - @django.db.transaction.atomic - def load_ghostbuster_requests(self, filename, parent = None): - """ - Whack IRDB to match ghostbusters.vcard. - - This accepts one or more vCards from a file. - """ - - self.resource_ca.ghostbuster_requests.filter(parent = parent).delete() + else: + turtles = [] + for parent in self.resource_ca.parents.all(): + try: + _ = parent.repository + except rpki.irdb.models.Repository.DoesNotExist: + turtles.append(parent) + try: + _ = self.resource_ca.rootd.repository + except rpki.irdb.models.Repository.DoesNotExist: + turtles.append(self.resource_ca.rootd) + except rpki.irdb.models.Rootd.DoesNotExist: + pass + if len(turtles) != 1: + self.log("No explicit parent_handle given and unable to guess") + raise CouldntFindRepoParent + turtle = turtles[0] + if isinstance(turtle, rpki.irdb.models.Rootd): + parent_handle = self.handle + else: + parent_handle = turtle.handle + self.log("No explicit parent_handle given, guessing parent {}".format(parent_handle)) + + rpki.irdb.models.Repository.objects.get_or_certify( + issuer = self.resource_ca, + handle = parent_handle, + client_handle = x.get("publisher_handle"), + service_uri = x.get("service_uri"), + sia_base = x.get("sia_base"), + rrdp_notification_uri = x.get("rrdp_notification_uri"), + ta = rpki.x509.X509(Base64 = x.findtext(tag_oob_repository_bpki_ta)), + turtle = turtle) + + + @django.db.transaction.atomic + def delete_repository(self, repository_handle): + """ + Delete a repository of this RPKI entity. + """ + + self.resource_ca.repositories.get(handle = repository_handle).delete() + + + @django.db.transaction.atomic + def renew_children(self, child_handle, valid_until = None): + """ + Update validity period for one child entity or, if child_handle is + None, for all child entities. + """ + + if child_handle is None: + children = self.resource_ca.children.all() + else: + children = self.resource_ca.children.filter(handle = child_handle) - vcard = [] + if valid_until is None: + valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 365) + else: + valid_until = rpki.sundial.datetime.fromXMLtime(valid_until) + if valid_until < rpki.sundial.now(): + raise PastExpiration("Specified new expiration time %s has passed" % valid_until) + + self.log("New validity date %s" % valid_until) + + for child in children: + child.valid_until = valid_until + child.save() + + + @django.db.transaction.atomic + def load_prefixes(self, filename, ignore_missing_children = False): + """ + Whack IRDB to match prefixes.csv. + """ + + grouped4 = {} + grouped6 = {} + + for handle, prefix in csv_reader(filename, columns = 2): + grouped = grouped6 if ":" in prefix else grouped4 + if handle not in grouped: + grouped[handle] = [] + grouped[handle].append(prefix) + + primary_keys = [] + + for version, grouped, rset in ((4, grouped4, rpki.resource_set.resource_set_ipv4), + (6, grouped6, rpki.resource_set.resource_set_ipv6)): + for handle, prefixes in grouped.iteritems(): + try: + child = self.resource_ca.children.get(handle = handle) + except rpki.irdb.models.Child.DoesNotExist: + if not ignore_missing_children: + raise + else: + for prefix in rset(",".join(prefixes)): + obj, created = rpki.irdb.models.ChildNet.objects.get_or_create( + child = child, + start_ip = str(prefix.min), + end_ip = str(prefix.max), + version = version) + primary_keys.append(obj.pk) + + q = rpki.irdb.models.ChildNet.objects + q = q.filter(child__issuer = self.resource_ca) + q = q.exclude(pk__in = primary_keys) + q.delete() + + + @django.db.transaction.atomic + def load_asns(self, filename, ignore_missing_children = False): + """ + Whack IRDB to match asns.csv. + """ + + grouped = {} + + for handle, asn in csv_reader(filename, columns = 2): + if handle not in grouped: + grouped[handle] = [] + grouped[handle].append(asn) + + primary_keys = [] + + for handle, asns in grouped.iteritems(): + try: + child = self.resource_ca.children.get(handle = handle) + except rpki.irdb.models.Child.DoesNotExist: + if not ignore_missing_children: + raise + else: + for asn in rpki.resource_set.resource_set_as(",".join(asns)): + obj, created = rpki.irdb.models.ChildASN.objects.get_or_create( + child = child, + start_as = str(asn.min), + end_as = str(asn.max)) + primary_keys.append(obj.pk) + + q = rpki.irdb.models.ChildASN.objects + q = q.filter(child__issuer = self.resource_ca) + q = q.exclude(pk__in = primary_keys) + q.delete() + + + @django.db.transaction.atomic + def load_roa_requests(self, filename): + """ + Whack IRDB to match roa.csv. + """ + + grouped = {} + + # format: p/n-m asn group + for pnm, asn, group in csv_reader(filename, columns = 3): + key = (asn, group) + if key not in grouped: + grouped[key] = [] + grouped[key].append(pnm) + + # Deleting and recreating all the ROA requests is inefficient, + # but rpkid's current representation of ROA requests is wrong + # (see #32), so it's not worth a lot of effort here as we're + # just going to have to rewrite this soon anyway. + + self.resource_ca.roa_requests.all().delete() + + for key, pnms in grouped.iteritems(): + asn, group = key + + roa_request = self.resource_ca.roa_requests.create(asn = asn) + + for pnm in pnms: + if ":" in pnm: + p = rpki.resource_set.roa_prefix_ipv6.parse_str(pnm) + v = 6 + else: + p = rpki.resource_set.roa_prefix_ipv4.parse_str(pnm) + v = 4 + roa_request.prefixes.create( + version = v, + prefix = str(p.prefix), + prefixlen = int(p.prefixlen), + max_prefixlen = int(p.max_prefixlen)) + + + @django.db.transaction.atomic + def load_ghostbuster_requests(self, filename, parent = None): + """ + Whack IRDB to match ghostbusters.vcard. + + This accepts one or more vCards from a file. + """ + + self.resource_ca.ghostbuster_requests.filter(parent = parent).delete() - for line in open(filename, "r"): - if not vcard and not line.upper().startswith("BEGIN:VCARD"): - continue - vcard.append(line) - if line.upper().startswith("END:VCARD"): - self.resource_ca.ghostbuster_requests.create(vcard = "".join(vcard), parent = parent) vcard = [] + for line in open(filename, "r"): + if not vcard and not line.upper().startswith("BEGIN:VCARD"): + continue + vcard.append(line) + if line.upper().startswith("END:VCARD"): + self.resource_ca.ghostbuster_requests.create(vcard = "".join(vcard), parent = parent) + vcard = [] - def call_rpkid(self, q_msg, suppress_error_check = False): - """ - Issue a call to rpkid, return result. - """ - url = "http://%s:%s/left-right" % ( - self.cfg.get("rpkid_server_host", section = myrpki_section), - self.cfg.get("rpkid_server_port", section = myrpki_section)) + def call_rpkid(self, q_msg, suppress_error_check = False): + """ + Issue a call to rpkid, return result. + """ - rpkid = self.server_ca.ee_certificates.get(purpose = "rpkid") - irbe = self.server_ca.ee_certificates.get(purpose = "irbe") + url = "http://%s:%s/left-right" % ( + self.cfg.get("rpkid_server_host", section = myrpki_section), + self.cfg.get("rpkid_server_port", section = myrpki_section)) - r_msg = rpki.http_simple.client( - proto_cms_msg = rpki.left_right.cms_msg, - client_key = irbe.private_key, - client_cert = irbe.certificate, - server_ta = self.server_ca.certificate, - server_cert = rpkid.certificate, - url = url, - q_msg = q_msg, - debug = self.show_xml) + rpkid = self.server_ca.ee_certificates.get(purpose = "rpkid") + irbe = self.server_ca.ee_certificates.get(purpose = "irbe") - if not suppress_error_check: - self.check_error_report(r_msg) - return r_msg + r_msg = rpki.http_simple.client( + proto_cms_msg = rpki.left_right.cms_msg, + client_key = irbe.private_key, + client_cert = irbe.certificate, + server_ta = self.server_ca.certificate, + server_cert = rpkid.certificate, + url = url, + q_msg = q_msg, + debug = self.show_xml) + if not suppress_error_check: + self.check_error_report(r_msg) + return r_msg - def _rpkid_tenant_control(self, *bools): - assert all(isinstance(b, str) for b in bools) - q_msg = self._compose_left_right_query() - q_pdu = SubElement(q_msg, rpki.left_right.tag_tenant, action = "set", tenant_handle = self.handle) - for b in bools: - q_pdu.set(b, "yes") - return self.call_rpkid(q_msg) + def _rpkid_tenant_control(self, *bools): + assert all(isinstance(b, str) for b in bools) + q_msg = self._compose_left_right_query() + q_pdu = SubElement(q_msg, rpki.left_right.tag_tenant, action = "set", tenant_handle = self.handle) + for b in bools: + q_pdu.set(b, "yes") + return self.call_rpkid(q_msg) - def run_rpkid_now(self): - """ - Poke rpkid to immediately run the cron job for the current handle. - This method is used by the GUI when a user has changed something in the - IRDB (ghostbuster, roa) which does not require a full synchronize() call, - to force the object to be immediately issued. - """ + def run_rpkid_now(self): + """ + Poke rpkid to immediately run the cron job for the current handle. - return self._rpkid_tenant_control("run_now") + This method is used by the GUI when a user has changed something in the + IRDB (ghostbuster, roa) which does not require a full synchronize() call, + to force the object to be immediately issued. + """ + return self._rpkid_tenant_control("run_now") - def publish_world_now(self): - """ - Poke rpkid to (re)publish everything for the current handle. - """ - return self._rpkid_tenant_control("publish_world_now") + def publish_world_now(self): + """ + Poke rpkid to (re)publish everything for the current handle. + """ + return self._rpkid_tenant_control("publish_world_now") - def reissue(self): - """ - Poke rpkid to reissue everything for the current handle. - """ - return self._rpkid_tenant_control("reissue") + def reissue(self): + """ + Poke rpkid to reissue everything for the current handle. + """ + return self._rpkid_tenant_control("reissue") - def rekey(self): - """ - Poke rpkid to rekey all RPKI certificates received for the current - handle. - """ - - return self._rpkid_tenant_control("rekey") - - - def revoke(self): - """ - Poke rpkid to revoke old RPKI keys for the current handle. - """ - - return self._rpkid_tenant_control("revoke") + def rekey(self): + """ + Poke rpkid to rekey all RPKI certificates received for the current + handle. + """ + + return self._rpkid_tenant_control("rekey") - def revoke_forgotten(self): - """ - Poke rpkid to revoke old forgotten RPKI keys for the current handle. - """ - return self._rpkid_tenant_control("revoke_forgotten") + def revoke(self): + """ + Poke rpkid to revoke old RPKI keys for the current handle. + """ + return self._rpkid_tenant_control("revoke") - def clear_all_sql_cms_replay_protection(self): - """ - Tell rpkid and pubd to clear replay protection for all SQL-based - entities. This is a fairly blunt instrument, but as we don't - expect this to be necessary except in the case of gross - misconfiguration, it should suffice. - """ - if self.run_rpkid: - q_msg = self._compose_left_right_query() - for ca in rpki.irdb.models.ResourceHolderCA.objects.all(): - SubElement(q_msg, rpki.left_right.tag_tenant, action = "set", - tenant_handle = ca.handle, clear_replay_protection = "yes") - self.call_rpkid(q_msg) + def revoke_forgotten(self): + """ + Poke rpkid to revoke old forgotten RPKI keys for the current handle. + """ + + return self._rpkid_tenant_control("revoke_forgotten") - if self.run_pubd: - q_msg = self._compose_publication_control_query() - for client in self.server_ca.clients.all(): - SubElement(q_msg, rpki.publication_control.tag_client, action = "set", - client_handle = client.handle, clear_reply_protection = "yes") - self.call_pubd(q_msg) + def clear_all_sql_cms_replay_protection(self): + """ + Tell rpkid and pubd to clear replay protection for all SQL-based + entities. This is a fairly blunt instrument, but as we don't + expect this to be necessary except in the case of gross + misconfiguration, it should suffice. + """ - def call_pubd(self, q_msg): - """ - Issue a call to pubd, return result. - """ + if self.run_rpkid: + q_msg = self._compose_left_right_query() + for ca in rpki.irdb.models.ResourceHolderCA.objects.all(): + SubElement(q_msg, rpki.left_right.tag_tenant, action = "set", + tenant_handle = ca.handle, clear_replay_protection = "yes") + self.call_rpkid(q_msg) - url = "http://%s:%s/control" % ( - self.cfg.get("pubd_server_host", section = myrpki_section), - self.cfg.get("pubd_server_port", section = myrpki_section)) + if self.run_pubd: + q_msg = self._compose_publication_control_query() + for client in self.server_ca.clients.all(): + SubElement(q_msg, rpki.publication_control.tag_client, action = "set", + client_handle = client.handle, clear_reply_protection = "yes") + self.call_pubd(q_msg) - pubd = self.server_ca.ee_certificates.get(purpose = "pubd") - irbe = self.server_ca.ee_certificates.get(purpose = "irbe") - r_msg = rpki.http_simple.client( - proto_cms_msg = rpki.publication_control.cms_msg, - client_key = irbe.private_key, - client_cert = irbe.certificate, - server_ta = self.server_ca.certificate, - server_cert = pubd.certificate, - url = url, - q_msg = q_msg, - debug = self.show_xml) + def call_pubd(self, q_msg): + """ + Issue a call to pubd, return result. + """ - self.check_error_report(r_msg) - return r_msg + url = "http://%s:%s/control" % ( + self.cfg.get("pubd_server_host", section = myrpki_section), + self.cfg.get("pubd_server_port", section = myrpki_section)) + pubd = self.server_ca.ee_certificates.get(purpose = "pubd") + irbe = self.server_ca.ee_certificates.get(purpose = "irbe") - def check_error_report(self, r_msg): - """ - Check a response from rpkid or pubd for error_report PDUs, log and - throw exceptions as needed. - """ + r_msg = rpki.http_simple.client( + proto_cms_msg = rpki.publication_control.cms_msg, + client_key = irbe.private_key, + client_cert = irbe.certificate, + server_ta = self.server_ca.certificate, + server_cert = pubd.certificate, + url = url, + q_msg = q_msg, + debug = self.show_xml) - failed = False - for r_pdu in r_msg.getiterator(rpki.left_right.tag_report_error): - failed = True - self.log("rpkid reported failure: %s" % r_pdu.get("error_code")) - if r_pdu.text: - self.log(r_pdu.text) - for r_pdu in r_msg.getiterator(rpki.publication_control.tag_report_error): - failed = True - self.log("pubd reported failure: %s" % r_pdu.get("error_code")) - if r_pdu.text: - self.log(r_pdu.text) - if failed: - raise CouldntTalkToDaemon - - - @django.db.transaction.atomic - def synchronize(self, *handles_to_poke): - """ - Configure RPKI daemons with the data built up by the other - commands in this program. Commands which modify the IRDB and want - to whack everything into sync should call this when they're done, - but be warned that this can be slow with a lot of CAs. + self.check_error_report(r_msg) + return r_msg - Any arguments given are handles of CAs which should be poked with a - operation. - """ - for ca in rpki.irdb.models.ResourceHolderCA.objects.all(): - self.synchronize_rpkid_one_ca_core(ca, ca.handle in handles_to_poke) - self.synchronize_pubd_core() - self.synchronize_rpkid_deleted_core() + def check_error_report(self, r_msg): + """ + Check a response from rpkid or pubd for error_report PDUs, log and + throw exceptions as needed. + """ + failed = False + for r_pdu in r_msg.getiterator(rpki.left_right.tag_report_error): + failed = True + self.log("rpkid reported failure: %s" % r_pdu.get("error_code")) + if r_pdu.text: + self.log(r_pdu.text) + for r_pdu in r_msg.getiterator(rpki.publication_control.tag_report_error): + failed = True + self.log("pubd reported failure: %s" % r_pdu.get("error_code")) + if r_pdu.text: + self.log(r_pdu.text) + if failed: + raise CouldntTalkToDaemon - @django.db.transaction.atomic - def synchronize_ca(self, ca = None, poke = False): - """ - Synchronize one CA. Most commands which modify a CA should call - this. CA to synchronize defaults to the current resource CA. - """ - if ca is None: - ca = self.resource_ca - self.synchronize_rpkid_one_ca_core(ca, poke) + @django.db.transaction.atomic + def synchronize(self, *handles_to_poke): + """ + Configure RPKI daemons with the data built up by the other + commands in this program. Commands which modify the IRDB and want + to whack everything into sync should call this when they're done, + but be warned that this can be slow with a lot of CAs. + + Any arguments given are handles of CAs which should be poked with a + operation. + """ + + for ca in rpki.irdb.models.ResourceHolderCA.objects.all(): + self.synchronize_rpkid_one_ca_core(ca, ca.handle in handles_to_poke) + self.synchronize_pubd_core() + self.synchronize_rpkid_deleted_core() - @django.db.transaction.atomic - def synchronize_deleted_ca(self): - """ - Delete CAs which are present in rpkid's database but not in the - IRDB. - """ + @django.db.transaction.atomic + def synchronize_ca(self, ca = None, poke = False): + """ + Synchronize one CA. Most commands which modify a CA should call + this. CA to synchronize defaults to the current resource CA. + """ + + if ca is None: + ca = self.resource_ca + self.synchronize_rpkid_one_ca_core(ca, poke) + + + @django.db.transaction.atomic + def synchronize_deleted_ca(self): + """ + Delete CAs which are present in rpkid's database but not in the + IRDB. + """ + + self.synchronize_rpkid_deleted_core() + + + @django.db.transaction.atomic + def synchronize_pubd(self): + """ + Synchronize pubd. Most commands which modify pubd should call this. + """ + + self.synchronize_pubd_core() + + + def synchronize_rpkid_one_ca_core(self, ca, poke = False): + """ + Synchronize one CA. This is the core synchronization code. Don't + call this directly, instead call one of the methods that calls + this inside a Django commit wrapper. + + This method configures rpkid with data built up by the other + commands in this program. Most commands which modify IRDB values + related to rpkid should call this when they're done. + + If poke is True, we append a left-right run_now operation for this + CA to the end of whatever other commands this method generates. + """ + + # We can use a single BSC for everything -- except BSC key + # rollovers. Drive off that bridge when we get to it. + + bsc_handle = "bsc" + + # A default RPKI CRL cycle time of six hours seems sane. One + # might make a case for a day instead, but we've been running with + # six hours for a while now and haven't seen a lot of whining. + + tenant_crl_interval = self.cfg.getint("tenant_crl_interval", 6 * 60 * 60, section = myrpki_section) + + # regen_margin now just controls how long before RPKI certificate + # expiration we should regenerate; it used to control the interval + # before RPKI CRL staleness at which to regenerate the CRL, but + # using the same timer value for both of these is hopeless. + # + # A default regeneration margin of two weeks gives enough time for + # humans to react. We add a two hour fudge factor in the hope + # that this will regenerate certificates just *before* the + # companion cron job warns of impending doom. + + tenant_regen_margin = self.cfg.getint("tenant_regen_margin", 14 * 24 * 60 * 60 + 2 * 60, section = myrpki_section) + + # See what rpkid already has on file for this entity. + + q_msg = self._compose_left_right_query() + SubElement(q_msg, rpki.left_right.tag_tenant, action = "get", tenant_handle = ca.handle) + SubElement(q_msg, rpki.left_right.tag_bsc, action = "list", tenant_handle = ca.handle) + SubElement(q_msg, rpki.left_right.tag_repository, action = "list", tenant_handle = ca.handle) + SubElement(q_msg, rpki.left_right.tag_parent, action = "list", tenant_handle = ca.handle) + SubElement(q_msg, rpki.left_right.tag_child, action = "list", tenant_handle = ca.handle) + + r_msg = self.call_rpkid(q_msg, suppress_error_check = True) + + self.check_error_report(r_msg) + + tenant_pdu = r_msg.find(rpki.left_right.tag_tenant) + + bsc_pdus = dict((r_pdu.get("bsc_handle"), r_pdu) + for r_pdu in r_msg.getiterator(rpki.left_right.tag_bsc)) + repository_pdus = dict((r_pdu.get("repository_handle"), r_pdu) + for r_pdu in r_msg.getiterator(rpki.left_right.tag_repository)) + parent_pdus = dict((r_pdu.get("parent_handle"), r_pdu) + for r_pdu in r_msg.getiterator(rpki.left_right.tag_parent)) + child_pdus = dict((r_pdu.get("child_handle"), r_pdu) + for r_pdu in r_msg.getiterator(rpki.left_right.tag_child)) + + q_msg = self._compose_left_right_query() + + tenant_cert, created = rpki.irdb.models.HostedCA.objects.get_or_certify( + issuer = self.server_ca, + hosted = ca) + + # There should be exactly one object per hosted entity, by definition + + if (tenant_pdu is None or + tenant_pdu.get("crl_interval") != str(tenant_crl_interval) or + tenant_pdu.get("regen_margin") != str(tenant_regen_margin) or + tenant_pdu.findtext(rpki.left_right.tag_bpki_cert, "").decode("base64") != tenant_cert.certificate.get_DER()): + q_pdu = SubElement(q_msg, rpki.left_right.tag_tenant, + action = "create" if tenant_pdu is None else "set", + tag = "tenant", + tenant_handle = ca.handle, + crl_interval = str(tenant_crl_interval), + regen_margin = str(tenant_regen_margin)) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = ca.certificate.get_Base64() + + # In general we only need one per . BSC objects + # are a little unusual in that the keypair and PKCS #10 + # subelement are generated by rpkid, so complete setup requires + # two round trips. + + bsc_pdu = bsc_pdus.pop(bsc_handle, None) + + if bsc_pdu is None or bsc_pdu.find(rpki.left_right.tag_pkcs10_request) is None: + SubElement(q_msg, rpki.left_right.tag_bsc, + action = "create" if bsc_pdu is None else "set", + tag = "bsc", + tenant_handle = ca.handle, + bsc_handle = bsc_handle, + generate_keypair = "yes") + + for bsc_handle in bsc_pdus: + SubElement(q_msg, rpki.left_right.tag_bsc, + action = "destroy", tenant_handle = ca.handle, bsc_handle = bsc_handle) + + # If we've already got actions queued up, run them now, so we + # can finish setting up the BSC before anything tries to use it. + + if len(q_msg) > 0: + SubElement(q_msg, rpki.left_right.tag_bsc, action = "list", tag = "bsc", tenant_handle = ca.handle) + r_msg = self.call_rpkid(q_msg) + bsc_pdus = dict((r_pdu.get("bsc_handle"), r_pdu) + for r_pdu in r_msg.getiterator(rpki.left_right.tag_bsc) + if r_pdu.get("action") == "list") + bsc_pdu = bsc_pdus.pop(bsc_handle, None) + + q_msg = self._compose_left_right_query() + + bsc_pkcs10 = bsc_pdu.find(rpki.left_right.tag_pkcs10_request) + assert bsc_pkcs10 is not None + + bsc, created = rpki.irdb.models.BSC.objects.get_or_certify( + issuer = ca, + handle = bsc_handle, + pkcs10 = rpki.x509.PKCS10(Base64 = bsc_pkcs10.text)) + + if (bsc_pdu.findtext(rpki.left_right.tag_signing_cert, "").decode("base64") != bsc.certificate.get_DER() or + bsc_pdu.findtext(rpki.left_right.tag_signing_cert_crl, "").decode("base64") != ca.latest_crl.get_DER()): + q_pdu = SubElement(q_msg, rpki.left_right.tag_bsc, + action = "set", + tag = "bsc", + tenant_handle = ca.handle, + bsc_handle = bsc_handle) + SubElement(q_pdu, rpki.left_right.tag_signing_cert).text = bsc.certificate.get_Base64() + SubElement(q_pdu, rpki.left_right.tag_signing_cert_crl).text = ca.latest_crl.get_Base64() + + # At present we need one per , not because + # rpkid requires that, but because pubd does. pubd probably should + # be fixed to support a single client allowed to update multiple + # trees, but for the moment the easiest way forward is just to + # enforce a 1:1 mapping between and objects + + for repository in ca.repositories.all(): + + repository_pdu = repository_pdus.pop(repository.handle, None) + + if (repository_pdu is None or + repository_pdu.get("bsc_handle") != bsc_handle or + repository_pdu.get("peer_contact_uri") != repository.service_uri or + repository_pdu.get("rrdp_notification_uri") != repository.rrdp_notification_uri or + repository_pdu.findtext(rpki.left_right.tag_bpki_cert, "").decode("base64") != repository.certificate.get_DER()): + q_pdu = SubElement(q_msg, rpki.left_right.tag_repository, + action = "create" if repository_pdu is None else "set", + tag = repository.handle, + tenant_handle = ca.handle, + repository_handle = repository.handle, + bsc_handle = bsc_handle, + peer_contact_uri = repository.service_uri) + if repository.rrdp_notification_uri: + q_pdu.set("rrdp_notification_uri", repository.rrdp_notification_uri) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = repository.certificate.get_Base64() + + for repository_handle in repository_pdus: + SubElement(q_msg, rpki.left_right.tag_repository, action = "destroy", + tenant_handle = ca.handle, repository_handle = repository_handle) + + # setup code currently assumes 1:1 mapping between + # and , and further assumes that the handles + # for an associated pair are the identical (that is: + # parent.repository_handle == parent.parent_handle). + # + # If no such repository exists, our choices are to ignore the + # parent entry or throw an error. For now, we ignore the parent. + + for parent in ca.parents.all(): + + try: + parent_pdu = parent_pdus.pop(parent.handle, None) + + if (parent_pdu is None or + parent_pdu.get("bsc_handle") != bsc_handle or + parent_pdu.get("repository_handle") != parent.handle or + parent_pdu.get("peer_contact_uri") != parent.service_uri or + parent_pdu.get("sia_base") != parent.repository.sia_base or + parent_pdu.get("sender_name") != parent.child_handle or + parent_pdu.get("recipient_name") != parent.parent_handle or + parent_pdu.findtext(rpki.left_right.tag_bpki_cert, "").decode("base64") != parent.certificate.get_DER()): + q_pdu = SubElement(q_msg, rpki.left_right.tag_parent, + action = "create" if parent_pdu is None else "set", + tag = parent.handle, + tenant_handle = ca.handle, + parent_handle = parent.handle, + bsc_handle = bsc_handle, + repository_handle = parent.handle, + peer_contact_uri = parent.service_uri, + sia_base = parent.repository.sia_base, + sender_name = parent.child_handle, + recipient_name = parent.parent_handle) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = parent.certificate.get_Base64() + + except rpki.irdb.models.Repository.DoesNotExist: + pass - self.synchronize_rpkid_deleted_core() - - - @django.db.transaction.atomic - def synchronize_pubd(self): - """ - Synchronize pubd. Most commands which modify pubd should call this. - """ - - self.synchronize_pubd_core() - - - def synchronize_rpkid_one_ca_core(self, ca, poke = False): - """ - Synchronize one CA. This is the core synchronization code. Don't - call this directly, instead call one of the methods that calls - this inside a Django commit wrapper. - - This method configures rpkid with data built up by the other - commands in this program. Most commands which modify IRDB values - related to rpkid should call this when they're done. - - If poke is True, we append a left-right run_now operation for this - CA to the end of whatever other commands this method generates. - """ - - # We can use a single BSC for everything -- except BSC key - # rollovers. Drive off that bridge when we get to it. - - bsc_handle = "bsc" - - # A default RPKI CRL cycle time of six hours seems sane. One - # might make a case for a day instead, but we've been running with - # six hours for a while now and haven't seen a lot of whining. - - tenant_crl_interval = self.cfg.getint("tenant_crl_interval", 6 * 60 * 60, section = myrpki_section) - - # regen_margin now just controls how long before RPKI certificate - # expiration we should regenerate; it used to control the interval - # before RPKI CRL staleness at which to regenerate the CRL, but - # using the same timer value for both of these is hopeless. - # - # A default regeneration margin of two weeks gives enough time for - # humans to react. We add a two hour fudge factor in the hope - # that this will regenerate certificates just *before* the - # companion cron job warns of impending doom. - - tenant_regen_margin = self.cfg.getint("tenant_regen_margin", 14 * 24 * 60 * 60 + 2 * 60, section = myrpki_section) - - # See what rpkid already has on file for this entity. - - q_msg = self._compose_left_right_query() - SubElement(q_msg, rpki.left_right.tag_tenant, action = "get", tenant_handle = ca.handle) - SubElement(q_msg, rpki.left_right.tag_bsc, action = "list", tenant_handle = ca.handle) - SubElement(q_msg, rpki.left_right.tag_repository, action = "list", tenant_handle = ca.handle) - SubElement(q_msg, rpki.left_right.tag_parent, action = "list", tenant_handle = ca.handle) - SubElement(q_msg, rpki.left_right.tag_child, action = "list", tenant_handle = ca.handle) - - r_msg = self.call_rpkid(q_msg, suppress_error_check = True) - - self.check_error_report(r_msg) - - tenant_pdu = r_msg.find(rpki.left_right.tag_tenant) - - bsc_pdus = dict((r_pdu.get("bsc_handle"), r_pdu) - for r_pdu in r_msg.getiterator(rpki.left_right.tag_bsc)) - repository_pdus = dict((r_pdu.get("repository_handle"), r_pdu) - for r_pdu in r_msg.getiterator(rpki.left_right.tag_repository)) - parent_pdus = dict((r_pdu.get("parent_handle"), r_pdu) - for r_pdu in r_msg.getiterator(rpki.left_right.tag_parent)) - child_pdus = dict((r_pdu.get("child_handle"), r_pdu) - for r_pdu in r_msg.getiterator(rpki.left_right.tag_child)) - - q_msg = self._compose_left_right_query() - - tenant_cert, created = rpki.irdb.models.HostedCA.objects.get_or_certify( - issuer = self.server_ca, - hosted = ca) - - # There should be exactly one object per hosted entity, by definition - - if (tenant_pdu is None or - tenant_pdu.get("crl_interval") != str(tenant_crl_interval) or - tenant_pdu.get("regen_margin") != str(tenant_regen_margin) or - tenant_pdu.findtext(rpki.left_right.tag_bpki_cert, "").decode("base64") != tenant_cert.certificate.get_DER()): - q_pdu = SubElement(q_msg, rpki.left_right.tag_tenant, - action = "create" if tenant_pdu is None else "set", - tag = "tenant", - tenant_handle = ca.handle, - crl_interval = str(tenant_crl_interval), - regen_margin = str(tenant_regen_margin)) - SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = ca.certificate.get_Base64() - - # In general we only need one per . BSC objects - # are a little unusual in that the keypair and PKCS #10 - # subelement are generated by rpkid, so complete setup requires - # two round trips. - - bsc_pdu = bsc_pdus.pop(bsc_handle, None) - - if bsc_pdu is None or bsc_pdu.find(rpki.left_right.tag_pkcs10_request) is None: - SubElement(q_msg, rpki.left_right.tag_bsc, - action = "create" if bsc_pdu is None else "set", - tag = "bsc", - tenant_handle = ca.handle, - bsc_handle = bsc_handle, - generate_keypair = "yes") - - for bsc_handle in bsc_pdus: - SubElement(q_msg, rpki.left_right.tag_bsc, - action = "destroy", tenant_handle = ca.handle, bsc_handle = bsc_handle) - - # If we've already got actions queued up, run them now, so we - # can finish setting up the BSC before anything tries to use it. - - if len(q_msg) > 0: - SubElement(q_msg, rpki.left_right.tag_bsc, action = "list", tag = "bsc", tenant_handle = ca.handle) - r_msg = self.call_rpkid(q_msg) - bsc_pdus = dict((r_pdu.get("bsc_handle"), r_pdu) - for r_pdu in r_msg.getiterator(rpki.left_right.tag_bsc) - if r_pdu.get("action") == "list") - bsc_pdu = bsc_pdus.pop(bsc_handle, None) - - q_msg = self._compose_left_right_query() - - bsc_pkcs10 = bsc_pdu.find(rpki.left_right.tag_pkcs10_request) - assert bsc_pkcs10 is not None - - bsc, created = rpki.irdb.models.BSC.objects.get_or_certify( - issuer = ca, - handle = bsc_handle, - pkcs10 = rpki.x509.PKCS10(Base64 = bsc_pkcs10.text)) - - if (bsc_pdu.findtext(rpki.left_right.tag_signing_cert, "").decode("base64") != bsc.certificate.get_DER() or - bsc_pdu.findtext(rpki.left_right.tag_signing_cert_crl, "").decode("base64") != ca.latest_crl.get_DER()): - q_pdu = SubElement(q_msg, rpki.left_right.tag_bsc, - action = "set", - tag = "bsc", - tenant_handle = ca.handle, - bsc_handle = bsc_handle) - SubElement(q_pdu, rpki.left_right.tag_signing_cert).text = bsc.certificate.get_Base64() - SubElement(q_pdu, rpki.left_right.tag_signing_cert_crl).text = ca.latest_crl.get_Base64() - - # At present we need one per , not because - # rpkid requires that, but because pubd does. pubd probably should - # be fixed to support a single client allowed to update multiple - # trees, but for the moment the easiest way forward is just to - # enforce a 1:1 mapping between and objects - - for repository in ca.repositories.all(): - - repository_pdu = repository_pdus.pop(repository.handle, None) - - if (repository_pdu is None or - repository_pdu.get("bsc_handle") != bsc_handle or - repository_pdu.get("peer_contact_uri") != repository.service_uri or - repository_pdu.get("rrdp_notification_uri") != repository.rrdp_notification_uri or - repository_pdu.findtext(rpki.left_right.tag_bpki_cert, "").decode("base64") != repository.certificate.get_DER()): - q_pdu = SubElement(q_msg, rpki.left_right.tag_repository, - action = "create" if repository_pdu is None else "set", - tag = repository.handle, - tenant_handle = ca.handle, - repository_handle = repository.handle, - bsc_handle = bsc_handle, - peer_contact_uri = repository.service_uri) - if repository.rrdp_notification_uri: - q_pdu.set("rrdp_notification_uri", repository.rrdp_notification_uri) - SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = repository.certificate.get_Base64() - - for repository_handle in repository_pdus: - SubElement(q_msg, rpki.left_right.tag_repository, action = "destroy", - tenant_handle = ca.handle, repository_handle = repository_handle) - - # setup code currently assumes 1:1 mapping between - # and , and further assumes that the handles - # for an associated pair are the identical (that is: - # parent.repository_handle == parent.parent_handle). - # - # If no such repository exists, our choices are to ignore the - # parent entry or throw an error. For now, we ignore the parent. - - for parent in ca.parents.all(): - - try: - parent_pdu = parent_pdus.pop(parent.handle, None) - - if (parent_pdu is None or - parent_pdu.get("bsc_handle") != bsc_handle or - parent_pdu.get("repository_handle") != parent.handle or - parent_pdu.get("peer_contact_uri") != parent.service_uri or - parent_pdu.get("sia_base") != parent.repository.sia_base or - parent_pdu.get("sender_name") != parent.child_handle or - parent_pdu.get("recipient_name") != parent.parent_handle or - parent_pdu.findtext(rpki.left_right.tag_bpki_cert, "").decode("base64") != parent.certificate.get_DER()): - q_pdu = SubElement(q_msg, rpki.left_right.tag_parent, - action = "create" if parent_pdu is None else "set", - tag = parent.handle, - tenant_handle = ca.handle, - parent_handle = parent.handle, - bsc_handle = bsc_handle, - repository_handle = parent.handle, - peer_contact_uri = parent.service_uri, - sia_base = parent.repository.sia_base, - sender_name = parent.child_handle, - recipient_name = parent.parent_handle) - SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = parent.certificate.get_Base64() - - except rpki.irdb.models.Repository.DoesNotExist: - pass - - try: - - parent_pdu = parent_pdus.pop(ca.handle, None) - - if (parent_pdu is None or - parent_pdu.get("bsc_handle") != bsc_handle or - parent_pdu.get("repository_handle") != ca.handle or - parent_pdu.get("peer_contact_uri") != ca.rootd.service_uri or - parent_pdu.get("sia_base") != ca.rootd.repository.sia_base or - parent_pdu.get("sender_name") != ca.handle or - parent_pdu.get("recipient_name") != ca.handle or - parent_pdu.findtext(rpki.left_right.tag_bpki_cert).decode("base64") != ca.rootd.certificate.get_DER()): - q_pdu = SubElement(q_msg, rpki.left_right.tag_parent, - action = "create" if parent_pdu is None else "set", - tag = ca.handle, - tenant_handle = ca.handle, - parent_handle = ca.handle, - bsc_handle = bsc_handle, - repository_handle = ca.handle, - peer_contact_uri = ca.rootd.service_uri, - sia_base = ca.rootd.repository.sia_base, - sender_name = ca.handle, - recipient_name = ca.handle) - SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = ca.rootd.certificate.get_Base64() - - except rpki.irdb.models.Rootd.DoesNotExist: - pass - - for parent_handle in parent_pdus: - SubElement(q_msg, rpki.left_right.tag_parent, action = "destroy", - tenant_handle = ca.handle, parent_handle = parent_handle) - - # Children are simpler than parents, because they call us, so no URL - # to construct and figuring out what certificate to use is their - # problem, not ours. - - for child in ca.children.all(): - - child_pdu = child_pdus.pop(child.handle, None) - - if (child_pdu is None or - child_pdu.get("bsc_handle") != bsc_handle or - child_pdu.findtext(rpki.left_right.tag_bpki_cert).decode("base64") != child.certificate.get_DER()): - q_pdu = SubElement(q_msg, rpki.left_right.tag_child, - action = "create" if child_pdu is None else "set", - tag = child.handle, - tenant_handle = ca.handle, - child_handle = child.handle, - bsc_handle = bsc_handle) - SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = child.certificate.get_Base64() - - for child_handle in child_pdus: - SubElement(q_msg, rpki.left_right.tag_child, action = "destroy", - tenant_handle = ca.handle, child_handle = child_handle) - - # If caller wants us to poke rpkid, add that to the very end of the message - - if poke: - SubElement(q_msg, rpki.left_right.tag_tenant, action = "set", tenant_handle = ca.handle, run_now = "yes") - - # If we changed anything, ship updates off to rpkid. - - if len(q_msg) > 0: - self.call_rpkid(q_msg) - - - def synchronize_pubd_core(self): - """ - Configure pubd with data built up by the other commands in this - program. This is the core synchronization code. Don't call this - directly, instead call a methods that calls this inside a Django - commit wrapper. - - This method configures pubd with data built up by the other - commands in this program. Commands which modify IRDB fields - related to pubd should call this when they're done. - """ - - # If we're not running pubd, the rest of this is a waste of time - - if not self.run_pubd: - return - - # See what pubd already has on file - - q_msg = self._compose_publication_control_query() - SubElement(q_msg, rpki.publication_control.tag_client, action = "list") - r_msg = self.call_pubd(q_msg) - client_pdus = dict((r_pdu.get("client_handle"), r_pdu) - for r_pdu in r_msg) + try: - # Check all clients + parent_pdu = parent_pdus.pop(ca.handle, None) + + if (parent_pdu is None or + parent_pdu.get("bsc_handle") != bsc_handle or + parent_pdu.get("repository_handle") != ca.handle or + parent_pdu.get("peer_contact_uri") != ca.rootd.service_uri or + parent_pdu.get("sia_base") != ca.rootd.repository.sia_base or + parent_pdu.get("sender_name") != ca.handle or + parent_pdu.get("recipient_name") != ca.handle or + parent_pdu.findtext(rpki.left_right.tag_bpki_cert).decode("base64") != ca.rootd.certificate.get_DER()): + q_pdu = SubElement(q_msg, rpki.left_right.tag_parent, + action = "create" if parent_pdu is None else "set", + tag = ca.handle, + tenant_handle = ca.handle, + parent_handle = ca.handle, + bsc_handle = bsc_handle, + repository_handle = ca.handle, + peer_contact_uri = ca.rootd.service_uri, + sia_base = ca.rootd.repository.sia_base, + sender_name = ca.handle, + recipient_name = ca.handle) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = ca.rootd.certificate.get_Base64() - q_msg = self._compose_publication_control_query() + except rpki.irdb.models.Rootd.DoesNotExist: + pass - for client in self.server_ca.clients.all(): + for parent_handle in parent_pdus: + SubElement(q_msg, rpki.left_right.tag_parent, action = "destroy", + tenant_handle = ca.handle, parent_handle = parent_handle) - client_pdu = client_pdus.pop(client.handle, None) + # Children are simpler than parents, because they call us, so no URL + # to construct and figuring out what certificate to use is their + # problem, not ours. - if (client_pdu is None or - client_pdu.get("base_uri") != client.sia_base or - client_pdu.findtext(rpki.publication_control.tag_bpki_cert, "").decode("base64") != client.certificate.get_DER()): - q_pdu = SubElement(q_msg, rpki.publication_control.tag_client, - action = "create" if client_pdu is None else "set", - client_handle = client.handle, - base_uri = client.sia_base) - SubElement(q_pdu, rpki.publication_control.tag_bpki_cert).text = client.certificate.get_Base64() + for child in ca.children.all(): - # rootd instances are also a weird sort of client + child_pdu = child_pdus.pop(child.handle, None) - for rootd in rpki.irdb.models.Rootd.objects.all(): + if (child_pdu is None or + child_pdu.get("bsc_handle") != bsc_handle or + child_pdu.findtext(rpki.left_right.tag_bpki_cert).decode("base64") != child.certificate.get_DER()): + q_pdu = SubElement(q_msg, rpki.left_right.tag_child, + action = "create" if child_pdu is None else "set", + tag = child.handle, + tenant_handle = ca.handle, + child_handle = child.handle, + bsc_handle = bsc_handle) + SubElement(q_pdu, rpki.left_right.tag_bpki_cert).text = child.certificate.get_Base64() - client_handle = rootd.issuer.handle + "-root" - client_pdu = client_pdus.pop(client_handle, None) - sia_base = "rsync://%s/%s/%s/" % (self.rsync_server, self.rsync_module, client_handle) + for child_handle in child_pdus: + SubElement(q_msg, rpki.left_right.tag_child, action = "destroy", + tenant_handle = ca.handle, child_handle = child_handle) - if (client_pdu is None or - client_pdu.get("base_uri") != sia_base or - client_pdu.findtext(rpki.publication_control.tag_bpki_cert, "").decode("base64") != rootd.issuer.certificate.get_DER()): - q_pdu = SubElement(q_msg, rpki.publication_control.tag_client, - action = "create" if client_pdu is None else "set", - client_handle = client_handle, - base_uri = sia_base) - SubElement(q_pdu, rpki.publication_control.tag_bpki_cert).text = rootd.issuer.certificate.get_Base64() + # If caller wants us to poke rpkid, add that to the very end of the message - # Delete any unknown clients + if poke: + SubElement(q_msg, rpki.left_right.tag_tenant, action = "set", tenant_handle = ca.handle, run_now = "yes") - for client_handle in client_pdus: - SubElement(q_msg, rpki.publication_control.tag_client, action = "destroy", client_handle = client_handle) + # If we changed anything, ship updates off to rpkid. - # If we changed anything, ship updates off to pubd + if len(q_msg) > 0: + self.call_rpkid(q_msg) - if len(q_msg) > 0: - self.call_pubd(q_msg) + def synchronize_pubd_core(self): + """ + Configure pubd with data built up by the other commands in this + program. This is the core synchronization code. Don't call this + directly, instead call a methods that calls this inside a Django + commit wrapper. - def synchronize_rpkid_deleted_core(self): - """ - Remove any objects present in rpkid's database but not - present in the IRDB. This is the core synchronization code. - Don't call this directly, instead call a methods that calls this - inside a Django commit wrapper. - """ + This method configures pubd with data built up by the other + commands in this program. Commands which modify IRDB fields + related to pubd should call this when they're done. + """ - q_msg = self._compose_left_right_query() - SubElement(q_msg, rpki.left_right.tag_tenant, action = "list") - self.call_rpkid(q_msg) + # If we're not running pubd, the rest of this is a waste of time - tenant_handles = set(s.get("tenant_handle") for s in q_msg) - ca_handles = set(ca.handle for ca in rpki.irdb.models.ResourceHolderCA.objects.all()) - assert ca_handles <= tenant_handles + if not self.run_pubd: + return - q_msg = self._compose_left_right_query() - for handle in (tenant_handles - ca_handles): - SubElement(q_msg, rpki.left_right.tag_tenant, action = "destroy", tenant_handle = handle) + # See what pubd already has on file - if len(q_msg) > 0: - self.call_rpkid(q_msg) + q_msg = self._compose_publication_control_query() + SubElement(q_msg, rpki.publication_control.tag_client, action = "list") + r_msg = self.call_pubd(q_msg) + client_pdus = dict((r_pdu.get("client_handle"), r_pdu) + for r_pdu in r_msg) + # Check all clients - @django.db.transaction.atomic - def add_ee_certificate_request(self, pkcs10, resources): - """ - Check a PKCS #10 request to see if it complies with the - specification for a RPKI EE certificate; if it does, add an - EECertificateRequest for it to the IRDB. + q_msg = self._compose_publication_control_query() - Not yet sure what we want for update and delete semantics here, so - for the moment this is straight addition. See methods like - .load_asns() and .load_prefixes() for other strategies. - """ + for client in self.server_ca.clients.all(): - pkcs10.check_valid_request_ee() - ee_request = self.resource_ca.ee_certificate_requests.create( - pkcs10 = pkcs10, - gski = pkcs10.gSKI(), - valid_until = resources.valid_until) - for r in resources.asn: - ee_request.asns.create(start_as = str(r.min), end_as = str(r.max)) - for r in resources.v4: - ee_request.address_ranges.create(start_ip = str(r.min), end_ip = str(r.max), version = 4) - for r in resources.v6: - ee_request.address_ranges.create(start_ip = str(r.min), end_ip = str(r.max), version = 6) - - - @django.db.transaction.atomic - def add_router_certificate_request(self, router_certificate_request_xml, valid_until = None): - """ - Read XML file containing one or more router certificate requests, - attempt to add request(s) to IRDB. + client_pdu = client_pdus.pop(client.handle, None) - Check each PKCS #10 request to see if it complies with the - specification for a router certificate; if it does, create an EE - certificate request for it along with the ASN resources and - router-ID supplied in the XML. - """ + if (client_pdu is None or + client_pdu.get("base_uri") != client.sia_base or + client_pdu.findtext(rpki.publication_control.tag_bpki_cert, "").decode("base64") != client.certificate.get_DER()): + q_pdu = SubElement(q_msg, rpki.publication_control.tag_client, + action = "create" if client_pdu is None else "set", + client_handle = client.handle, + base_uri = client.sia_base) + SubElement(q_pdu, rpki.publication_control.tag_bpki_cert).text = client.certificate.get_Base64() - x = etree_read(router_certificate_request_xml, schema = rpki.relaxng.router_certificate) + # rootd instances are also a weird sort of client - for x in x.getiterator(tag_router_certificate_request): + for rootd in rpki.irdb.models.Rootd.objects.all(): - pkcs10 = rpki.x509.PKCS10(Base64 = x.text) - router_id = long(x.get("router_id")) - asns = rpki.resource_set.resource_set_as(x.get("asn")) - if not valid_until: - valid_until = x.get("valid_until") + client_handle = rootd.issuer.handle + "-root" + client_pdu = client_pdus.pop(client_handle, None) + sia_base = "rsync://%s/%s/%s/" % (self.rsync_server, self.rsync_module, client_handle) - if valid_until and isinstance(valid_until, (str, unicode)): - valid_until = rpki.sundial.datetime.fromXMLtime(valid_until) + if (client_pdu is None or + client_pdu.get("base_uri") != sia_base or + client_pdu.findtext(rpki.publication_control.tag_bpki_cert, "").decode("base64") != rootd.issuer.certificate.get_DER()): + q_pdu = SubElement(q_msg, rpki.publication_control.tag_client, + action = "create" if client_pdu is None else "set", + client_handle = client_handle, + base_uri = sia_base) + SubElement(q_pdu, rpki.publication_control.tag_bpki_cert).text = rootd.issuer.certificate.get_Base64() - if not valid_until: - valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 365) - elif valid_until < rpki.sundial.now(): - raise PastExpiration("Specified expiration date %s has already passed" % valid_until) + # Delete any unknown clients - pkcs10.check_valid_request_router() + for client_handle in client_pdus: + SubElement(q_msg, rpki.publication_control.tag_client, action = "destroy", client_handle = client_handle) - cn = "ROUTER-%08x" % asns[0].min - sn = "%08x" % router_id + # If we changed anything, ship updates off to pubd - ee_request = self.resource_ca.ee_certificate_requests.create( - pkcs10 = pkcs10, - gski = pkcs10.gSKI(), - valid_until = valid_until, - cn = cn, - sn = sn, - eku = rpki.oids.id_kp_bgpsec_router) + if len(q_msg) > 0: + self.call_pubd(q_msg) - for r in asns: - ee_request.asns.create(start_as = str(r.min), end_as = str(r.max)) + def synchronize_rpkid_deleted_core(self): + """ + Remove any objects present in rpkid's database but not + present in the IRDB. This is the core synchronization code. + Don't call this directly, instead call a methods that calls this + inside a Django commit wrapper. + """ - @django.db.transaction.atomic - def delete_router_certificate_request(self, gski): - """ - Delete a router certificate request from this RPKI entity. - """ + q_msg = self._compose_left_right_query() + SubElement(q_msg, rpki.left_right.tag_tenant, action = "list") + self.call_rpkid(q_msg) - self.resource_ca.ee_certificate_requests.get(gski = gski).delete() + tenant_handles = set(s.get("tenant_handle") for s in q_msg) + ca_handles = set(ca.handle for ca in rpki.irdb.models.ResourceHolderCA.objects.all()) + assert ca_handles <= tenant_handles + + q_msg = self._compose_left_right_query() + for handle in (tenant_handles - ca_handles): + SubElement(q_msg, rpki.left_right.tag_tenant, action = "destroy", tenant_handle = handle) + + if len(q_msg) > 0: + self.call_rpkid(q_msg) + + + @django.db.transaction.atomic + def add_ee_certificate_request(self, pkcs10, resources): + """ + Check a PKCS #10 request to see if it complies with the + specification for a RPKI EE certificate; if it does, add an + EECertificateRequest for it to the IRDB. + + Not yet sure what we want for update and delete semantics here, so + for the moment this is straight addition. See methods like + .load_asns() and .load_prefixes() for other strategies. + """ + + pkcs10.check_valid_request_ee() + ee_request = self.resource_ca.ee_certificate_requests.create( + pkcs10 = pkcs10, + gski = pkcs10.gSKI(), + valid_until = resources.valid_until) + for r in resources.asn: + ee_request.asns.create(start_as = str(r.min), end_as = str(r.max)) + for r in resources.v4: + ee_request.address_ranges.create(start_ip = str(r.min), end_ip = str(r.max), version = 4) + for r in resources.v6: + ee_request.address_ranges.create(start_ip = str(r.min), end_ip = str(r.max), version = 6) + + + @django.db.transaction.atomic + def add_router_certificate_request(self, router_certificate_request_xml, valid_until = None): + """ + Read XML file containing one or more router certificate requests, + attempt to add request(s) to IRDB. + + Check each PKCS #10 request to see if it complies with the + specification for a router certificate; if it does, create an EE + certificate request for it along with the ASN resources and + router-ID supplied in the XML. + """ + + x = etree_read(router_certificate_request_xml, schema = rpki.relaxng.router_certificate) + + for x in x.getiterator(tag_router_certificate_request): + + pkcs10 = rpki.x509.PKCS10(Base64 = x.text) + router_id = long(x.get("router_id")) + asns = rpki.resource_set.resource_set_as(x.get("asn")) + if not valid_until: + valid_until = x.get("valid_until") + + if valid_until and isinstance(valid_until, (str, unicode)): + valid_until = rpki.sundial.datetime.fromXMLtime(valid_until) + + if not valid_until: + valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 365) + elif valid_until < rpki.sundial.now(): + raise PastExpiration("Specified expiration date %s has already passed" % valid_until) + + pkcs10.check_valid_request_router() + + cn = "ROUTER-%08x" % asns[0].min + sn = "%08x" % router_id + + ee_request = self.resource_ca.ee_certificate_requests.create( + pkcs10 = pkcs10, + gski = pkcs10.gSKI(), + valid_until = valid_until, + cn = cn, + sn = sn, + eku = rpki.oids.id_kp_bgpsec_router) + + for r in asns: + ee_request.asns.create(start_as = str(r.min), end_as = str(r.max)) + + + @django.db.transaction.atomic + def delete_router_certificate_request(self, gski): + """ + Delete a router certificate request from this RPKI entity. + """ + + self.resource_ca.ee_certificate_requests.get(gski = gski).delete() diff --git a/rpki/irdbd.py b/rpki/irdbd.py index 96757477..91859f5d 100644 --- a/rpki/irdbd.py +++ b/rpki/irdbd.py @@ -41,183 +41,183 @@ logger = logging.getLogger(__name__) class main(object): - def handle_list_resources(self, q_pdu, r_msg): - tenant_handle = q_pdu.get("tenant_handle") - child_handle = q_pdu.get("child_handle") - child = rpki.irdb.models.Child.objects.get(issuer__handle = tenant_handle, handle = child_handle) - resources = child.resource_bag - r_pdu = SubElement(r_msg, rpki.left_right.tag_list_resources, tenant_handle = tenant_handle, child_handle = child_handle, - valid_until = child.valid_until.strftime("%Y-%m-%dT%H:%M:%SZ")) - for k, v in (("asn", resources.asn), - ("ipv4", resources.v4), - ("ipv6", resources.v6), - ("tag", q_pdu.get("tag"))): - if v: - r_pdu.set(k, str(v)) - - def handle_list_roa_requests(self, q_pdu, r_msg): - tenant_handle = q_pdu.get("tenant_handle") - for request in rpki.irdb.models.ROARequest.objects.raw(""" - SELECT irdb_roarequest.* - FROM irdb_roarequest, irdb_resourceholderca - WHERE irdb_roarequest.issuer_id = irdb_resourceholderca.id - AND irdb_resourceholderca.handle = %s - """, [tenant_handle]): - prefix_bag = request.roa_prefix_bag - r_pdu = SubElement(r_msg, rpki.left_right.tag_list_roa_requests, tenant_handle = tenant_handle, asn = str(request.asn)) - for k, v in (("ipv4", prefix_bag.v4), - ("ipv6", prefix_bag.v6), - ("tag", q_pdu.get("tag"))): - if v: - r_pdu.set(k, str(v)) - - def handle_list_ghostbuster_requests(self, q_pdu, r_msg): - tenant_handle = q_pdu.get("tenant_handle") - parent_handle = q_pdu.get("parent_handle") - ghostbusters = rpki.irdb.models.GhostbusterRequest.objects.filter(issuer__handle = tenant_handle, parent__handle = parent_handle) - if ghostbusters.count() == 0: - ghostbusters = rpki.irdb.models.GhostbusterRequest.objects.filter(issuer__handle = tenant_handle, parent = None) - for ghostbuster in ghostbusters: - r_pdu = SubElement(r_msg, q_pdu.tag, tenant_handle = tenant_handle, parent_handle = parent_handle) - if q_pdu.get("tag"): - r_pdu.set("tag", q_pdu.get("tag")) - r_pdu.text = ghostbuster.vcard - - def handle_list_ee_certificate_requests(self, q_pdu, r_msg): - tenant_handle = q_pdu.get("tenant_handle") - for ee_req in rpki.irdb.models.EECertificateRequest.objects.filter(issuer__handle = tenant_handle): - resources = ee_req.resource_bag - r_pdu = SubElement(r_msg, q_pdu.tag, tenant_handle = tenant_handle, gski = ee_req.gski, - valid_until = ee_req.valid_until.strftime("%Y-%m-%dT%H:%M:%SZ"), - cn = ee_req.cn, sn = ee_req.sn) - for k, v in (("asn", resources.asn), - ("ipv4", resources.v4), - ("ipv6", resources.v6), - ("eku", ee_req.eku), - ("tag", q_pdu.get("tag"))): - if v: - r_pdu.set(k, str(v)) - SubElement(r_pdu, rpki.left_right.tag_pkcs10).text = ee_req.pkcs10.get_Base64() - - def handler(self, request, q_der): - try: - from django.db import connection - connection.cursor() # Reconnect to mysqld if necessary - self.start_new_transaction() - serverCA = rpki.irdb.models.ServerCA.objects.get() - rpkid = serverCA.ee_certificates.get(purpose = "rpkid") - irdbd = serverCA.ee_certificates.get(purpose = "irdbd") - q_cms = rpki.left_right.cms_msg(DER = q_der) - q_msg = q_cms.unwrap((serverCA.certificate, rpkid.certificate)) - self.cms_timestamp = q_cms.check_replay(self.cms_timestamp, request.path) - if q_msg.get("type") != "query": - raise rpki.exceptions.BadQuery("Message type is %s, expected query" % q_msg.get("type")) - r_msg = Element(rpki.left_right.tag_msg, nsmap = rpki.left_right.nsmap, - type = "reply", version = rpki.left_right.version) - try: - for q_pdu in q_msg: - getattr(self, "handle_" + q_pdu.tag[len(rpki.left_right.xmlns):])(q_pdu, r_msg) - - except Exception, e: - logger.exception("Exception processing PDU %r", q_pdu) - r_pdu = SubElement(r_msg, rpki.left_right.tag_report_error, error_code = e.__class__.__name__) - r_pdu.text = str(e) - if q_pdu.get("tag") is not None: - r_pdu.set("tag", q_pdu.get("tag")) - - request.send_cms_response(rpki.left_right.cms_msg().wrap(r_msg, irdbd.private_key, irdbd.certificate)) - - except Exception, e: - logger.exception("Unhandled exception while processing HTTP request") - request.send_error(500, "Unhandled exception %s: %s" % (e.__class__.__name__, e)) - - def __init__(self, **kwargs): - - global rpki # pylint: disable=W0602 - - os.environ.update(TZ = "UTC", - DJANGO_SETTINGS_MODULE = "rpki.django_settings.irdb") - time.tzset() - - parser = argparse.ArgumentParser(description = __doc__) - parser.add_argument("-c", "--config", - help = "override default location of configuration file") - parser.add_argument("-f", "--foreground", action = "store_true", - help = "do not daemonize") - parser.add_argument("--pidfile", - help = "override default location of pid file") - parser.add_argument("--profile", - help = "enable profiling, saving data to PROFILE") - rpki.log.argparse_setup(parser) - args = parser.parse_args() - - rpki.log.init("irdbd", args) - - self.cfg = rpki.config.parser(set_filename = args.config, section = "irdbd") - self.cfg.set_global_flags() - - if not args.foreground: - rpki.daemonize.daemon(pidfile = args.pidfile) - - if args.profile: - import cProfile - prof = cProfile.Profile() - try: - prof.runcall(self.main) - finally: - prof.dump_stats(args.profile) - logger.info("Dumped profile data to %s", args.profile) - else: - self.main() - - def main(self): - - startup_msg = self.cfg.get("startup-message", "") - if startup_msg: - logger.info(startup_msg) - - # Now that we know which configuration file to use, it's OK to - # load modules that require Django's settings module. - - import django - django.setup() - - global rpki # pylint: disable=W0602 - import rpki.irdb # pylint: disable=W0621 - - self.http_server_host = self.cfg.get("server-host", "") - self.http_server_port = self.cfg.getint("server-port") - - self.cms_timestamp = None - - rpki.http_simple.server( - host = self.http_server_host, - port = self.http_server_port, - handlers = self.handler) - - def start_new_transaction(self): - - # Entirely too much fun with read-only access to transactional databases. - # - # http://stackoverflow.com/questions/3346124/how-do-i-force-django-to-ignore-any-caches-and-reload-data - # http://devblog.resolversystems.com/?p=439 - # http://groups.google.com/group/django-users/browse_thread/thread/e25cec400598c06d - # http://stackoverflow.com/questions/1028671/python-mysqldb-update-query-fails - # http://dev.mysql.com/doc/refman/5.0/en/set-transaction.html - # - # It turns out that MySQL is doing us a favor with this weird - # transactional behavior on read, because without it there's a - # race condition if multiple updates are committed to the IRDB - # while we're in the middle of processing a query. Note that - # proper transaction management by the committers doesn't protect - # us, this is a transactional problem on read. So we need to use - # explicit transaction management. Since irdbd is a read-only - # consumer of IRDB data, this means we need to commit an empty - # transaction at the beginning of processing each query, to reset - # the transaction isolation snapshot. - - import django.db.transaction - - with django.db.transaction.atomic(): - #django.db.transaction.commit() - pass + def handle_list_resources(self, q_pdu, r_msg): + tenant_handle = q_pdu.get("tenant_handle") + child_handle = q_pdu.get("child_handle") + child = rpki.irdb.models.Child.objects.get(issuer__handle = tenant_handle, handle = child_handle) + resources = child.resource_bag + r_pdu = SubElement(r_msg, rpki.left_right.tag_list_resources, tenant_handle = tenant_handle, child_handle = child_handle, + valid_until = child.valid_until.strftime("%Y-%m-%dT%H:%M:%SZ")) + for k, v in (("asn", resources.asn), + ("ipv4", resources.v4), + ("ipv6", resources.v6), + ("tag", q_pdu.get("tag"))): + if v: + r_pdu.set(k, str(v)) + + def handle_list_roa_requests(self, q_pdu, r_msg): + tenant_handle = q_pdu.get("tenant_handle") + for request in rpki.irdb.models.ROARequest.objects.raw(""" + SELECT irdb_roarequest.* + FROM irdb_roarequest, irdb_resourceholderca + WHERE irdb_roarequest.issuer_id = irdb_resourceholderca.id + AND irdb_resourceholderca.handle = %s + """, [tenant_handle]): + prefix_bag = request.roa_prefix_bag + r_pdu = SubElement(r_msg, rpki.left_right.tag_list_roa_requests, tenant_handle = tenant_handle, asn = str(request.asn)) + for k, v in (("ipv4", prefix_bag.v4), + ("ipv6", prefix_bag.v6), + ("tag", q_pdu.get("tag"))): + if v: + r_pdu.set(k, str(v)) + + def handle_list_ghostbuster_requests(self, q_pdu, r_msg): + tenant_handle = q_pdu.get("tenant_handle") + parent_handle = q_pdu.get("parent_handle") + ghostbusters = rpki.irdb.models.GhostbusterRequest.objects.filter(issuer__handle = tenant_handle, parent__handle = parent_handle) + if ghostbusters.count() == 0: + ghostbusters = rpki.irdb.models.GhostbusterRequest.objects.filter(issuer__handle = tenant_handle, parent = None) + for ghostbuster in ghostbusters: + r_pdu = SubElement(r_msg, q_pdu.tag, tenant_handle = tenant_handle, parent_handle = parent_handle) + if q_pdu.get("tag"): + r_pdu.set("tag", q_pdu.get("tag")) + r_pdu.text = ghostbuster.vcard + + def handle_list_ee_certificate_requests(self, q_pdu, r_msg): + tenant_handle = q_pdu.get("tenant_handle") + for ee_req in rpki.irdb.models.EECertificateRequest.objects.filter(issuer__handle = tenant_handle): + resources = ee_req.resource_bag + r_pdu = SubElement(r_msg, q_pdu.tag, tenant_handle = tenant_handle, gski = ee_req.gski, + valid_until = ee_req.valid_until.strftime("%Y-%m-%dT%H:%M:%SZ"), + cn = ee_req.cn, sn = ee_req.sn) + for k, v in (("asn", resources.asn), + ("ipv4", resources.v4), + ("ipv6", resources.v6), + ("eku", ee_req.eku), + ("tag", q_pdu.get("tag"))): + if v: + r_pdu.set(k, str(v)) + SubElement(r_pdu, rpki.left_right.tag_pkcs10).text = ee_req.pkcs10.get_Base64() + + def handler(self, request, q_der): + try: + from django.db import connection + connection.cursor() # Reconnect to mysqld if necessary + self.start_new_transaction() + serverCA = rpki.irdb.models.ServerCA.objects.get() + rpkid = serverCA.ee_certificates.get(purpose = "rpkid") + irdbd = serverCA.ee_certificates.get(purpose = "irdbd") + q_cms = rpki.left_right.cms_msg(DER = q_der) + q_msg = q_cms.unwrap((serverCA.certificate, rpkid.certificate)) + self.cms_timestamp = q_cms.check_replay(self.cms_timestamp, request.path) + if q_msg.get("type") != "query": + raise rpki.exceptions.BadQuery("Message type is %s, expected query" % q_msg.get("type")) + r_msg = Element(rpki.left_right.tag_msg, nsmap = rpki.left_right.nsmap, + type = "reply", version = rpki.left_right.version) + try: + for q_pdu in q_msg: + getattr(self, "handle_" + q_pdu.tag[len(rpki.left_right.xmlns):])(q_pdu, r_msg) + + except Exception, e: + logger.exception("Exception processing PDU %r", q_pdu) + r_pdu = SubElement(r_msg, rpki.left_right.tag_report_error, error_code = e.__class__.__name__) + r_pdu.text = str(e) + if q_pdu.get("tag") is not None: + r_pdu.set("tag", q_pdu.get("tag")) + + request.send_cms_response(rpki.left_right.cms_msg().wrap(r_msg, irdbd.private_key, irdbd.certificate)) + + except Exception, e: + logger.exception("Unhandled exception while processing HTTP request") + request.send_error(500, "Unhandled exception %s: %s" % (e.__class__.__name__, e)) + + def __init__(self, **kwargs): + + global rpki # pylint: disable=W0602 + + os.environ.update(TZ = "UTC", + DJANGO_SETTINGS_MODULE = "rpki.django_settings.irdb") + time.tzset() + + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument("-c", "--config", + help = "override default location of configuration file") + parser.add_argument("-f", "--foreground", action = "store_true", + help = "do not daemonize") + parser.add_argument("--pidfile", + help = "override default location of pid file") + parser.add_argument("--profile", + help = "enable profiling, saving data to PROFILE") + rpki.log.argparse_setup(parser) + args = parser.parse_args() + + rpki.log.init("irdbd", args) + + self.cfg = rpki.config.parser(set_filename = args.config, section = "irdbd") + self.cfg.set_global_flags() + + if not args.foreground: + rpki.daemonize.daemon(pidfile = args.pidfile) + + if args.profile: + import cProfile + prof = cProfile.Profile() + try: + prof.runcall(self.main) + finally: + prof.dump_stats(args.profile) + logger.info("Dumped profile data to %s", args.profile) + else: + self.main() + + def main(self): + + startup_msg = self.cfg.get("startup-message", "") + if startup_msg: + logger.info(startup_msg) + + # Now that we know which configuration file to use, it's OK to + # load modules that require Django's settings module. + + import django + django.setup() + + global rpki # pylint: disable=W0602 + import rpki.irdb # pylint: disable=W0621 + + self.http_server_host = self.cfg.get("server-host", "") + self.http_server_port = self.cfg.getint("server-port") + + self.cms_timestamp = None + + rpki.http_simple.server( + host = self.http_server_host, + port = self.http_server_port, + handlers = self.handler) + + def start_new_transaction(self): + + # Entirely too much fun with read-only access to transactional databases. + # + # http://stackoverflow.com/questions/3346124/how-do-i-force-django-to-ignore-any-caches-and-reload-data + # http://devblog.resolversystems.com/?p=439 + # http://groups.google.com/group/django-users/browse_thread/thread/e25cec400598c06d + # http://stackoverflow.com/questions/1028671/python-mysqldb-update-query-fails + # http://dev.mysql.com/doc/refman/5.0/en/set-transaction.html + # + # It turns out that MySQL is doing us a favor with this weird + # transactional behavior on read, because without it there's a + # race condition if multiple updates are committed to the IRDB + # while we're in the middle of processing a query. Note that + # proper transaction management by the committers doesn't protect + # us, this is a transactional problem on read. So we need to use + # explicit transaction management. Since irdbd is a read-only + # consumer of IRDB data, this means we need to commit an empty + # transaction at the beginning of processing each query, to reset + # the transaction isolation snapshot. + + import django.db.transaction + + with django.db.transaction.atomic(): + #django.db.transaction.commit() + pass diff --git a/rpki/left_right.py b/rpki/left_right.py index 387e908f..3572ee98 100644 --- a/rpki/left_right.py +++ b/rpki/left_right.py @@ -71,9 +71,9 @@ allowed_content_types = (content_type,) class cms_msg(rpki.x509.XML_CMS_object): - """ - CMS-signed left-right PDU. - """ + """ + CMS-signed left-right PDU. + """ - encoding = "us-ascii" - schema = rpki.relaxng.left_right + encoding = "us-ascii" + schema = rpki.relaxng.left_right diff --git a/rpki/log.py b/rpki/log.py index 828982da..8afee4ba 100644 --- a/rpki/log.py +++ b/rpki/log.py @@ -30,12 +30,12 @@ import argparse import traceback as tb try: - have_setproctitle = False - if os.getenv("DISABLE_SETPROCTITLE") is None: - import setproctitle # pylint: disable=F0401 - have_setproctitle = True + have_setproctitle = False + if os.getenv("DISABLE_SETPROCTITLE") is None: + import setproctitle # pylint: disable=F0401 + have_setproctitle = True except ImportError: - pass + pass logger = logging.getLogger(__name__) @@ -67,234 +67,234 @@ proctitle_extra = os.path.basename(os.getcwd()) class Formatter(object): - """ - Reimplementation (easier than subclassing in this case) of - logging.Formatter. - - It turns out that the logging code only cares about this class's - .format(record) method, everything else is internal; so long as - .format() converts a record into a properly formatted string, the - logging code is happy. - - So, rather than mess around with dynamically constructing and - deconstructing and tweaking format strings and ten zillion options - we don't use, we just provide our own implementation that supports - what we do need. - """ - - converter = time.gmtime - - def __init__(self, ident, handler): - self.ident = ident - self.is_syslog = isinstance(handler, logging.handlers.SysLogHandler) - - def format(self, record): - return "".join(self.coformat(record)).rstrip("\n") - - def coformat(self, record): - - try: - if not self.is_syslog: - yield time.strftime("%Y-%m-%d %H:%M:%S ", time.gmtime(record.created)) - except: # pylint: disable=W0702 - yield "[$!$Time format failed]" - - try: - yield "%s[%d]: " % (self.ident, record.process) - except: # pylint: disable=W0702 - yield "[$!$ident format failed]" - - try: - if isinstance(record.context, (str, unicode)): - yield record.context + " " - else: - yield repr(record.context) + " " - except AttributeError: - pass - except: # pylint: disable=W0702 - yield "[$!$context format failed]" - - try: - yield record.getMessage() - except: # pylint: disable=W0702 - yield "[$!$record.getMessage() failed]" - - try: - if record.exc_info: - if self.is_syslog or not enable_tracebacks: - lines = tb.format_exception_only(record.exc_info[0], record.exc_info[1]) - lines.insert(0, ": ") - else: - lines = tb.format_exception(record.exc_info[0], record.exc_info[1], record.exc_info[2]) - lines.insert(0, "\n") - for line in lines: - yield line - except: # pylint: disable=W0702 - yield "[$!$exception formatting failed]" + """ + Reimplementation (easier than subclassing in this case) of + logging.Formatter. + + It turns out that the logging code only cares about this class's + .format(record) method, everything else is internal; so long as + .format() converts a record into a properly formatted string, the + logging code is happy. + + So, rather than mess around with dynamically constructing and + deconstructing and tweaking format strings and ten zillion options + we don't use, we just provide our own implementation that supports + what we do need. + """ + + converter = time.gmtime + + def __init__(self, ident, handler): + self.ident = ident + self.is_syslog = isinstance(handler, logging.handlers.SysLogHandler) + + def format(self, record): + return "".join(self.coformat(record)).rstrip("\n") + + def coformat(self, record): + + try: + if not self.is_syslog: + yield time.strftime("%Y-%m-%d %H:%M:%S ", time.gmtime(record.created)) + except: # pylint: disable=W0702 + yield "[$!$Time format failed]" + + try: + yield "%s[%d]: " % (self.ident, record.process) + except: # pylint: disable=W0702 + yield "[$!$ident format failed]" + + try: + if isinstance(record.context, (str, unicode)): + yield record.context + " " + else: + yield repr(record.context) + " " + except AttributeError: + pass + except: # pylint: disable=W0702 + yield "[$!$context format failed]" + + try: + yield record.getMessage() + except: # pylint: disable=W0702 + yield "[$!$record.getMessage() failed]" + + try: + if record.exc_info: + if self.is_syslog or not enable_tracebacks: + lines = tb.format_exception_only(record.exc_info[0], record.exc_info[1]) + lines.insert(0, ": ") + else: + lines = tb.format_exception(record.exc_info[0], record.exc_info[1], record.exc_info[2]) + lines.insert(0, "\n") + for line in lines: + yield line + except: # pylint: disable=W0702 + yield "[$!$exception formatting failed]" def argparse_setup(parser, default_thunk = None): - """ - Set up argparse stuff for functionality in this module. + """ + Set up argparse stuff for functionality in this module. - Default logging destination is syslog, but you can change this - by setting default_thunk to a callable which takes no arguments - and which returns a instance of a logging.Handler subclass. + Default logging destination is syslog, but you can change this + by setting default_thunk to a callable which takes no arguments + and which returns a instance of a logging.Handler subclass. - Also see rpki.log.init(). - """ + Also see rpki.log.init(). + """ - class LogLevelAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string = None): - setattr(namespace, self.dest, getattr(logging, values.upper())) + class LogLevelAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string = None): + setattr(namespace, self.dest, getattr(logging, values.upper())) - parser.add_argument("--log-level", default = logging.WARNING, action = LogLevelAction, - choices = ("debug", "info", "warning", "error", "critical"), - help = "how verbosely to log") + parser.add_argument("--log-level", default = logging.WARNING, action = LogLevelAction, + choices = ("debug", "info", "warning", "error", "critical"), + help = "how verbosely to log") - group = parser.add_mutually_exclusive_group() + group = parser.add_mutually_exclusive_group() - syslog_address = "/dev/log" if os.path.exists("/dev/log") else ("localhost", logging.handlers.SYSLOG_UDP_PORT) + syslog_address = "/dev/log" if os.path.exists("/dev/log") else ("localhost", logging.handlers.SYSLOG_UDP_PORT) - class SyslogAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string = None): - namespace.log_handler = lambda: logging.handlers.SysLogHandler(address = syslog_address, facility = values) + class SyslogAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string = None): + namespace.log_handler = lambda: logging.handlers.SysLogHandler(address = syslog_address, facility = values) - group.add_argument("--log-syslog", nargs = "?", const = "daemon", action = SyslogAction, - choices = sorted(logging.handlers.SysLogHandler.facility_names.keys()), - help = "send logging to syslog") + group.add_argument("--log-syslog", nargs = "?", const = "daemon", action = SyslogAction, + choices = sorted(logging.handlers.SysLogHandler.facility_names.keys()), + help = "send logging to syslog") - class StreamAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string = None): - namespace.log_handler = lambda: logging.StreamHandler(stream = self.const) + class StreamAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string = None): + namespace.log_handler = lambda: logging.StreamHandler(stream = self.const) - group.add_argument("--log-stderr", nargs = 0, action = StreamAction, const = sys.stderr, - help = "send logging to standard error") + group.add_argument("--log-stderr", nargs = 0, action = StreamAction, const = sys.stderr, + help = "send logging to standard error") - group.add_argument("--log-stdout", nargs = 0, action = StreamAction, const = sys.stdout, - help = "send logging to standard output") + group.add_argument("--log-stdout", nargs = 0, action = StreamAction, const = sys.stdout, + help = "send logging to standard output") - class WatchedFileAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string = None): - namespace.log_handler = lambda: logging.handlers.WatchedFileHandler(filename = values) + class WatchedFileAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string = None): + namespace.log_handler = lambda: logging.handlers.WatchedFileHandler(filename = values) - group.add_argument("--log-file", action = WatchedFileAction, - help = "send logging to a file, reopening if rotated away") + group.add_argument("--log-file", action = WatchedFileAction, + help = "send logging to a file, reopening if rotated away") - class RotatingFileAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string = None): - namespace.log_handler = lambda: logging.handlers.RotatingFileHandler( - filename = values[0], - maxBytes = int(values[1]) * 1024, - backupCount = int(values[2])) + class RotatingFileAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string = None): + namespace.log_handler = lambda: logging.handlers.RotatingFileHandler( + filename = values[0], + maxBytes = int(values[1]) * 1024, + backupCount = int(values[2])) - group.add_argument("--log-rotating-file", action = RotatingFileAction, - nargs = 3, metavar = ("FILENAME", "KBYTES", "COUNT"), - help = "send logging to rotating file") + group.add_argument("--log-rotating-file", action = RotatingFileAction, + nargs = 3, metavar = ("FILENAME", "KBYTES", "COUNT"), + help = "send logging to rotating file") - class TimedRotatingFileAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string = None): - namespace.log_handler = lambda: logging.handlers.TimedRotatingFileHandler( - filename = values[0], - interval = int(values[1]), - backupCount = int(values[2]), - when = "H", - utc = True) + class TimedRotatingFileAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string = None): + namespace.log_handler = lambda: logging.handlers.TimedRotatingFileHandler( + filename = values[0], + interval = int(values[1]), + backupCount = int(values[2]), + when = "H", + utc = True) - group.add_argument("--log-timed-rotating-file", action = TimedRotatingFileAction, - nargs = 3, metavar = ("FILENAME", "HOURS", "COUNT"), - help = "send logging to timed rotating file") + group.add_argument("--log-timed-rotating-file", action = TimedRotatingFileAction, + nargs = 3, metavar = ("FILENAME", "HOURS", "COUNT"), + help = "send logging to timed rotating file") - if default_thunk is None: - default_thunk = lambda: logging.handlers.SysLogHandler(address = syslog_address, facility = "daemon") + if default_thunk is None: + default_thunk = lambda: logging.handlers.SysLogHandler(address = syslog_address, facility = "daemon") - parser.set_defaults(log_handler = default_thunk) + parser.set_defaults(log_handler = default_thunk) def init(ident = None, args = None): - """ - Initialize logging system. + """ + Initialize logging system. - Default logging destination is stderr if "args" is not specified. - """ + Default logging destination is stderr if "args" is not specified. + """ - # pylint: disable=E1103 + # pylint: disable=E1103 - if ident is None: - ident = os.path.basename(sys.argv[0]) + if ident is None: + ident = os.path.basename(sys.argv[0]) - if args is None: - args = argparse.Namespace(log_level = logging.WARNING, - log_handler = logging.StreamHandler) + if args is None: + args = argparse.Namespace(log_level = logging.WARNING, + log_handler = logging.StreamHandler) - handler = args.log_handler() - handler.setFormatter(Formatter(ident, handler)) + handler = args.log_handler() + handler.setFormatter(Formatter(ident, handler)) - root_logger = logging.getLogger() - root_logger.addHandler(handler) - root_logger.setLevel(args.log_level) + root_logger = logging.getLogger() + root_logger.addHandler(handler) + root_logger.setLevel(args.log_level) - if ident and have_setproctitle and use_setproctitle: - if proctitle_extra: - setproctitle.setproctitle("%s (%s)" % (ident, proctitle_extra)) - else: - setproctitle.setproctitle(ident) + if ident and have_setproctitle and use_setproctitle: + if proctitle_extra: + setproctitle.setproctitle("%s (%s)" % (ident, proctitle_extra)) + else: + setproctitle.setproctitle(ident) def class_logger(module_logger, attribute = "logger"): - """ - Class decorator to add a class-level Logger object as a class - attribute. This allows control of debugging messages at the class - level rather than just the module level. + """ + Class decorator to add a class-level Logger object as a class + attribute. This allows control of debugging messages at the class + level rather than just the module level. - This decorator takes the module logger as an argument. - """ + This decorator takes the module logger as an argument. + """ - def decorator(cls): - setattr(cls, attribute, module_logger.getChild(cls.__name__)) - return cls - return decorator + def decorator(cls): + setattr(cls, attribute, module_logger.getChild(cls.__name__)) + return cls + return decorator def log_repr(obj, *tokens): - """ - Constructor for __repr__() strings, handles suppression of Python - IDs as needed, includes tenant_handle when available. - """ + """ + Constructor for __repr__() strings, handles suppression of Python + IDs as needed, includes tenant_handle when available. + """ - # pylint: disable=W0702 + # pylint: disable=W0702 - words = ["%s.%s" % (obj.__class__.__module__, obj.__class__.__name__)] - try: - words.append("{%s}" % obj.tenant.tenant_handle) - except: - pass + words = ["%s.%s" % (obj.__class__.__module__, obj.__class__.__name__)] + try: + words.append("{%s}" % obj.tenant.tenant_handle) + except: + pass - for token in tokens: - if token is not None: - try: - s = str(token) - except: - s = "???" - logger.exception("Failed to generate repr() string for object of type %r", type(token)) - if s: - words.append(s) + for token in tokens: + if token is not None: + try: + s = str(token) + except: + s = "???" + logger.exception("Failed to generate repr() string for object of type %r", type(token)) + if s: + words.append(s) - if show_python_ids: - words.append(" at %#x" % id(obj)) + if show_python_ids: + words.append(" at %#x" % id(obj)) - return "<" + " ".join(words) + ">" + return "<" + " ".join(words) + ">" def show_stack(stack_logger = None): - """ - Log a stack trace. - """ + """ + Log a stack trace. + """ - if stack_logger is None: - stack_logger = logger + if stack_logger is None: + stack_logger = logger - for frame in tb.format_stack(): - for line in frame.split("\n"): - if line: - stack_logger.debug("%s", line.rstrip()) + for frame in tb.format_stack(): + for line in frame.split("\n"): + if line: + stack_logger.debug("%s", line.rstrip()) diff --git a/rpki/myrpki.py b/rpki/myrpki.py index 2ae912f0..929c2a70 100644 --- a/rpki/myrpki.py +++ b/rpki/myrpki.py @@ -19,5 +19,5 @@ This is a tombstone for a program that no longer exists. """ if __name__ != "__main__": # sic -- don't break regression tests - import sys - sys.exit('"myrpki" is obsolete. Please use "rpkic" instead.') + import sys + sys.exit('"myrpki" is obsolete. Please use "rpkic" instead.') diff --git a/rpki/mysql_import.py b/rpki/mysql_import.py index 538e1916..bbb7ac22 100644 --- a/rpki/mysql_import.py +++ b/rpki/mysql_import.py @@ -52,11 +52,11 @@ from __future__ import with_statement import warnings if hasattr(warnings, "catch_warnings"): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - import MySQLdb + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + import MySQLdb else: - import MySQLdb + import MySQLdb import _mysql_exceptions diff --git a/rpki/oids.py b/rpki/oids.py index afb95020..abc928bc 100644 --- a/rpki/oids.py +++ b/rpki/oids.py @@ -82,22 +82,22 @@ id_sha256 = "2.16.840.1.101.3.4.2.1" _oid2name = {} for _sym in dir(): - if not _sym.startswith("_"): - _val = globals()[_sym] - if not isinstance(_val, str) or not all(_v.isdigit() for _v in _val.split(".")): - raise ValueError("Bad OID definition: %s = %r" % (_sym, _val)) - _oid2name[_val] = _sym.replace("_", "-") + if not _sym.startswith("_"): + _val = globals()[_sym] + if not isinstance(_val, str) or not all(_v.isdigit() for _v in _val.split(".")): + raise ValueError("Bad OID definition: %s = %r" % (_sym, _val)) + _oid2name[_val] = _sym.replace("_", "-") # pylint: disable=W0631 del _sym del _val def oid2name(oid): - """ - Translate an OID into a string suitable for printing. - """ + """ + Translate an OID into a string suitable for printing. + """ - if not isinstance(oid, (str, unicode)) or not all(o.isdigit() for o in oid.split(".")): - raise ValueError("Parameter does not look like an OID string: " + repr(oid)) + if not isinstance(oid, (str, unicode)) or not all(o.isdigit() for o in oid.split(".")): + raise ValueError("Parameter does not look like an OID string: " + repr(oid)) - return _oid2name.get(oid, oid) + return _oid2name.get(oid, oid) diff --git a/rpki/old_irdbd.py b/rpki/old_irdbd.py index 9294ee84..fca1f1d9 100644 --- a/rpki/old_irdbd.py +++ b/rpki/old_irdbd.py @@ -46,270 +46,270 @@ logger = logging.getLogger(__name__) class main(object): - def handle_list_resources(self, q_pdu, r_msg): - - r_pdu = rpki.left_right.list_resources_elt() - r_pdu.tag = q_pdu.tag - r_pdu.self_handle = q_pdu.self_handle - r_pdu.child_handle = q_pdu.child_handle - - self.cur.execute( - """ - SELECT registrant_id, valid_until - FROM registrant - WHERE registry_handle = %s AND registrant_handle = %s - """, - (q_pdu.self_handle, q_pdu.child_handle)) - - if self.cur.rowcount != 1: - raise rpki.exceptions.NotInDatabase( - "This query should have produced a single exact match, something's messed up" - " (rowcount = %d, self_handle = %s, child_handle = %s)" - % (self.cur.rowcount, q_pdu.self_handle, q_pdu.child_handle)) - - registrant_id, valid_until = self.cur.fetchone() - - r_pdu.valid_until = valid_until.strftime("%Y-%m-%dT%H:%M:%SZ") - - r_pdu.asn = rpki.resource_set.resource_set_as.from_sql( - self.cur, - """ - SELECT start_as, end_as - FROM registrant_asn - WHERE registrant_id = %s - """, - (registrant_id,)) - - r_pdu.ipv4 = rpki.resource_set.resource_set_ipv4.from_sql( - self.cur, - """ - SELECT start_ip, end_ip - FROM registrant_net - WHERE registrant_id = %s AND version = 4 - """, - (registrant_id,)) - - r_pdu.ipv6 = rpki.resource_set.resource_set_ipv6.from_sql( - self.cur, - """ - SELECT start_ip, end_ip - FROM registrant_net - WHERE registrant_id = %s AND version = 6 - """, - (registrant_id,)) - - r_msg.append(r_pdu) - - - def handle_list_roa_requests(self, q_pdu, r_msg): - - self.cur.execute( - "SELECT roa_request_id, asn FROM roa_request WHERE self_handle = %s", - (q_pdu.self_handle,)) - - for roa_request_id, asn in self.cur.fetchall(): - - r_pdu = rpki.left_right.list_roa_requests_elt() - r_pdu.tag = q_pdu.tag - r_pdu.self_handle = q_pdu.self_handle - r_pdu.asn = asn - - r_pdu.ipv4 = rpki.resource_set.roa_prefix_set_ipv4.from_sql( - self.cur, - """ - SELECT prefix, prefixlen, max_prefixlen - FROM roa_request_prefix - WHERE roa_request_id = %s AND version = 4 - """, - (roa_request_id,)) - - r_pdu.ipv6 = rpki.resource_set.roa_prefix_set_ipv6.from_sql( - self.cur, - """ - SELECT prefix, prefixlen, max_prefixlen - FROM roa_request_prefix - WHERE roa_request_id = %s AND version = 6 - """, - (roa_request_id,)) - - r_msg.append(r_pdu) - - - def handle_list_ghostbuster_requests(self, q_pdu, r_msg): - - self.cur.execute( - """ - SELECT vcard - FROM ghostbuster_request - WHERE self_handle = %s AND parent_handle = %s - """, - (q_pdu.self_handle, q_pdu.parent_handle)) - - vcards = [result[0] for result in self.cur.fetchall()] - - if not vcards: - - self.cur.execute( - """ - SELECT vcard - FROM ghostbuster_request - WHERE self_handle = %s AND parent_handle IS NULL - """, - (q_pdu.self_handle,)) - - vcards = [result[0] for result in self.cur.fetchall()] - - for vcard in vcards: - r_pdu = rpki.left_right.list_ghostbuster_requests_elt() - r_pdu.tag = q_pdu.tag - r_pdu.self_handle = q_pdu.self_handle - r_pdu.parent_handle = q_pdu.parent_handle - r_pdu.vcard = vcard - r_msg.append(r_pdu) - - - def handle_list_ee_certificate_requests(self, q_pdu, r_msg): - - self.cur.execute( - """ - SELECT ee_certificate_id, pkcs10, gski, cn, sn, eku, valid_until - FROM ee_certificate - WHERE self_handle = %s - """, - (q_pdu.self_handle,)) - - for ee_certificate_id, pkcs10, gski, cn, sn, eku, valid_until in self.cur.fetchall(): - - r_pdu = rpki.left_right.list_ee_certificate_requests_elt() - r_pdu.tag = q_pdu.tag - r_pdu.self_handle = q_pdu.self_handle - r_pdu.valid_until = valid_until.strftime("%Y-%m-%dT%H:%M:%SZ") - r_pdu.pkcs10 = rpki.x509.PKCS10(DER = pkcs10) - r_pdu.gski = gski - r_pdu.cn = cn - r_pdu.sn = sn - r_pdu.eku = eku - - r_pdu.asn = rpki.resource_set.resource_set_as.from_sql( - self.cur, - """ - SELECT start_as, end_as - FROM ee_certificate_asn - WHERE ee_certificate_id = %s - """, - (ee_certificate_id,)) + def handle_list_resources(self, q_pdu, r_msg): + + r_pdu = rpki.left_right.list_resources_elt() + r_pdu.tag = q_pdu.tag + r_pdu.self_handle = q_pdu.self_handle + r_pdu.child_handle = q_pdu.child_handle + + self.cur.execute( + """ + SELECT registrant_id, valid_until + FROM registrant + WHERE registry_handle = %s AND registrant_handle = %s + """, + (q_pdu.self_handle, q_pdu.child_handle)) + + if self.cur.rowcount != 1: + raise rpki.exceptions.NotInDatabase( + "This query should have produced a single exact match, something's messed up" + " (rowcount = %d, self_handle = %s, child_handle = %s)" + % (self.cur.rowcount, q_pdu.self_handle, q_pdu.child_handle)) + + registrant_id, valid_until = self.cur.fetchone() + + r_pdu.valid_until = valid_until.strftime("%Y-%m-%dT%H:%M:%SZ") + + r_pdu.asn = rpki.resource_set.resource_set_as.from_sql( + self.cur, + """ + SELECT start_as, end_as + FROM registrant_asn + WHERE registrant_id = %s + """, + (registrant_id,)) + + r_pdu.ipv4 = rpki.resource_set.resource_set_ipv4.from_sql( + self.cur, + """ + SELECT start_ip, end_ip + FROM registrant_net + WHERE registrant_id = %s AND version = 4 + """, + (registrant_id,)) + + r_pdu.ipv6 = rpki.resource_set.resource_set_ipv6.from_sql( + self.cur, + """ + SELECT start_ip, end_ip + FROM registrant_net + WHERE registrant_id = %s AND version = 6 + """, + (registrant_id,)) + + r_msg.append(r_pdu) + + + def handle_list_roa_requests(self, q_pdu, r_msg): + + self.cur.execute( + "SELECT roa_request_id, asn FROM roa_request WHERE self_handle = %s", + (q_pdu.self_handle,)) + + for roa_request_id, asn in self.cur.fetchall(): + + r_pdu = rpki.left_right.list_roa_requests_elt() + r_pdu.tag = q_pdu.tag + r_pdu.self_handle = q_pdu.self_handle + r_pdu.asn = asn + + r_pdu.ipv4 = rpki.resource_set.roa_prefix_set_ipv4.from_sql( + self.cur, + """ + SELECT prefix, prefixlen, max_prefixlen + FROM roa_request_prefix + WHERE roa_request_id = %s AND version = 4 + """, + (roa_request_id,)) + + r_pdu.ipv6 = rpki.resource_set.roa_prefix_set_ipv6.from_sql( + self.cur, + """ + SELECT prefix, prefixlen, max_prefixlen + FROM roa_request_prefix + WHERE roa_request_id = %s AND version = 6 + """, + (roa_request_id,)) + + r_msg.append(r_pdu) + + + def handle_list_ghostbuster_requests(self, q_pdu, r_msg): + + self.cur.execute( + """ + SELECT vcard + FROM ghostbuster_request + WHERE self_handle = %s AND parent_handle = %s + """, + (q_pdu.self_handle, q_pdu.parent_handle)) + + vcards = [result[0] for result in self.cur.fetchall()] + + if not vcards: + + self.cur.execute( + """ + SELECT vcard + FROM ghostbuster_request + WHERE self_handle = %s AND parent_handle IS NULL + """, + (q_pdu.self_handle,)) + + vcards = [result[0] for result in self.cur.fetchall()] + + for vcard in vcards: + r_pdu = rpki.left_right.list_ghostbuster_requests_elt() + r_pdu.tag = q_pdu.tag + r_pdu.self_handle = q_pdu.self_handle + r_pdu.parent_handle = q_pdu.parent_handle + r_pdu.vcard = vcard + r_msg.append(r_pdu) + + + def handle_list_ee_certificate_requests(self, q_pdu, r_msg): + + self.cur.execute( + """ + SELECT ee_certificate_id, pkcs10, gski, cn, sn, eku, valid_until + FROM ee_certificate + WHERE self_handle = %s + """, + (q_pdu.self_handle,)) + + for ee_certificate_id, pkcs10, gski, cn, sn, eku, valid_until in self.cur.fetchall(): + + r_pdu = rpki.left_right.list_ee_certificate_requests_elt() + r_pdu.tag = q_pdu.tag + r_pdu.self_handle = q_pdu.self_handle + r_pdu.valid_until = valid_until.strftime("%Y-%m-%dT%H:%M:%SZ") + r_pdu.pkcs10 = rpki.x509.PKCS10(DER = pkcs10) + r_pdu.gski = gski + r_pdu.cn = cn + r_pdu.sn = sn + r_pdu.eku = eku + + r_pdu.asn = rpki.resource_set.resource_set_as.from_sql( + self.cur, + """ + SELECT start_as, end_as + FROM ee_certificate_asn + WHERE ee_certificate_id = %s + """, + (ee_certificate_id,)) + + r_pdu.ipv4 = rpki.resource_set.resource_set_ipv4.from_sql( + self.cur, + """ + SELECT start_ip, end_ip + FROM ee_certificate_net + WHERE ee_certificate_id = %s AND version = 4 + """, + (ee_certificate_id,)) + + r_pdu.ipv6 = rpki.resource_set.resource_set_ipv6.from_sql( + self.cur, + """ + SELECT start_ip, end_ip + FROM ee_certificate_net + WHERE ee_certificate_id = %s AND version = 6 + """, + (ee_certificate_id,)) + + r_msg.append(r_pdu) + + + handle_dispatch = { + rpki.left_right.list_resources_elt : handle_list_resources, + rpki.left_right.list_roa_requests_elt : handle_list_roa_requests, + rpki.left_right.list_ghostbuster_requests_elt : handle_list_ghostbuster_requests, + rpki.left_right.list_ee_certificate_requests_elt : handle_list_ee_certificate_requests } + + def handler(self, request, q_der): + try: + + self.db.ping(True) + + r_msg = rpki.left_right.msg.reply() - r_pdu.ipv4 = rpki.resource_set.resource_set_ipv4.from_sql( - self.cur, - """ - SELECT start_ip, end_ip - FROM ee_certificate_net - WHERE ee_certificate_id = %s AND version = 4 - """, - (ee_certificate_id,)) - - r_pdu.ipv6 = rpki.resource_set.resource_set_ipv6.from_sql( - self.cur, - """ - SELECT start_ip, end_ip - FROM ee_certificate_net - WHERE ee_certificate_id = %s AND version = 6 - """, - (ee_certificate_id,)) - - r_msg.append(r_pdu) - - - handle_dispatch = { - rpki.left_right.list_resources_elt : handle_list_resources, - rpki.left_right.list_roa_requests_elt : handle_list_roa_requests, - rpki.left_right.list_ghostbuster_requests_elt : handle_list_ghostbuster_requests, - rpki.left_right.list_ee_certificate_requests_elt : handle_list_ee_certificate_requests } - - def handler(self, request, q_der): - try: - - self.db.ping(True) + try: - r_msg = rpki.left_right.msg.reply() + q_msg = rpki.left_right.cms_msg_saxify(DER = q_der).unwrap((self.bpki_ta, self.rpkid_cert)) - try: + if not isinstance(q_msg, rpki.left_right.msg) or not q_msg.is_query(): + raise rpki.exceptions.BadQuery("Unexpected %r PDU" % q_msg) - q_msg = rpki.left_right.cms_msg_saxify(DER = q_der).unwrap((self.bpki_ta, self.rpkid_cert)) - - if not isinstance(q_msg, rpki.left_right.msg) or not q_msg.is_query(): - raise rpki.exceptions.BadQuery("Unexpected %r PDU" % q_msg) - - for q_pdu in q_msg: + for q_pdu in q_msg: - try: + try: - try: - h = self.handle_dispatch[type(q_pdu)] - except KeyError: - raise rpki.exceptions.BadQuery("Unexpected %r PDU" % q_pdu) - else: - h(self, q_pdu, r_msg) + try: + h = self.handle_dispatch[type(q_pdu)] + except KeyError: + raise rpki.exceptions.BadQuery("Unexpected %r PDU" % q_pdu) + else: + h(self, q_pdu, r_msg) - except Exception, e: - logger.exception("Exception serving PDU %r", q_pdu) - r_msg.append(rpki.left_right.report_error_elt.from_exception(e, q_pdu.self_handle, q_pdu.tag)) + except Exception, e: + logger.exception("Exception serving PDU %r", q_pdu) + r_msg.append(rpki.left_right.report_error_elt.from_exception(e, q_pdu.self_handle, q_pdu.tag)) - except Exception, e: - logger.exception("Exception decoding query") - r_msg.append(rpki.left_right.report_error_elt.from_exception(e)) + except Exception, e: + logger.exception("Exception decoding query") + r_msg.append(rpki.left_right.report_error_elt.from_exception(e)) - request.send_cms_response(rpki.left_right.cms_msg_saxify().wrap(r_msg, self.irdbd_key, self.irdbd_cert)) + request.send_cms_response(rpki.left_right.cms_msg_saxify().wrap(r_msg, self.irdbd_key, self.irdbd_cert)) - except Exception, e: - logger.exception("Unhandled exception, returning HTTP failure") - request.send_error(500, "Unhandled exception %s: %s" % (e.__class__.__name__, e)) + except Exception, e: + logger.exception("Unhandled exception, returning HTTP failure") + request.send_error(500, "Unhandled exception %s: %s" % (e.__class__.__name__, e)) - def __init__(self): + def __init__(self): - os.environ["TZ"] = "UTC" - time.tzset() + os.environ["TZ"] = "UTC" + time.tzset() - parser = argparse.ArgumentParser(description = __doc__) - parser.add_argument("-c", "--config", - help = "override default location of configuration file") - parser.add_argument("-f", "--foreground", action = "store_true", - help = "do not daemonize (ignored, old_irdbd never daemonizes)") - rpki.log.argparse_setup(parser) - args = parser.parse_args() + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument("-c", "--config", + help = "override default location of configuration file") + parser.add_argument("-f", "--foreground", action = "store_true", + help = "do not daemonize (ignored, old_irdbd never daemonizes)") + rpki.log.argparse_setup(parser) + args = parser.parse_args() - rpki.log.init("irdbd", args) + rpki.log.init("irdbd", args) - self.cfg = rpki.config.parser(set_filename = args.config, section = "irdbd") + self.cfg = rpki.config.parser(set_filename = args.config, section = "irdbd") - startup_msg = self.cfg.get("startup-message", "") - if startup_msg: - logger.info(startup_msg) + startup_msg = self.cfg.get("startup-message", "") + if startup_msg: + logger.info(startup_msg) - self.cfg.set_global_flags() + self.cfg.set_global_flags() - self.db = MySQLdb.connect(user = self.cfg.get("sql-username"), - db = self.cfg.get("sql-database"), - passwd = self.cfg.get("sql-password")) + self.db = MySQLdb.connect(user = self.cfg.get("sql-username"), + db = self.cfg.get("sql-database"), + passwd = self.cfg.get("sql-password")) - self.cur = self.db.cursor() - self.db.autocommit(True) + self.cur = self.db.cursor() + self.db.autocommit(True) - self.bpki_ta = rpki.x509.X509(Auto_update = self.cfg.get("bpki-ta")) - self.rpkid_cert = rpki.x509.X509(Auto_update = self.cfg.get("rpkid-cert")) - self.irdbd_cert = rpki.x509.X509(Auto_update = self.cfg.get("irdbd-cert")) - self.irdbd_key = rpki.x509.RSA( Auto_update = self.cfg.get("irdbd-key")) + self.bpki_ta = rpki.x509.X509(Auto_update = self.cfg.get("bpki-ta")) + self.rpkid_cert = rpki.x509.X509(Auto_update = self.cfg.get("rpkid-cert")) + self.irdbd_cert = rpki.x509.X509(Auto_update = self.cfg.get("irdbd-cert")) + self.irdbd_key = rpki.x509.RSA( Auto_update = self.cfg.get("irdbd-key")) - u = urlparse.urlparse(self.cfg.get("http-url")) + u = urlparse.urlparse(self.cfg.get("http-url")) - assert u.scheme in ("", "http") and \ - u.username is None and \ - u.password is None and \ - u.params == "" and \ - u.query == "" and \ - u.fragment == "" + assert u.scheme in ("", "http") and \ + u.username is None and \ + u.password is None and \ + u.params == "" and \ + u.query == "" and \ + u.fragment == "" - rpki.http_simple.server(host = u.hostname or "localhost", - port = u.port or 443, - handlers = ((u.path, self.handler),)) + rpki.http_simple.server(host = u.hostname or "localhost", + port = u.port or 443, + handlers = ((u.path, self.handler),)) diff --git a/rpki/pubd.py b/rpki/pubd.py index f917c18d..ee258f26 100644 --- a/rpki/pubd.py +++ b/rpki/pubd.py @@ -45,252 +45,252 @@ logger = logging.getLogger(__name__) class main(object): - """ - Main program for pubd. - """ - - def __init__(self): - - os.environ.update(TZ = "UTC", - DJANGO_SETTINGS_MODULE = "rpki.django_settings.pubd") - time.tzset() - - self.irbe_cms_timestamp = None - - parser = argparse.ArgumentParser(description = __doc__) - parser.add_argument("-c", "--config", - help = "override default location of configuration file") - parser.add_argument("-f", "--foreground", action = "store_true", - help = "do not daemonize") - parser.add_argument("--pidfile", - help = "override default location of pid file") - parser.add_argument("--profile", - help = "enable profiling, saving data to PROFILE") - rpki.log.argparse_setup(parser) - args = parser.parse_args() - - self.profile = args.profile - - rpki.log.init("pubd", args) - - self.cfg = rpki.config.parser(set_filename = args.config, section = "pubd") - self.cfg.set_global_flags() - - if not args.foreground: - rpki.daemonize.daemon(pidfile = args.pidfile) - - if self.profile: - import cProfile - prof = cProfile.Profile() - try: - prof.runcall(self.main) - finally: - prof.dump_stats(self.profile) - logger.info("Dumped profile data to %s", self.profile) - else: - self.main() - - def main(self): - - if self.profile: - logger.info("Running in profile mode with output to %s", self.profile) - - import django - django.setup() - - global rpki # pylint: disable=W0602 - import rpki.pubdb # pylint: disable=W0621 - - self.bpki_ta = rpki.x509.X509(Auto_update = self.cfg.get("bpki-ta")) - self.irbe_cert = rpki.x509.X509(Auto_update = self.cfg.get("irbe-cert")) - self.pubd_cert = rpki.x509.X509(Auto_update = self.cfg.get("pubd-cert")) - self.pubd_key = rpki.x509.RSA( Auto_update = self.cfg.get("pubd-key")) - self.pubd_crl = rpki.x509.CRL( Auto_update = self.cfg.get("pubd-crl")) - - self.http_server_host = self.cfg.get("server-host", "") - self.http_server_port = self.cfg.getint("server-port") - - self.publication_base = self.cfg.get("publication-base", "publication/") - - self.rrdp_uri_base = self.cfg.get("rrdp-uri-base", - "http://%s/rrdp/" % socket.getfqdn()) - self.rrdp_expiration_interval = rpki.sundial.timedelta.parse(self.cfg.get("rrdp-expiration-interval", "6h")) - self.rrdp_publication_base = self.cfg.get("rrdp-publication-base", - "rrdp-publication/") - - try: - self.session = rpki.pubdb.models.Session.objects.get() - except rpki.pubdb.models.Session.DoesNotExist: - self.session = rpki.pubdb.models.Session.objects.create(uuid = str(uuid.uuid4()), serial = 0) - - rpki.http_simple.server( - host = self.http_server_host, - port = self.http_server_port, - handlers = (("/control", self.control_handler), - ("/client/", self.client_handler))) - - - def control_handler(self, request, q_der): - """ - Process one PDU from the IRBE. """ - - from django.db import transaction, connection - - try: - connection.cursor() # Reconnect to mysqld if necessary - q_cms = rpki.publication_control.cms_msg(DER = q_der) - q_msg = q_cms.unwrap((self.bpki_ta, self.irbe_cert)) - self.irbe_cms_timestamp = q_cms.check_replay(self.irbe_cms_timestamp, "control") - if q_msg.get("type") != "query": - raise rpki.exceptions.BadQuery("Message type is %s, expected query" % q_msg.get("type")) - r_msg = Element(rpki.publication_control.tag_msg, nsmap = rpki.publication_control.nsmap, - type = "reply", version = rpki.publication_control.version) - - try: - q_pdu = None - with transaction.atomic(): - - for q_pdu in q_msg: - if q_pdu.tag != rpki.publication_control.tag_client: - raise rpki.exceptions.BadQuery("PDU is %s, expected client" % q_pdu.tag) - client_handle = q_pdu.get("client_handle") - action = q_pdu.get("action") - if client_handle is None: - logger.info("Control %s request", action) - else: - logger.info("Control %s request for %s", action, client_handle) - - if action in ("get", "list"): - if action == "get": - clients = rpki.pubdb.models.Client.objects.get(client_handle = client_handle), - else: - clients = rpki.pubdb.models.Client.objects.all() - for client in clients: - r_pdu = SubElement(r_msg, q_pdu.tag, action = action, - client_handle = client.client_handle, base_uri = client.base_uri) - if q_pdu.get("tag"): - r_pdu.set("tag", q_pdu.get("tag")) - SubElement(r_pdu, rpki.publication_control.tag_bpki_cert).text = client.bpki_cert.get_Base64() - if client.bpki_glue is not None: - SubElement(r_pdu, rpki.publication_control.tag_bpki_glue).text = client.bpki_glue.get_Base64() - - if action in ("create", "set"): - if action == "create": - client = rpki.pubdb.models.Client(client_handle = client_handle) - else: - client = rpki.pubdb.models.Client.objects.get(client_handle = client_handle) - if q_pdu.get("base_uri"): - client.base_uri = q_pdu.get("base_uri") - bpki_cert = q_pdu.find(rpki.publication_control.tag_bpki_cert) - if bpki_cert is not None: - client.bpki_cert = rpki.x509.X509(Base64 = bpki_cert.text) - bpki_glue = q_pdu.find(rpki.publication_control.tag_bpki_glue) - if bpki_glue is not None: - client.bpki_glue = rpki.x509.X509(Base64 = bpki_glue.text) - if q_pdu.get("clear_replay_protection") == "yes": - client.last_cms_timestamp = None - client.save() - logger.debug("Stored client_handle %s, base_uri %s, bpki_cert %r, bpki_glue %r, last_cms_timestamp %s", - client.client_handle, client.base_uri, client.bpki_cert, client.bpki_glue, - client.last_cms_timestamp) - r_pdu = SubElement(r_msg, q_pdu.tag, action = action, client_handle = client_handle) - if q_pdu.get("tag"): - r_pdu.set("tag", q_pdu.get("tag")) - - if action == "destroy": - rpki.pubdb.models.Client.objects.filter(client_handle = client_handle).delete() - r_pdu = SubElement(r_msg, q_pdu.tag, action = action, client_handle = client_handle) - if q_pdu.get("tag"): - r_pdu.set("tag", q_pdu.get("tag")) - - except Exception, e: - logger.exception("Exception processing PDU %r", q_pdu) - r_pdu = SubElement(r_msg, rpki.publication_control.tag_report_error, error_code = e.__class__.__name__) - r_pdu.text = str(e) - if q_pdu.get("tag") is not None: - r_pdu.set("tag", q_pdu.get("tag")) - - request.send_cms_response(rpki.publication_control.cms_msg().wrap(r_msg, self.pubd_key, self.pubd_cert)) - - except Exception, e: - logger.exception("Unhandled exception processing control query, path %r", request.path) - request.send_error(500, "Unhandled exception %s: %s" % (e.__class__.__name__, e)) - - - client_url_regexp = re.compile("/client/([-A-Z0-9_/]+)$", re.I) - - def client_handler(self, request, q_der): - """ - Process one PDU from a client. + Main program for pubd. """ - from django.db import transaction, connection - - try: - connection.cursor() # Reconnect to mysqld if necessary - match = self.client_url_regexp.search(request.path) - if match is None: - raise rpki.exceptions.BadContactURL("Bad path: %s" % request.path) - client = rpki.pubdb.models.Client.objects.get(client_handle = match.group(1)) - q_cms = rpki.publication.cms_msg(DER = q_der) - q_msg = q_cms.unwrap((self.bpki_ta, client.bpki_cert, client.bpki_glue)) - client.last_cms_timestamp = q_cms.check_replay(client.last_cms_timestamp, client.client_handle) - client.save() - if q_msg.get("type") != "query": - raise rpki.exceptions.BadQuery("Message type is %s, expected query" % q_msg.get("type")) - r_msg = Element(rpki.publication.tag_msg, nsmap = rpki.publication.nsmap, - type = "reply", version = rpki.publication.version) - delta = None - try: - with transaction.atomic(): - for q_pdu in q_msg: - if q_pdu.get("uri"): - logger.info("Client %s request for %s", q_pdu.tag, q_pdu.get("uri")) - else: - logger.info("Client %s request", q_pdu.tag) + def __init__(self): - if q_pdu.tag == rpki.publication.tag_list: - for obj in client.publishedobject_set.all(): - r_pdu = SubElement(r_msg, q_pdu.tag, uri = obj.uri, hash = obj.hash) + os.environ.update(TZ = "UTC", + DJANGO_SETTINGS_MODULE = "rpki.django_settings.pubd") + time.tzset() + + self.irbe_cms_timestamp = None + + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument("-c", "--config", + help = "override default location of configuration file") + parser.add_argument("-f", "--foreground", action = "store_true", + help = "do not daemonize") + parser.add_argument("--pidfile", + help = "override default location of pid file") + parser.add_argument("--profile", + help = "enable profiling, saving data to PROFILE") + rpki.log.argparse_setup(parser) + args = parser.parse_args() + + self.profile = args.profile + + rpki.log.init("pubd", args) + + self.cfg = rpki.config.parser(set_filename = args.config, section = "pubd") + self.cfg.set_global_flags() + + if not args.foreground: + rpki.daemonize.daemon(pidfile = args.pidfile) + + if self.profile: + import cProfile + prof = cProfile.Profile() + try: + prof.runcall(self.main) + finally: + prof.dump_stats(self.profile) + logger.info("Dumped profile data to %s", self.profile) + else: + self.main() + + def main(self): + + if self.profile: + logger.info("Running in profile mode with output to %s", self.profile) + + import django + django.setup() + + global rpki # pylint: disable=W0602 + import rpki.pubdb # pylint: disable=W0621 + + self.bpki_ta = rpki.x509.X509(Auto_update = self.cfg.get("bpki-ta")) + self.irbe_cert = rpki.x509.X509(Auto_update = self.cfg.get("irbe-cert")) + self.pubd_cert = rpki.x509.X509(Auto_update = self.cfg.get("pubd-cert")) + self.pubd_key = rpki.x509.RSA( Auto_update = self.cfg.get("pubd-key")) + self.pubd_crl = rpki.x509.CRL( Auto_update = self.cfg.get("pubd-crl")) + + self.http_server_host = self.cfg.get("server-host", "") + self.http_server_port = self.cfg.getint("server-port") + + self.publication_base = self.cfg.get("publication-base", "publication/") + + self.rrdp_uri_base = self.cfg.get("rrdp-uri-base", + "http://%s/rrdp/" % socket.getfqdn()) + self.rrdp_expiration_interval = rpki.sundial.timedelta.parse(self.cfg.get("rrdp-expiration-interval", "6h")) + self.rrdp_publication_base = self.cfg.get("rrdp-publication-base", + "rrdp-publication/") + + try: + self.session = rpki.pubdb.models.Session.objects.get() + except rpki.pubdb.models.Session.DoesNotExist: + self.session = rpki.pubdb.models.Session.objects.create(uuid = str(uuid.uuid4()), serial = 0) + + rpki.http_simple.server( + host = self.http_server_host, + port = self.http_server_port, + handlers = (("/control", self.control_handler), + ("/client/", self.client_handler))) + + + def control_handler(self, request, q_der): + """ + Process one PDU from the IRBE. + """ + + from django.db import transaction, connection + + try: + connection.cursor() # Reconnect to mysqld if necessary + q_cms = rpki.publication_control.cms_msg(DER = q_der) + q_msg = q_cms.unwrap((self.bpki_ta, self.irbe_cert)) + self.irbe_cms_timestamp = q_cms.check_replay(self.irbe_cms_timestamp, "control") + if q_msg.get("type") != "query": + raise rpki.exceptions.BadQuery("Message type is %s, expected query" % q_msg.get("type")) + r_msg = Element(rpki.publication_control.tag_msg, nsmap = rpki.publication_control.nsmap, + type = "reply", version = rpki.publication_control.version) + + try: + q_pdu = None + with transaction.atomic(): + + for q_pdu in q_msg: + if q_pdu.tag != rpki.publication_control.tag_client: + raise rpki.exceptions.BadQuery("PDU is %s, expected client" % q_pdu.tag) + client_handle = q_pdu.get("client_handle") + action = q_pdu.get("action") + if client_handle is None: + logger.info("Control %s request", action) + else: + logger.info("Control %s request for %s", action, client_handle) + + if action in ("get", "list"): + if action == "get": + clients = rpki.pubdb.models.Client.objects.get(client_handle = client_handle), + else: + clients = rpki.pubdb.models.Client.objects.all() + for client in clients: + r_pdu = SubElement(r_msg, q_pdu.tag, action = action, + client_handle = client.client_handle, base_uri = client.base_uri) + if q_pdu.get("tag"): + r_pdu.set("tag", q_pdu.get("tag")) + SubElement(r_pdu, rpki.publication_control.tag_bpki_cert).text = client.bpki_cert.get_Base64() + if client.bpki_glue is not None: + SubElement(r_pdu, rpki.publication_control.tag_bpki_glue).text = client.bpki_glue.get_Base64() + + if action in ("create", "set"): + if action == "create": + client = rpki.pubdb.models.Client(client_handle = client_handle) + else: + client = rpki.pubdb.models.Client.objects.get(client_handle = client_handle) + if q_pdu.get("base_uri"): + client.base_uri = q_pdu.get("base_uri") + bpki_cert = q_pdu.find(rpki.publication_control.tag_bpki_cert) + if bpki_cert is not None: + client.bpki_cert = rpki.x509.X509(Base64 = bpki_cert.text) + bpki_glue = q_pdu.find(rpki.publication_control.tag_bpki_glue) + if bpki_glue is not None: + client.bpki_glue = rpki.x509.X509(Base64 = bpki_glue.text) + if q_pdu.get("clear_replay_protection") == "yes": + client.last_cms_timestamp = None + client.save() + logger.debug("Stored client_handle %s, base_uri %s, bpki_cert %r, bpki_glue %r, last_cms_timestamp %s", + client.client_handle, client.base_uri, client.bpki_cert, client.bpki_glue, + client.last_cms_timestamp) + r_pdu = SubElement(r_msg, q_pdu.tag, action = action, client_handle = client_handle) + if q_pdu.get("tag"): + r_pdu.set("tag", q_pdu.get("tag")) + + if action == "destroy": + rpki.pubdb.models.Client.objects.filter(client_handle = client_handle).delete() + r_pdu = SubElement(r_msg, q_pdu.tag, action = action, client_handle = client_handle) + if q_pdu.get("tag"): + r_pdu.set("tag", q_pdu.get("tag")) + + except Exception, e: + logger.exception("Exception processing PDU %r", q_pdu) + r_pdu = SubElement(r_msg, rpki.publication_control.tag_report_error, error_code = e.__class__.__name__) + r_pdu.text = str(e) if q_pdu.get("tag") is not None: - r_pdu.set("tag", q_pdu.get("tag")) + r_pdu.set("tag", q_pdu.get("tag")) + + request.send_cms_response(rpki.publication_control.cms_msg().wrap(r_msg, self.pubd_key, self.pubd_cert)) + + except Exception, e: + logger.exception("Unhandled exception processing control query, path %r", request.path) + request.send_error(500, "Unhandled exception %s: %s" % (e.__class__.__name__, e)) + + + client_url_regexp = re.compile("/client/([-A-Z0-9_/]+)$", re.I) + + def client_handler(self, request, q_der): + """ + Process one PDU from a client. + """ + + from django.db import transaction, connection + + try: + connection.cursor() # Reconnect to mysqld if necessary + match = self.client_url_regexp.search(request.path) + if match is None: + raise rpki.exceptions.BadContactURL("Bad path: %s" % request.path) + client = rpki.pubdb.models.Client.objects.get(client_handle = match.group(1)) + q_cms = rpki.publication.cms_msg(DER = q_der) + q_msg = q_cms.unwrap((self.bpki_ta, client.bpki_cert, client.bpki_glue)) + client.last_cms_timestamp = q_cms.check_replay(client.last_cms_timestamp, client.client_handle) + client.save() + if q_msg.get("type") != "query": + raise rpki.exceptions.BadQuery("Message type is %s, expected query" % q_msg.get("type")) + r_msg = Element(rpki.publication.tag_msg, nsmap = rpki.publication.nsmap, + type = "reply", version = rpki.publication.version) + delta = None + try: + with transaction.atomic(): + for q_pdu in q_msg: + if q_pdu.get("uri"): + logger.info("Client %s request for %s", q_pdu.tag, q_pdu.get("uri")) + else: + logger.info("Client %s request", q_pdu.tag) + + if q_pdu.tag == rpki.publication.tag_list: + for obj in client.publishedobject_set.all(): + r_pdu = SubElement(r_msg, q_pdu.tag, uri = obj.uri, hash = obj.hash) + if q_pdu.get("tag") is not None: + r_pdu.set("tag", q_pdu.get("tag")) + + else: + assert q_pdu.tag in (rpki.publication.tag_publish, rpki.publication.tag_withdraw) + if delta is None: + delta = self.session.new_delta(rpki.sundial.now() + self.rrdp_expiration_interval) + client.check_allowed_uri(q_pdu.get("uri")) + if q_pdu.tag == rpki.publication.tag_publish: + der = q_pdu.text.decode("base64") + logger.info("Publishing %s", rpki.x509.uri_dispatch(q_pdu.get("uri"))(DER = der).tracking_data(q_pdu.get("uri"))) + delta.publish(client, der, q_pdu.get("uri"), q_pdu.get("hash")) + else: + logger.info("Withdrawing %s", q_pdu.get("uri")) + delta.withdraw(client, q_pdu.get("uri"), q_pdu.get("hash")) + r_pdu = SubElement(r_msg, q_pdu.tag, uri = q_pdu.get("uri")) + if q_pdu.get("tag") is not None: + r_pdu.set("tag", q_pdu.get("tag")) + + if delta is not None: + delta.activate() + self.session.generate_snapshot() + self.session.expire_deltas() + + except Exception, e: + logger.exception("Exception processing PDU %r", q_pdu) + r_pdu = SubElement(r_msg, rpki.publication.tag_report_error, error_code = e.__class__.__name__) + r_pdu.text = str(e) + if q_pdu.get("tag") is not None: + r_pdu.set("tag", q_pdu.get("tag")) else: - assert q_pdu.tag in (rpki.publication.tag_publish, rpki.publication.tag_withdraw) - if delta is None: - delta = self.session.new_delta(rpki.sundial.now() + self.rrdp_expiration_interval) - client.check_allowed_uri(q_pdu.get("uri")) - if q_pdu.tag == rpki.publication.tag_publish: - der = q_pdu.text.decode("base64") - logger.info("Publishing %s", rpki.x509.uri_dispatch(q_pdu.get("uri"))(DER = der).tracking_data(q_pdu.get("uri"))) - delta.publish(client, der, q_pdu.get("uri"), q_pdu.get("hash")) - else: - logger.info("Withdrawing %s", q_pdu.get("uri")) - delta.withdraw(client, q_pdu.get("uri"), q_pdu.get("hash")) - r_pdu = SubElement(r_msg, q_pdu.tag, uri = q_pdu.get("uri")) - if q_pdu.get("tag") is not None: - r_pdu.set("tag", q_pdu.get("tag")) - - if delta is not None: - delta.activate() - self.session.generate_snapshot() - self.session.expire_deltas() - - except Exception, e: - logger.exception("Exception processing PDU %r", q_pdu) - r_pdu = SubElement(r_msg, rpki.publication.tag_report_error, error_code = e.__class__.__name__) - r_pdu.text = str(e) - if q_pdu.get("tag") is not None: - r_pdu.set("tag", q_pdu.get("tag")) - - else: - if delta is not None: - self.session.synchronize_rrdp_files(self.rrdp_publication_base, self.rrdp_uri_base) - delta.update_rsync_files(self.publication_base) - - request.send_cms_response(rpki.publication.cms_msg().wrap(r_msg, self.pubd_key, self.pubd_cert, self.pubd_crl)) - - except Exception, e: - logger.exception("Unhandled exception processing client query, path %r", request.path) - request.send_error(500, "Could not process PDU: %s" % e) + if delta is not None: + self.session.synchronize_rrdp_files(self.rrdp_publication_base, self.rrdp_uri_base) + delta.update_rsync_files(self.publication_base) + + request.send_cms_response(rpki.publication.cms_msg().wrap(r_msg, self.pubd_key, self.pubd_cert, self.pubd_crl)) + + except Exception, e: + logger.exception("Unhandled exception processing client query, path %r", request.path) + request.send_error(500, "Could not process PDU: %s" % e) diff --git a/rpki/pubdb/models.py b/rpki/pubdb/models.py index 2b6d67e4..46dcf493 100644 --- a/rpki/pubdb/models.py +++ b/rpki/pubdb/models.py @@ -48,266 +48,266 @@ rrdp_tag_withdraw = rrdp_xmlns + "withdraw" # sure quite where to put it at the moment. def DERSubElement(elt, name, der, attrib = None, **kwargs): - """ - Convenience wrapper around SubElement for use with Base64 text. - """ + """ + Convenience wrapper around SubElement for use with Base64 text. + """ - se = SubElement(elt, name, attrib, **kwargs) - se.text = rpki.x509.base64_with_linebreaks(der) - se.tail = "\n" - return se + se = SubElement(elt, name, attrib, **kwargs) + se.text = rpki.x509.base64_with_linebreaks(der) + se.tail = "\n" + return se class Client(models.Model): - client_handle = models.CharField(unique = True, max_length = 255) - base_uri = models.TextField() - bpki_cert = CertificateField() - bpki_glue = CertificateField(null = True) - last_cms_timestamp = SundialField(blank = True, null = True) + client_handle = models.CharField(unique = True, max_length = 255) + base_uri = models.TextField() + bpki_cert = CertificateField() + bpki_glue = CertificateField(null = True) + last_cms_timestamp = SundialField(blank = True, null = True) - def check_allowed_uri(self, uri): - """ - Make sure that a target URI is within this client's allowed URI space. - """ + def check_allowed_uri(self, uri): + """ + Make sure that a target URI is within this client's allowed URI space. + """ - if not uri.startswith(self.base_uri): - raise rpki.exceptions.ForbiddenURI + if not uri.startswith(self.base_uri): + raise rpki.exceptions.ForbiddenURI class Session(models.Model): - uuid = models.CharField(unique = True, max_length=36) - serial = models.BigIntegerField() - snapshot = models.TextField(blank = True) - hash = models.CharField(max_length = 64, blank = True) - - ## @var keep_all_rrdp_files - # Debugging flag to prevent expiration of old RRDP files. - # This simplifies debugging delta code. Need for this - # may go away once RRDP is fully integrated into rcynic. - keep_all_rrdp_files = False - - def new_delta(self, expires): - """ - Construct a new delta associated with this session. - """ + uuid = models.CharField(unique = True, max_length=36) + serial = models.BigIntegerField() + snapshot = models.TextField(blank = True) + hash = models.CharField(max_length = 64, blank = True) + + ## @var keep_all_rrdp_files + # Debugging flag to prevent expiration of old RRDP files. + # This simplifies debugging delta code. Need for this + # may go away once RRDP is fully integrated into rcynic. + keep_all_rrdp_files = False + + def new_delta(self, expires): + """ + Construct a new delta associated with this session. + """ + + delta = Delta(session = self, + serial = self.serial + 1, + expires = expires) + delta.elt = Element(rrdp_tag_delta, + nsmap = rrdp_nsmap, + version = rrdp_version, + session_id = self.uuid, + serial = str(delta.serial)) + return delta + + + def expire_deltas(self): + """ + Delete deltas whose expiration date has passed. + """ + + self.delta_set.filter(expires__lt = rpki.sundial.now()).delete() + + + def generate_snapshot(self): + """ + Generate an XML snapshot of this session. + """ + + xml = Element(rrdp_tag_snapshot, nsmap = rrdp_nsmap, + version = rrdp_version, + session_id = self.uuid, + serial = str(self.serial)) + xml.text = "\n" + for obj in self.publishedobject_set.all(): + DERSubElement(xml, rrdp_tag_publish, + der = obj.der, + uri = obj.uri) + rpki.relaxng.rrdp.assertValid(xml) + self.snapshot = ElementToString(xml, pretty_print = True) + self.hash = rpki.x509.sha256(self.snapshot).encode("hex") + self.save() + + + @property + def snapshot_fn(self): + return "%s/snapshot/%s.xml" % (self.uuid, self.serial) + + + @property + def notification_fn(self): + return "notify.xml" + + + @staticmethod + def _write_rrdp_file(fn, text, rrdp_publication_base, overwrite = False): + if overwrite or not os.path.exists(os.path.join(rrdp_publication_base, fn)): + tn = os.path.join(rrdp_publication_base, fn + ".%s.tmp" % os.getpid()) + if not os.path.isdir(os.path.dirname(tn)): + os.makedirs(os.path.dirname(tn)) + with open(tn, "w") as f: + f.write(text) + os.rename(tn, os.path.join(rrdp_publication_base, fn)) + + + @staticmethod + def _rrdp_filename_to_uri(fn, rrdp_uri_base): + return "%s/%s" % (rrdp_uri_base.rstrip("/"), fn) + + + def _generate_update_xml(self, rrdp_uri_base): + xml = Element(rrdp_tag_notification, nsmap = rrdp_nsmap, + version = rrdp_version, + session_id = self.uuid, + serial = str(self.serial)) + SubElement(xml, rrdp_tag_snapshot, + uri = self._rrdp_filename_to_uri(self.snapshot_fn, rrdp_uri_base), + hash = self.hash) + for delta in self.delta_set.all(): + SubElement(xml, rrdp_tag_delta, + uri = self._rrdp_filename_to_uri(delta.fn, rrdp_uri_base), + hash = delta.hash, + serial = str(delta.serial)) + rpki.relaxng.rrdp.assertValid(xml) + return ElementToString(xml, pretty_print = True) + + + def synchronize_rrdp_files(self, rrdp_publication_base, rrdp_uri_base): + """ + Write current RRDP files to disk, clean up old files and directories. + """ + + current_filenames = set() + + for delta in self.delta_set.all(): + self._write_rrdp_file(delta.fn, delta.xml, rrdp_publication_base) + current_filenames.add(delta.fn) + + self._write_rrdp_file(self.snapshot_fn, self.snapshot, rrdp_publication_base) + current_filenames.add(self.snapshot_fn) + + self._write_rrdp_file(self.notification_fn, self._generate_update_xml(rrdp_uri_base), + rrdp_publication_base, overwrite = True) + current_filenames.add(self.notification_fn) + + if not self.keep_all_rrdp_files: + for root, dirs, files in os.walk(rrdp_publication_base, topdown = False): + for fn in files: + fn = os.path.join(root, fn) + if fn[len(rrdp_publication_base):].lstrip("/") not in current_filenames: + os.remove(fn) + for dn in dirs: + try: + os.rmdir(os.path.join(root, dn)) + except OSError: + pass - delta = Delta(session = self, - serial = self.serial + 1, - expires = expires) - delta.elt = Element(rrdp_tag_delta, - nsmap = rrdp_nsmap, - version = rrdp_version, - session_id = self.uuid, - serial = str(delta.serial)) - return delta +class Delta(models.Model): + serial = models.BigIntegerField() + xml = models.TextField() + hash = models.CharField(max_length = 64) + expires = SundialField() + session = models.ForeignKey(Session) - def expire_deltas(self): - """ - Delete deltas whose expiration date has passed. - """ - self.delta_set.filter(expires__lt = rpki.sundial.now()).delete() + @staticmethod + def _uri_to_filename(uri, publication_base): + if not uri.startswith("rsync://"): + raise rpki.exceptions.BadURISyntax(uri) + path = uri.split("/")[4:] + path.insert(0, publication_base.rstrip("/")) + filename = "/".join(path) + if "/../" in filename or filename.endswith("/.."): + raise rpki.exceptions.BadURISyntax(filename) + return filename - def generate_snapshot(self): - """ - Generate an XML snapshot of this session. - """ + @property + def fn(self): + return "%s/deltas/%s.xml" % (self.session.uuid, self.serial) - xml = Element(rrdp_tag_snapshot, nsmap = rrdp_nsmap, - version = rrdp_version, - session_id = self.uuid, - serial = str(self.serial)) - xml.text = "\n" - for obj in self.publishedobject_set.all(): - DERSubElement(xml, rrdp_tag_publish, - der = obj.der, - uri = obj.uri) - rpki.relaxng.rrdp.assertValid(xml) - self.snapshot = ElementToString(xml, pretty_print = True) - self.hash = rpki.x509.sha256(self.snapshot).encode("hex") - self.save() - - - @property - def snapshot_fn(self): - return "%s/snapshot/%s.xml" % (self.uuid, self.serial) - - - @property - def notification_fn(self): - return "notify.xml" - - - @staticmethod - def _write_rrdp_file(fn, text, rrdp_publication_base, overwrite = False): - if overwrite or not os.path.exists(os.path.join(rrdp_publication_base, fn)): - tn = os.path.join(rrdp_publication_base, fn + ".%s.tmp" % os.getpid()) - if not os.path.isdir(os.path.dirname(tn)): - os.makedirs(os.path.dirname(tn)) - with open(tn, "w") as f: - f.write(text) - os.rename(tn, os.path.join(rrdp_publication_base, fn)) - - - @staticmethod - def _rrdp_filename_to_uri(fn, rrdp_uri_base): - return "%s/%s" % (rrdp_uri_base.rstrip("/"), fn) - - - def _generate_update_xml(self, rrdp_uri_base): - xml = Element(rrdp_tag_notification, nsmap = rrdp_nsmap, - version = rrdp_version, - session_id = self.uuid, - serial = str(self.serial)) - SubElement(xml, rrdp_tag_snapshot, - uri = self._rrdp_filename_to_uri(self.snapshot_fn, rrdp_uri_base), - hash = self.hash) - for delta in self.delta_set.all(): - SubElement(xml, rrdp_tag_delta, - uri = self._rrdp_filename_to_uri(delta.fn, rrdp_uri_base), - hash = delta.hash, - serial = str(delta.serial)) - rpki.relaxng.rrdp.assertValid(xml) - return ElementToString(xml, pretty_print = True) - - - def synchronize_rrdp_files(self, rrdp_publication_base, rrdp_uri_base): - """ - Write current RRDP files to disk, clean up old files and directories. - """ - current_filenames = set() - - for delta in self.delta_set.all(): - self._write_rrdp_file(delta.fn, delta.xml, rrdp_publication_base) - current_filenames.add(delta.fn) - - self._write_rrdp_file(self.snapshot_fn, self.snapshot, rrdp_publication_base) - current_filenames.add(self.snapshot_fn) - - self._write_rrdp_file(self.notification_fn, self._generate_update_xml(rrdp_uri_base), - rrdp_publication_base, overwrite = True) - current_filenames.add(self.notification_fn) - - if not self.keep_all_rrdp_files: - for root, dirs, files in os.walk(rrdp_publication_base, topdown = False): - for fn in files: - fn = os.path.join(root, fn) - if fn[len(rrdp_publication_base):].lstrip("/") not in current_filenames: - os.remove(fn) - for dn in dirs: - try: - os.rmdir(os.path.join(root, dn)) - except OSError: - pass + def activate(self): + rpki.relaxng.rrdp.assertValid(self.elt) + self.xml = ElementToString(self.elt, pretty_print = True) + self.hash = rpki.x509.sha256(self.xml).encode("hex") + self.save() + self.session.serial += 1 + self.session.save() -class Delta(models.Model): - serial = models.BigIntegerField() - xml = models.TextField() - hash = models.CharField(max_length = 64) - expires = SundialField() - session = models.ForeignKey(Session) - - - @staticmethod - def _uri_to_filename(uri, publication_base): - if not uri.startswith("rsync://"): - raise rpki.exceptions.BadURISyntax(uri) - path = uri.split("/")[4:] - path.insert(0, publication_base.rstrip("/")) - filename = "/".join(path) - if "/../" in filename or filename.endswith("/.."): - raise rpki.exceptions.BadURISyntax(filename) - return filename - - - @property - def fn(self): - return "%s/deltas/%s.xml" % (self.session.uuid, self.serial) - - - def activate(self): - rpki.relaxng.rrdp.assertValid(self.elt) - self.xml = ElementToString(self.elt, pretty_print = True) - self.hash = rpki.x509.sha256(self.xml).encode("hex") - self.save() - self.session.serial += 1 - self.session.save() - - - def publish(self, client, der, uri, obj_hash): - try: - obj = client.publishedobject_set.get(session = self.session, uri = uri) - if obj.hash == obj_hash: - obj.delete() - elif obj_hash is None: - raise rpki.exceptions.ExistingObjectAtURI("Object already published at %s" % uri) - else: - raise rpki.exceptions.DifferentObjectAtURI("Found different object at %s (old %s, new %s)" % (uri, obj.hash, obj_hash)) - except rpki.pubdb.models.PublishedObject.DoesNotExist: - pass - logger.debug("Publishing %s", uri) - PublishedObject.objects.create(session = self.session, client = client, der = der, uri = uri, - hash = rpki.x509.sha256(der).encode("hex")) - se = DERSubElement(self.elt, rrdp_tag_publish, der = der, uri = uri) - if obj_hash is not None: - se.set("hash", obj_hash) - rpki.relaxng.rrdp.assertValid(self.elt) - - - def withdraw(self, client, uri, obj_hash): - obj = client.publishedobject_set.get(session = self.session, uri = uri) - if obj.hash != obj_hash: - raise rpki.exceptions.DifferentObjectAtURI("Found different object at %s (old %s, new %s)" % (uri, obj.hash, obj_hash)) - logger.debug("Withdrawing %s", uri) - obj.delete() - SubElement(self.elt, rrdp_tag_withdraw, uri = uri, hash = obj_hash).tail = "\n" - rpki.relaxng.rrdp.assertValid(self.elt) - - - def update_rsync_files(self, publication_base): - from errno import ENOENT - min_path_len = len(publication_base.rstrip("/")) - for pdu in self.elt: - assert pdu.tag in (rrdp_tag_publish, rrdp_tag_withdraw) - fn = self._uri_to_filename(pdu.get("uri"), publication_base) - if pdu.tag == rrdp_tag_publish: - tn = fn + ".tmp" - dn = os.path.dirname(fn) - if not os.path.isdir(dn): - os.makedirs(dn) - with open(tn, "wb") as f: - f.write(pdu.text.decode("base64")) - os.rename(tn, fn) - else: + def publish(self, client, der, uri, obj_hash): try: - os.remove(fn) - except OSError, e: - if e.errno != ENOENT: - raise - dn = os.path.dirname(fn) - while len(dn) > min_path_len: - try: - os.rmdir(dn) - except OSError: - break - else: - dn = os.path.dirname(dn) - del self.elt + obj = client.publishedobject_set.get(session = self.session, uri = uri) + if obj.hash == obj_hash: + obj.delete() + elif obj_hash is None: + raise rpki.exceptions.ExistingObjectAtURI("Object already published at %s" % uri) + else: + raise rpki.exceptions.DifferentObjectAtURI("Found different object at %s (old %s, new %s)" % (uri, obj.hash, obj_hash)) + except rpki.pubdb.models.PublishedObject.DoesNotExist: + pass + logger.debug("Publishing %s", uri) + PublishedObject.objects.create(session = self.session, client = client, der = der, uri = uri, + hash = rpki.x509.sha256(der).encode("hex")) + se = DERSubElement(self.elt, rrdp_tag_publish, der = der, uri = uri) + if obj_hash is not None: + se.set("hash", obj_hash) + rpki.relaxng.rrdp.assertValid(self.elt) + + + def withdraw(self, client, uri, obj_hash): + obj = client.publishedobject_set.get(session = self.session, uri = uri) + if obj.hash != obj_hash: + raise rpki.exceptions.DifferentObjectAtURI("Found different object at %s (old %s, new %s)" % (uri, obj.hash, obj_hash)) + logger.debug("Withdrawing %s", uri) + obj.delete() + SubElement(self.elt, rrdp_tag_withdraw, uri = uri, hash = obj_hash).tail = "\n" + rpki.relaxng.rrdp.assertValid(self.elt) + + + def update_rsync_files(self, publication_base): + from errno import ENOENT + min_path_len = len(publication_base.rstrip("/")) + for pdu in self.elt: + assert pdu.tag in (rrdp_tag_publish, rrdp_tag_withdraw) + fn = self._uri_to_filename(pdu.get("uri"), publication_base) + if pdu.tag == rrdp_tag_publish: + tn = fn + ".tmp" + dn = os.path.dirname(fn) + if not os.path.isdir(dn): + os.makedirs(dn) + with open(tn, "wb") as f: + f.write(pdu.text.decode("base64")) + os.rename(tn, fn) + else: + try: + os.remove(fn) + except OSError, e: + if e.errno != ENOENT: + raise + dn = os.path.dirname(fn) + while len(dn) > min_path_len: + try: + os.rmdir(dn) + except OSError: + break + else: + dn = os.path.dirname(dn) + del self.elt class PublishedObject(models.Model): - uri = models.CharField(max_length = 255) - der = models.BinaryField() - hash = models.CharField(max_length = 64) - client = models.ForeignKey(Client) - session = models.ForeignKey(Session) - - class Meta: # pylint: disable=C1001,W0232 - unique_together = (("session", "hash"), - ("session", "uri")) + uri = models.CharField(max_length = 255) + der = models.BinaryField() + hash = models.CharField(max_length = 64) + client = models.ForeignKey(Client) + session = models.ForeignKey(Session) + + class Meta: # pylint: disable=C1001,W0232 + unique_together = (("session", "hash"), + ("session", "uri")) diff --git a/rpki/publication.py b/rpki/publication.py index 16824d05..393e078e 100644 --- a/rpki/publication.py +++ b/rpki/publication.py @@ -51,34 +51,34 @@ allowed_content_types = (content_type,) def raise_if_error(pdu): - """ - Raise an appropriate error if this is a PDU. - - As a convenience, this will also accept a PDU and raise an - appropriate error if it contains any PDUs or if - the is not a reply. - """ - - if pdu.tag == tag_report_error: - code = pdu.get("error_code") - logger.debug(" code %r", code) - e = getattr(rpki.exceptions, code, None) - if e is not None and issubclass(e, rpki.exceptions.RPKI_Exception): - raise e(pdu.text) - else: - raise rpki.exceptions.BadPublicationReply("Unexpected response from pubd: %r, %r" % (code, pdu)) - - if pdu.tag == tag_msg: - if pdu.get("type") != "reply": - raise rpki.exceptions.BadPublicationReply("Unexpected response from pubd: expected reply, got %r" % pdu.get("type")) - for p in pdu: - raise_if_error(p) + """ + Raise an appropriate error if this is a PDU. + + As a convenience, this will also accept a PDU and raise an + appropriate error if it contains any PDUs or if + the is not a reply. + """ + + if pdu.tag == tag_report_error: + code = pdu.get("error_code") + logger.debug(" code %r", code) + e = getattr(rpki.exceptions, code, None) + if e is not None and issubclass(e, rpki.exceptions.RPKI_Exception): + raise e(pdu.text) + else: + raise rpki.exceptions.BadPublicationReply("Unexpected response from pubd: %r, %r" % (code, pdu)) + + if pdu.tag == tag_msg: + if pdu.get("type") != "reply": + raise rpki.exceptions.BadPublicationReply("Unexpected response from pubd: expected reply, got %r" % pdu.get("type")) + for p in pdu: + raise_if_error(p) class cms_msg(rpki.x509.XML_CMS_object): - """ - CMS-signed publication PDU. - """ + """ + CMS-signed publication PDU. + """ - encoding = "us-ascii" - schema = rpki.relaxng.publication + encoding = "us-ascii" + schema = rpki.relaxng.publication diff --git a/rpki/publication_control.py b/rpki/publication_control.py index ddb9d417..b0668eef 100644 --- a/rpki/publication_control.py +++ b/rpki/publication_control.py @@ -44,31 +44,31 @@ tag_report_error = rpki.relaxng.publication_control.xmlns + "report_error" def raise_if_error(pdu): - """ - Raise an appropriate error if this is a PDU. + """ + Raise an appropriate error if this is a PDU. - As a convience, this will also accept a PDU and raise an - appropriate error if it contains any PDUs. - """ + As a convience, this will also accept a PDU and raise an + appropriate error if it contains any PDUs. + """ - if pdu.tag == tag_report_error: - code = pdu.get("error_code") - logger.debug(" code %r", code) - e = getattr(rpki.exceptions, code, None) - if e is not None and issubclass(e, rpki.exceptions.RPKI_Exception): - raise e(pdu.text) - else: - raise rpki.exceptions.BadPublicationReply("Unexpected response from pubd: %r, %r" % (code, pdu)) + if pdu.tag == tag_report_error: + code = pdu.get("error_code") + logger.debug(" code %r", code) + e = getattr(rpki.exceptions, code, None) + if e is not None and issubclass(e, rpki.exceptions.RPKI_Exception): + raise e(pdu.text) + else: + raise rpki.exceptions.BadPublicationReply("Unexpected response from pubd: %r, %r" % (code, pdu)) - if pdu.tag == tag_msg: - for p in pdu: - raise_if_error(p) + if pdu.tag == tag_msg: + for p in pdu: + raise_if_error(p) class cms_msg(rpki.x509.XML_CMS_object): - """ - CMS-signed publication control PDU. - """ + """ + CMS-signed publication control PDU. + """ - encoding = "us-ascii" - schema = rpki.relaxng.publication_control + encoding = "us-ascii" + schema = rpki.relaxng.publication_control diff --git a/rpki/rcynic.py b/rpki/rcynic.py index a36e4a4e..3307e926 100644 --- a/rpki/rcynic.py +++ b/rpki/rcynic.py @@ -25,142 +25,142 @@ import rpki.resource_set from xml.etree.ElementTree import ElementTree class UnknownObject(rpki.exceptions.RPKI_Exception): - """ - Unrecognized object in rcynic result cache. - """ + """ + Unrecognized object in rcynic result cache. + """ class NotRsyncURI(rpki.exceptions.RPKI_Exception): - """ - URI is not an rsync URI. - """ + """ + URI is not an rsync URI. + """ class rcynic_object(object): - """ - An object read from rcynic cache. - """ + """ + An object read from rcynic cache. + """ - def __init__(self, filename, **kwargs): - self.filename = filename - for k, v in kwargs.iteritems(): - setattr(self, k, v) - self.obj = self.obj_class(DER_file = filename) + def __init__(self, filename, **kwargs): + self.filename = filename + for k, v in kwargs.iteritems(): + setattr(self, k, v) + self.obj = self.obj_class(DER_file = filename) - def __repr__(self): - return "<%s %s %s at 0x%x>" % (self.__class__.__name__, self.uri, self.resources, id(self)) + def __repr__(self): + return "<%s %s %s at 0x%x>" % (self.__class__.__name__, self.uri, self.resources, id(self)) - def show_attrs(self, *attrs): - """ - Print a bunch of object attributes, quietly ignoring any that - might be missing. - """ + def show_attrs(self, *attrs): + """ + Print a bunch of object attributes, quietly ignoring any that + might be missing. + """ - for a in attrs: - try: - print "%s: %s" % (a.capitalize(), getattr(self, a)) - except AttributeError: - pass + for a in attrs: + try: + print "%s: %s" % (a.capitalize(), getattr(self, a)) + except AttributeError: + pass - def show(self): - """ - Print common object attributes. - """ + def show(self): + """ + Print common object attributes. + """ - self.show_attrs("filename", "uri", "status", "timestamp") + self.show_attrs("filename", "uri", "status", "timestamp") class rcynic_certificate(rcynic_object): - """ - A certificate from rcynic cache. - """ - - obj_class = rpki.x509.X509 - - def __init__(self, filename, **kwargs): - rcynic_object.__init__(self, filename, **kwargs) - self.notBefore = self.obj.getNotBefore() - self.notAfter = self.obj.getNotAfter() - self.aia_uri = self.obj.get_aia_uri() - self.sia_directory_uri = self.obj.get_sia_directory_uri() - self.manifest_uri = self.obj.get_sia_manifest_uri() - self.resources = self.obj.get_3779resources() - self.is_ca = self.obj.is_CA() - self.serial = self.obj.getSerial() - self.issuer = self.obj.getIssuer() - self.subject = self.obj.getSubject() - self.ski = self.obj.hSKI() - self.aki = self.obj.hAKI() - - def show(self): """ - Print certificate attributes. + A certificate from rcynic cache. """ - rcynic_object.show(self) - self.show_attrs("notBefore", "notAfter", "aia_uri", "sia_directory_uri", "resources") + obj_class = rpki.x509.X509 + + def __init__(self, filename, **kwargs): + rcynic_object.__init__(self, filename, **kwargs) + self.notBefore = self.obj.getNotBefore() + self.notAfter = self.obj.getNotAfter() + self.aia_uri = self.obj.get_aia_uri() + self.sia_directory_uri = self.obj.get_sia_directory_uri() + self.manifest_uri = self.obj.get_sia_manifest_uri() + self.resources = self.obj.get_3779resources() + self.is_ca = self.obj.is_CA() + self.serial = self.obj.getSerial() + self.issuer = self.obj.getIssuer() + self.subject = self.obj.getSubject() + self.ski = self.obj.hSKI() + self.aki = self.obj.hAKI() + + def show(self): + """ + Print certificate attributes. + """ + + rcynic_object.show(self) + self.show_attrs("notBefore", "notAfter", "aia_uri", "sia_directory_uri", "resources") class rcynic_roa(rcynic_object): - """ - A ROA from rcynic cache. - """ - - obj_class = rpki.x509.ROA - - def __init__(self, filename, **kwargs): - rcynic_object.__init__(self, filename, **kwargs) - self.obj.extract() - self.asID = self.obj.get_POW().getASID() - self.prefix_sets = [] - v4, v6 = self.obj.get_POW().getPrefixes() - if v4: - self.prefix_sets.append(rpki.resource_set.roa_prefix_set_ipv4([ - rpki.resource_set.roa_prefix_ipv4(p[0], p[1], p[2]) for p in v4])) - if v6: - self.prefix_sets.append(rpki.resource_set.roa_prefix_set_ipv6([ - rpki.resource_set.roa_prefix_ipv6(p[0], p[1], p[2]) for p in v6])) - self.ee = rpki.x509.X509(POW = self.obj.get_POW().certs()[0]) - self.notBefore = self.ee.getNotBefore() - self.notAfter = self.ee.getNotAfter() - self.aia_uri = self.ee.get_aia_uri() - self.resources = self.ee.get_3779resources() - self.issuer = self.ee.getIssuer() - self.serial = self.ee.getSerial() - self.subject = self.ee.getSubject() - self.aki = self.ee.hAKI() - self.ski = self.ee.hSKI() - - def show(self): """ - Print ROA attributes. + A ROA from rcynic cache. """ - rcynic_object.show(self) - self.show_attrs("notBefore", "notAfter", "aia_uri", "resources", "asID") - if self.prefix_sets: - print "Prefixes:", ",".join(str(i) for i in self.prefix_sets) + obj_class = rpki.x509.ROA + + def __init__(self, filename, **kwargs): + rcynic_object.__init__(self, filename, **kwargs) + self.obj.extract() + self.asID = self.obj.get_POW().getASID() + self.prefix_sets = [] + v4, v6 = self.obj.get_POW().getPrefixes() + if v4: + self.prefix_sets.append(rpki.resource_set.roa_prefix_set_ipv4([ + rpki.resource_set.roa_prefix_ipv4(p[0], p[1], p[2]) for p in v4])) + if v6: + self.prefix_sets.append(rpki.resource_set.roa_prefix_set_ipv6([ + rpki.resource_set.roa_prefix_ipv6(p[0], p[1], p[2]) for p in v6])) + self.ee = rpki.x509.X509(POW = self.obj.get_POW().certs()[0]) + self.notBefore = self.ee.getNotBefore() + self.notAfter = self.ee.getNotAfter() + self.aia_uri = self.ee.get_aia_uri() + self.resources = self.ee.get_3779resources() + self.issuer = self.ee.getIssuer() + self.serial = self.ee.getSerial() + self.subject = self.ee.getSubject() + self.aki = self.ee.hAKI() + self.ski = self.ee.hSKI() + + def show(self): + """ + Print ROA attributes. + """ + + rcynic_object.show(self) + self.show_attrs("notBefore", "notAfter", "aia_uri", "resources", "asID") + if self.prefix_sets: + print "Prefixes:", ",".join(str(i) for i in self.prefix_sets) class rcynic_ghostbuster(rcynic_object): - """ - Ghostbuster record from the rcynic cache. - """ - - obj_class = rpki.x509.Ghostbuster - - def __init__(self, *args, **kwargs): - rcynic_object.__init__(self, *args, **kwargs) - self.obj.extract() - self.vcard = self.obj.get_content() - self.ee = rpki.x509.X509(POW = self.obj.get_POW().certs()[0]) - self.notBefore = self.ee.getNotBefore() - self.notAfter = self.ee.getNotAfter() - self.aia_uri = self.ee.get_aia_uri() - self.issuer = self.ee.getIssuer() - self.serial = self.ee.getSerial() - self.subject = self.ee.getSubject() - self.aki = self.ee.hAKI() - self.ski = self.ee.hSKI() - - def show(self): - rcynic_object.show(self) - self.show_attrs("notBefore", "notAfter", "vcard") + """ + Ghostbuster record from the rcynic cache. + """ + + obj_class = rpki.x509.Ghostbuster + + def __init__(self, *args, **kwargs): + rcynic_object.__init__(self, *args, **kwargs) + self.obj.extract() + self.vcard = self.obj.get_content() + self.ee = rpki.x509.X509(POW = self.obj.get_POW().certs()[0]) + self.notBefore = self.ee.getNotBefore() + self.notAfter = self.ee.getNotAfter() + self.aia_uri = self.ee.get_aia_uri() + self.issuer = self.ee.getIssuer() + self.serial = self.ee.getSerial() + self.subject = self.ee.getSubject() + self.aki = self.ee.hAKI() + self.ski = self.ee.hSKI() + + def show(self): + rcynic_object.show(self) + self.show_attrs("notBefore", "notAfter", "vcard") file_name_classes = { ".cer" : rcynic_certificate, @@ -168,112 +168,112 @@ file_name_classes = { ".roa" : rcynic_roa } class rcynic_file_iterator(object): - """ - Iterate over files in an rcynic output tree, yielding a Python - representation of each object found. - """ - - def __init__(self, rcynic_root, - authenticated_subdir = "authenticated"): - self.rcynic_dir = os.path.join(rcynic_root, authenticated_subdir) - - def __iter__(self): - for root, dirs, files in os.walk(self.rcynic_dir): # pylint: disable=W0612 - for filename in files: - filename = os.path.join(root, filename) - ext = os.path.splitext(filename)[1] - if ext in file_name_classes: - yield file_name_classes[ext](filename) + """ + Iterate over files in an rcynic output tree, yielding a Python + representation of each object found. + """ + + def __init__(self, rcynic_root, + authenticated_subdir = "authenticated"): + self.rcynic_dir = os.path.join(rcynic_root, authenticated_subdir) + + def __iter__(self): + for root, dirs, files in os.walk(self.rcynic_dir): # pylint: disable=W0612 + for filename in files: + filename = os.path.join(root, filename) + ext = os.path.splitext(filename)[1] + if ext in file_name_classes: + yield file_name_classes[ext](filename) class validation_status_element(object): - def __init__(self, *args, **kwargs): - self.attrs = [] - for k, v in kwargs.iteritems(): - setattr(self, k, v) - # attribute names are saved so that the __repr__ method can - # display the subset of attributes the user specified - self.attrs.append(k) - self._obj = None - - def get_obj(self): - if not self._obj: - self._obj = self.file_class(filename=self.filename, uri=self.uri) - return self._obj - - def __repr__(self): - v = [self.__class__.__name__, 'id=%s' % str(id(self))] - v.extend(['%s=%s' % (x, getattr(self, x)) for x in self.attrs]) - return '<%s>' % (' '.join(v),) - - obj = property(get_obj) + def __init__(self, *args, **kwargs): + self.attrs = [] + for k, v in kwargs.iteritems(): + setattr(self, k, v) + # attribute names are saved so that the __repr__ method can + # display the subset of attributes the user specified + self.attrs.append(k) + self._obj = None + + def get_obj(self): + if not self._obj: + self._obj = self.file_class(filename=self.filename, uri=self.uri) + return self._obj + + def __repr__(self): + v = [self.__class__.__name__, 'id=%s' % str(id(self))] + v.extend(['%s=%s' % (x, getattr(self, x)) for x in self.attrs]) + return '<%s>' % (' '.join(v),) + + obj = property(get_obj) class rcynic_xml_iterator(object): - """ - Iterate over validation_status entries in the XML output from an - rcynic run. Yields a tuple for each entry: - - timestamp, generation, status, object - - where URI, status, and timestamp are the corresponding values from - the XML element, OK is a boolean indicating whether validation was - considered succesful, and object is a Python representation of the - object in question. If OK is True, object will be from rcynic's - authenticated output tree; otherwise, object will be from rcynic's - unauthenticated output tree. - - Note that it is possible for the same URI to appear in more than one - validation_status element; in such cases, the succesful case (OK - True) should be the last entry (as rcynic will stop trying once it - gets a good copy), but there may be multiple failures, which might - or might not have different status codes. - """ - - def __init__(self, rcynic_root, xml_file, - authenticated_old_subdir = "authenticated.old", - unauthenticated_subdir = "unauthenticated"): - self.rcynic_root = rcynic_root - self.xml_file = xml_file - self.authenticated_subdir = os.path.join(rcynic_root, 'authenticated') - self.authenticated_old_subdir = os.path.join(rcynic_root, authenticated_old_subdir) - self.unauthenticated_subdir = os.path.join(rcynic_root, unauthenticated_subdir) - - base_uri = "rsync://" - - def uri_to_filename(self, uri): - if uri.startswith(self.base_uri): - return uri[len(self.base_uri):] - else: - raise NotRsyncURI("Not an rsync URI %r" % uri) - - def __iter__(self): - for validation_status in ElementTree(file=self.xml_file).getroot().getiterator("validation_status"): - timestamp = validation_status.get("timestamp") - status = validation_status.get("status") - uri = validation_status.text.strip() - generation = validation_status.get("generation") - - # determine the path to this object - if status == 'object_accepted': - d = self.authenticated_subdir - elif generation == 'backup': - d = self.authenticated_old_subdir - else: - d = self.unauthenticated_subdir - - filename = os.path.join(d, self.uri_to_filename(uri)) - - ext = os.path.splitext(filename)[1] - if ext in file_name_classes: - yield validation_status_element(timestamp = timestamp, generation = generation, - uri=uri, status = status, filename = filename, - file_class = file_name_classes[ext]) + """ + Iterate over validation_status entries in the XML output from an + rcynic run. Yields a tuple for each entry: + + timestamp, generation, status, object + + where URI, status, and timestamp are the corresponding values from + the XML element, OK is a boolean indicating whether validation was + considered succesful, and object is a Python representation of the + object in question. If OK is True, object will be from rcynic's + authenticated output tree; otherwise, object will be from rcynic's + unauthenticated output tree. + + Note that it is possible for the same URI to appear in more than one + validation_status element; in such cases, the succesful case (OK + True) should be the last entry (as rcynic will stop trying once it + gets a good copy), but there may be multiple failures, which might + or might not have different status codes. + """ + + def __init__(self, rcynic_root, xml_file, + authenticated_old_subdir = "authenticated.old", + unauthenticated_subdir = "unauthenticated"): + self.rcynic_root = rcynic_root + self.xml_file = xml_file + self.authenticated_subdir = os.path.join(rcynic_root, 'authenticated') + self.authenticated_old_subdir = os.path.join(rcynic_root, authenticated_old_subdir) + self.unauthenticated_subdir = os.path.join(rcynic_root, unauthenticated_subdir) + + base_uri = "rsync://" + + def uri_to_filename(self, uri): + if uri.startswith(self.base_uri): + return uri[len(self.base_uri):] + else: + raise NotRsyncURI("Not an rsync URI %r" % uri) + + def __iter__(self): + for validation_status in ElementTree(file=self.xml_file).getroot().getiterator("validation_status"): + timestamp = validation_status.get("timestamp") + status = validation_status.get("status") + uri = validation_status.text.strip() + generation = validation_status.get("generation") + + # determine the path to this object + if status == 'object_accepted': + d = self.authenticated_subdir + elif generation == 'backup': + d = self.authenticated_old_subdir + else: + d = self.unauthenticated_subdir + + filename = os.path.join(d, self.uri_to_filename(uri)) + + ext = os.path.splitext(filename)[1] + if ext in file_name_classes: + yield validation_status_element(timestamp = timestamp, generation = generation, + uri=uri, status = status, filename = filename, + file_class = file_name_classes[ext]) def label_iterator(xml_file): - """ - Returns an iterator which contains all defined labels from an rcynic XML - output file. Each item is a tuple of the form - (label, kind, description). - """ - - for label in ElementTree(file=xml_file).find("labels"): - yield label.tag, label.get("kind"), label.text.strip() + """ + Returns an iterator which contains all defined labels from an rcynic XML + output file. Each item is a tuple of the form + (label, kind, description). + """ + + for label in ElementTree(file=xml_file).find("labels"): + yield label.tag, label.get("kind"), label.text.strip() diff --git a/rpki/relaxng.py b/rpki/relaxng.py index 566be90f..49ea88d8 100644 --- a/rpki/relaxng.py +++ b/rpki/relaxng.py @@ -7,17 +7,17 @@ from rpki.relaxng_parser import RelaxNGParser left_right = RelaxNGParser(r''' @@ -1945,29 +1945,29 @@ publication_control = RelaxNGParser(r''' publication = RelaxNGParser(r'''