diff options
Diffstat (limited to 'potpourri')
| -rw-r--r-- | potpourri/django-legacy-database.README | 4 | ||||
| -rw-r--r-- | potpourri/django-legacy-database.tar.xz | bin | 0 -> 10076 bytes | |||
| -rwxr-xr-x | potpourri/rrdp-fetch-from-tal | 229 | ||||
| -rwxr-xr-x | potpourri/rrdp-fetch.py | 68 | ||||
| -rwxr-xr-x | potpourri/rrdp-test-tool | 135 | ||||
| -rw-r--r-- | potpourri/upgrade-add-ghostbusters.py | 2 |
6 files changed, 437 insertions, 1 deletions
diff --git a/potpourri/django-legacy-database.README b/potpourri/django-legacy-database.README new file mode 100644 index 00000000..41a3b911 --- /dev/null +++ b/potpourri/django-legacy-database.README @@ -0,0 +1,4 @@ +Snapshot of work in progress on converting our existing databases into +Django using South 1.0 migrations. This will probably need rewriting +to address changes in how we deal with Django settings and multiple +databases, this snapshot is just to get it into the subversion archive. diff --git a/potpourri/django-legacy-database.tar.xz b/potpourri/django-legacy-database.tar.xz Binary files differnew file mode 100644 index 00000000..762dde7d --- /dev/null +++ b/potpourri/django-legacy-database.tar.xz diff --git a/potpourri/rrdp-fetch-from-tal b/potpourri/rrdp-fetch-from-tal new file mode 100755 index 00000000..08d245dd --- /dev/null +++ b/potpourri/rrdp-fetch-from-tal @@ -0,0 +1,229 @@ +#!/usr/bin/env python +# $Id$ +# +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +Fetch RPKI data using RRDP starting from a TAL. + +Work in progress, don't be too surprised by anything this does or +doesn't do. +""" + +import rpki.relaxng +import rpki.x509 +import lxml.etree +import argparse +import urlparse +import urllib2 +import sys +import os + + +class Tags(object): + def __init__(self, *tags): + for tag in tags: + setattr(self, tag, rpki.relaxng.rrdp.xmlns + tag) + +tags = Tags("notification", "delta", "snapshot", "publish", "withdraw") + + +class RSyncHandler(urllib2.BaseHandler): + """ + Jam support for rsync:// URIs into urllib2 framework. + Very basic, probably not paranoid enough. + """ + + _n = 0 + + def rsync_open(self, req): + import subprocess, mimetools + u = req.get_full_url() + if u.endswith("/"): + raise urllib2.URLError("rsync directory URI not allowed") + t = "/tmp/rrdp-fetch-from-tal.%d.%d" % (os.getpid(), self._n) + self._n += 1 + subprocess.check_call(("rsync", u, t)) + h = mimetools.Message(open("/dev/null")) + h["Content-type"] = "text/plain" + h["Content-length"] = str(os.stat(t).st_size) + f = open(t, "rb") + os.unlink(t) + return urllib2.addinfourl(f, h, u) + +urllib2.install_opener(urllib2.build_opener(RSyncHandler)) + + +class main(object): + + def __init__(self): + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument("--rcynic-tree", default = "rcynic-data/unauthenticated", + help = "directory tree in which to write extracted RPKI objects") + parser.add_argument("--serial-filename", # default handled later + help = "file name in which to store RRDP serial number") + parser.add_argument("tal", help = "trust anchor locator") + self.args = parser.parse_args() + if not os.path.isdir(self.args.rcynic_tree): + os.makedirs(self.args.rcynic_tree) + self.urls = set() + self.ta = self.ta_fetch() + url = self.ta.get_sia_rrdp_notify() + if url is None: + sys.exit("Couldn't get RRDP URI from trust anchor") + self.rrdp_fetch(url) + self.write_ta() + + def rrdp_fetch(self, url): + if url in self.urls: + print "Already fetched %s, skipping" % url + return + self.urls.add(url) + xml = lxml.etree.ElementTree(file = urllib2.urlopen(url)).getroot() + rpki.relaxng.rrdp.assertValid(xml) + if xml.tag[len(rpki.relaxng.rrdp.xmlns):] != "notification": + sys.exit("Expected notification at %s, found %s" % (url, xml.tag)) + self.prettyprint_notification(xml) + + # We should be checking session_id here, but we're not storing it yet + + old_serial = self.get_serial() + new_serial = int(xml.get("serial")) + deltas = dict((int(elt.get("serial")), elt) + for elt in xml.iterchildren(tags.delta)) + if old_serial == 0 or not all(serial + 1 in deltas + for serial in xrange(old_serial, new_serial)): + return self.snapshot_fetch(xml.iterchildren(tags.snapshot).next()) + for serial in sorted(deltas): + if serial > old_serial: + self.delta_fetch(deltas[serial]) + + def prettyprint_notification(self, xml): + print "Notification version %s session %s serial %s" % ( + xml.get("version"), xml.get("session_id"), xml.get("serial")) + elt = xml.iterchildren(tags.snapshot).next() + print " Snapshot URI %s hash %s" % ( + elt.get("uri"), elt.get("hash")) + for elt in xml.iterchildren(tags.delta): + print " Delta %6s URI %s hash %s" % ( + elt.get("serial"), elt.get("uri"), elt.get("hash")) + + def ta_fetch(self): + with open(self.args.tal, "r") as f: + tal = f.read() + uris, key = tal.split("\n\n", 2) + key = rpki.x509.PublicKey(Base64 = key) + for uri in uris.split(): + ta = rpki.x509.X509(DER = urllib2.urlopen(uri).read()) + if ta.getPublicKey() == key: + return ta + print "TAL key mismatch for certificate", url + sys.exit("Could not fetch trust anchor") + + @property + def serial_filename(self): + return self.args.serial_filename or os.path.join(self.args.rcynic_tree, "serial") + + def get_serial(self): + try: + with open(self.serial_filename, "r") as f: + return int(f.read().strip()) + except: + return 0 + + def set_serial(self, value): + with open(self.serial_filename, "w") as f: + f.write("%s\n" % value) + + def uri_to_filename(self, uri): + assert uri.startswith("rsync://") + return os.path.join(self.args.rcynic_tree, uri[len("rsync://"):]) + + def add_obj(self, uri, obj): + fn = self.uri_to_filename(uri) + dn = os.path.dirname(fn) + if not os.path.isdir(dn): + os.makedirs(dn) + with open(fn, "wb") as f: + f.write(obj) + + def del_obj(self, uri, hash): + fn = self.uri_to_filename(uri) + with open(fn, "rb") as f: + if hash.lower() != rpki.x509.sha256(f.read()).encode("hex"): + raise RuntimeError("Hash mismatch for URI %s" % uri) + os.unlink(fn) + dn = os.path.dirname(fn) + while True: + try: + os.rmdir(dn) + except OSError: + break + else: + dn = os.path.dirname(dn) + + def xml_fetch(self, elt): + url = elt.get("uri") + hash = elt.get("hash").lower() + print "Fetching", url + text = urllib2.urlopen(url).read() + h = rpki.x509.sha256(text).encode("hex") + if h != hash: + sys.exit("Bad hash for %s: expected %s got %s" % (url, hash, h)) + xml = lxml.etree.XML(text) + rpki.relaxng.rrdp.schema.assertValid(xml) + return xml + + def snapshot_fetch(self, xml): + xml = self.xml_fetch(xml) + print "Unpacking snapshot version %s session %s serial %6s" % ( + xml.get("version"), xml.get("session_id"), xml.get("serial")) + for elt in xml.iterchildren(tags.publish): + print " ", elt.get("uri") + self.add_obj(elt.get("uri"), elt.text.decode("base64")) + self.set_serial(xml.get("serial")) + + def delta_fetch(self, xml): + xml = self.xml_fetch(xml) + old_serial = int(self.get_serial()) + new_serial = int(xml.get("serial")) + print "Unpacking deltas version %s session %s serial %s" % ( + xml.get("version"), xml.get("session_id"), new_serial) + if old_serial != new_serial - 1: + raise RuntimeError("Can't apply deltas: old serial %s new serial %s" % (old_serial, new_serial)) + for i, elt in enumerate(xml.iterchildren(tags.withdraw)): + uri = elt.get("uri") + hash = elt.get("hash") + print " %3d withdraw URI %s hash %s" % (i, uri, hash) + self.del_obj(uri, hash) + for i, elt in enumerate(xml.iterchildren(tags.publish)): + uri = elt.get("uri") + hash = elt.get("hash", None) + print " %3d publish URI %s hash %s" % (i, uri, hash) + if hash is not None: + self.del_obj(uri, hash) + self.add_obj(elt.get("uri"), elt.text.decode("base64")) + self.set_serial(new_serial) + + def write_ta(self): + der = self.ta.get_DER() + fn = rpki.x509.sha256(der).encode("hex") + ".cer" + if not os.path.exists(fn): + print "Writing", fn + with open(fn, "wb") as f: + f.write(der) + +if __name__ == "__main__": + main() diff --git a/potpourri/rrdp-fetch.py b/potpourri/rrdp-fetch.py new file mode 100755 index 00000000..469c0c9f --- /dev/null +++ b/potpourri/rrdp-fetch.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# $Id$ +# +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +Fetch an RRDP notifcation file and follow all the links. Should be +merged into rrdp-test-tool eventually, but one thing at a time. +""" + +from urllib2 import urlopen +from lxml.etree import ElementTree, XML +from socket import getfqdn +from rpki.x509 import sha256 +from rpki.relaxng import rrdp +from urlparse import urlparse +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter + +class BadHash(Exception): + "Calculated hash value doesn't match expected hash value." + +def fetch(elt): + uri = elt.get("uri") + hash = elt.get("hash").lower() + print "Fetching", uri + + text = urlopen(uri).read() + h = sha256(text).encode("hex") + if h != hash: + raise BadHash("Bad hash for %s: expected %s got %s" % (uri, hash, h)) + + xml = XML(text) + rrdp.schema.assertValid(xml) + + u = urlparse(uri) + fn = u.netloc + u.path + + return elt, xml, fn + +parser = ArgumentParser(description = __doc__, formatter_class = ArgumentDefaultsHelpFormatter) +parser.add_argument("uri", nargs = "?", + default = "http://" + getfqdn() + "/rrdp/updates.xml", + help = "RRDP notification file to fetch") +args = parser.parse_args() + +updates = ElementTree(file = urlopen(args.uri)) +rrdp.schema.assertValid(updates) + +snapshot = fetch(updates.find(rrdp.xmlns + "snapshot")) + +deltas = [fetch(elt) for elt in updates.findall(rrdp.xmlns + "delta")] + +print updates +print snapshot +for delta in deltas: + print delta diff --git a/potpourri/rrdp-test-tool b/potpourri/rrdp-test-tool new file mode 100755 index 00000000..ccf17960 --- /dev/null +++ b/potpourri/rrdp-test-tool @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# $Id$ +# +# Copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +""" +Test tool for prototype RRDP implementation. Eventually some of this +code will likely be refactored into more user-friendly form, but for +the moment this just does whatever insane thing I need to do this week +for testing. +""" + +import rpki.relaxng +import rpki.x509 +import lxml.etree +import argparse +import os + +class Tags(object): + def __init__(self, *tags): + for tag in tags: + setattr(self, tag, rpki.relaxng.rrdp.xmlns + tag) + +tags = Tags("notification", "delta", "snapshot", "publish", "withdraw") + +class main(object): + + def __init__(self): + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument("--rcynic-tree", default = "rcynic-data/unauthenticated", + help = "directory tree in which to write extracted RPKI objects") + parser.add_argument("--serial-filename", + help = "file name in which to store RRDP serial number") + parser.add_argument("rrdp_file", nargs = "+", + help = "RRDP snapshot or deltas file") + self.args = parser.parse_args() + if not os.path.isdir(self.args.rcynic_tree): + os.makedirs(self.args.rcynic_tree) + for rrdp_file in self.args.rrdp_file: + xml = lxml.etree.ElementTree(file = rrdp_file).getroot() + rpki.relaxng.rrdp.assertValid(xml) + getattr(self, "handle_" + xml.tag[len(rpki.relaxng.rrdp.xmlns):])(xml) + + @property + def serial_filename(self): + return self.args.serial_filename or os.path.join(self.args.rcynic_tree, "serial") + + def get_serial(self): + with open(self.serial_filename, "r") as f: + return f.read().strip() + + def set_serial(self, value): + with open(self.serial_filename, "w") as f: + f.write("%s\n" % value) + + def handle_notification(self, xml): + print "Notification version %s session %s serial %s" % ( + xml.get("version"), xml.get("session_id"), xml.get("serial")) + assert xml[0].tag == tags.snapshot + print " Snapshot URI %s hash %s" % ( + xml[0].get("uri"), xml[0].get("hash")) + for i, elt in enumerate(xml.iterchildren(tags.delta)): + print " Delta %3d serial %6s URI %s hash %s" % ( + i, elt.get("serial"), elt.get("uri"), elt.get("hash")) + + def uri_to_filename(self, uri): + assert uri.startswith("rsync://") + return os.path.join(self.args.rcynic_tree, uri[len("rsync://"):]) + + def add_obj(self, uri, obj): + fn = self.uri_to_filename(uri) + dn = os.path.dirname(fn) + if not os.path.isdir(dn): + os.makedirs(dn) + with open(fn, "wb") as f: + f.write(obj) + + def del_obj(self, uri, hash): + fn = self.uri_to_filename(uri) + with open(fn, "rb") as f: + if hash.lower() != rpki.x509.sha256(f.read()).encode("hex"): + raise RuntimeError("Hash mismatch for URI %s" % uri) + os.unlink(fn) + dn = os.path.dirname(fn) + while True: + try: + os.rmdir(dn) + except OSError: + break + else: + dn = os.path.dirname(dn) + + def handle_snapshot(self, xml): + print "Unpacking snapshot version %s session %s serial %6s" % ( + xml.get("version"), xml.get("session_id"), xml.get("serial")) + for elt in xml.iterchildren(tags.publish): + print " ", elt.get("uri") + self.add_obj(elt.get("uri"), elt.text.decode("base64")) + self.set_serial(xml.get("serial")) + + def handle_delta(self, xml): + old_serial = int(self.get_serial()) + new_serial = int(xml.get("serial")) + print "Unpacking deltas version %s session %s serial %s" % ( + xml.get("version"), xml.get("session_id"), new_serial) + if old_serial != new_serial - 1: + raise RuntimeError("Can't apply deltas: old serial %s new serial %s" % (old_serial, new_serial)) + for i, elt in enumerate(xml.iterchildren(tags.withdraw)): + uri = elt.get("uri") + hash = elt.get("hash") + print " %3d withdraw URI %s hash %s" % (i, uri, hash) + self.del_obj(uri, hash) + for i, elt in enumerate(xml.iterchildren(tags.publish)): + uri = elt.get("uri") + hash = elt.get("hash", None) + print " %3d publish URI %s hash %s" % (i, uri, hash) + if hash is not None: + self.del_obj(uri, hash) + self.add_obj(elt.get("uri"), elt.text.decode("base64")) + self.set_serial(new_serial) + +if __name__ == "__main__": + main() diff --git a/potpourri/upgrade-add-ghostbusters.py b/potpourri/upgrade-add-ghostbusters.py index a8c8a92b..2548487c 100644 --- a/potpourri/upgrade-add-ghostbusters.py +++ b/potpourri/upgrade-add-ghostbusters.py @@ -43,7 +43,7 @@ for o, a in opts: if o in ("-c", "--config"): cfg_file = a -cfg = rpki.config.parser(cfg_file, "myrpki") +cfg = rpki.config.parser(filename = cfg_file, section = "myrpki") fix("irdbd", """ CREATE TABLE ghostbuster_request ( |