6 files changed, 411 insertions, 1 deletions
diff --git a/potpourri/django-legacy-database.README b/potpourri/django-legacy-database.README
new file mode 100644
index 00000000..41a3b911
--- /dev/null
+++ b/potpourri/django-legacy-database.README
@@ -0,0 +1,4 @@
+Snapshot of work in progress on converting our existing databases into
+Django using South 1.0 migrations.  This will probably need rewriting
+to address changes in how we deal with Django settings and multiple
+databases, this snapshot is just to get it into the subversion archive.
diff --git a/potpourri/django-legacy-database.tar.xz b/potpourri/django-legacy-database.tar.xz
new file mode 100644
index 00000000..762dde7d
--- /dev/null
+++ b/potpourri/django-legacy-database.tar.xz
diff --git a/potpourri/rrdp-fetch-from-tal b/potpourri/rrdp-fetch-from-tal
new file mode 100755
index 00000000..0a97955e
--- /dev/null
+++ b/potpourri/rrdp-fetch-from-tal
@@ -0,0 +1,203 @@
+#!/usr/bin/env python
+# $Id$
+#
+# Copyright (C) 2014  Dragon Research Labs ("DRL")
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH
+# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS.  IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+
+"""
+Fetch RPKI data using RRDP starting from a TAL.
+
+Work in progress, don't be too surprised by anything this does or
+doesn't do.
+"""
+
+import rpki.relaxng
+import rpki.x509
+import lxml.etree
+import argparse
+import urlparse
+import urllib2            
+import sys
+import os
+
+
+class Tags(object):
+  def __init__(self, *tags):
+    for tag in tags:
+      setattr(self, tag, rpki.relaxng.rrdp.xmlns + tag)
+
+tags = Tags("notification", "delta", "snapshot", "publish", "withdraw")
+
+
+class main(object):
+
+  def __init__(self):
+    parser = argparse.ArgumentParser(description = __doc__)
+    parser.add_argument("--rcynic-tree", default = "rcynic-data/unauthenticated",
+                        help = "directory tree in which to write extracted RPKI objects")
+    parser.add_argument("--serial-filename", # default handled later
+                        help = "file name in which to store RRDP serial number")
+    parser.add_argument("tal", help = "trust anchor locator")
+    self.args = parser.parse_args()
+    if not os.path.isdir(self.args.rcynic_tree):
+      os.makedirs(self.args.rcynic_tree)
+    self.urls = set()
+    self.ta = self.ta_fetch()
+    url = self.ta.get_sia_rrdp_notify()
+    if url is None:
+      sys.exit("Couldn't get RRDP URI from trust anchor")
+    self.rrdp_fetch(url)
+    self.write_ta()
+
+  def rrdp_fetch(self, url):
+    if url in self.urls:
+      print "Already fetched %s, skipping" % url
+      return
+    self.urls.add(url)
+    xml = lxml.etree.ElementTree(file = urllib2.urlopen(url)).getroot()
+    rpki.relaxng.rrdp.assertValid(xml)
+    if xml.tag[len(rpki.relaxng.rrdp.xmlns):] != "notification":
+      sys.exit("Expected notification at %s, found %s" % (url, xml.tag))
+    self.prettyprint_notification(xml)
+
+    # We should be checking session_id here, but we're not storing it yet
+
+    old_serial = self.get_serial()
+    new_serial = int(xml.get("serial"))
+    deltas = dict((int(elt.get("serial")), elt)
+                  for elt in xml.iterchildren(tags.delta))
+    if old_serial == 0 or not all(serial + 1 in deltas
+                                  for serial in xrange(old_serial, new_serial)):
+      return self.snapshot_fetch(xml.iterchildren(tags.snapshot).next())
+    for serial in sorted(deltas):
+      if serial > old_serial:
+        self.delta_fetch(deltas[serial])
+
+  def prettyprint_notification(self, xml):
+    print "Notification version %s session %s serial %s" % (
+      xml.get("version"), xml.get("session_id"), xml.get("serial"))
+    elt = xml.iterchildren(tags.snapshot).next()
+    print " Snapshot URI %s hash %s" % (
+      elt.get("uri"), elt.get("hash"))
+    for elt in xml.iterchildren(tags.delta):
+      print " Delta %6s URI %s hash %s" % (
+        elt.get("serial"), elt.get("uri"), elt.get("hash"))
+
+  def ta_fetch(self):
+    with open(self.args.tal, "r") as f:
+      tal = f.read()
+    uris, key = tal.split("\n\n", 2)
+    key = rpki.x509.PublicKey(Base64 = key)
+    for uri in uris.split():
+      ta = rpki.x509.X509(DER = urllib2.urlopen(uri).read())
+      if ta.getPublicKey() == key:
+        return ta
+      print "TAL key mismatch for certificate", url
+    sys.exit("Could not fetch trust anchor")
+
+  @property
+  def serial_filename(self):
+    return self.args.serial_filename or os.path.join(self.args.rcynic_tree, "serial")
+
+  def get_serial(self):
+    try:
+      with open(self.serial_filename, "r") as f:
+        return int(f.read().strip())
+    except:
+      return 0
+
+  def set_serial(self, value):
+    with open(self.serial_filename, "w") as f:
+      f.write("%s\n" % value)
+
+  def uri_to_filename(self, uri):
+    assert uri.startswith("rsync://")
+    return os.path.join(self.args.rcynic_tree, uri[len("rsync://"):])
+
+  def add_obj(self, uri, obj):
+    fn = self.uri_to_filename(uri)
+    dn = os.path.dirname(fn)
+    if not os.path.isdir(dn):
+      os.makedirs(dn)
+    with open(fn, "wb") as f:
+      f.write(obj)
+
+  def del_obj(self, uri, hash):
+    fn = self.uri_to_filename(uri)
+    with open(fn, "rb") as f:
+      if hash.lower() != rpki.x509.sha256(f.read()).encode("hex"):
+        raise RuntimeError("Hash mismatch for URI %s" % uri)
+    os.unlink(fn)
+    dn = os.path.dirname(fn)
+    while True:
+      try:
+        os.rmdir(dn)
+      except OSError:
+        break
+      else:
+        dn = os.path.dirname(dn)
+
+  def xml_fetch(self, elt):
+    url = elt.get("uri")
+    hash = elt.get("hash").lower()
+    print "Fetching", url
+    text = urllib2.urlopen(url).read()
+    h = rpki.x509.sha256(text).encode("hex")
+    if h != hash:
+      sys.exit("Bad hash for %s: expected %s got %s" % (url, hash, h))
+    xml = lxml.etree.XML(text)
+    rpki.relaxng.rrdp.schema.assertValid(xml)
+    return xml
+
+  def snapshot_fetch(self, xml):
+    xml = self.xml_fetch(xml)
+    print "Unpacking snapshot version %s session %s serial %6s" % (
+      xml.get("version"), xml.get("session_id"), xml.get("serial"))
+    for elt in xml.iterchildren(tags.publish):
+      print " ", elt.get("uri")
+      self.add_obj(elt.get("uri"), elt.text.decode("base64"))
+    self.set_serial(xml.get("serial"))
+
+  def delta_fetch(self, xml):
+    xml = self.xml_fetch(xml)
+    old_serial = int(self.get_serial())
+    new_serial = int(xml.get("serial"))
+    print "Unpacking deltas version %s session %s serial %s" % (
+      xml.get("version"), xml.get("session_id"), new_serial)
+    if old_serial != new_serial - 1:
+      raise RuntimeError("Can't apply deltas: old serial %s new serial %s" % (old_serial, new_serial))
+    for i, elt in enumerate(xml.iterchildren(tags.withdraw)):
+      uri = elt.get("uri")
+      hash = elt.get("hash")
+      print "  %3d withdraw URI %s hash %s" % (i, uri, hash)
+      self.del_obj(uri, hash)
+    for i, elt in enumerate(xml.iterchildren(tags.publish)):
+      uri = elt.get("uri")
+      hash = elt.get("hash", None)
+      print "  %3d publish  URI %s hash %s" % (i, uri, hash)
+      if hash is not None:
+        self.del_obj(uri, hash)
+      self.add_obj(elt.get("uri"), elt.text.decode("base64"))
+    self.set_serial(new_serial)
+
+  def write_ta(self):
+    der = self.ta.get_DER()
+    fn = rpki.x509.sha256(der).encode("hex") + ".cer"
+    if not os.path.exists(fn):
+      print "Writing", fn
+      with open(fn, "wb") as f:
+        f.write(der)
+
+if __name__ == "__main__":
+  main()
diff --git a/potpourri/rrdp-fetch.py b/potpourri/rrdp-fetch.py
new file mode 100755
index 00000000..aa5b762b
--- /dev/null
+++ b/potpourri/rrdp-fetch.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# $Id$
+#
+# Copyright (C) 2014  Dragon Research Labs ("DRL")
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH
+# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS.  IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+
+"""
+Fetch an RRDP notifcation file and follow all the links.  Should be
+merged into rrdp-test-tool eventually, but one thing at a time.
+"""
+
+from urllib2            import urlopen
+from lxml.etree         import ElementTree, XML
+from socket             import getfqdn
+from rpki.x509          import sha256
+from rpki.relaxng       import rrdp
+from urlparse           import urlparse
+from argparse           import ArgumentParser, ArgumentDefaultsHelpFormatter
+
+class BadHash(Exception):
+  "Calculated hash value doesn't match expected hash value."
+
+def fetch(elt):
+  uri = elt.get("uri")
+  hash = elt.get("hash")
+  print "Fetching", uri
+
+  text = urlopen(uri).read()
+  h = sha256(text).encode("hex")
+  if h != hash:
+    raise BadHash("Bad hash for %s: expected %s got %s" % (uri, hash, h))
+
+  xml = XML(text)
+  rrdp.schema.assertValid(xml)
+
+  u = urlparse(uri)
+  fn = u.netloc + u.path
+
+  return elt, xml, fn
+
+parser = ArgumentParser(description = __doc__, formatter_class = ArgumentDefaultsHelpFormatter)
+parser.add_argument("uri", nargs = "?",
+                    default = "http://" + getfqdn() + "/rrdp/updates.xml",
+                    help = "RRDP notification file to fetch")
+args = parser.parse_args()
+
+updates = ElementTree(file = urlopen(args.uri))
+rrdp.schema.assertValid(updates)
+
+snapshot = fetch(updates.find(rrdp.xmlns + "snapshot"))
+
+deltas = [fetch(elt) for elt in updates.findall(rrdp.xmlns + "delta")]
+
+print updates
+print snapshot
+for delta in deltas:
+  print delta
diff --git a/potpourri/rrdp-test-tool b/potpourri/rrdp-test-tool
new file mode 100755
index 00000000..8ea90f17
--- /dev/null
+++ b/potpourri/rrdp-test-tool
@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+# $Id$
+#
+# Copyright (C) 2014  Dragon Research Labs ("DRL")
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH
+# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS.  IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+
+"""
+Test tool for prototype RRDP implementation.  Eventually some of this
+code will likely be refactored into more user-friendly form, but for
+the moment this just does whatever insane thing I need to do this week
+for testing.
+"""
+
+import rpki.relaxng
+import rpki.x509
+import lxml.etree
+import argparse
+import os
+
+class Tags(object):
+  def __init__(self, *tags):
+    for tag in tags:
+      setattr(self, tag, rpki.relaxng.rrdp.xmlns + tag)
+
+tags = Tags("notification", "delta", "snapshot", "publish", "withdraw")
+
+class main(object):
+
+  def __init__(self):
+    parser = argparse.ArgumentParser(description = __doc__)
+    parser.add_argument("--rcynic-tree", default = "rcynic-data/unauthenticated",
+                        help = "directory tree in which to write extracted RPKI objects")
+    parser.add_argument("--serial-filename",
+                        help = "file name in which to store RRDP serial number")
+    parser.add_argument("rrdp_file", nargs = "+",
+                        help = "RRDP snapshot or deltas file")
+    self.args = parser.parse_args()
+    if not os.path.isdir(self.args.rcynic_tree):
+      os.makedirs(self.args.rcynic_tree)
+    for rrdp_file in self.args.rrdp_file:
+      xml = lxml.etree.ElementTree(file = rrdp_file).getroot()
+      rpki.relaxng.rrdp.assertValid(xml)
+      getattr(self, "handle_" + xml.tag[len(rpki.relaxng.rrdp.xmlns):])(xml)
+
+  @property
+  def serial_filename(self):
+    return self.args.serial_filename or os.path.join(self.args.rcynic_tree, "serial")
+
+  def get_serial(self):
+    with open(self.serial_filename, "r") as f:
+      return f.read().strip()
+
+  def set_serial(self, value):
+    with open(self.serial_filename, "w") as f:
+      f.write("%s\n" % value)
+
+  def handle_notification(self, xml):
+    print "Notification version %s session %s serial %s" % (
+      xml.get("version"), xml.get("session_id"), xml.get("serial"))
+    assert xml[0].tag == tags.snapshot
+    print " Snapshot URI %s hash %s" % (
+      xml[0].get("uri"), xml[0].get("hash"))
+    for i, elt in enumerate(xml.iterchildren(tags.delta)):
+      print " Delta %3d serial %6s URI %s hash %s" % (
+        i, elt.get("serial"), elt.get("uri"), elt.get("hash"))
+
+  def uri_to_filename(self, uri):
+    assert uri.startswith("rsync://")
+    return os.path.join(self.args.rcynic_tree, uri[len("rsync://"):])
+
+  def add_obj(self, uri, obj):
+    fn = self.uri_to_filename(uri)
+    dn = os.path.dirname(fn)
+    if not os.path.isdir(dn):
+      os.makedirs(dn)
+    with open(fn, "wb") as f:
+      f.write(obj)
+
+  def del_obj(self, uri, hash):
+    fn = self.uri_to_filename(uri)
+    with open(fn, "rb") as f:
+      if hash != rpki.x509.sha256(f.read()).encode("hex"):
+        raise RuntimeError("Hash mismatch for URI %s" % uri)
+    os.unlink(fn)
+    dn = os.path.dirname(fn)
+    while True:
+      try:
+        os.rmdir(dn)
+      except OSError:
+        break
+      else:
+        dn = os.path.dirname(dn)
+
+  def handle_snapshot(self, xml):
+    print "Unpacking snapshot version %s session %s serial %6s" % (
+      xml.get("version"), xml.get("session_id"), xml.get("serial"))
+    for elt in xml.iterchildren(tags.publish):
+      print " ", elt.get("uri")
+      self.add_obj(elt.get("uri"), elt.text.decode("base64"))
+    self.set_serial(xml.get("serial"))
+
+  def handle_delta(self, xml):
+    old_serial = int(self.get_serial())
+    new_serial = int(xml.get("serial"))
+    print "Unpacking deltas version %s session %s serial %s" % (
+      xml.get("version"), xml.get("session_id"), new_serial)
+    if old_serial != new_serial - 1:
+      raise RuntimeError("Can't apply deltas: old serial %s new serial %s" % (old_serial, new_serial))
+    for i, elt in enumerate(xml.iterchildren(tags.withdraw)):
+      uri = elt.get("uri")
+      hash = elt.get("hash")
+      print "  %3d withdraw URI %s hash %s" % (i, uri, hash)
+      self.del_obj(uri, hash)
+    for i, elt in enumerate(xml.iterchildren(tags.publish)):
+      uri = elt.get("uri")
+      hash = elt.get("hash", None)
+      print "  %3d publish  URI %s hash %s" % (i, uri, hash)
+      if hash is not None:
+        self.del_obj(uri, hash)
+      self.add_obj(elt.get("uri"), elt.text.decode("base64"))
+    self.set_serial(new_serial)
+
+if __name__ == "__main__":
+  main()
diff --git a/potpourri/upgrade-add-ghostbusters.py b/potpourri/upgrade-add-ghostbusters.py
index a8c8a92b..2548487c 100644
--- a/potpourri/upgrade-add-ghostbusters.py
+++ b/potpourri/upgrade-add-ghostbusters.py
@@ -43,7 +43,7 @@ for o, a in opts:
   if o in ("-c", "--config"):
     cfg_file = a
 
-cfg = rpki.config.parser(cfg_file, "myrpki")
+cfg = rpki.config.parser(filename = cfg_file, section = "myrpki")
 
 fix("irdbd", """
     CREATE TABLE ghostbuster_request (