Add rcynic-dump as a blunt instrument to let existing file-based

programs use rcynicng's output.  Long term, such programs should read
the database instead, but being able to diff directory trees is useful
while testing.

svn path=/branches/tk705/; revision=6220

3 files changed, 117 insertions, 8 deletions

diff --git a/rp/rcynic/rcynic-dump b/rp/rcynic/rcynic-dump
new file mode 100755
index 00000000..ae685d05
--- /dev/null
+++ b/rp/rcynic/rcynic-dump
@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+
+# $Id$
+
+"""
+Dump rcynicng database to old-style disk files.
+
+This is a slow operation due to blocking operations in the underlying
+filesystem, so in the long run we will almost certainly want to
+rewrite the RP toolkit to use the database directly, but it's (much)
+easier to compare results between the old and new validation engines
+when they use the same data representation.
+"""
+
+import os
+import sys
+import time
+import shutil
+import logging
+import argparse
+
+import rpki.config
+import rpki.autoconf
+
+logger = logging.getLogger("rcynic-dump")
+
+os.environ.update(TZ = "UTC",
+                  DJANGO_SETTINGS_MODULE = "rpki.django_settings.rcynic")
+time.tzset()
+
+logging.basicConfig(level = logging.DEBUG, format = "%(asctime)s  %(message)s", datefmt = "%Y-%m-%d %H:%M:%S")
+
+parser = argparse.ArgumentParser(description = __doc__)
+parser.add_argument("-c", "--config")
+parser.add_argument("output_tree", nargs = "?", default = "rcynic-data")
+args = parser.parse_args()
+
+rpki.config.rpki_conf_envname = "RCYNIC_CONF"
+rpki.config.default_filename = os.path.join(rpki.autoconf.sysconfdir, "rcynic.conf")
+rpki.config.parser(set_filename = args.config, section = "rcynic")
+
+import django
+django.setup()
+
+import rpki.rcynicdb
+
+def uri_to_filename(obj, base):
+    return os.path.join(args.output_tree, base, obj.uri[obj.uri.index("://") + 3:])
+
+def sha256_to_filename(obj):
+    return os.path.join(args.output_tree, "sha256",
+                        obj.sha256[0:8], obj.sha256[8:16], obj.sha256[16:24], obj.sha256[24:])
+
+def authenticated_to_dirname(authenticated):
+    return "authenticated-{}".format(authenticated.started.strftime("%Y-%m-%dT%H:%M:%SZ"))
+
+seen = set()
+
+def check_der(fn, der):
+    with open(fn, "rb") as f:
+        return der == f.read()
+
+def mkdir_maybe(fn):
+    dn = os.path.dirname(fn)
+    if not os.path.exists(dn):
+        os.makedirs(dn)
+
+for obj in rpki.rcynicdb.models.RPKIObject.objects.all():
+
+    hfn = sha256_to_filename(obj)
+    ufn = uri_to_filename(obj, "unauthenticated")
+
+    if not os.path.exists(hfn) or not check_der(hfn, obj.der):
+        mkdir_maybe(hfn)
+        with open(hfn, "wb") as f:
+            f.write(obj.der)
+
+    seen.add(hfn)
+    seen.add(ufn)
+
+    for auth in obj.authenticated.all():
+        afn = uri_to_filename(obj, authenticated_to_dirname(auth))
+        mkdir_maybe(afn)
+        if not os.path.exists(afn):
+            os.link(hfn, afn)
+        elif not check_der(afn, obj.der):
+            os.unlink(afn)
+            os.link(hfn, afn)
+        seen.add(afn)
+
+auth = rpki.rcynicdb.models.Authenticated.objects.order_by("-started").first()
+
+if auth is not None:
+    src = authenticated_to_dirname(auth)
+    dst = os.path.join(args.output_tree, "authenticated")
+    if os.path.exists(dst):
+        os.unlink(dst)
+    os.symlink(src, dst)
diff --git a/rp/rcynic/rcynicng b/rp/rcynic/rcynicng
index 760aa6ec..d89fda3a 100755
--- a/rp/rcynic/rcynicng
+++ b/rp/rcynic/rcynicng
@@ -13,7 +13,6 @@ import shutil
 import errno
 import logging
 import argparse
-import datetime
 import subprocess
 
 import tornado.gen
@@ -24,6 +23,8 @@ import tornado.process
 
 import rpki.POW
 import rpki.sundial
+import rpki.config
+import rpki.autoconf
 
 from rpki.oids import id_kp_bgpsec_router
 
@@ -848,8 +849,8 @@ class Fetcher(object):
 
             retrieval = rpki.rcynicdb.models.Retrieval.objects.create(
                 uri        = self.uri,
-                started    = datetime.datetime.fromtimestamp(t0),
-                finished   = datetime.datetime.fromtimestamp(t1),
+                started    = rpki.sundial.datetime.fromtimestamp(t0),
+                finished   = rpki.sundial.datetime.fromtimestamp(t1),
                 successful = self.status == 0)
 
             for fn in self._rsync_walk(path):
@@ -967,12 +968,16 @@ class posint(int):
 
 
 def main():
+    global rpki
+
     os.environ.update(TZ = "UTC",
                       DJANGO_SETTINGS_MODULE = "rpki.django_settings.rcynic")
     time.tzset()
 
     parser = argparse.ArgumentParser(description = __doc__)
 
+    parser.add_argument("-c", "--config", help = "override default location of configuration file")
+
     parser.add_argument("--authenticated",      default = "rcynic-data/authenticated")
     parser.add_argument("--unauthenticated",    default = "rcynic-data/unauthenticated")
     parser.add_argument("--xml-file",           default = "rcynicng.xml",       type = argparse.FileType("w"))
@@ -988,6 +993,14 @@ def main():
     global args
     args = parser.parse_args()
 
+    rpki.config.rpki_conf_envname = "RCYNIC_CONF"
+    rpki.config.default_filename = os.path.join(rpki.autoconf.sysconfdir, "rcynic.conf")
+    rpki.config.parser(set_filename = args.config, section = "rcynic")
+
+    # If we're going to allow some options to be controlled from both
+    # config file and command line, this is where we sort that out.
+    # Doesn't play well with action=store_true.
+
     import django
     django.setup()
 
@@ -996,13 +1009,12 @@ def main():
         import django.core.management
         django.core.management.call_command("migrate", verbosity = 0, interactive = False)
 
-    global rpki
     import rpki.rcynicdb
 
     logging.basicConfig(level = logging.DEBUG, format = "%(asctime)s  %(message)s", datefmt = "%Y-%m-%d %H:%M:%S")
 
     global authenticated
-    authenticated = rpki.rcynicdb.models.Authenticated.objects.create(started  = datetime.datetime.now())
+    authenticated = rpki.rcynicdb.models.Authenticated.objects.create(started  = rpki.sundial.datetime.now())
 
     global task_queue
     task_queue = tornado.queues.Queue()
@@ -1010,7 +1022,7 @@ def main():
 
     final_report()
 
-    authenticated.finished = datetime.datetime.now()
+    authenticated.finished = rpki.sundial.datetime.now()
     authenticated.save()
 
 
diff --git a/rp/rcynic/validation_status b/rp/rcynic/validation_status
index f961f473..d8e2c8ae 100755
--- a/rp/rcynic/validation_status
+++ b/rp/rcynic/validation_status
@@ -29,8 +29,7 @@ except ImportError:
 
 for filename in ([sys.stdin] if len(sys.argv) < 2 else sys.argv[1:]):
     for elt in ElementTree(file = filename).findall("validation_status"):
-        print "%s %8s %-40s %s" % (
+        print "%s %-40s %s" % (
             elt.get("timestamp"),
-            elt.get("generation"),
             elt.get("status"),
             elt.text.strip())