Add (mostly) backwards-compatible iterator which handles finding

authenticated RP data via either os.walk() or Django ORM database
traversal.

svn path=/branches/tk705/; revision=6236

3 files changed, 42 insertions, 52 deletions

diff --git a/rp/utils/hashdir b/rp/utils/hashdir
index c7c18350..21619ce4 100755
--- a/rp/utils/hashdir
+++ b/rp/utils/hashdir
@@ -27,41 +27,39 @@ distributed as part of the repository system.
 import os
 import sys
 import argparse
-import rpki.POW
+
+from rpki.rcynicdb.iterator import authenticated_objects
 
 def check_dir(s):
     if os.path.isdir(s):
         return os.path.abspath(s)
     else:
-        raise argparse.ArgumentTypeError("%r is not a directory" % s)
+        raise argparse.ArgumentTypeError("{!r} is not a directory".format(s))
 
 parser = argparse.ArgumentParser(description = __doc__)
 parser.add_argument("-v", "--verbose", action = "store_true", help = "whistle while you work")
-parser.add_argument("rcynic_dir", type = check_dir, help = "rcynic authenticated output directory")
+parser.add_argument("rcynic_dir", nargs = "?", type = check_dir, help = "rcynic authenticated output directory")
 parser.add_argument("output_dir", help = "name of output directory to create")
 args = parser.parse_args()
 
 if not os.path.isdir(args.output_dir):
     os.makedirs(args.output_dir)
 
-for root, dirs, files in os.walk(args.rcynic_dir):
-    for ifn in files:
-        ifn = os.path.join(root, ifn)
-        if ifn.endswith(".cer"):
-            obj = rpki.POW.X509.derReadFile(ifn)
-            fmt = "%08x.%%d" % obj.getSubjectHash()
-        elif ifn.endswith(".crl"):
-            obj = rpki.POW.CRL.derReadFile(ifn)
-            fmt = "%08x.r%%d" % obj.getIssuerHash()
-        else:
+def store(uri, obj, fmt):
+    for i in xrange(1000000):
+        fn = os.path.join(args.output_dir, fmt.format(i))
+        if os.path.exists(fn):
             continue
-        for i in xrange(1000000):
-            ofn = os.path.join(args.output_dir, fmt % i)
-            if not os.path.exists(ofn):
-                with open(ofn, "w") as f:
-                    f.write(obj.pemWrite())
-                    if args.verbose:
-                        print ofn, "<=", ifn
-                break
-        else:
-            sys.exit("No path name available for %s (%s)" % (ifn, ofn))
+        with open(fn, "w") as f:
+            f.write(obj.pemWrite())
+            if args.verbose:
+                print fn, "<=", uri
+            return
+    else:
+        sys.exit("No path name available for {} ({})".format(uri, fn))
+
+for uri, cer in authenticated_objects(uri_suffix = ".cer"):
+    store(uri, cer, "{:08x}.{{:d}}".format(cer.getSubjectHash()))
+
+for uri, crl in authenticated_objects(uri_suffix = ".crl"):
+    store(uri, crl, "{:08x}.r{{:d}}".format(crl.getIssuerHash()))
diff --git a/rp/utils/scan_roas b/rp/utils/scan_roas
index 689d1365..740d6628 100755
--- a/rp/utils/scan_roas
+++ b/rp/utils/scan_roas
@@ -26,6 +26,8 @@ import os
 import argparse
 import rpki.POW
 
+from rpki.rcynicdb.iterator import authenticated_objects
+
 def check_dir(d):
     if not os.path.isdir(d):
         raise argparse.ArgumentTypeError("%r is not a directory" % d)
@@ -33,12 +35,6 @@ def check_dir(d):
 
 class ROA(rpki.POW.ROA):                # pylint: disable=W0232
 
-    @classmethod
-    def parse(cls, fn):                 # pylint: disable=W0621
-        self = cls.derReadFile(fn)      # pylint: disable=E1101
-        self.extractWithoutVerifying()
-        return self
-
     @property
     def prefixes(self):
         v4, v6 = self.getPrefixes()     # pylint: disable=E1101
@@ -53,12 +49,12 @@ class ROA(rpki.POW.ROA):                # pylint: disable=W0232
         return "%s %s %s" % (self.signingTime(), self.getASID(), " ".join(self.prefixes))
 
 parser = argparse.ArgumentParser(description = __doc__)
-parser.add_argument("rcynic_dir", nargs = "+", type = check_dir,
+parser.add_argument("rcynic_dir", nargs = "?", type = check_dir,
                     help = "rcynic authenticated output directory")
 args = parser.parse_args()
 
-for rcynic_dir in args.rcynic_dir:
-    for root, dirs, files in os.walk(rcynic_dir):
-        for fn in files:
-            if fn.endswith(".roa"):
-                print ROA.parse(os.path.join(root, fn))
+for uri, roa in authenticated_objects(args.rcynic_dir,
+                                      uri_suffix = ".roa",
+                                      class_map = dict(roa = ROA)):
+    roa.extractWithoutVerifying()
+    print roa
diff --git a/rp/utils/scan_routercerts b/rp/utils/scan_routercerts
index 9a13d7a9..74cd2b69 100755
--- a/rp/utils/scan_routercerts
+++ b/rp/utils/scan_routercerts
@@ -27,31 +27,27 @@ import argparse
 import rpki.POW
 import rpki.oids
 
+from rpki.rcynicdb.iterator import authenticated_objects
+
 def check_dir(s):
     if not os.path.isdir(s):
-        raise argparse.ArgumentTypeError("%r is not a directory" % s)
+        raise argparse.ArgumentTypeError("{!r} is not a directory".format(s))
     return s
 
 parser = argparse.ArgumentParser(description = __doc__)
-parser.add_argument("rcynic_dir", type = check_dir, help = "rcynic authenticated output directory")
+parser.add_argument("rcynic_dir", nargs = "?", type = check_dir,
+                    help = "rcynic authenticated output directory")
 args = parser.parse_args()
 
-for root, dirs, files in os.walk(args.rcynic_dir):
-
-    for fn in files:
-
-        if not fn.endswith(".cer"):
-            continue
-
-        x = rpki.POW.X509.derReadFile(os.path.join(root, fn))
+for uri, cer in authenticated_objects(args.rcynic_dir, uri_suffix = ".cer"):
 
-        if rpki.oids.id_kp_bgpsec_router not in (x.getEKU() or ()):
-            continue
+    if rpki.oids.id_kp_bgpsec_router not in (cer.getEKU() or ()):
+        continue
 
-        sys.stdout.write(base64.urlsafe_b64encode(x.getSKI()).rstrip("="))
+    sys.stdout.write(base64.urlsafe_b64encode(cer.getSKI()).rstrip("="))
 
-        for min_asn, max_asn in x.getRFC3779()[0]:
-            for asn in xrange(min_asn, max_asn + 1):
-                sys.stdout.write(" %s" % asn)
+    for min_asn, max_asn in cer.getRFC3779()[0]:
+        for asn in xrange(min_asn, max_asn + 1):
+            sys.stdout.write(" {}".format(asn))
 
-        sys.stdout.write(" %s\n" % base64.b64encode(x.getPublicKey().derWritePublic()))
+    sys.stdout.write(" {}\n".format(base64.b64encode(cer.getPublicKey().derWritePublic())))