Starting to look like a validator. Still gets different somewhat

results from the original rcynic, still some loose ends and unfinished
bits, and no support for RRDP yet (which was sort of the ultimate
point of the exercise), but approaching the point of being usable, and
about an order of magnitude shorter than the C original.

svn path=/branches/tk705/; revision=6194

1 files changed, 173 insertions, 71 deletions

diff --git a/rp/rcynic/rcynicng b/rp/rcynic/rcynicng
index 59c9b71a..4c5e7b7f 100755
--- a/rp/rcynic/rcynicng
+++ b/rp/rcynic/rcynicng
@@ -15,6 +15,7 @@ RP code.  Gotta start somewhere.
 import os
 import sys
 import time
+import shutil
 import logging
 import argparse
 import subprocess
@@ -103,9 +104,56 @@ class Status(object):
         for code in codes:
             status.add(code)
 
+    @classmethod
+    def test(cls, uri, generation, code):
+        key = (uri, generation)
+        return key in cls.db and code in cls.db[key].status
+
+
+def install_object(obj):
+    fn = uri_to_filename(obj.uri, new_authenticated)
+    dn = os.path.dirname(fn)
+    #logger.debug("Installing %r by linking %s to %s", obj, obj.fn, fn)
+    if not os.path.isdir(dn):
+        os.makedirs(dn)
+    os.link(obj.fn, fn)
+
+
+def final_install():
+
+    real_old = os.path.realpath(old_authenticated).rstrip("/")
+    real_new = os.path.realpath(new_authenticated).rstrip("/")
+
+    fn = args.authenticated.rstrip("/") + ".new"
+    logger.debug("Symlinking %s to %s", os.path.basename(real_new), args.authenticated)
+    if os.path.exists(fn):
+        os.unlink(fn)
+    os.symlink(os.path.basename(real_new), fn)
+    os.rename(fn, args.authenticated)
+
+    if os.path.isdir(real_old):
+        fn = args.authenticated.rstrip("/") + ".old"
+        logger.debug("Symlinking %s to %s", os.path.basename(real_old), fn)
+        if os.path.exists(fn):
+            os.unlink(fn)
+        os.symlink(os.path.basename(real_old), fn)
+
+    dn = os.path.dirname(args.authenticated.rstrip("/"))
+    for fn in os.listdir(dn):
+        fn = os.path.join(dn, fn)
+        if fn.startswith(args.authenticated.rstrip("/") + ".") and os.path.realpath(fn) not in (real_new, real_old):
+            logger.debug("Removing %s", fn)
+            shutil.rmtree(fn)
+
 
 class X509(rpki.POW.X509):
 
+    def __repr__(self):
+        try:
+            return "<X509 \"{}\" {} at 0x{:x}>".format(self.uri, self.generation, id(self))
+        except:
+            return "<X509 at 0x{:x}>".format(id(self))
+
     @classmethod
     def derReadURI(cls, uri, generation, cms = None):
         fn = uri_to_filename(uri, generation.tree)
@@ -145,7 +193,7 @@ class X509(rpki.POW.X509):
                     count += 1
         return count
 
-    def check(self, trusted = None, crl = None):
+    def check(self, trusted, crl):
         status = Status.update(self.uri, self.generation)
         is_ta = trusted is None
         is_routercert = (self.eku is not None and id_kp_bgpsec_router in self.eku and
@@ -186,7 +234,8 @@ class X509(rpki.POW.X509):
             status.add(codes.MALFORMED_CRLDP_EXTENSION)
         try:
             self.verify(trusted = [self] if trusted is None else trusted, crl = crl, status = status)
-        except rpki.POW.ValidationError:
+        except rpki.POW.ValidationError as e:
+            logger.debug("%r rejected: %s", self, e)
             status.add(codes.OBJECT_REJECTED)
         codes.normalize(status)
         return not any(s.kind == "bad" for s in status)
@@ -194,6 +243,12 @@ class X509(rpki.POW.X509):
 
 class CRL(rpki.POW.CRL):
 
+    def __repr__(self):
+        try:
+            return "<CRL \"{}\" {} at 0x{:x}>".format(self.uri, self.generation, id(self))
+        except:
+            return "<CRL at 0x{:x}>".format(id(self))
+
     @classmethod
     def derReadURI(cls, uri, generation):
         fn = uri_to_filename(uri, generation.tree)
@@ -217,7 +272,8 @@ class CRL(rpki.POW.CRL):
         status = Status.update(self.uri, self.generation)
         try:
             self.verify(issuer, status)
-        except rpki.POW.ValidationError:
+        except rpki.POW.ValidationError as e:
+            logger.debug("%r rejected: %s", self, e)
             status.add(codes.OBJECT_REJECTED)
         codes.normalize(status)
         return not any(s.kind == "bad" for s in status)
@@ -225,6 +281,12 @@ class CRL(rpki.POW.CRL):
 
 class Ghostbuster(rpki.POW.CMS):
 
+    def __repr__(self):
+        try:
+            return "<Ghostbuster \"{}\" {} at 0x{:x}>".format(self.uri, self.generation, id(self))
+        except:
+            return "<Ghostbuster at 0x{:x}>".format(id(self))
+
     @classmethod
     def derReadURI(cls, uri, generation):
         fn = uri_to_filename(uri, generation.tree)
@@ -242,12 +304,13 @@ class Ghostbuster(rpki.POW.CMS):
         self.vcard  = None
         return self
 
-    def check(self, trusted = None, crl = None):
+    def check(self, trusted, crl):
         status = Status.update(self.uri, self.generation)
         self.ee.check(trusted = trusted, crl = crl)
         try:
             self.vcard = self.verify()
-        except rpki.POW.ValidationError:
+        except rpki.POW.ValidationError as e:
+            logger.debug("%r rejected: %s", self, e)
             status.add(codes.OBJECT_REJECTED)
         codes.normalize(status)
         return not any(s.kind == "bad" for s in status)
@@ -255,6 +318,12 @@ class Ghostbuster(rpki.POW.CMS):
 
 class Manifest(rpki.POW.Manifest):
 
+    def __repr__(self):
+        try:
+            return "<Manifest \"{}\" {} at 0x{:x}>".format(self.uri, self.generation, id(self))
+        except:
+            return "<Manifest at 0x{:x}>".format(id(self))
+
     @classmethod
     def derReadURI(cls, uri, generation):
         fn = uri_to_filename(uri, generation.tree)
@@ -274,12 +343,13 @@ class Manifest(rpki.POW.Manifest):
         self.number     = None
         return self
 
-    def check(self, trusted = None, crl = None):
+    def check(self, trusted, crl):
         status = Status.update(self.uri, self.generation)
         self.ee.check(trusted = trusted, crl = crl)
         try:
             self.verify()
-        except rpki.POW.ValidationError:
+        except rpki.POW.ValidationError as e:
+            logger.debug("%r rejected: %s", self, e)
             status.add(codes.OBJECT_REJECTED)
         self.thisUpdate = self.getThisUpdate()
         self.nextUpdate = self.getNextUpdate()
@@ -297,6 +367,12 @@ class Manifest(rpki.POW.Manifest):
 
 class ROA(rpki.POW.ROA):
 
+    def __repr__(self):
+        try:
+            return "<ROA \"{}\" {} at 0x{:x}>".format(self.uri, self.generation, id(self))
+        except:
+            return "<ROA at 0x{:x}>".format(id(self))
+
     @classmethod
     def derReadURI(cls, uri, generation):
         fn = uri_to_filename(uri, generation.tree)
@@ -315,7 +391,7 @@ class ROA(rpki.POW.ROA):
         self.prefixes   = None
         return self
 
-    def check(self, trusted = None, crl = None):
+    def check(self, trusted, crl):
         status = Status.update(self.uri, self.generation)
         self.ee.check(trusted = trusted, crl = crl)
         try:
@@ -374,7 +450,7 @@ class  WalkFrame(object):
     def ready(self, wsk):
         self.trusted = wsk.trusted()
 
-        logger.debug("%r scanning products", self)
+        #logger.debug("%r scanning products", self)
 
         mft_uri = first_rsync_uri(self.cer.rpkiManifest)
 
@@ -383,13 +459,13 @@ class  WalkFrame(object):
         # NB: CRL checks on manifest EE certificates deferred until we've picked a CRL.
 
         self.current_mft = Manifest.derReadURI(mft_uri, Generation.current)
-        if self.current_mft is not None and self.current_mft.check(trusted = self.trusted):
+        if self.current_mft is not None and self.current_mft.check(trusted = self.trusted, crl = None):
             crl_candidates.extend(self.current_mft.find_crl_uris())
         else:
             self.current_mft = None
 
         self.backup_mft  = Manifest.derReadURI(mft_uri, Generation.backup)
-        if self.backup_mft is not None and self.backup_mft.check(trusted = self.trusted):
+        if self.backup_mft is not None and self.backup_mft.check(trusted = self.trusted, crl = None):
             crl_candidates.extend(self.backup_mft.find_crl_uris())
         else:
             self.backup_mft = None
@@ -412,7 +488,7 @@ class  WalkFrame(object):
                 if crl is None or crl == self.crl:
                     continue
                 if crl.sha256 != digest:
-                    Status.add(uri, generation, codes.DIGEST_MISMATCH)
+                    #Status.add(uri, generation, codes.DIGEST_MISMATCH)
                     continue
                 if not crl.check(self.trusted[0]) or (self.crl is not None and crl.number < self.crl.number):
                     continue
@@ -423,6 +499,9 @@ class  WalkFrame(object):
             wsk.pop()
             return
 
+        install_object(self.crl)
+        Status.add(self.crl.uri, self.crl.generation, codes.OBJECT_ACCEPTED)
+
         #logger.debug("Picked %s CRL %s", self.crl.generation, self.crl.uri)
 
         if self.current_mft is not None and self.crl.isRevoked(self.current_mft.ee):
@@ -451,6 +530,8 @@ class  WalkFrame(object):
 
         self.state = self.loop
 
+    fns2 = dict(cer = X509, gbr = Ghostbuster, roa = ROA)
+
     @tornado.gen.coroutine
     def loop(self, wsk):
 
@@ -467,36 +548,38 @@ class  WalkFrame(object):
                 counter = 0
 
             uri = self.diruri + fn
+            cls = self.fns2.get(uri[-3:])
+
+            # Need general URI validator here?
 
             if uri == self.crl.uri:
                 continue
 
-            if fn.endswith(".roa"):
-                roa = ROA.derReadURI(uri, self.generation)
-                roa.check()             # XXX Do something with result
+            if self.generation is Generation.backup and Status.test(uri, Generation.current, codes.OBJECT_ACCEPTED):
+                logger.debug("Current version of %s already accepted, skipping", uri)
                 continue
 
-            if fn.endswith(".gbr"):
-                gbr = Ghostbuster.derReadURI(uri, self.generation)
-                gbr.check()             # XXX Do something with result
+            if uri[-4] != "." or cls is None:
+                Status.add(uri, self.generation, codes.UNKNOWN_OBJECT_TYPE_SKIPPED)
                 continue
 
-            if fn.endswith(".cer"):
-                cer = X509.derReadURI(uri, self.generation)
-                cer.check()             # XXX Do something with result
-                if cer.is_ca:
-                    wsk.push(cer)
+            obj = cls.derReadURI(uri, self.generation)
 
-                    # XXX Temporary: Need to integrate with FSM
-                    # looping, rsync fetching, etc -- this is just a
-                    # hack to preserve old walk_tree() behavior
-                    # temporarily for testing.
+            ok  = obj.check(trusted = self.trusted, crl = self.crl)
 
-                    return
+            if obj.sha256 != digest:
+                Status.add(uri, generation, codes.DIGEST_MISMATCH)
+                ok = False
 
-                continue
+            if ok:
+                install_object(obj)
+                Status.add(uri, self.generation, codes.OBJECT_ACCEPTED)
+            else:
+                Status.add(uri, self.generation, codes.OBJECT_REJECTED)
 
-            Status.add(uri, self.generation, codes.UNKNOWN_OBJECT_TYPE_SKIPPED)
+            if ok and cls is X509 and obj.is_ca:
+                wsk.push(obj)
+                return
 
         if self.generation is Generation.current and self.backup_mft is not None:
             self.mft_iterator = iter(self.backup_mft.getFiles())
@@ -540,31 +623,6 @@ class WalkTask(object):
         return stack
 
 
-def parse_arguments():
-
-    def check_dir(s):
-        if not os.path.isdir(s):
-            raise argparse.ArgumentTypeError("%r is not a directory" % s)
-        return s
-
-    def posint(s):
-        i = int(s)
-        if i <= 0:
-            raise argparse.ArgumentTypeError("%r is not a positive integer " % s)
-        return i
-
-    parser = argparse.ArgumentParser(description = __doc__)
-    parser.add_argument("--unauthenticated",   type = check_dir, default = "rcynic-data/unauthenticated")
-    parser.add_argument("--old-authenticated", type = check_dir, default = "rcynic-data/authenticated.old")
-    parser.add_argument("--tals",              type = check_dir, default = "sample-trust-anchors")
-    parser.add_argument("--output",                              default = "rcynic-data/rcynicng-output")
-    parser.add_argument("--workers",           type = posint,    default = 10)
-    parser.add_argument("--no-fetch",                            action  = "store_true")
-    parser.add_argument("--xml-file", type = argparse.FileType("w"), default = "rcynicng.xml")
-    parser.add_argument("--spawn-on-fetch",                      action = "store_true")
-    return parser.parse_args()
-
-
 def read_tals():
     for root, dirs, files in os.walk(args.tals):
         for fn in files:
@@ -747,19 +805,19 @@ class CheckTALTask(object):
 
     @tornado.gen.coroutine
     def check(self, generation):
-        status = Status.update(self.uri, generation)
         self.cer = X509.derReadURI(self.uri, generation)
         ok = False
         if self.cer is None:
-            status.add(codes.UNREADABLE_TRUST_ANCHOR)
-            status.add(codes.OBJECT_REJECTED)
+            Status.add(self.uri, generation, codes.UNREADABLE_TRUST_ANCHOR)
         elif self.key.derWritePublic() != self.cer.getPublicKey().derWritePublic():
-            status.add(codes.TRUST_ANCHOR_KEY_MISMATCH)
-            status.add(codes.OBJECT_REJECTED)
-        elif not self.cer.check():
-            status.add(codes.OBJECT_REJECTED)
+            Status.add(self.uri, generation, codes.TRUST_ANCHOR_KEY_MISMATCH)
         else:
-            ok = True
+            ok = self.cer.check(trusted = None, crl = None)
+        if ok:
+            install_object(self.cer)
+            Status.add(self.uri, generation, codes.OBJECT_ACCEPTED)
+        else:
+            Status.add(self.uri, generation, codes.OBJECT_REJECTED)
         raise tornado.gen.Return(ok)
 
 
@@ -782,6 +840,14 @@ def worker(meself):
 
 
 def final_report():
+    # Clean up a bit to avoid confusing the user unnecessarily.
+    for s in Status.db.itervalues():
+        if codes.OBJECT_ACCEPTED in s.status:
+            s.status.discard(codes.OBJECT_REJECTED)
+        if s.generation is Generation.backup:
+            if Status.test(s.uri, Generation.current, codes.OBJECT_ACCEPTED):
+                s.status.discard(codes.OBJECT_REJECTED)
+            s.status.discard(codes.OBJECT_NOT_FOUND)
     doc = Element("rcynic-summary") # rcynic-version = "", summary-version = "", reporting-hostname = ""
     labels = SubElement(doc, "labels")
     for code in codes.all():
@@ -800,20 +866,56 @@ def final_report():
 
 
 @tornado.gen.coroutine
-def main():
+def launcher():
     for i in xrange(args.workers):
         tornado.ioloop.IOLoop.current().spawn_callback(worker, i)
+
     yield [task_queue.put(CheckTALTask(uri, key)) for uri, key in read_tals()]
     yield task_queue.join()
-    final_report()
 
 
-if __name__ == "__main__":
+class posint(int):
+    def __init__(self, value):
+        if self <= 0:
+            raise ValueError
+
+
+def main():
     os.putenv("TZ", "UTC")
     time.tzset()
-    task_queue = tornado.queues.Queue()
-    args = parse_arguments()
-    logging.basicConfig(level = logging.DEBUG, format = "%(asctime)s  %(message)s", datefmt = "%Y-%m-%d %H:%M:%S")
+
+    parser = argparse.ArgumentParser(description = __doc__)
+
+    parser.add_argument("--authenticated",      default = "rcynic-data/authenticated")
+    parser.add_argument("--unauthenticated",    default = "rcynic-data/unauthenticated")
+    parser.add_argument("--xml-file",           default = "rcynicng.xml",       type = argparse.FileType("w"))
+
+    parser.add_argument("--tals",               default = "sample-trust-anchors")
+
+    parser.add_argument("--workers", type = posint, default = 10)
+    parser.add_argument("--no-fetch", action  = "store_true")
+    parser.add_argument("--spawn-on-fetch",     action = "store_true")
+
+    global args
+    args = parser.parse_args()
+
+    global new_authenticated, old_authenticated
+    new_authenticated = args.authenticated.rstrip("/") + time.strftime(".%Y-%m-%dT%H:%M:%SZ")
+    old_authenticated = args.authenticated
+
     Generation("current", args.unauthenticated)
-    Generation("backup",  args.old_authenticated)
-    tornado.ioloop.IOLoop.current().run_sync(main)
+    Generation("backup",  old_authenticated)
+
+    logging.basicConfig(level = logging.DEBUG, format = "%(asctime)s  %(message)s", datefmt = "%Y-%m-%d %H:%M:%S")
+
+    global task_queue
+    task_queue = tornado.queues.Queue()
+    tornado.ioloop.IOLoop.current().run_sync(launcher)
+
+    final_report()
+
+    final_install()
+
+
+if __name__ == "__main__":
+    main()