First test of new validation code. No major surprises. Much testing

left to do, still need to add in stuff that we pushed out to Python
rather than trying to do in C (eg, a lot of the URI tests), but basics
seem to work.  Checkpointing before attempting a major simplification
of the StatusCode mechanism.

svn path=/branches/tk705/; revision=6179

2 files changed, 192 insertions, 113 deletions

diff --git a/ext/POW.c b/ext/POW.c
index 16dd070d..c37ff1e7 100644
--- a/ext/POW.c
+++ b/ext/POW.c
@@ -1210,6 +1210,17 @@ whack_ec_key_to_namedCurve(EVP_PKEY *pkey)
  */
 
 /*
+ * Arguably most of this awful code could be replaced by a tiny
+ * separate module written in Python and imported here using
+ * PyImport_ImportModule(), after which we could grab objects from
+ * that module using PyObject_GetAttrString().  If necessary, that
+ * module could also contain code which, when invoked as a script
+ * rather than imported, dumped out a .h file we could import here,
+ * but it's probably not necessary, plain C strings here as names for
+ * objects in that module would probably be fine for our purposes.
+ */
+
+/*
  * There's some ugly C preprocessor junk here.  Sorry, but it's the
  * simplest way to keep all the definitions in a single place and
  * expand them into all the forms we need in both C and Python.  We
@@ -3937,7 +3948,7 @@ x509_object_verify(x509_object *self, PyObject *args, PyObject *kwds)
   PyObject *crl       = Py_None;
   PyObject *status    = Py_None;
   X509 *issuer = NULL;
-  int ok = 0, is_ta;
+  int ok = 0, is_ta = 0;
 
   if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOOO!O", kwlist,
                                    &trusted, &untrusted, &crl, &PySet_Type, &status, &ctxclass))
@@ -3971,49 +3982,60 @@ x509_object_verify(x509_object *self, PyObject *args, PyObject *kwds)
   if (crl != Py_None)
     X509_VERIFY_PARAM_set_flags(ctx->ctx->param, X509_V_FLAG_CRL_CHECK);
 
-  /*
-   * Tedious search for issuer.  Should we even be doing this?  rcynic
-   * knows which cert it thinks is the issuer, so it's a waste of time
-   * there, and we don't need to do this when we're not doing detailed
-   * RPKI checking, so the answer is probably no, we don't need this.
-   *
-   * Except that it seems to work better.  Which may just mean that I
-   * hnorked the ordering of the trusted chain when passing it in
-   * during testing.
-   *
-   * For the moment, keep options open, clean up later.
-   */
+  if (status != Py_None) {
+
+    /*
+     * Tedious search for issuer.  Should we even be doing this?  rcynic
+     * knows which cert it thinks is the issuer, so it's a waste of time
+     * there, and we don't need to do this when we're not doing detailed
+     * RPKI checking, so the answer is probably no, we don't need this.
+     *
+     * Except that it seems to work better when we do this.  Which may
+     * just mean that I hnorked the ordering of the trusted chain when
+     * passing it in during testing.
+     *
+     * For the moment, keep options open, clean up later.
+     */
 
 #if 1
 
-  {
-    int i;
-    for (i = 0; issuer == NULL && i < sk_X509_num(trusted_stack); i++)
-      if (X509_check_issued((issuer = sk_X509_value(trusted_stack, i)), self->x509) != 0)
-        issuer = NULL;
-    for (i = 0; issuer == NULL && i < sk_X509_num(untrusted_stack); i++)
-      if (X509_check_issued((issuer = sk_X509_value(untrusted_stack, i)), self->x509) != 0)
-        issuer = NULL;
-  }
+      int i;
+      for (i = 0; issuer == NULL && i < sk_X509_num(trusted_stack); i++)
+        if (X509_check_issued((issuer = sk_X509_value(trusted_stack, i)), self->x509) != 0)
+          issuer = NULL;
+      for (i = 0; issuer == NULL && i < sk_X509_num(untrusted_stack); i++)
+        if (X509_check_issued((issuer = sk_X509_value(untrusted_stack, i)), self->x509) != 0)
+          issuer = NULL;
 
-  is_ta = (sk_X509_num(trusted_stack) == 1 && 
-           sk_X509_num(untrusted_stack) == 0 &&
-           X509_cmp(issuer, self->x509) == 0);
+    is_ta = (sk_X509_num(trusted_stack) == 1 && 
+             sk_X509_num(untrusted_stack) == 0 &&
+             X509_cmp(issuer, self->x509) == 0);
 
 #else
 #warning Do we need to do something about picking issuer out of trusted_stack?
 
-  is_ta = (sk_X509_num(trusted_stack) == 1 &&
-           sk_X509_num(untrusted_stack) == 0 &&
-           X509_cmp(sk_X509_value(trusted_stack, 0), self->x509) == 0 &&
-           X509_check_issued(self->x509, self->x509) == 0);
+    is_ta = (sk_X509_num(trusted_stack) == 1 &&
+             sk_X509_num(untrusted_stack) == 0 &&
+             X509_cmp(sk_X509_value(trusted_stack, 0), self->x509) == 0 &&
+             X509_check_issued(self->x509, self->x509) == 0);
 
 #endif
 
-  if (issuer == NULL)
-    issuer = sk_X509_value(trusted_stack, 0);
+    if (issuer == NULL)
+      issuer = sk_X509_value(trusted_stack, 0);
+  }
 
 #warning Need to do something about check_object_type_* mess
+  /*
+   * Original concept was reasonable, but now that we're making the
+   * Python caller responsible for validation of embedded
+   * certificates, we don't have a handle on the CMS object for the
+   * encapsulated cases.  So either we need to do the
+   * signed-object-type-specific checks at the Python level, or we
+   * need the caller to tell us what kind of certificate this is
+   * supposed to be, whether by passing us a filename extension or
+   * some other means.
+   */
 
   if (status != Py_None && !check_x509(self->x509, issuer, status, is_ta, check_object_type_cer, ctx->ctx))
     goto error;
diff --git a/rp/rcynic/rcynicng b/rp/rcynic/rcynicng
index b9d23380..61d83dc8 100755
--- a/rp/rcynic/rcynicng
+++ b/rp/rcynic/rcynicng
@@ -21,40 +21,79 @@ import rpki.POW
 
 from lxml.etree import ElementTree, Element, SubElement, Comment
 
-args = None
+class Status(object):
+    """
+    Validation status database, like validation_status_t in rcynic:tos.
+    """
+
+    db = dict()
+
+    def __init__(self, uri, generation = None):
+        assert generation in ("current", "backup", None)
+        self.uri = uri
+        self.generation = generation
+        self.timestamp = None
+        self.status = set()
+
+    def __str__(self):
+        return "{time} {self.uri} {status} {self.generation}".format(
+            self   = self,
+            time   = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(self.timestamp)),
+            status = ",".join(s.name for s in sorted(self.status)))
+
+    @classmethod
+    def update(cls, uri, generation = None):
+        try:
+            key  = (uri, generation)
+            self = cls.db[key]
+        except KeyError:
+            self = cls.db[key] = cls(uri, generation)
+        self.timestamp = time.time()
+        return self.status
+
 
-def check_dir(s):
-    if not os.path.isdir(s):
-        raise argparse.ArgumentTypeError("%r is not a directory" % s)
-    return s
+def parse_arguments():
+
+    def check_dir(s):
+        if not os.path.isdir(s):
+            raise argparse.ArgumentTypeError("%r is not a directory" % s)
+        return s
 
-def parse_options():
-    global args                         # pylint: disable=W0603
     parser = argparse.ArgumentParser(description = __doc__)
     parser.add_argument("--unauthenticated",   type = check_dir, default = "rcynic-data/unauthenticated")
     parser.add_argument("--old-authenticated", type = check_dir, default = "rcynic-data/authenticated.old")
     parser.add_argument("--tals",              type = check_dir, default = "sample-trust-anchors")
     parser.add_argument("--output",                              default = "rcynic-data/rcynicng-output")
-    args = parser.parse_args()
+    return parser.parse_args()
 
 
 def read_tals():
     for root, dirs, files in os.walk(args.tals):
         for fn in files:
             if fn.endswith(".tal"):
-                with open(os.path.join(root, fn), "r") as f:
-                    lines = f.readlines()
-                uri = lines.pop(0).strip()
-                b64 = "".join(lines[lines.index("\n"):])
-                key = rpki.POW.Asymmetric.derReadPublic(b64.decode("base64"))
-                yield uri, key
-
-def uri_to_fn(uri, base = None):
+                furi = "file://" + os.path.abspath(os.path.join(root, fn))
+                try:
+                    with open(os.path.join(root, fn), "r") as f:
+                        lines = f.readlines()
+                    uri = lines.pop(0).strip()
+                    b64 = "".join(lines[lines.index("\n"):])
+                    key = rpki.POW.Asymmetric.derReadPublic(b64.decode("base64"))
+                    if not uri.endswith(".cer"):
+                        Status.update(furi).add(rpki.POW.validation_status.MALFORMED_TAL_URI)
+                    yield uri, key
+                except:
+                    Status.update(furi).add(rpki.POW.validation_status.UNREADABLE_TRUST_ANCHOR_LOCATOR)
+
+
+def uri_to_filename(uri, base = None):
     fn = uri[uri.index("://")+3:]
     if base is not None:
         fn = os.path.join(base, fn)
     return fn
 
+def uri_to_basename(uri):
+    return uri.rpartition("/")[2]
+
 def first_uri(uris, scheme):
     for uri in uris:
         if uri.startswith(scheme):
@@ -64,89 +103,107 @@ def first_uri(uris, scheme):
 def first_rsync_uri(uris):
     return first_uri(uris, "rsync://")
 
+def sha256(bytes):
+    d = rpki.POW.Digest(rpki.POW.SHA256_DIGEST)
+    d.update(bytes)
+    return d.digest()
 
-def walk_tree(ca, trusted, crl, basedir):
-    ca_status = set()
-    ca.verify(trusted = trusted, crl = crl, status = ca_status)
+
+def walk_tree(cauri, ca, trusted, crl, basedir):
     trusted.insert(0, ca)
-    diruri, mfturi = [first_rsync_uri(uri) for uri in  ca.getSIA()[:2]]
-    mft = rpki.POW.Manifest.derReadFile(uri_to_fn(mfturi, basedir))
-    ee = mft.certs()[0]
-    crldp = first_rsync_uri(ee.getCRLDP())
-    crl = rpki.POW.CRL.derReadFile(uri_to_fn(crldp, basedir))
-    crl_status = set()
-    mft_status = set()
+
+    sia    = ca.getSIA()
+    diruri = first_rsync_uri(sia[0])
+    mfturi = first_rsync_uri(sia[1])
+    try:
+        mft = rpki.POW.Manifest.derReadFile(uri_to_filename(mfturi, basedir))
+    except rpki.POW.Error as e:
+        print mfturi, e
+        return
+    ee     = mft.certs()[0]
+    crldp  = ee.getCRLDP()
+    crluri = first_rsync_uri(crldp)
+    try:
+        crl = rpki.POW.CRL.derReadFile(uri_to_filename(crluri, basedir))
+    except rpki.POW.Error as e:
+        print crluri, e
+        return
+
+    crl_status = Status.update(crluri)
     crl.verify(ca, crl_status)
+
+    mft_status = Status.update(mfturi)
     ee.verify(trusted = trusted, crl = crl, status = mft_status)
     mft.verify(status = mft_status)
 
-    print "CA status: ", ", ".join(str(s) for s in ca_status)
-    print "CRL status:", ", ".join(str(s) for s in crl_status)
-    print "MFT status:", ", ".join(str(s) for s in mft_status)
+    crl_status.add(rpki.POW.validation_status.CRL_NOT_IN_MANIFEST)
 
     for fn, digest in mft.getFiles():
+        uri = diruri + fn
+        status = Status.update(uri)
 
-        with open(os.path.join(uri_to_fn(diruri, basedir), fn), "rb") as f:
-            obj = f.read()
-        dgst = rpki.POW.Digest(rpki.POW.SHA256_DIGEST)
-        dgst.update(obj)
-        print fn, digest.encode("hex"), "OK hash" if dgst.digest() == digest else "Bad hash"
-
-        if fn.endswith(".crl") and obj != crl.derWrite():
-            print "CRL mismatch"
-        if fn.endswith(".crl"):
+        if uri == crluri:
+            if digest != sha256(crl.derWrite()):
+                status.add(rpki.POW.validation_status.DIGEST_MISMATCH)
+            status.remove(rpki.POW.validation_status.CRL_NOT_IN_MANIFEST)
             continue
 
+        with open(os.path.join(uri_to_filename(diruri, basedir), fn), "rb") as f:
+            der = f.read()
+        if sha256(der) != digest:
+            status.add(rpki.POW.validation_status.DIGEST_MISMATCH)
+
         if fn.endswith(".roa"):
-            roa = rpki.POW.ROA.derRead(obj)
-            roa_status = set()
+            roa = rpki.POW.ROA.derRead(der)
             ee = roa.certs()[0]
-            ee.verify(trusted = trusted, crl = crl, status = roa_status)
-            roa.verify(status = roa_status)
+            ee.verify(trusted = trusted, crl = crl, status = status)
+            roa.verify(status = status)
             continue
 
         if fn.endswith(".gbr"):
-            gbr = rpki.POW.CMS.derRead(obj)
-            gbr_status = set()
+            gbr = rpki.POW.CMS.derRead(der)
             ee = gbr.certs()[0]
-            ee.verify(trusted = trusted, crl = crl, status = gbr_status)
-            vcard = gbr.verify(status = gbr_status)
-            print vcard
+            ee.verify(trusted = trusted, crl = crl, status = status)
+            vcard = gbr.verify(status = status)
             continue
 
-        cer = rpki.POW.X509.derRead(obj)
-        bc = cer.getBasicConstraints()
-        if bc and bc[0]:
-            try:
-                walk_tree(cer, trusted, crl, basedir)
-            except rpki.POW.Error as e:
-                print "CA", diruri + fn, "failed:", e
-        else:
-            cer_status = set()
-            cer.verify(trusted = trusted, crl = crl, status = cer_status)
-
-
-def main():
-
-    os.putenv("TZ", "UTC")
-    time.tzset()
-
-    parse_options()
-
-    basedir = args.unauthenticated
+        if fn.endswith(".cer"):
+            cer = rpki.POW.X509.derRead(der)
+            cer.verify(trusted = trusted, crl = crl, status = status)
+            is_ca = (cer.getBasicConstraints() or (False, None))[0]
+            if is_ca:
+                walk_tree(diruri + fn, cer, trusted, crl, basedir)
+            continue
 
-    for uri, pk in read_tals():
-        print
-        try:
-            x = rpki.POW.X509.derReadFile(uri_to_fn(uri, basedir))
-        except rpki.POW.OpenSSLError:
-            print "Couldn't open TA {}".format(uri)
-        else:
-            ok = pk.derWritePublic() == x.getPublicKey().derWritePublic()
-            print "OK " if ok else "Bad", uri
-            if ok:
-                walk_tree(x, [x], None, basedir)
-
-
-if __name__ == "__main__":
-    main()
+        status.add(rpki.POW.validation_status.UNKNOWN_OBJECT_TYPE_SKIPPED)
+
+
+os.putenv("TZ", "UTC")
+time.tzset()
+
+args = parse_arguments()
+
+basedir = args.unauthenticated
+
+for uri, key in read_tals():
+    status = Status.update(uri)
+    status.add(rpki.POW.validation_status.OBJECT_REJECTED)
+    try:
+        cer = rpki.POW.X509.derReadFile(uri_to_filename(uri, basedir))
+    except rpki.POW.OpenSSLError:
+        status.add(rpki.POW.validation_status.UNREADABLE_TRUST_ANCHOR)
+        continue
+    if key.derWritePublic() != cer.getPublicKey().derWritePublic():
+        status.add(rpki.POW.validation_status.TRUST_ANCHOR_KEY_MISMATCH)
+        continue
+    trusted = [cer]
+    try:
+        cer.verify(trusted = trusted, status = status)
+    except:
+        continue
+    else:
+        status.remove(rpki.POW.validation_status.OBJECT_REJECTED)
+    walk_tree(uri, cer, trusted, None, basedir)
+
+for uri in sorted(Status.db):
+    print Status.db[uri]