#!/usr/local/bin/python # $Id$ # Copyright (C) 2013 Dragon Research Labs ("DRL") # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. # # THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE # OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR # PERFORMANCE OF THIS SOFTWARE. ######################################################################## # # DANGER WILL ROBINSON # # This is a PROTOTYPE of a local trust anchor mechanism. At the # moment, it DOES NOT WORK by any sane standard of measurement. It # produces output, but there is no particular reason to believe said # output is useful, and fairly good reason to believe that it is not. # # With luck, this may eventually mutate into something useful. For # now, just leave it alone unless you really know what you are doing, # in which case, on your head be it. # # YOU HAVE BEEN WARNED # ######################################################################## import os import sys import yaml import glob import time import shutil import base64 import socket import sqlite3 import weakref import rpki.POW import rpki.x509 import rpki.sundial import rpki.resource_set # Lots of icky global variables, clean this up later. tal_directory = None constraints = None rcynic_input = None rcynic_output = None tals = None keyfile = None serial = long(time.time()) << 32 ltakey = None ltacer = None ltauri = "rsync://localhost/lta" ltasia = ltauri + "/" ltaaia = ltauri + ".cer" ltamft = ltauri + "/lta.mft" ltacrl = ltauri + "/lta.crl" cer_delta = rpki.sundial.timedelta(days = 7) crl_delta = rpki.sundial.timedelta(hours = 1) all_mentioned_resources = rpki.resource_set.resource_bag() # Teach SQLite3 about our data types. sqlite3.register_adapter(rpki.POW.IPAddress, lambda x: buffer("_" + x.toBytes())) sqlite3.register_converter("RangeVal", lambda s: long(s) if s.isdigit() else rpki.POW.IPAddress.fromBytes(s[1:])) sqlite3.register_adapter(rpki.x509.X501DN, str) def main(): print "Parsing YAML" parse_yaml() print print "Parsing TALs" parse_tals() print print "Creating DB" rpdb = RPDB() print print "Creating CA" create_ca(rpdb) print print "Loading DB" rpdb.load() print print "Compute resources we need to prune from input forest" compute_all_mentioned_resources() print print "Processing deletions" process_constraint_deletions(rpdb) print print "Re-parenting TAs" re_parent_tas(rpdb) print print "Generating CRL and manifest" generate_crl_and_manifest(rpdb) print print "Committing final changes to DB" rpdb.commit() print print "Dumping para-objects" rpdb.dump_paras() print print "Closing DB" rpdb.close() def create_ca(rpdb): global serial global ltakey global ltacer if os.path.exists(keyfile): ltakey = rpki.x509.RSA(Auto_file = keyfile) else: ltakey = rpki.x509.RSA.generate(quiet = True) with os.fdopen(os.open(keyfile, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0400), "w") as f: f.write(ltakey.get_PEM()) cer = X509.self_certify( cn = "%s LTA Root Certificate" % socket.getfqdn(), keypair = ltakey, subject_key = ltakey.get_RSApublic(), serial = serial, sia = (ltasia, ltamft, None), notAfter = rpki.sundial.now() + cer_delta, resources = rpki.resource_set.resource_bag.from_str("0-4294967295,0.0.0.0/0,::/0")) subject_id = rpdb.find_keyname(cer.getSubject(), cer.get_SKI()) rpdb.cur.execute("INSERT INTO object (der, fn2, subject, issuer, para) " "VALUES (?, 'cer', ?, ?, 1)", (buffer(cer.get_DER()), subject_id, subject_id)) rowid = rpdb.cur.lastrowid rpdb.cur.execute("INSERT INTO uri (id, uri) VALUES (?, ?)", (rowid, ltaaia)) ltacer = rpdb.find_by_id(rowid) class Constraint(object): roa_asn = None roa_maxlen = None router_cert_key = None router_cert_subject = None def __init__(self, y): self.prefixes = rpki.resource_set.resource_bag.from_str(str(y.get("prefix", ""))) self.asns = rpki.resource_set.resource_bag.from_str(str(y.get("asn", ""))) self.ghostbuster = y.get("ghostbuster") if "roa" in y: self.roa_asn = long(y["roa"]["asn"]) if "maxlen" in y["roa"]: self.roa_maxlen = long(y["roa"]["maxlen"]) if "router-cert" in y: self.router_cert_key = y["router-cert"]["key"] self.router_cert_subject = y["router-cert"]["subject"] @property def mentioned_resources(self): return self.prefixes | self.asns def parse_yaml(fn = "rcynic-lta.yaml"): global tal_directory global constraints global rcynic_input global rcynic_output global keyfile y = yaml.safe_load(open(fn, "r")) tal_directory = y["tal-directory"] rcynic_input = y["rcynic-input"] rcynic_output = y["rcynic-output"] keyfile = y["keyfile"] constraints = [Constraint(yy) for yy in y["constraints"]] def parse_tals(): global tals tals = {} for fn in glob.iglob(os.path.join(tal_directory, "*.tal")): with open(fn, "r") as f: uri = f.readline().strip() key = rpki.POW.Asymmetric.derReadPublic(base64.b64decode(f.read())) tals[uri] = key def compute_all_mentioned_resources(): global all_mentioned_resources for constraint in constraints: all_mentioned_resources |= constraint.mentioned_resources def process_constraint_deletions(rpdb): for obj in rpdb.find_by_resource_bag(all_mentioned_resources): rpdb.add_para(obj, obj.resources - all_mentioned_resources) def re_parent_tas(rpdb): for uri, key in tals.iteritems(): for ta in rpdb.find_by_ski_or_uri(key.calculateSKI(), uri): if ta.para_obj is None: rpdb.add_para(ta, ta.resources - all_mentioned_resources) def generate_crl_and_manifest(rpdb): thisUpdate = rpki.sundial.now() nextUpdate = thisUpdate + crl_delta serial = long(time.time()) issuer = ltacer.getSubject() aki = buffer(ltacer.get_SKI()) crl = CRL.generate( keypair = ltakey, issuer = ltacer, serial = serial, thisUpdate = thisUpdate, nextUpdate = nextUpdate, revokedCertificates = ()) issuer_id = rpdb.find_keyname(issuer, aki) rpdb.cur.execute("INSERT INTO object (der, fn2, subject, issuer, para) " "VALUES (?, 'crl', NULL, ?, 1)", (buffer(crl.get_DER()), issuer_id)) rowid = rpdb.cur.lastrowid rpdb.cur.execute("INSERT INTO uri (id, uri) VALUES (?, ?)", (rowid, ltacrl)) crl = rpdb.find_by_id(rowid) key = rpki.x509.RSA.generate(quiet = True) cer = ltacer.issue( keypair = ltakey, subject_key = key.get_RSApublic(), serial = serial, sia = (None, None, ltamft), aia = ltaaia, crldp = ltacrl, resources = rpki.resource_set.resource_bag.from_inheritance(), notAfter = ltacer.getNotAfter(), is_ca = False) mft = rpki.x509.SignedManifest.build( serial = serial, thisUpdate = thisUpdate, nextUpdate = nextUpdate, names_and_objs = [(obj.uri, obj) for obj in rpdb.find_by_aki(ltacer.get_SKI())], keypair = key, certs = cer) subject_id = rpdb.find_keyname(cer.getSubject(), cer.get_SKI()) rpdb.cur.execute("INSERT INTO object (der, fn2, subject, issuer, para) " "VALUES (?, 'mft', ?, ?, 1)", (buffer(mft.get_DER()), subject_id, issuer_id)) rowid = rpdb.cur.lastrowid rpdb.cur.execute("INSERT INTO uri (id, uri) VALUES (?, ?)", (rowid, ltamft)) class DER_object_mixin(object): """ Mixin to add some SQL-related methods to classes derived from rpki.x509.DER_object. """ _rpdb = None _rowid = None _para = False _para_id = None _orig_id = None @property def rowid(self): return self._rowid @property def resources(self): return self.get_3779resources() @property def para_resources(self): return self.resources if self.para_obj is None else self.para_obj.resources @property def para_obj(self): return None if self._para_id is None else self._rpdb.find_by_id(self._para_id) @para_obj.setter def para_obj(self, value): assert value is None if self._para_id is not None: self._rpdb.cur.execute("DELETE FROM object WHERE id = ?", (self._para_id,)) self._para_id = None @property def orig_obj(self): return None if self._orig_id is None else self._rpdb.find_by_id(self._orig_id) def _update_bool(self, name, value): assert self._rpdb is not None and self._rowid is not None and isinstance(value, bool) self._rpdb.cur.execute("UPDATE object SET %s = ? WHERE id = ?" % name, (value, self._rowid)) setattr(self, "_" + name, value) #self._rpdb.db.commit() @property def para(self): return self._para @para.setter def para(self, value): self._update_bool("para", value) class X509 (rpki.x509.X509, DER_object_mixin): pass class CRL (rpki.x509.CRL, DER_object_mixin): pass class SignedManifest (rpki.x509.SignedManifest, DER_object_mixin): pass class ROA (rpki.x509.ROA, DER_object_mixin): pass class Ghostbuster (rpki.x509.Ghostbuster, DER_object_mixin): pass class VerifyContextNoRFC3779(rpki.POW.X509StoreCTX): """ Provide callback for OpenSSL certificate verification. Ignores RFC 3779 nesting errors and warnings about self-signed TA certificates. """ def verify_callback(self, ok): return ok or self.getError() in (rpki.POW.X509_V_ERR_UNNESTED_RESOURCE, rpki.POW.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT) class RPDB(object): """ Relying party database. For now just wire in the database name and rcynic root, fix this later if overall approach seems usable. Might even end up just being an in-memory SQL database, who knows? """ fn2map = dict(cer = X509, crl = CRL, mft = SignedManifest, roa = ROA, gbr = Ghostbuster) mapfn2 = dict((v, k) for k, v in fn2map.iteritems()) object_fields = " %s " % ", ".join("object.%s" % field for field in ( "id", "fn2", "der", "para", "para_id", "orig_id")) def __init__(self, db_name = "rcynic-lta.db"): try: os.unlink(db_name) except: pass self.db = sqlite3.connect(db_name, detect_types = sqlite3.PARSE_DECLTYPES) self.db.text_factory = str self.cur = self.db.cursor() self.cache = weakref.WeakValueDictionary() self.cur.executescript(''' PRAGMA foreign_keys = on; CREATE TABLE keyname ( id INTEGER PRIMARY KEY NOT NULL, name TEXT NOT NULL, keyid BLOB NOT NULL, UNIQUE (name, keyid)); CREATE TABLE object ( id INTEGER PRIMARY KEY NOT NULL, der BLOB NOT NULL, fn2 TEXT NOT NULL CHECK (fn2 IN ('cer', 'crl', 'mft', 'roa', 'gbr')), subject INTEGER REFERENCES keyname(id) ON DELETE RESTRICT ON UPDATE RESTRICT, issuer INTEGER NOT NULL REFERENCES keyname(id) ON DELETE RESTRICT ON UPDATE RESTRICT, para BOOLEAN NOT NULL DEFAULT 0, para_id INTEGER REFERENCES object(id) ON DELETE SET NULL ON UPDATE SET NULL, orig_id INTEGER REFERENCES object(id) ON DELETE CASCADE ON UPDATE CASCADE, UNIQUE (der), CHECK ((subject IS NULL) == (fn2 == 'crl'))); CREATE TABLE uri ( id INTEGER NOT NULL REFERENCES object(id) ON DELETE CASCADE ON UPDATE CASCADE, uri TEXT NOT NULL, UNIQUE (uri)); CREATE TABLE range ( id INTEGER NOT NULL REFERENCES object(id) ON DELETE CASCADE ON UPDATE CASCADE, min RangeVal NOT NULL, max RangeVal NOT NULL, UNIQUE (id, min, max)); ''') def load(self, spinner = 100): start = rpki.sundial.now() nobj = 0 for root, dirs, files in os.walk(rcynic_input): for fn in files: fn = os.path.join(root, fn) fn2 = os.path.splitext(fn)[1][1:] try: obj = self.fn2map[fn2](DER_file = fn) except: continue if spinner and nobj % spinner == 0: sys.stderr.write("\r%s %d %s..." % ("|\\-/"[(nobj/spinner) & 3], nobj, rpki.sundial.now() - start)) nobj += 1 if fn2 == "crl": ski = None aki = buffer(obj.get_AKI()) cer = None bag = None issuer = obj.getIssuer() subject = None else: if fn2 == "cer": cer = obj else: cer = rpki.x509.X509(POW = obj.get_POW().certs()[0]) issuer = cer.getIssuer() subject = cer.getSubject() ski = buffer(cer.get_SKI()) aki = cer.get_AKI() if aki is None: assert subject == issuer aki = ski else: aki = buffer(aki) bag = cer.get_3779resources() der = buffer(obj.get_DER()) uri = "rsync://" + fn[len(rcynic_input) + 1:] self.cur.execute("SELECT id FROM object WHERE der = ?", (der,)) r = self.cur.fetchone() if r is not None: rowid = r[0] else: subject_id = None if ski is None else self.find_keyname(subject, ski) issuer_id = self.find_keyname(issuer, aki) self.cur.execute("INSERT INTO object (der, fn2, subject, issuer) VALUES (?, ?, ?, ?)", (der, fn2, subject_id, issuer_id)) rowid = self.cur.lastrowid if bag is not None: for rset in (bag.asn, bag.v4, bag.v6): if rset is not None: self.cur.executemany("REPLACE INTO range (id, min, max) VALUES (?, ?, ?)", ((rowid, i.min, i.max) for i in rset)) self.cur.execute("INSERT INTO uri (id, uri) VALUES (?, ?)", (rowid, uri)) if spinner: sys.stderr.write("\r= %d objects in %s, committing..." % (nobj, rpki.sundial.now() - start)) self.db.commit() if spinner: sys.stderr.write("done.\n") def add_para(self, obj, resources): # At least some of the following is probably wrong at this point. # Under the new scheme we're going to need to generate signed # objects too. # # As far as I can tell at the moment, we only generate # paracertificates for CA certificates, never for EE certificates. # # At present, ROAs are the only signed objects that specify # resources explictly rather than using inheritance, and EE # certificates for ROAs are supposed to be an exact match for the # address resources in the ROA anyway, so this is likely not a # serious restriction, at least for now. # # Fixing this, if it's a problem, would require extending POW.c to # allow us to whack the certificate(s) bundled into a CMS object. # There's no documentation on how we would even do that, although # I suspect that the OpenSSL library routine # CMS_set1_signers_certs() might do the trick. Ignore for now. assert isinstance(obj, X509) assert not obj.para if obj.para_obj is not None: resources &= obj.para_obj.resources obj.para_obj = None if not resources: return global serial serial += 1 pow = obj.get_POW() x = rpki.POW.X509() x.setVersion( pow.getVersion()) x.setSubject( pow.getSubject()) x.setNotBefore( pow.getNotBefore()) x.setNotAfter( pow.getNotAfter()) x.setPublicKey( pow.getPublicKey()) x.setSKI( pow.getSKI()) x.setBasicConstraints( pow.getBasicConstraints()) x.setKeyUsage( pow.getKeyUsage()) x.setCertificatePolicies( pow.getCertificatePolicies()) x.setSIA( *pow.getSIA()) x.setIssuer( ltacer.get_POW().getIssuer()) x.setAKI( ltacer.get_POW().getSKI()) x.setAIA( (ltaaia,)) x.setCRLDP( (ltacrl,)) x.setSerial( serial) x.setRFC3779( asn = ((r.min, r.max) for r in resources.asn), ipv4 = ((r.min, r.max) for r in resources.v4), ipv6 = ((r.min, r.max) for r in resources.v6)) x.sign(ltakey.get_POW(), rpki.POW.SHA256_DIGEST) cer = X509(POW = x) ski = buffer(cer.get_SKI()) aki = buffer(cer.get_AKI()) bag = cer.get_3779resources() issuer = cer.getIssuer() subject = cer.getSubject() der = buffer(cer.get_DER()) uri = ltasia + cer.gSKI() + ".cer" subject_id = self.find_keyname(subject, ski) issuer_id = self.find_keyname(issuer, aki) self.cur.execute("INSERT INTO object (der, fn2, subject, issuer, para, orig_id) " "VALUES (?, 'cer', ?, ?, 1, ?)", (der, subject_id, issuer_id, obj.rowid)) rowid = self.cur.lastrowid self.cur.execute("UPDATE object SET para_id = ? WHERE id = ?", (rowid, obj.rowid)) obj._para_id = rowid self.cur.execute("INSERT INTO uri (id, uri) VALUES (?, ?)", (rowid, uri)) #self.db.commit() def dump_paras(self): shutil.rmtree(rcynic_output, ignore_errors = True) rsync = "rsync://" for obj in self.find_paras(): assert obj.uri.startswith(rsync) fn = os.path.join(rcynic_output, obj.uri[len(rsync):]) dn = os.path.dirname(fn) if not os.path.exists(dn): os.makedirs(dn) with open(fn, "wb") as f: f.write(obj.get_DER()) def find_keyname(self, name, keyid): keys = (name, buffer(keyid)) self.cur.execute("SELECT id FROM keyname WHERE name = ? AND keyid = ?", keys) result = self.cur.fetchone() if result is None: self.cur.execute("INSERT INTO keyname (name, keyid) VALUES (?, ?)", keys) result = self.cur.lastrowid else: result = result[0] #print "keyname %s >> %s %r" % (result, base64.urlsafe_b64encode(keyid).rstrip("="), name) return result def find_by_id(self, rowid): r = self._find_results(None, "SELECT" + self.object_fields + "FROM object WHERE id = ?", [rowid]) assert len(r) < 2 return r[0] if r else None def find_by_ski_or_uri(self, ski, uri): if not ski and not uri: return [] elif ski and uri: return self._find_results( None, "SELECT" + self.object_fields + "FROM object " + "JOIN uri ON object.id = uri.id " + "JOIN keyname ON object.subject = keyname.id " + "WHERE object.para = 0 AND keyname.keyid = ? AND uri.uri = ?", [buffer(ski), uri]) elif ski: return self._find_results( None, "SELECT" + self.object_fields + "FROM object " + "WHERE para = 0 AND ski = ?", [buffer(ski)]) else: return self._find_results( None, "SELECT" + self.object_fields + "FROM object, uri " + "WHERE para = 0 AND uri.uri = ? AND object.id = uri.id", [uri]) def find_by_ski(self, ski, fn2 = None): if ski is None: return self._find_results(fn2, "SELECT" + self.object_fields + "FROM object WHERE ski IS NULL") else: return self._find_results(fn2, "SELECT" + self.object_fields + "FROM object WHERE ski = ?", [buffer(ski)]) def find_by_aki(self, aki, fn2 = None): if aki is None: return self._find_results(fn2, "SELECT" + self.object_fields + "FROM object WHERE aki IS NULL") else: return self._find_results(fn2, "SELECT" + self.object_fields + "FROM object " + "JOIN keyname ON object.issuer = keyname.id " + "WHERE keyname.keyid = ?", [buffer(aki)]) def find_paras(self, fn2 = None): return self._find_results(fn2, "SELECT" + self.object_fields + "FROM object WHERE para <> 0") def find_parent(self, child): return self._find_results(None, "SELECT" + self.object_fields + "FROM object WHERE subject = ? AND SKI = ?", [child.getIssuer(), buffer(child.get_AKI())]) def find_products(self, aki, issuer, fn2 = None): return self._find_results(fn2, "SELECT" + self.object_fields + "FROM object WHERE aki = ? AND issuer = ?", [buffer(aki), issuer]) def find_by_uri(self, uri): return self._find_results(None, "SELECT" + self.object_fields + "FROM object, uri WHERE uri.uri = ? AND object.id = uri.id", [uri]) # It's easiest to understand overlap conditions by understanding # non-overlap then inverting and and applying De Morgan's law. Ranges # A and B do not overlap if either A.min > B.max or A.max < B.min; # therefore they do overlap if A.min <= B.max and A.max >= B.min. def find_by_range(self, range_min, range_max = None, fn2 = None): if range_max is None: range_max = range_min if isinstance(range_min, (str, unicode)): range_min = long(range_min) if range_min.isdigit() else rpki.POW.IPAddress(range_min) if isinstance(range_max, (str, unicode)): range_max = long(range_max) if range_max.isdigit() else rpki.POW.IPAddress(range_max) assert isinstance(range_min, (int, long, rpki.POW.IPAddress)) assert isinstance(range_max, (int, long, rpki.POW.IPAddress)) return self._find_results( fn2, """ SELECT %s FROM object, range WHERE ? <= max AND ? >= min AND object.id = range.id """ % self.object_fields, [range_min, range_max]) def find_by_resource_bag(self, bag, fn2 = None): assert bag.asn or bag.v4 or bag.v6 qset = [] aset = [] for rset in (bag.asn, bag.v4, bag.v6): if rset: for r in rset: qset.append("(? <= max AND ? >= min)") aset.append(r.min) aset.append(r.max) return self._find_results( fn2, """ SELECT %s FROM object, range WHERE object.id = range.id AND (%s) """ % (self.object_fields, " OR ".join(qset)), aset) def find_uris(self, rowid): self.cur.execute("SELECT uri FROM uri WHERE id = ?", (rowid,)) return [u[0] for u in self.cur.fetchall()] def _find_results(self, fn2, query, args = None): if args is None: args = [] if fn2 is not None: assert fn2 in self.fn2map query += " AND fn2 = ?" args.append(fn2) query += " GROUP BY object.id" results = [] self.cur.execute(query, args) selections = self.cur.fetchall() for rowid, fn2, der, para, para_id, orig_id in selections: if rowid in self.cache: obj = self.cache[rowid] assert obj._rowid == rowid assert obj._para == para assert obj._para_id == para_id, "Assertion failure: obj._para_id %s para_id %s" % (obj._para_id, para_id) assert obj._orig_id == orig_id, "Assertion failure: obj._orig_id %s orig_id %s" % (obj._orig_id, orig_id) else: obj = self.fn2map[fn2](DER = der) self.cur.execute("SELECT uri FROM uri WHERE id = ?", (rowid,)) obj.uris = [u[0] for u in self.cur.fetchall()] obj.uri = obj.uris[0] if len(obj.uris) == 1 else None obj._rpdb = self obj._rowid = rowid obj._para = para obj._para_id = para_id obj._orig_id = orig_id self.cache[rowid] = obj results.append(obj) return results def commit(self): self.db.commit() def close(self): self.commit() self.cur.close() self.db.close() def parse_xki(s): """ Parse text form of an SKI or AKI. We accept two encodings: colon-delimited hexadecimal, and URL-safe Base64. The former is what OpenSSL prints in its text representation of SKI and AKI extensions; the latter is the g(SKI) value that some RPKI CA engines (including rpkid) use when constructing filenames. In either case, we check that the decoded result contains the right number of octets to be a SHA-1 hash. """ if ":" in s: b = "".join(chr(int(c, 16)) for c in s.split(":")) else: b = base64.urlsafe_b64decode(s + ("=" * (4 - len(s) % 4))) if len(b) != 20: raise RuntimeError("Bad length for SHA1 xKI value: %r" % s) return b if __name__ == "__main__": #profile = None profile = "rcynic-lta.prof" if profile: import cProfile prof = cProfile.Profile() try: prof.runcall(main) finally: prof.dump_stats(profile) sys.stderr.write("Dumped profile data to %s\n" % profile) else: main()