#!/usr/local/bin/python # $Id$ # Copyright (C) 2013 Dragon Research Labs ("DRL") # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. # # THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE # OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR # PERFORMANCE OF THIS SOFTWARE. import os import sys import yaml import glob import time import base64 import socket import sqlite3 import weakref import rpki.POW import rpki.x509 import rpki.sundial import rpki.resource_set # Lots of icky global variables, clean this up later. tals = {} constraints = None serial = long(time.time()) << 32 ltakey = None ltacer = None ltauri = "rsync://localhost/lta" ltasia = ltauri + "/" ltaaia = ltauri + ".cer" ltamft = ltauri + "/lta.mft" ltacrl = ltauri + "/lta.crl" # Teach SQLite3 about our data types. sqlite3.register_adapter(rpki.POW.IPAddress, lambda x: buffer("_" + x.toBytes())) sqlite3.register_converter("RangeVal", lambda s: long(s) if s.isdigit() else rpki.POW.IPAddress.fromBytes(s[1:])) sqlite3.register_adapter(rpki.x509.X501DN, str) def main(): print "Parsing YAML" parse_yaml() print print "Creating CA" create_ca() print print "Creating DB" rpdb = RPDB() print print "Loading DB" rpdb.load() if False: print print "Initializing nochain attributes" rpdb.initialize_chains() print print "Processing targets" process_targets(rpdb) print print "Processing ancestors" process_ancestors(rpdb) print print "Processing tree" process_tree(rpdb) print print "Closing DB" rpdb.close() def parse_xki(s): """ Parse text form of an SKI or AKI. We accept two encodings: colon-delimited hexadecimal, and URL-safe Base64. The former is what OpenSSL prints in its text representation of SKI and AKI extensions; the latter is the g(SKI) value that some RPKI CA engines (including rpkid) use when constructing filenames. In either case, we check that the decoded result contains the right number of octets to be a SHA-1 hash. """ if ":" in s: b = "".join(chr(int(c, 16)) for c in s.split(":")) else: b = base64.urlsafe_b64decode(s + ("=" * (4 - len(s) % 4))) if len(b) != 20: raise RuntimeError("Bad length for SHA1 xKI value: %r" % s) return b def create_ca(): global serial global ltakey global ltacer fn = "rcynic-lta.key" if os.path.exists(fn): ltakey = rpki.x509.RSA(Auto_file = fn) else: ltakey = rpki.x509.RSA.generate(quiet = True) with os.fdopen(os.open(fn, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0400), "w") as f: f.write(ltakey.get_PEM()) ltacer = rpki.x509.X509.self_certify( keypair = ltakey, subject_key = ltakey.get_RSApublic(), serial = serial, sia = (ltasia, ltamft, None), notAfter = rpki.sundial.now() + rpki.sundial.timedelta(days = 7), resources = rpki.resource_set.resource_bag.from_str("0-4294967295,0.0.0.0/0,::/0")) with open("rcynic-lta.cer", "wb") as f: f.write(ltacer.get_DER()) def parse_yaml(fn = "rcynic-lta.yaml"): global tals global constraints y = yaml.safe_load(open(fn, "r")) constraints = y["constraints"] for fn in glob.iglob(os.path.join(y["tal-directory"], "*.tal")): with open(fn, "r") as f: uri = f.readline().strip() key = rpki.x509.RSApublic(Base64 = f.read()) tals[uri] = key def process_targets(rpdb): for y in constraints: found = rpdb.find_by_ski_or_uri(parse_xki(y["ski"]) if "ski" in y else None, y.get("uri", None)) if len(found) != 1: if found: print "Constraint entry matched multiple objects, skipping (%s %s %r)" % ( y.get("ski", ""), y.get("uri", ""), found) else: print "Constraint entry matched nothing, skipping (%s %s)" % ( y.get("ski", ""), y.get("uri", "")) continue obj = found.pop() new_resources = old_resources = obj.get_3779resources() if "set" in y: new_resources = rpki.resource_set.resource_bag.from_str(y["set"]) if "add" in y: new_resources = new_resources | rpki.resource_set.resource_bag.from_str(y["add"]) if "sub" in y: new_resources = new_resources - rpki.resource_set.resource_bag.from_str(y["sub"]) if False: print "SKI:", obj.hSKI() print "URI:", obj.uri print "Old:", old_resources print "New:", new_resources print "Add:", new_resources - old_resources print "Sub:", old_resources - new_resources obj.original = True obj.target = True rpdb.add_para(obj, new_resources) # Not really sure what to do about the conflict detection requirement # in the last paragraph of 4.2.3. I think it stems at least in part # from the rather arbitrary way that the draft mixes user control with # required side effects. I guess one could detect the cases that this # paragraph talks about by checking for an existing paracertificate # before for every node during ancestor processing, and doing some # kind of set operation like: # # ((original_ancestor ^ para_ancestor) & target) != null # # Ignore for now, I think, until the rest of this is working. def process_ancestors(rpdb): for target in rpdb.find_targets(): target_resources = target.get_3779resources() if True: print print "Target %r" % target #print "Resources", str(target_resources) child = target while child.get_AKI() is not None: parents = rpdb.find_parent(child) print "Parents %r" % parents if len(parents) == 1: parent_to_modify = parent_to_follow = parents[0] elif len(parents) == 2: parents.sort(key = lambda p: p.para) parent_to_modify = parents[1] parent_to_follow = parents[0] else: assert len(parents) in (1, 2) if True: print "Same %s, modify %r, follow %r" % (parent_to_modify == parent_to_follow, parent_to_modify, parent_to_follow) assert not parent_to_follow.para old_resources = parent_to_modify.get_3779resources() new_resources = old_resources - target_resources if False: print "Old:", old_resources print "New:", new_resources if True: print "Add:", new_resources - old_resources print "Sub:", old_resources - new_resources parent_to_modify.original = True rpdb.add_para(parent_to_modify, new_resources) child = parent_to_follow def process_tree(rpdb): for target in rpdb.find_targets(): # CONTINUE HERE raise NotImplementedError class DER_object_mixin(object): """ Mixin to add some SQL-related methods to classes derived from rpki.x509.DER_object. """ _rpdb = None _rowid = None _nochain = True _original = False _para = False _target = False @property def rowid(self): return self._rowid def _update_bool(self, name, value): assert self._rpdb is not None and self._rowid is not None and isinstance(value, bool) self._rpdb.cur.execute("UPDATE object SET %s = ? WHERE id = ?" % name, (value, self._rowid)) setattr(self, "_" + name, value) self._rpdb.db.commit() @property def nochain(self): return self._nochain @nochain.setter def nochain(self, value): self._update_bool("nochain", value) @property def original(self): return self._original @original.setter def original(self, value): self._update_bool("original", value) @property def para(self): return self._para @para.setter def para(self, value): self._update_bool("para", value) @property def target(self): return self._target @target.setter def target(self, value): self._update_bool("target", value) class X509 (rpki.x509.X509, DER_object_mixin): pass class CRL (rpki.x509.CRL, DER_object_mixin): pass class SignedManifest (rpki.x509.SignedManifest, DER_object_mixin): pass class ROA (rpki.x509.ROA, DER_object_mixin): pass class Ghostbuster (rpki.x509.Ghostbuster, DER_object_mixin): pass class RPDB(object): """ Relying party database. For now just wire in the database name and rcynic root, fix this later if overall approach seems usable. Might even end up just being an in-memory SQL database, who knows? """ fn2map = dict(cer = X509, crl = CRL, mft = SignedManifest, roa = ROA, gbr = Ghostbuster) mapfn2 = dict((v, k) for k, v in fn2map.iteritems()) object_fields = " object.id, fn2, der, nochain, original, para, target " def __init__(self, db_name = "rcynic-lta.db"): try: os.unlink(db_name) except: pass self.db = sqlite3.connect(db_name, detect_types = sqlite3.PARSE_DECLTYPES) self.db.text_factory = str self.cur = self.db.cursor() self.cache = weakref.WeakValueDictionary() self.cur.executescript(''' PRAGMA foreign_keys = on; CREATE TABLE object ( id INTEGER PRIMARY KEY NOT NULL, der BLOB NOT NULL, fn2 TEXT NOT NULL, ski BLOB, aki BLOB, issuer TEXT, subject TEXT, nochain BOOLEAN NOT NULL DEFAULT 0, original BOOLEAN NOT NULL DEFAULT 0, para BOOLEAN NOT NULL DEFAULT 0, target BOOLEAN NOT NULL DEFAULT 0, UNIQUE (der)); CREATE TABLE uri ( id INTEGER NOT NULL, uri TEXT NOT NULL, UNIQUE (uri), FOREIGN KEY (id) REFERENCES object(id) ON DELETE CASCADE ON UPDATE CASCADE); CREATE INDEX uri_index ON uri(id); CREATE TABLE range ( id INTEGER NOT NULL, min RangeVal NOT NULL, max RangeVal NOT NULL, UNIQUE (id, min, max), FOREIGN KEY (id) REFERENCES object(id) ON DELETE CASCADE ON UPDATE CASCADE); CREATE INDEX range_index ON range(min, max); ''') # Note that we need to read the authenticated tree, not the # unauthenticated tree, as the draft says it assumes that its input # certificates have already passed some kind of validation. def load(self, rcynic_root = os.path.expanduser("~/rpki/subvert-rpki.hactrn.net/trunk/rcynic/rcynic-data/authenticated"), spinner = 100): nobj = 0 for root, dirs, files in os.walk(rcynic_root): for fn in files: fn = os.path.join(root, fn) fn2 = os.path.splitext(fn)[1][1:] try: obj = self.fn2map[fn2](DER_file = fn) except: continue if spinner and nobj % spinner == 0: sys.stderr.write("\r%s %d..." % ("|\\-/"[(nobj/spinner) & 3], nobj)) nobj += 1 if fn2 == "crl": ski = None aki = buffer(obj.get_AKI()) cer = None bag = None issuer = obj.getIssuer() subject = None else: if fn2 == "cer": cer = obj else: cer = rpki.x509.X509(POW = obj.get_POW().certs()[0]) ski = buffer(cer.get_SKI()) try: aki = buffer(cer.get_AKI()) except: aki = None bag = cer.get_3779resources() issuer = cer.getIssuer() subject = cer.getSubject() der = buffer(obj.get_DER()) uri = "rsync://" + fn[len(rcynic_root) + 1:] try: self.cur.execute("INSERT INTO object (der, fn2, ski, aki, issuer, subject) " "VALUES (?, ?, ?, ?, ?, ?)", (der, fn2, ski, aki, issuer, subject)) rowid = self.cur.lastrowid except sqlite3.IntegrityError: self.cur.execute("SELECT id FROM object WHERE der = ? AND fn2 = ?", (der, fn2)) rows = self.cur.fetchall() rowid = rows[0][0] assert len(rows) == 1 else: if bag is not None: for rset in (bag.asn, bag.v4, bag.v6): if rset is not None: self.cur.executemany("REPLACE INTO range (id, min, max) VALUES (?, ?, ?)", ((rowid, i.min, i.max) for i in rset)) self.cur.execute("INSERT INTO uri (id, uri) VALUES (?, ?)", (rowid, uri)) if spinner: sys.stderr.write("\r= %d objects, committing..." % nobj) self.db.commit() if spinner: sys.stderr.write("done.\n") def add_para(self, obj, resources): """ As far as I can tell at the moment, we only generate paracertificates for CA certificates, never for EE certificates. At present, ROAs are the only signed objects that specify resources explictly rather than using inheritance, and EE certificates for ROAs are supposed to be an exact match for the address resources in the ROA anyway, so this is likely not a serious restriction, at least for now. Fixing this, if it's a problem, would require extending POW.c to allow us to whack the certificate(s) bundled into a CMS object. There's no documentation on how we would even do that, although I suspect that the OpenSSL library routine CMS_set1_signers_certs() might do the trick. Ignore for now. """ assert isinstance(obj, X509) global serial serial += 1 pow = obj.get_POW() x = rpki.POW.X509() x.setVersion( pow.getVersion()) x.setSubject( pow.getSubject()) x.setNotBefore( pow.getNotBefore()) x.setNotAfter( pow.getNotAfter()) x.setPublicKey( pow.getPublicKey()) x.setSKI( pow.getSKI()) x.setBasicConstraints( pow.getBasicConstraints()) x.setKeyUsage( pow.getKeyUsage()) x.setCertificatePolicies( pow.getCertificatePolicies()) x.setSIA( *pow.getSIA()) x.setIssuer( ltacer.get_POW().getIssuer()) x.setAKI( ltacer.get_POW().getSKI()) x.setAIA( (ltaaia,)) x.setCRLDP( (ltacrl,)) x.setSerial( serial) x.setRFC3779( asn = ((r.min, r.max) for r in resources.asn), ipv4 = ((r.min, r.max) for r in resources.v4), ipv6 = ((r.min, r.max) for r in resources.v6)) x.sign(ltakey.get_POW(), rpki.POW.SHA256_DIGEST) cer = X509(POW = x) ski = buffer(cer.get_SKI()) aki = buffer(cer.get_AKI()) bag = cer.get_3779resources() issuer = cer.getIssuer() subject = cer.getSubject() der = buffer(cer.get_DER()) uri = ltasia + cer.gSKI() + ".cer" if obj.para: self.cur.execute("DELETE FROM object WHERE id = ?", (obj.rowid,)) self.cur.execute("INSERT INTO object (der, fn2, ski, aki, issuer, subject, para) " "VALUES (?, 'cer', ?, ?, ?, ?, 1)", (der, ski, aki, issuer, subject)) rowid = self.cur.lastrowid for rset in (bag.asn, bag.v4, bag.v6): if rset is not None: self.cur.executemany("REPLACE INTO range (id, min, max) VALUES (?, ?, ?)", ((rowid, i.min, i.max) for i in rset)) self.cur.execute("INSERT INTO uri (id, uri) VALUES (?, ?)", (rowid, uri)) self.db.commit() return self.find_by_id(rowid) def find_by_id(self, rowid): return self._find_results(None, "SELECT" + self.object_fields + "FROM object WHERE id = ?", [rowid]) def find_by_ski_or_uri(self, ski, uri): if not ski and not uri: return [] elif ski and uri: return self._find_results( None, "SELECT" + self.object_fields + "FROM object, uri " + "WHERE para = 0 AND ski = ? AND uri.uri = ? AND object.id = uri.id", [buffer(ski), uri]) elif ski: return self._find_results( None, "SELECT" + self.object_fields + "FROM object " + "WHERE para = 0 AND ski = ?", [buffer(ski)]) else: return self._find_results( None, "SELECT" + self.object_fields + "FROM object, uri " + "WHERE para = 0 AND uri.uri = ? AND object.id = uri.id", [uri]) def find_by_ski(self, ski, fn2 = None): if ski is None: return self._find_results(fn2, "SELECT" + self.object_fields + "FROM object WHERE ski IS NULL") else: return self._find_results(fn2, "SELECT" + self.object_fields + "FROM object WHERE ski = ?", [buffer(ski)]) def find_by_aki(self, aki, fn2 = None): if aki is None: return self._find_results(fn2, "SELECT" + self.object_fields + "FROM object WHERE aki IS NULL") else: return self._find_results(fn2, "SELECT" + self.object_fields + "FROM object WHERE aki = ?", [buffer(aki)]) def find_targets(self, fn2 = None): return self._find_results(fn2, "SELECT" + self.object_fields + "FROM object WHERE target <> 0 AND nochain = 0") def find_parent(self, child): return self._find_results(None, "SELECT" + self.object_fields + "FROM object WHERE subject = ? AND SKI = ?", [child.getIssuer(), buffer(child.get_AKI())]) def find_products(self, aki, issuer, fn2 = None): return self._find_results(fn2, "SELECT" + self.object_fields + "FROM object WHERE aki = ? AND issuer = ?", [buffer(aki), issuer]) def find_by_uri(self, uri): return self._find_results(None, "SELECT" + self.object_fields + "FROM object, uri WHERE uri.uri = ? AND object.id = uri.id", [uri]) # It's easiest to understand overlap conditions by understanding # non-overlap then inverting and and applying De Morgan's law. Ranges # A and B do not overlap if either A.min > B.max or A.max < B.min; # therefore they do overlap if A.min <= B.max and A.max >= B.min. def find_by_range(self, range_min, range_max = None, fn2 = None): if range_max is None: range_max = range_min if isinstance(range_min, (str, unicode)): range_min = long(range_min) if range_min.isdigit() else rpki.POW.IPAddress(range_min) if isinstance(range_max, (str, unicode)): range_max = long(range_max) if range_max.isdigit() else rpki.POW.IPAddress(range_max) assert isinstance(range_min, (int, long, rpki.POW.IPAddress)) assert isinstance(range_max, (int, long, rpki.POW.IPAddress)) return self._find_results( fn2, """ SELECT %s FROM object, range WHERE ? <= max AND ? >= min AND object.id = range.id """ % self.object_fields, [range_min, range_max]) def find_by_resource_bag(self, bag, fn2 = None): assert bag.asn or bag.v4 or bag.v6 qset = [] aset = [] for rset in (bag.asn, bag.v4, bag.v6): if rset: for r in rset: qset.append("(? <= max AND ? >= min)") aset.append(r.min) aset.append(r.max) return self._find_results( fn2, """ SELECT %s FROM object, range WHERE object.id = range.id AND (%s) """ % (self.object_fields, " OR ".join(qset)), aset) def _find_results(self, fn2, query, args = None): if args is None: args = [] if fn2 is not None: assert fn2 in self.fn2map query += " AND fn2 = ?" args.append(fn2) query += " GROUP BY object.id" results = [] self.cur.execute(query, args) selections = self.cur.fetchall() for rowid, fn2, der, nochain, original, para, target in selections: if rowid in self.cache: obj = self.cache[rowid] assert obj.rowid == rowid assert obj._nochain == nochain assert obj._original == original assert obj._para == para assert obj._target == target else: obj = self.fn2map[fn2](DER = der) self.cur.execute("SELECT uri FROM uri WHERE id = ?", (rowid,)) obj.uris = [u[0] for u in self.cur.fetchall()] obj.uri = obj.uris[0] if len(obj.uris) == 1 else None obj._rpdb = self obj._rowid = rowid obj._nochain = nochain obj._original = original obj._para = para obj._target = target self.cache[rowid] = obj results.append(obj) return results def close(self): self.cur.close() self.db.close() def initialize_chains(self): for uri, key in tals.iteritems(): cer = self.find_by_uri(uri)[0] if cer.getPublicKey() == key: cer.nochain = False else: print "TAL public key mismatch for %s, skipping: %s %s" % (uri, key.hSKI(), cer.hSKI()) before = after = None while before is None or before != after: before = after self.cur.execute( """ UPDATE object SET nochain = 0 WHERE aki || issuer IN (SELECT ski || subject FROM object WHERE fn2 = 'cer' AND nochain = 0) """) self.cur.execute("SELECT SUM(nochain) FROM object") after = self.cur.fetchone()[0] self.db.commit() def test(rpdb): fn2s = [None] + rpdb.fn2map.keys() print print "Looking for certificates without AKI" for r in rpdb.find_by_aki(None, "cer"): print r, r.uris print print "Testing range functions" for fn2 in fn2s: if fn2 is not None: print print "Restricting search to type", fn2 print print "Looking for range that should include adrilankha and psg again" for r in rpdb.find_by_range("147.28.0.19", "147.28.0.62", fn2): print r, r.uris print print "Looking for range that should include adrilankha" for r in rpdb.find_by_range("147.28.0.19", "147.28.0.19", fn2): print r, r.uris print print "Looking for range that should include ASN 3130" for r in rpdb.find_by_range(3130, 3130, fn2): print r, r.uris print print "Moving on to resource sets" for fn2 in fn2s: if fn2 is not None: print print "Restricting search to type", fn2 for expr in ("147.28.0.19-147.28.0.62", "3130", "2001:418:1::19/128", "147.28.0.19-147.28.0.62,198.180.150.50/32", "3130,147.28.0.19-147.28.0.62,198.180.150.50/32", "2001:418:1::62/128,198.180.150.50/32,2001:418:8006::50/128", "147.28.0.19-147.28.0.62,2001:418:1::19/128,2001:418:1::62/128,198.180.150.50/32,2001:418:8006::50/128"): print print "Trying", expr for r in rpdb.find_by_resource_bag(rpki.resource_set.resource_bag.from_str(expr), fn2): print r, r.uris if __name__ == "__main__": main()