From fe0bf509f528dbdc50c7182f81057c6a4e15e4bd Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Sat, 5 Apr 2014 22:42:12 +0000 Subject: Source tree reorg, phase 1. Almost everything moved, no file contents changed. svn path=/branches/tk685/; revision=5757 --- scripts/rp-sqlite | 425 ------------------------------------------------------ 1 file changed, 425 deletions(-) delete mode 100755 scripts/rp-sqlite (limited to 'scripts/rp-sqlite') diff --git a/scripts/rp-sqlite b/scripts/rp-sqlite deleted file mode 100755 index ee43096d..00000000 --- a/scripts/rp-sqlite +++ /dev/null @@ -1,425 +0,0 @@ -#!/usr/local/bin/python - -# $Id$ - -# Copyright (C) 2013 Dragon Research Labs ("DRL") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -# Preliminary script to work out what's involved in building an -# SQLite3 database of RP objects. We haven't bothered with this until -# now in rcynic, because we mostly just walk the filesystem tree, but -# LTA and some of the ideas Tim is playing with require a lot of -# lookups based on things that are not the URIs we use as filenames, -# so some kind of indexing may become necessary. Given the complexity -# of building any kind of real index over RFC 3779 resources, -# otherwise fine lightweight tools like the Python shelve library -# probably won't cut it here, and I don't want to add a dependency on -# MySQL on the RP side (yet?), so let's see what we can do with SQLite3. - -import os -import sys -import yaml -import base64 -import sqlite3 -import weakref -import rpki.POW -import rpki.x509 -import rpki.resource_set - -sqlite3.register_adapter(rpki.POW.IPAddress, - lambda x: buffer("_" + x.toBytes())) - -sqlite3.register_converter("RangeVal", - lambda s: long(s) if s.isdigit() else rpki.POW.IPAddress.fromBytes(s[1:])) - - -def main(): - rpdb = RPDB() - rpdb.load() - test(rpdb) - rpdb.close() - -def test(rpdb): - fn2s = [None] + rpdb.fn2map.keys() - - print - print "Testing YAML parsing" - parse_yaml(rpdb) - - print - print "Looking for certificates without AKI" - for r in rpdb.find_by_aki(None, "cer"): - print r, r.uris - print - print "Testing range functions" - for fn2 in fn2s: - if fn2 is not None: - print - print "Restricting search to type", fn2 - print - print "Looking for range that should include adrilankha and psg again" - for r in rpdb.find_by_range("147.28.0.19", "147.28.0.62", fn2): - print r, r.uris - print - print "Looking for range that should include adrilankha" - for r in rpdb.find_by_range("147.28.0.19", "147.28.0.19", fn2): - print r, r.uris - print - print "Looking for range that should include ASN 3130" - for r in rpdb.find_by_range(3130, 3130, fn2): - print r, r.uris - print - print "Moving on to resource sets" - for fn2 in fn2s: - if fn2 is not None: - print - print "Restricting search to type", fn2 - for expr in ("147.28.0.19-147.28.0.62", - "3130", - "2001:418:1::19/128", - "147.28.0.19-147.28.0.62,198.180.150.50/32", - "3130,147.28.0.19-147.28.0.62,198.180.150.50/32", - "2001:418:1::62/128,198.180.150.50/32,2001:418:8006::50/128", - "147.28.0.19-147.28.0.62,2001:418:1::19/128,2001:418:1::62/128,198.180.150.50/32,2001:418:8006::50/128"): - print - print "Trying", expr - for r in rpdb.find_by_resource_bag(rpki.resource_set.resource_bag.from_str(expr), fn2): - print r, r.uris - - -def parse_xki(s): - """ - Parse text form of an SKI or AKI. We accept two encodings: - colon-delimited hexadecimal, and URL-safe Base64. The former is - what OpenSSL prints in its text representation of SKI and AKI - extensions; the latter is the g(SKI) value that some RPKI CA engines - (including rpkid) use when constructing filenames. - - In either case, we check that the decoded result contains the right - number of octets to be a SHA-1 hash. - """ - - if ":" in s: - b = "".join(chr(int(c, 16)) for c in s.split(":")) - else: - b = base64.urlsafe_b64decode(s + ("=" * (4 - len(s) % 4))) - if len(b) != 20: - raise RuntimeError("Bad length for SHA1 xKI value: %r" % s) - return b - - -def parse_yaml(rpdb, fn = "rp-sqlite.yaml"): - yy = yaml.safe_load(open(fn, "r")) - for y in yy: - - ski = None - uri = None - obj = set() - - print - - if "ski" in y: - ski = parse_xki(y["ski"]) - obj.update(rpdb.find_by_ski(ski)) - if "uri" in y: - uri = y["uri"] - obj.update(rpdb.find_by_uri(uri)) - if len(obj) == 1: - obj = obj.pop() - else: - raise RuntimeError("Constraint entry must name a unique object using SKI, URI, or both (%r, %r, %r)" % ( - ski, uri, obj)) - - print "URI:", uri - print "SKI:", " ".join("%02X" % ord(c) for c in ski), "(" + y["ski"] + ")" - - new_resources = old_resources = obj.get_3779resources() - - if "set" in y: - new_resources = rpki.resource_set.resource_bag.from_str(y["set"]) - - if "add" in y: - new_resources = new_resources | rpki.resource_set.resource_bag.from_str(y["add"]) - - if "sub" in y: - new_resources = new_resources - rpki.resource_set.resource_bag.from_str(y["sub"]) - - if new_resources == old_resources: - print "No resource change, skipping" - continue - - print "Old:", old_resources - print "New:", new_resources - print "Add:", new_resources - old_resources - print "Sub:", old_resources - new_resources - - # See draft-ietf-sidr-ltamgmt-08.txt for real processing details, but overview: - # - # - Process constraints file as above to determine list of target - # certificates (2.1). May need to add more fields to YAML hash - # for things like CP, CRLDP, etc, although I'm not entirely sure - # yet which of those it really makes sense to tweak via - # constraints. - # - # - Use resources from selected target certificates to determine - # which additional certificates we need to reissue to remove those - # resources (2.2, "perforation"). In theory we already have SQL - # that will just locate all of these for us. - # - # - Figure out which trust anchors to process (2.3, TA - # re-parenting); we can look in SQL for NULL AKI, but that's just - # a hint, we either have to verify that rcynic accepted those TAs - # or we have to look at the TALs. Looking at TALs is probably - # easier. - # - # At some point we probably need to parse the constraints file into - # Constraints objects or something like that, except that we may - # really need something more general that will accomodate - # perforation and TA reparenting as well. Figure out and refactor - # as we go along, most likely. - - -class RPDB(object): - """ - Relying party database. - - For now just wire in the database name and rcynic root, fix this - later if overall approach seems usable. Might even end up just - being an in-memory SQL database, who knows? - """ - - fn2map = dict(cer = rpki.x509.X509, - crl = rpki.x509.CRL, - mft = rpki.x509.SignedManifest, - roa = rpki.x509.ROA, - gbr = rpki.x509.Ghostbuster) - - def __init__(self, db_name = "rp-sqlite.db", delete_old_db = True): - - if delete_old_db: - try: - os.unlink(db_name) - except: - pass - - exists = os.path.exists(db_name) - - self.db = sqlite3.connect(db_name, detect_types = sqlite3.PARSE_DECLTYPES) - self.db.text_factory = str - self.cur = self.db.cursor() - - self.cache = weakref.WeakValueDictionary() - - if exists: - return - - self.cur.executescript(''' - PRAGMA foreign_keys = on; - - CREATE TABLE object ( - id INTEGER PRIMARY KEY NOT NULL, - der BLOB NOT NULL, - fn2 TEXT NOT NULL, - ski BLOB, - aki BLOB, - inherits BOOLEAN NOT NULL, - UNIQUE (der)); - - CREATE TABLE uri ( - id INTEGER NOT NULL, - uri TEXT NOT NULL, - UNIQUE (uri), - FOREIGN KEY (id) REFERENCES object(id) - ON DELETE CASCADE - ON UPDATE CASCADE); - - CREATE INDEX uri_index ON uri(id); - - CREATE TABLE range ( - id INTEGER NOT NULL, - min RangeVal NOT NULL, - max RangeVal NOT NULL, - UNIQUE (id, min, max), - FOREIGN KEY (id) REFERENCES object(id) - ON DELETE CASCADE - ON UPDATE CASCADE); - - CREATE INDEX range_index ON range(min, max); - ''') - - def load(self, - rcynic_root = os.path.expanduser("~/rpki/subvert-rpki.hactrn.net/trunk/" - "rcynic/rcynic-data/unauthenticated"), - spinner = 100): - - nobj = 0 - - for root, dirs, files in os.walk(rcynic_root): - for fn in files: - fn = os.path.join(root, fn) - fn2 = os.path.splitext(fn)[1][1:] - - try: - obj = self.fn2map[fn2](DER_file = fn) - except: - continue - - if spinner and nobj % spinner == 0: - sys.stderr.write("\r%s %d..." % ("|\\-/"[(nobj/spinner) & 3], nobj)) - - nobj += 1 - - inherits = False - - if fn2 == "crl": - ski = None - aki = buffer(obj.get_AKI()) - cer = None - bag = None - - else: - if fn2 == "cer": - cer = obj - else: - cer = rpki.x509.X509(POW = obj.get_POW().certs()[0]) - ski = buffer(cer.get_SKI()) - try: - aki = buffer(cer.get_AKI()) - except: - aki = None - bag = cer.get_3779resources() - inherits = bag.asn.inherit or bag.v4.inherit or bag.v6.inherit - - der = buffer(obj.get_DER()) - uri = "rsync://" + fn[len(rcynic_root) + 1:] - - try: - self.cur.execute("INSERT INTO object (der, fn2, ski, aki, inherits) VALUES (?, ?, ?, ?, ?)", - (der, fn2, ski, aki, inherits)) - rowid = self.cur.lastrowid - - except sqlite3.IntegrityError: - self.cur.execute("SELECT id FROM object WHERE der = ? AND fn2 = ?", (der, fn2)) - rows = self.cur.fetchall() - rowid = rows[0][0] - assert len(rows) == 1 - - else: - if bag is not None: - for rset in (bag.asn, bag.v4, bag.v6): - if rset is not None: - self.cur.executemany("REPLACE INTO range (id, min, max) VALUES (?, ?, ?)", - ((rowid, i.min, i.max) for i in rset)) - - self.cur.execute("INSERT INTO uri (id, uri) VALUES (?, ?)", - (rowid, uri)) - - if spinner: - sys.stderr.write("\r= %d objects, committing..." % nobj) - - self.db.commit() - - if spinner: - sys.stderr.write("done.\n") - - - def find_by_ski(self, ski, fn2 = None): - if ski is None: - return self._find_results(fn2, "SELECT id, fn2, der FROM object WHERE ski IS NULL") - else: - return self._find_results(fn2, "SELECT id, fn2, der FROM object WHERE ski = ?", [buffer(ski)]) - - def find_by_aki(self, aki, fn2 = None): - if aki is None: - return self._find_results(fn2, "SELECT id, fn2, der FROM object WHERE aki IS NULL") - else: - return self._find_results(fn2, "SELECT id, fn2, der FROM object WHERE aki = ?", [buffer(aki)]) - - def find_by_uri(self, uri): - return self._find_results(None, "SELECT object.id, fn2, der FROM object, uri WHERE uri.uri = ? AND object.id = uri.id", [uri]) - - - # It's easiest to understand overlap conditions by understanding - # non-overlap then inverting and and applying De Morgan's law. Ranges - # A and B do not overlap if either A.min > B.max or A.max < B.min; - # therefore they do overlap if A.min <= B.max and A.max >= B.min. - - def find_by_range(self, range_min, range_max = None, fn2 = None): - if range_max is None: - range_max = range_min - if isinstance(range_min, (str, unicode)): - range_min = long(range_min) if range_min.isdigit() else rpki.POW.IPAddress(range_min) - if isinstance(range_max, (str, unicode)): - range_max = long(range_max) if range_max.isdigit() else rpki.POW.IPAddress(range_max) - assert isinstance(range_min, (int, long, rpki.POW.IPAddress)) - assert isinstance(range_max, (int, long, rpki.POW.IPAddress)) - return self._find_results( - fn2, - """ - SELECT object.id, fn2, der FROM object, range - WHERE ? <= max AND ? >= min AND object.id = range.id - """, - [range_min, range_max]) - - - def find_by_resource_bag(self, bag, fn2 = None): - assert bag.asn or bag.v4 or bag.v6 - qset = [] - aset = [] - for rset in (bag.asn, bag.v4, bag.v6): - if rset: - for r in rset: - qset.append("(? <= max AND ? >= min)") - aset.append(r.min) - aset.append(r.max) - return self._find_results( - fn2, - """ - SELECT object.id, fn2, der FROM object, range - WHERE object.id = range.id AND (%s) - """ % (" OR ".join(qset)), - aset) - - - def _find_results(self, fn2, query, args = None): - if args is None: - args = [] - if fn2 is not None: - assert fn2 in self.fn2map - query += " AND fn2 = ?" - args.append(fn2) - query += " GROUP BY object.id" - results = [] - self.cur.execute(query, args) - selections = self.cur.fetchall() - for rowid, fn2, der in selections: - if rowid in self.cache: - obj = self.cache[rowid] - else: - obj = self.fn2map[fn2](DER = der) - self.cur.execute("SELECT uri FROM uri WHERE id = ?", (rowid,)) - obj.uris = [u[0] for u in self.cur.fetchall()] - obj.uri = obj.uris[0] if len(obj.uris) == 1 else None - self.cache[rowid] = obj - results.append(obj) - return results - - - def close(self): - self.cur.close() - self.db.close() - - -if __name__ == "__main__": - main() -- cgit v1.2.3