From ae65819856649f96d5ac91391606eff91552db39 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Tue, 19 Aug 2014 21:08:13 +0000 Subject: Add tarball filenames to session table so we don't have to do all the work of extracting and parsing before discovering that we've hit a duplicate. Not sure what equivalent would be for Maildir (maybe Message-ID?) so deferring that for now. svn path=/trunk/; revision=5925 --- potpourri/validation-status-sql.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/potpourri/validation-status-sql.py b/potpourri/validation-status-sql.py index d37cb4ba..62e3089e 100755 --- a/potpourri/validation-status-sql.py +++ b/potpourri/validation-status-sql.py @@ -60,23 +60,20 @@ if creating: db.executescript(''' CREATE TABLE sessions ( id INTEGER PRIMARY KEY NOT NULL, - session DATETIME NOT NULL, - UNIQUE (session)); + session DATETIME UNIQUE NOT NULL, + filename TEXT UNIQUE); CREATE TABLE uris ( id INTEGER PRIMARY KEY NOT NULL, - uri TEXT NOT NULL, - UNIQUE (uri)); + uri TEXT UNIQUE NOT NULL); CREATE TABLE codes ( id INTEGER PRIMARY KEY NOT NULL, - code TEXT NOT NULL, - UNIQUE (code)); + code TEXT UNIQUE NOT NULL); CREATE TABLE generations ( id INTEGER PRIMARY KEY NOT NULL, - generation TEXT, - UNIQUE (generation)); + generation TEXT UNIQUE); CREATE TABLE events ( id INTEGER PRIMARY KEY NOT NULL, @@ -108,9 +105,10 @@ def string_id(table, value): return db.execute("INSERT INTO %s (%s) VALUES (?)" % (table, field), (value,)).lastrowid -def parse_xml(xml): +def parse_xml(xml, fn = None): try: - session_id = db.execute("INSERT INTO sessions (session) VALUES (datetime(?))", (xml.get("date"),)).lastrowid + session_id = db.execute("INSERT INTO sessions (session, filename) VALUES (datetime(?), ?)", + (xml.get("date"), fn)).lastrowid except sqlite3.IntegrityError: return @@ -125,9 +123,11 @@ def parse_xml(xml): def parse_tarball(fn): + if db.execute("SELECT filename FROM sessions WHERE filename = ?", (fn,)).fetchone(): + return print "Processing", fn - parse_xml(lxml.etree.ElementTree( - file = subprocess.Popen(("tar", "Oxf", fn, args.path_within_tarball), stdout = subprocess.PIPE).stdout).getroot()) + pipe = subprocess.Popen(("tar", "Oxf", fn, args.path_within_tarball), stdout = subprocess.PIPE).stdout + parse_xml(lxml.etree.ElementTree(file = pipe).getroot(), fn) if args.mailbox: -- cgit v1.2.3 From 19aaabec6feb5038e75f3c0868415bccb015b430 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Wed, 20 Aug 2014 03:25:26 +0000 Subject: Restructure, use Message-ID instead of filename when reading Maildir. svn path=/trunk/; revision=5927 --- potpourri/validation-status-sql.py | 270 +++++++++++++++++++++---------------- 1 file changed, 157 insertions(+), 113 deletions(-) diff --git a/potpourri/validation-status-sql.py b/potpourri/validation-status-sql.py index 62e3089e..858c8a10 100755 --- a/potpourri/validation-status-sql.py +++ b/potpourri/validation-status-sql.py @@ -32,120 +32,164 @@ import argparse import lxml.etree import subprocess -parser = argparse.ArgumentParser( - description = __doc__, - formatter_class = argparse.ArgumentDefaultsHelpFormatter) -group = parser.add_mutually_exclusive_group(required = True) -group.add_argument("--mailbox", "--mb", - help = "Maildir mailbox containing rcynic XML output") -group.add_argument("--tarballs", - help = "directory tree of tar files containing containing rcynic XML output") -parser.add_argument("--database", "--db", - default = "validation-status-sql.db", - help = "name for SQLite3 database") -parser.add_argument("--path-within-tarball", - default = "var/rcynic/data/rcynic.xml", - help = "name of file to extract from tarball(s)") -parser.add_argument("--tar-extensions", nargs = "+", - default = ".tar .tar.gz .tgz .tar.bz2 .tbz .tar.xz .txz".split(), - help = "extensions to recognize as indicating tar files") -args = parser.parse_args() - -creating = not os.path.exists(args.database) -db = sqlite3.connect(args.database) -db.text_factory = str -db.execute("PRAGMA foreign_keys = on") - -if creating: - db.executescript(''' - CREATE TABLE sessions ( - id INTEGER PRIMARY KEY NOT NULL, - session DATETIME UNIQUE NOT NULL, - filename TEXT UNIQUE); - - CREATE TABLE uris ( - id INTEGER PRIMARY KEY NOT NULL, - uri TEXT UNIQUE NOT NULL); - - CREATE TABLE codes ( - id INTEGER PRIMARY KEY NOT NULL, - code TEXT UNIQUE NOT NULL); - - CREATE TABLE generations ( - id INTEGER PRIMARY KEY NOT NULL, - generation TEXT UNIQUE); - - CREATE TABLE events ( - id INTEGER PRIMARY KEY NOT NULL, - timestamp DATETIME NOT NULL, - session_id INTEGER NOT NULL REFERENCES sessions(id) ON DELETE RESTRICT ON UPDATE RESTRICT, - generation_id INTEGER NOT NULL REFERENCES generations(id) ON DELETE RESTRICT ON UPDATE RESTRICT, - code_id INTEGER NOT NULL REFERENCES codes(id) ON DELETE RESTRICT ON UPDATE RESTRICT, - uri_id INTEGER NOT NULL REFERENCES uris(id) ON DELETE RESTRICT ON UPDATE RESTRICT, - UNIQUE (timestamp, generation_id, code_id, uri_id)); - - CREATE VIEW status AS - SELECT events.id, session, timestamp, generation, uri, code - FROM events - JOIN sessions ON sessions.id = events.session_id - JOIN uris ON uris.id = events.uri_id - JOIN codes ON codes.id = events.code_id - JOIN generations ON generations.id = events.generation_id; - ''') - - -def string_id(table, value): - field = table.rstrip("s") - try: - if value is None: - return db.execute("SELECT id FROM %s WHERE %s IS NULL" % (table, field)).fetchone()[0] +class Parser(object): + + @staticmethod + def main(): + parser = argparse.ArgumentParser( + description = __doc__, + formatter_class = argparse.ArgumentDefaultsHelpFormatter) + group = parser.add_mutually_exclusive_group(required = True) + group.add_argument("--mailbox", "--mb", + help = "Maildir mailbox containing rcynic XML output") + group.add_argument("--tarballs", + help = "directory tree of tar files containing containing rcynic XML output") + parser.add_argument("--database", "--db", + default = "validation-status-sql.db", + help = "SQLite3 database") + parser.add_argument("--path-within-tarball", + default = "var/rcynic/data/rcynic.xml", + help = "rcynic.xml path name within tarball(s)") + parser.add_argument("--tar-extensions", nargs = "+", + default = ".tar .tar.gz .tgz .tar.bz2 .tbz .tar.xz .txz".split(), + help = "extensions to recognize as indicating tar files") + args = parser.parse_args() + if args.mailbox: + ParserMailbox(args) + else: + ParserTarball(args) + + def __init__(self, args): + self.args = args + self.init_sql() + self.init_hook() + self.parsed = 1 + for self.current, self.iterval in enumerate(self.iterator, 1): + self.parse_xml() + if self.parsed > 1: + sys.stderr.write("\n") + db.close() + + + def init_sql(self): + creating = not os.path.exists(self.args.database) + self.db = sqlite3.connect(self.args.database) + self.db.text_factory = str + self.db.execute("PRAGMA foreign_keys = on") + + if creating: + self.db.executescript(''' + CREATE TABLE sessions ( + id INTEGER PRIMARY KEY NOT NULL, + session DATETIME UNIQUE NOT NULL, + handle TEXT UNIQUE NOT NULL); + + CREATE TABLE uris ( + id INTEGER PRIMARY KEY NOT NULL, + uri TEXT UNIQUE NOT NULL); + + CREATE TABLE codes ( + id INTEGER PRIMARY KEY NOT NULL, + code TEXT UNIQUE NOT NULL); + + CREATE TABLE generations ( + id INTEGER PRIMARY KEY NOT NULL, + generation TEXT UNIQUE); + + CREATE TABLE events ( + id INTEGER PRIMARY KEY NOT NULL, + timestamp DATETIME NOT NULL, + session_id INTEGER NOT NULL REFERENCES sessions(id) ON DELETE RESTRICT ON UPDATE RESTRICT, + generation_id INTEGER NOT NULL REFERENCES generations(id) ON DELETE RESTRICT ON UPDATE RESTRICT, + code_id INTEGER NOT NULL REFERENCES codes(id) ON DELETE RESTRICT ON UPDATE RESTRICT, + uri_id INTEGER NOT NULL REFERENCES uris(id) ON DELETE RESTRICT ON UPDATE RESTRICT, + UNIQUE (timestamp, generation_id, code_id, uri_id)); + + CREATE VIEW status AS + SELECT events.id, handle, session, timestamp, generation, uri, code + FROM events + JOIN sessions ON sessions.id = events.session_id + JOIN uris ON uris.id = events.uri_id + JOIN codes ON codes.id = events.code_id + JOIN generations ON generations.id = events.generation_id; + ''') + + def string_id(self, table, value): + field = table.rstrip("s") + try: + if value is None: + return self.db.execute("SELECT id FROM %s WHERE %s IS NULL" % (table, field)).fetchone()[0] + else: + return self.db.execute("SELECT id FROM %s WHERE %s = ?" % (table, field), (value,)).fetchone()[0] + except: + return self.db.execute("INSERT INTO %s (%s) VALUES (?)" % (table, field), (value,)).lastrowid + + + def parse_xml(self): + sys.stderr.write("\r%s %d/%d/%d...%s " % ("|\\-/"[self.current & 3], + self.current, self.parsed, self.total, self.handle)) + if self.db.execute("SELECT handle FROM sessions WHERE handle = ?", (self.handle,)).fetchone(): + return + xml = self.read_xml() + with self.db: + session_id = self.db.execute("INSERT INTO sessions (session, handle) VALUES (datetime(?), ?)", + (xml.get("date"), self.handle)).lastrowid + self.db.executemany("INSERT INTO events (session_id, timestamp, generation_id, code_id, uri_id) " + "VALUES (?, datetime(?), ?, ?, ?)", + ((session_id, + x.get("timestamp"), + self.string_id("generations", x.get("generation")), + self.string_id("codes", x.get("status")), + self.string_id("uris", x.text.strip())) + for x in xml.findall("validation_status"))) + self.parsed += 1 + + +class ParserTarball(Parser): + + def init_hook(self): + self.total = 0 + for fn in self.iter_tarball_names(): + self.total += 1 + self.iterator = self.iter_tarball_names() + + @property + def handle(self): + return self.iterval + + def read_xml(self): + return lxml.etree.ElementTree( + file = subprocess.Popen(("tar", "Oxf", self.iterval, self.args.path_within_tarball), + stdout = subprocess.PIPE).stdout).getroot() + + def iter_tarball_names(self): + if os.path.isdir(self.args.tarballs): + for root, dirs, files in os.walk(self.args.tarballs): + for fn in files: + if any(fn.endswith(ext) for ext in self.args.tar_extensions): + yield os.path.join(root, fn) else: - return db.execute("SELECT id FROM %s WHERE %s = ?" % (table, field), (value,)).fetchone()[0] - except: - return db.execute("INSERT INTO %s (%s) VALUES (?)" % (table, field), (value,)).lastrowid + yield self.args.tarballs + + +class ParserMailbox(Parser): + def init_hook(self): + self.mb = mailbox.Maildir(self.args.mailbox, factory = None, create = False) + self.total = len(self.mb) + self.iterator = self.mb.iterkeys() -def parse_xml(xml, fn = None): + @property + def handle(self): + return self.mb[self.iterval].get("Message-ID") + + def read_xml(self): + return lxml.etree.XML(self.mb[self.iterval].get_payload()) + + +if __name__ == "__main__": try: - session_id = db.execute("INSERT INTO sessions (session, filename) VALUES (datetime(?), ?)", - (xml.get("date"), fn)).lastrowid - except sqlite3.IntegrityError: - return - - with db: - db.executemany("INSERT INTO events (session_id, timestamp, generation_id, code_id, uri_id) VALUES (?, datetime(?), ?, ?, ?)", - ((session_id, - x.get("timestamp"), - string_id("generations", x.get("generation")), - string_id("codes", x.get("status")), - string_id("uris", x.text.strip())) - for x in xml.findall("validation_status"))) - - -def parse_tarball(fn): - if db.execute("SELECT filename FROM sessions WHERE filename = ?", (fn,)).fetchone(): - return - print "Processing", fn - pipe = subprocess.Popen(("tar", "Oxf", fn, args.path_within_tarball), stdout = subprocess.PIPE).stdout - parse_xml(lxml.etree.ElementTree(file = pipe).getroot(), fn) - - -if args.mailbox: - mb = mailbox.Maildir(args.mailbox, factory = None, create = False) - for i, key in enumerate(mb.iterkeys(), 1): - sys.stderr.write("\r%s %d/%d..." % ("|\\-/"[i & 3], i, len(mb))) - parse_xml(lxml.etree.XML(mb[key].get_payload())) - sys.stderr.write("\n") - -elif not os.path.isdir(args.tarballs): - parse_tarball(args.tarballs) - -else: - if os.path.isdir(args.tarballs): - for root, dirs, files in os.walk(args.tarballs): - for fn in files: - if any(fn.endswith(ext) for ext in args.tar_extensions): - parse_tarball(os.path.join(root, fn)) - - -db.close() + Parser.main() + except KeyboardInterrupt: + pass + -- cgit v1.2.3 From 117944e68f33063e1881c026c0b427dcc5d283dd Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Wed, 20 Aug 2014 03:25:57 +0000 Subject: Typo in copyright line. svn path=/trunk/; revision=5928 --- potpourri/validation-status-sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/potpourri/validation-status-sql.py b/potpourri/validation-status-sql.py index 858c8a10..90d66435 100755 --- a/potpourri/validation-status-sql.py +++ b/potpourri/validation-status-sql.py @@ -2,7 +2,7 @@ # $Id$ # -# Copyright (C) 2013-2014 2014 Dragon Research Labs ("DRL") +# Copyright (C) 2013-2014 Dragon Research Labs ("DRL") # Portions copyright (C) 2011-2012 Internet Systems Consortium ("ISC") # # Permission to use, copy, modify, and distribute this software for any -- cgit v1.2.3 From d3671dfe4db2b60e5ad7b574e31dde3bdd79a00a Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Wed, 20 Aug 2014 13:05:52 +0000 Subject: Get final db.close() right, even if it is unnecessary. svn path=/trunk/; revision=5930 --- potpourri/validation-status-sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/potpourri/validation-status-sql.py b/potpourri/validation-status-sql.py index 90d66435..81d79b48 100755 --- a/potpourri/validation-status-sql.py +++ b/potpourri/validation-status-sql.py @@ -68,7 +68,7 @@ class Parser(object): self.parse_xml() if self.parsed > 1: sys.stderr.write("\n") - db.close() + self.db.close() def init_sql(self): -- cgit v1.2.3 From 63c933922c2a89f7612fabc7fc96a3c5ae359556 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Mon, 25 Aug 2014 04:48:26 +0000 Subject: Solve several minor problems at once by storing timestamps as seconds-since-epoch. svn path=/trunk/; revision=5933 --- potpourri/validation-status-sql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/potpourri/validation-status-sql.py b/potpourri/validation-status-sql.py index 81d79b48..9e68d4ba 100755 --- a/potpourri/validation-status-sql.py +++ b/potpourri/validation-status-sql.py @@ -132,10 +132,10 @@ class Parser(object): return xml = self.read_xml() with self.db: - session_id = self.db.execute("INSERT INTO sessions (session, handle) VALUES (datetime(?), ?)", + session_id = self.db.execute("INSERT INTO sessions (session, handle) VALUES (strftime('%s', ?), ?)", (xml.get("date"), self.handle)).lastrowid self.db.executemany("INSERT INTO events (session_id, timestamp, generation_id, code_id, uri_id) " - "VALUES (?, datetime(?), ?, ?, ?)", + "VALUES (?, strftime('%s', ?), ?, ?, ?)", ((session_id, x.get("timestamp"), self.string_id("generations", x.get("generation")), -- cgit v1.2.3 From 005a8b3b6575eca08abfdf2569564b1e8895de97 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Tue, 26 Aug 2014 01:31:48 +0000 Subject: Use named indexes to make it possible to add and remove them later. May have finally gotten the right balance of indexes for basic use. Use various optimizations to let us load large data sets before the heat death of the universe. Some of these optimizations are dangerous, in the sense that if this script crashes while constructing the database, you'll have to rebuild the database from scratch. Probably ought to offer both this and the slow-but-safe approach as command line options, but: - The speed improvements look to be worth at least an order of magnitude in the runtime, - The speed improvements also prevent all the fsync() calls in the safe approach from turning the underlying filesystem into cream cheese while the script is running, and - This script is just a research anlysis tool to begin with. So I think the risk is justified in this case. svn path=/trunk/; revision=5934 --- potpourri/validation-status-sql.py | 76 +++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 26 deletions(-) diff --git a/potpourri/validation-status-sql.py b/potpourri/validation-status-sql.py index 9e68d4ba..fc52e64b 100755 --- a/potpourri/validation-status-sql.py +++ b/potpourri/validation-status-sql.py @@ -63,11 +63,13 @@ class Parser(object): self.args = args self.init_sql() self.init_hook() + self.index1() self.parsed = 1 for self.current, self.iterval in enumerate(self.iterator, 1): self.parse_xml() if self.parsed > 1: sys.stderr.write("\n") + self.index2() self.db.close() @@ -75,52 +77,74 @@ class Parser(object): creating = not os.path.exists(self.args.database) self.db = sqlite3.connect(self.args.database) self.db.text_factory = str - self.db.execute("PRAGMA foreign_keys = on") + self.db.executescript(''' + PRAGMA foreign_keys = off; + PRAGMA synchronous = off; + PRAGMA count_changes = off; + ''') if creating: self.db.executescript(''' CREATE TABLE sessions ( - id INTEGER PRIMARY KEY NOT NULL, - session DATETIME UNIQUE NOT NULL, - handle TEXT UNIQUE NOT NULL); + session_id INTEGER PRIMARY KEY NOT NULL, + session DATETIME NOT NULL, + handle TEXT NOT NULL + ); CREATE TABLE uris ( - id INTEGER PRIMARY KEY NOT NULL, - uri TEXT UNIQUE NOT NULL); + uri_id INTEGER PRIMARY KEY NOT NULL, + uri TEXT NOT NULL + ); CREATE TABLE codes ( - id INTEGER PRIMARY KEY NOT NULL, - code TEXT UNIQUE NOT NULL); + code_id INTEGER PRIMARY KEY NOT NULL, + code TEXT NOT NULL + ); CREATE TABLE generations ( - id INTEGER PRIMARY KEY NOT NULL, - generation TEXT UNIQUE); + generation_id INTEGER PRIMARY KEY NOT NULL, + generation TEXT NOT NULL + ); CREATE TABLE events ( id INTEGER PRIMARY KEY NOT NULL, timestamp DATETIME NOT NULL, - session_id INTEGER NOT NULL REFERENCES sessions(id) ON DELETE RESTRICT ON UPDATE RESTRICT, - generation_id INTEGER NOT NULL REFERENCES generations(id) ON DELETE RESTRICT ON UPDATE RESTRICT, - code_id INTEGER NOT NULL REFERENCES codes(id) ON DELETE RESTRICT ON UPDATE RESTRICT, - uri_id INTEGER NOT NULL REFERENCES uris(id) ON DELETE RESTRICT ON UPDATE RESTRICT, - UNIQUE (timestamp, generation_id, code_id, uri_id)); + session_id INTEGER NOT NULL REFERENCES sessions (session_id) ON DELETE RESTRICT ON UPDATE RESTRICT, + generation_id INTEGER NOT NULL REFERENCES generations (generation_id) ON DELETE RESTRICT ON UPDATE RESTRICT, + code_id INTEGER NOT NULL REFERENCES codes (code_id) ON DELETE RESTRICT ON UPDATE RESTRICT, + uri_id INTEGER NOT NULL REFERENCES uris (uri_id) ON DELETE RESTRICT ON UPDATE RESTRICT + ); CREATE VIEW status AS - SELECT events.id, handle, session, timestamp, generation, uri, code - FROM events - JOIN sessions ON sessions.id = events.session_id - JOIN uris ON uris.id = events.uri_id - JOIN codes ON codes.id = events.code_id - JOIN generations ON generations.id = events.generation_id; + SELECT id, handle, session, timestamp, generation, code, uri + FROM events + NATURAL JOIN sessions + NATURAL JOIN uris + NATURAL JOIN codes + NATURAL JOIN generations; ''') + + def index1(self): + self.db.executescript(''' + CREATE UNIQUE INDEX IF NOT EXISTS sessions_index ON sessions (session); + CREATE UNIQUE INDEX IF NOT EXISTS handles_index ON sessions (handle); + CREATE UNIQUE INDEX IF NOT EXISTS uris_index ON uris (uri); + CREATE UNIQUE INDEX IF NOT EXISTS codes_index ON codes (code); + CREATE UNIQUE INDEX IF NOT EXISTS generations_index ON generations (generation); + ''') + + + def index2(self): + self.db.executescript(''' + CREATE UNIQUE INDEX IF NOT EXISTS events_index ON events (uri_id, timestamp, code_id, generation_id); + ''') + + def string_id(self, table, value): field = table.rstrip("s") try: - if value is None: - return self.db.execute("SELECT id FROM %s WHERE %s IS NULL" % (table, field)).fetchone()[0] - else: - return self.db.execute("SELECT id FROM %s WHERE %s = ?" % (table, field), (value,)).fetchone()[0] + return self.db.execute("SELECT %s_id FROM %s WHERE %s = ?" % (field, table, field), (value,)).fetchone()[0] except: return self.db.execute("INSERT INTO %s (%s) VALUES (?)" % (table, field), (value,)).lastrowid @@ -138,7 +162,7 @@ class Parser(object): "VALUES (?, strftime('%s', ?), ?, ?, ?)", ((session_id, x.get("timestamp"), - self.string_id("generations", x.get("generation")), + self.string_id("generations", x.get("generation", "none")), self.string_id("codes", x.get("status")), self.string_id("uris", x.text.strip())) for x in xml.findall("validation_status"))) -- cgit v1.2.3 From e123d08c9962f58525d98864b572d7987684dff0 Mon Sep 17 00:00:00 2001 From: RPKI Documentation Robot Date: Fri, 12 Sep 2014 00:00:45 +0000 Subject: Automatic pull of documentation from Wiki. svn path=/trunk/; revision=5945 --- doc/doc.RPKI.RP | 14 ++--- doc/doc.RPKI.RP.HierarchicalRsync | 8 +-- doc/doc.RPKI.RP.RunningUnderCron | 14 ++--- doc/doc.RPKI.RP.rpki-rtr | 120 +++++++++++++++++++++----------------- doc/manual.pdf | Bin 757140 -> 757928 bytes 5 files changed, 83 insertions(+), 73 deletions(-) diff --git a/doc/doc.RPKI.RP b/doc/doc.RPKI.RP index 1f6774ee..107e878b 100644 --- a/doc/doc.RPKI.RP +++ b/doc/doc.RPKI.RP @@ -8,7 +8,7 @@ BGP security. See the CA tools for programs to help you generate RPKI objects, if you need to do that. -The RP main tools are rcynic and rtr-origin, each of which is discussed below. +The RP main tools are rcynic and rpki-rtr, each of which is discussed below. The installation process sets up everything you need for a basic RPKI validation installation. You will, however, need to think at least briefly @@ -30,14 +30,14 @@ documentation if you need to know more. See the discussion of trust anchors. -***** rtr-origin ***** +***** rpki-rtr ***** -rtr-origin is an implementation of the rpki-rtr protocol, using rcynic's output -as its data source. rtr-origin includes the rpki-rtr server, a test client, and -a utiltity for examining the content of the database rtr-origin generates from -the data supplied by rcynic. +rpki-rtr is an implementation of the rpki-rtr protocol, using rcynic's output +as its data source. rpki-rtr includes the rpki-rtr server, a test client, and a +utiltity for examining the content of the database rpki-rtr generates from the +data supplied by rcynic. -See the rtr-origin documentation for further details. +See the rpki-rtr documentation for further details. ***** rcynic-cron ***** diff --git a/doc/doc.RPKI.RP.HierarchicalRsync b/doc/doc.RPKI.RP.HierarchicalRsync index 06aeefeb..c3825556 100644 --- a/doc/doc.RPKI.RP.HierarchicalRsync +++ b/doc/doc.RPKI.RP.HierarchicalRsync @@ -50,8 +50,8 @@ Script for a downstream relying party using ssh might look like this: data/rcynic.${host} done cd /var/rcynic/rpki-rtr - /usr/bin/su -m rcynic -c '/usr/local/bin/rtr-origin --cronjob /var/rcynic/ - data/authenticated' + /usr/bin/su -m rcynic -c '/usr/local/bin/rpki-rtr cronjob /var/rcynic/data/ + authenticated' where /root/rpki_ssh_id_rsa is an SSH private key authorized to log in as user "rpkisync" on the gatherer machines. If you want to lock this down a little @@ -81,8 +81,8 @@ configuration would look more like this: data/rcynic.${host} done cd /var/rcynic/rpki-rtr - /usr/bin/su -m rcynic -c '/usr/local/bin/rtr-origin --cronjob /var/rcynic/ - data/authenticated' + /usr/bin/su -m rcynic -c '/usr/local/bin/rpki-rtr cronjob /var/rcynic/data/ + authenticated' where "unauthenticated" here is an rsync module pointing at /var/rcynic/data/ unauthenticated on each of the gatherer machines. Configuration for such a diff --git a/doc/doc.RPKI.RP.RunningUnderCron b/doc/doc.RPKI.RP.RunningUnderCron index efb6defd..3fc2da71 100644 --- a/doc/doc.RPKI.RP.RunningUnderCron +++ b/doc/doc.RPKI.RP.RunningUnderCron @@ -6,8 +6,8 @@ under the cron daemon, so that they can make use of rcynic's output immediately after rcynic finishes a validation run. rcynic-cron runs the basic set of relying party tools (rcynic, rcynic-html, and -`rtr-origin --cronjob`); if this suffices for your purposes, you don't need to -do anything else. This section is a discussion of alternative approaches. +rpki-rtr cronjob); if this suffices for your purposes, you don't need to do +anything else. This section is a discussion of alternative approaches. Which tools you want to run depends on how you intend to use the relying party tools. Here we assume a typical case in which you want to gather and validate @@ -39,9 +39,8 @@ On FreeBSD or MacOSX, this script might look like this: rcynic.conf || exit /var/rcynic/bin/rcynic-html /var/rcynic/data/rcynic.xml /usr/local/www/data/ rcynic - cd /var/rcynic/rpki-rtr - /usr/bin/su -m rcynic -c '/usr/local/bin/rtr-origin --cronjob /var/rcynic/ - data/authenticated' + /usr/bin/su -m rcynic -c '/usr/local/bin/rpki-rtr cronjob /var/rcynic/data/ + authenticated /var/rcynic/rpki-rtr' This assumes that you have done @@ -54,9 +53,8 @@ program: #!/bin/sh - /usr/bin/chrootuid /var/rcynic rcynic /bin/rcynic -c /etc/rcynic.conf || exit /var/rcynic/bin/rcynic-html /var/rcynic/data/rcynic.xml /var/www/rcynic - cd /var/rcynic/rpki-rtr - /usr/bin/su -m rcynic -c '/usr/local/bin/rtr-origin --cronjob /var/rcynic/ - data/authenticated' + /usr/bin/su -m rcynic -c '/usr/local/bin/rpki-rtr cronjob /var/rcynic/data/ + authenticated /var/rcynic/rpki-rtr' If you use the chroot program instead of chrootuid, change the line that invokes rcynic to: diff --git a/doc/doc.RPKI.RP.rpki-rtr b/doc/doc.RPKI.RP.rpki-rtr index af91b4a9..bb27ea4e 100644 --- a/doc/doc.RPKI.RP.rpki-rtr +++ b/doc/doc.RPKI.RP.rpki-rtr @@ -1,73 +1,75 @@ ****** rpki-rtr ****** -rtr-origin is an implementation of the "RPKI-router" protocol (RFC-6810). +rpki-rtr is an implementation of the "RPKI-router" protocol (RFC-6810). -rtr-origin depends on `rcynic` to collect and validate the RPKI data. rtr- -origin's's job is to serve up that data in a lightweight format suitable for +rpki-rtr depends on `rcynic` to collect and validate the RPKI data. rpki- +rtr's's job is to serve up that data in a lightweight format suitable for routers that want to do prefix origin authentication. -To use rtr-origin, you need to do two things beyond just running rcynic: +To use rpki-rtr, you need to do two things beyond just running rcynic: 1. You need to post-process `rcynic`'s output into the data files used by - rtr-origin. The rcynic-cron script handles this automatically, so the + rpki-rtr. The rcynic-cron script handles this automatically, so the default installation should already be taking care of this for you. - 2. You need to set up a listener for the rtr-origin server, using the - generated data files. The platform-specific packages for FreeBSD, Debian, - and Ubuntu automatically set up a plain TCP listener, but you will have to - do something on other platforms, or if you're using a transport protocol + 2. You need to set up a listener for the rpki-rtr server, using the generated + data files. The platform-specific packages for FreeBSD, Debian, and Ubuntu + automatically set up a plain TCP listener, but you will have to do + something on other platforms, or if you're using a transport protocol other than plain TCP. ***** Post-processing rcynic's output ***** -rtr-origin is designed to do the translation from raw RPKI data into the rpki- -rtr protocol only once. It does this by pre-computing the answers to all the +rpki-rtr is designed to do the translation from raw RPKI data into the rpki-rtr +protocol only once. It does this by pre-computing the answers to all the queries it is willing to answer for a given data set, and storing them on disk. -rtr-origin's --cronjob mode handles this computation. +rpki-rtr's cronjob command handles this computation. -To set this up, add an invocation of rtr-origin --cronjob to the cron job -you're already running to run rcynic. As mentioned above, if you're running the +To set this up, add an invocation of rpki-rtr cronjob to the cron job you're +already running to run rcynic. As mentioned above, if you're running the rcynic-cron script, this is already being done for you automatically, so you don't need to do anything. If you've written your own cron script, you'll need to add something like this to your script: - cd /var/rcynic/rpki-rtr - /usr/local/bin/rtr-origin --cronjob /var/rcynic/data/authenticated + /usr/local/bin/rpki-rtr cronjob /var/rcynic/data/authenticated /var/rcynic/ + rpki-rtr -In --cronjob mode, rtr-origin, needs write access to a directory where it can -store pre-digested versions of the data it pulls from rcynic. In the example -above, the directory /var/rcynic/rpki-rtr should be writable by the user ID -that is executing the cron script. +rpki-rtr cronjob needs write access to a directory where it can store pre- +digested versions of the data it pulls from rcynic. In the example above, the +directory /var/rcynic/rpki-rtr should be writable by the user ID that is +executing the cron script. -rtr-origin creates a collection of data files, as well as a subdirectory in -which each instance of the program running in --server mode can write a PF_UNIX -socket file. At present, rtr-origin creates these files under the directory in -which you run it, hence the cd command shown above. +rpki-rtr creates a collection of data files, as well as a subdirectory in which +each instance of rpki-rtr server can place a PF_UNIX socket file. By default, +rpki-rtr creates these files under the directory in which you run it, but you +can change that by specifying the target directory as a second command line +argument, as shown above. -You should make sure that rtr-origin --cronjob runs at least once before -attempting to configure --server mode. Nothing terrible will happen if you -don't do this, but --server invocations started before the first --cronjob run -may behave oddly. +You should make sure that rpki-rtr cronjob runs at least once before attempting +to configure rpki-rtr server. Nothing terrible will happen if you don't do +this, but rpki-rtr server invocations started before the first rpki-rtr cronjob +run may behave oddly. ***** Setting up the rpki-rtr server ***** -You need to to set up a server listener that invokes rtr-origin in --server -mode. What kind of server listener you set up depends on which network protocol -you're using to transport this protocol. rtr-origin is happy to run under -inetd, xinetd, sshd, or pretty much anything -- rtr-origin doesn't really care, -it just reads from stdin and writes to stdout. +You need to to set up a server listener that invokes `rpki-rtr server`. What +kind of server listener you set up depends on which network protocol you're +using to transport this protocol. rpki-rtr is happy to run under inetd, xinetd, +sshd, or pretty much anything -- rpki-rtr doesn't really care, it just reads +from stdin and writes to stdout. ---server mode should be run as a non-privileged user (it is read-only for a +rpki-rtr server should be run as a non-privileged user (it is read-only for a reason). You may want to set up a separate UNIX userid for this purpose. ---server mode takes an optional argument specifying the path to its data +rpki-rtr server takes an optional argument specifying the path to its data directory; if you omit this argument, it uses the directory in in which you run it. The details of how you set up a listener for this vary depending on the network -protocol and the operating system on which you run it. Here are two examples, -one for running under inetd on FreeBSD, the other for running under sshd. +protocol and the operating system on which you run it. Here are three examples, +for running under inetd on FreeBSD, under sshd, or as a free-standing server +using rpki-rtr listener. -**** Running rtr-origin --server under inetd **** +**** Running rpki-rtr server under inetd **** Running under inetd with plain TCP is insecure and should only be done for testing, but you can also run it with TCP-MD5 or TCP-AO, or over IPsec. The @@ -87,8 +89,8 @@ To run under inetd, you need to: 1. Add the service line to /etc/inetd.conf: - rpki-rtr stream tcp nowait nobody /usr/local/bin/rtr-origin rtr-origin -- - server /var/rcynic/rpki-rtr + rpki-rtr stream tcp nowait nobody /usr/local/bin/rpki-rtr rpki-rtr server / + var/rcynic/rpki-rtr This assumes that you want the server to run as user "nobody", which is generally a safe choice, or you could create a new non-priviledged @@ -96,12 +98,12 @@ To run under inetd, you need to: anything bad, but it's a network server that doesn't need root access, therefore it shouldn't have root access. -**** Running rtr-origin --server under sshd **** +**** Running rpki-rtr server under sshd **** -To run rtr-origin under sshd, you need to: +To run rpki-rtr server under sshd, you need to: 1. Decide whether to run a new instance of sshd on a separate port or use the - standard port. rtr-origin doesn't care, but some people seem to think that + standard port. rpki-rtr doesn't care, but some people seem to think that it's somehow more secure to run this service on a different port. Setting up sshd in general is beyond the scope of this documention, but most likely you can copy the bulk of your configuration from the standard @@ -109,12 +111,12 @@ To run rtr-origin under sshd, you need to: 2. Configure sshd to know about the rpki-rtr subsystem. Add something like this to your sshd.conf: - Subsystem rpki-rtr /usr/local/bin/rtr-origin + Subsystem rpki-rtr /usr/local/bin/rpki-rtr 1. Configure the userid(s) you expect SSH clients to use to connect to the server. For operational use you almost certainly do NOT want this user to have a normal shell, instead you should configure its shell to be the - server (/usr/local/bin/rtr-origin or wherever you've installed it on your + server (/usr/local/bin/rpki-rtr or wherever you've installed it on your system) and its home directory to be the rpki-rtr data directory (/var/ rcynic/rpki-rtr or whatever you're using). If you're using passwords to authenticate instead of ssh keys (not recommended) you will always need to @@ -140,6 +142,19 @@ To run rtr-origin under sshd, you need to: Subsystem line, which runs the server.sh script as the "rpki-rtr" service, as required by the protocol specification. +**** Running rpki-rtr listener **** + +rpki-rtr listener is a free-standing plain TCP server which just listens on a +TCP socket then forks a child process running rpki-rtr server. + +All of the caveats regarding plain TCP apply to rpki-rtr listener. + +rpki-rtr listener takes one required argument, the TCP port number on which to +listen; it also accepts a second argument which specifies the rcynic output +directory, like rpki-rtr server. + + /usr/local/bin/rpki-rtr listener 42420 /var/rcynic/rpki-rtr + **** Other transports **** You can also run this code under xinetd, or the netpipes "faucet" program, or @@ -151,19 +166,16 @@ doesn't make a lot of sense to go to all the trouble of checking RPKI data then let the bad guys feed bad data into your routers anyway because you were running the rpki-rtr link over an unsecured TCP connection. -***** Other modes ***** +***** Other commands ***** -rtr-origin has two other modes which might be useful for debugging: +rpki-rtr has two other commands which might be useful for debugging: - 1. --client mode implements a dumb client program for this protocol, over - SSH, raw TCP, or by invoking --server mode directly in a subprocess. The + 1. rpki-rtr client implements a dumb client program for this protocol, over + SSH, raw TCP, or by invoking rpki-rtr server directly in a subprocess. The output is not expected to be useful except for debugging. Either run it locally where you run the cron job, or run it anywhere on the net, as in - $ rtr-origin --client tcp + $ rpki-rtr client tcp - 2. --show mode will display a text dump of pre-digested data files in the + 2. rpki-rtr show will display a text dump of pre-digested data files in the current directory. - -rtr-origin has a few other modes intended to support specific research -projects, but they're not intended for general use. diff --git a/doc/manual.pdf b/doc/manual.pdf index 2985cc6a..656f1223 100644 Binary files a/doc/manual.pdf and b/doc/manual.pdf differ -- cgit v1.2.3 From 429adae788694f109174e35467c49d13b9533fe2 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Sat, 13 Sep 2014 03:56:54 +0000 Subject: Groundwork for Django ORM world conquest: sort out settings.py mess. svn path=/branches/tk713/; revision=5948 --- ca/irbe_cli | 2 +- ca/rpki-confgen.xml | 32 ++--- ca/rpki-manage | 17 ++- ca/rpki-sql-backup | 2 +- ca/rpki-sql-setup | 4 +- ca/rpki-start-servers | 4 +- ca/rpki.wsgi | 3 +- ca/tests/smoketest.py | 2 +- ca/tests/sql-cleaner.py | 2 +- ca/tests/sql-dumper.py | 2 +- ca/tests/yamlconf.py | 4 +- ca/tests/yamltest.py | 14 +- potpourri/upgrade-add-ghostbusters.py | 2 +- rpki/config.py | 43 ++++-- rpki/db_router.py | 57 ++++++++ rpki/django_settings.py | 238 ++++++++++++++++++++++++++++++++++ rpki/fields.py | 192 +++++++++++++++++++++++++++ rpki/gui/default_settings.py | 171 ------------------------ rpki/irdb/models.py | 120 +++-------------- rpki/irdbd.py | 29 +---- rpki/old_irdbd.py | 2 +- rpki/pubd.py | 2 +- rpki/pubdb/__init__.py | 3 + rpki/rootd.py | 2 +- rpki/rpkic.py | 2 +- rpki/rpkid.py | 2 +- rpki/rpkidb/__init__.py | 3 + 27 files changed, 596 insertions(+), 360 deletions(-) create mode 100644 rpki/db_router.py create mode 100644 rpki/django_settings.py create mode 100644 rpki/fields.py delete mode 100644 rpki/gui/default_settings.py create mode 100644 rpki/pubdb/__init__.py create mode 100644 rpki/rpkidb/__init__.py diff --git a/ca/irbe_cli b/ca/irbe_cli index 1becd403..c38cf93b 100755 --- a/ca/irbe_cli +++ b/ca/irbe_cli @@ -305,7 +305,7 @@ for o, a in opts: if not argv: usage(1) -cfg = rpki.config.parser(cfg_file, "irbe_cli") +cfg = rpki.config.parser(set_filename = cfg_file, section = "irbe_cli") q_msg_left_right = [] q_msg_publication = [] diff --git a/ca/rpki-confgen.xml b/ca/rpki-confgen.xml index e0ed273a..a29ad8cd 100644 --- a/ca/rpki-confgen.xml +++ b/ca/rpki-confgen.xml @@ -816,30 +816,18 @@
- Glue to allow the Django application to pull user configuration - from this file rather than directly editing settings.py. + Glue to allow Django to pull user configuration from this file + rather than requiring the user to edit settings.py. - - - - - +