diff options
Diffstat (limited to 'scripts')
51 files changed, 0 insertions, 5629 deletions
diff --git a/scripts/analyze-rcynic-history.py b/scripts/analyze-rcynic-history.py deleted file mode 100644 index b72d0741..00000000 --- a/scripts/analyze-rcynic-history.py +++ /dev/null @@ -1,290 +0,0 @@ -# $Id$ -# -# Copyright (C) 2011-2012 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Parse traffic data out of rynic XML output, whack it a bit, print some -summaries and run gnuplot to draw some pictures. -""" - -plot_all_hosts = False - -window_hours = 72 - -import mailbox -import sys -import urlparse -import os -import datetime -import subprocess -import shelve - -from xml.etree.cElementTree import (ElementTree as ElementTree, - fromstring as ElementTreeFromString) - -def parse_utc(s): - return datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%SZ") - -class Rsync_History(object): - """ - An Rsync_History object represents one rsync connection. - """ - - def __init__(self, elt): - self.error = elt.get("error") - self.uri = elt.text.strip() - self.hostname = urlparse.urlparse(self.uri).hostname or None - self.elapsed = parse_utc(elt.get("finished")) - parse_utc(elt.get("started")) - -class Host(object): - """ - A host object represents all the data collected for one host. Note - that it (usually) contains a list of all the sessions in which this - host appears. - """ - - def __init__(self, hostname, session_id): - self.hostname = hostname - self.session_id = session_id - self.elapsed = datetime.timedelta(0) - self.connection_count = 0 - self.dead_connections = 0 - self.uris = set() - self.total_connection_time = datetime.timedelta(0) - - def add_rsync_history(self, h): - self.connection_count += 1 - self.elapsed += h.elapsed - self.dead_connections += int(h.error is not None) - self.total_connection_time += h.elapsed - - def add_uri(self, u): - self.uris.add(u) - - def finalize(self): - self.object_count = len(self.uris) - del self.uris - - @property - def failed(self): - return 1 if self.dead_connections else 0 - - @property - def seconds_per_object(self): - if self.failed: - return None - else: - return float(self.elapsed.days * 24 * 60 * 60 + - self.elapsed.seconds + - self.elapsed.microseconds / 10**6) / float(self.object_count) - - @property - def objects_per_connection(self): - if self.failed: - return None - else: - return float(self.object_count) / float(self.connection_count) - - @property - def average_connection_time(self): - return float(self.total_connection_time.days * 24 * 60 * 60 + - self.total_connection_time.seconds + - self.total_connection_time.microseconds / 10**6) / float(self.connection_count) - - class Format(object): - - def __init__(self, attr, title, fmt, ylabel = ""): - self.attr = attr - self.title = title - self.width = len(title) - int("%" in fmt) - self.fmt = "%%%d%s" % (self.width, fmt) - self.oops = "*" * self.width - self.ylabel = ylabel - - def __call__(self, obj): - try: - value = getattr(obj, self.attr) - return None if value is None else self.fmt % value - except ZeroDivisionError: - return self.oops - - format = (Format("connection_count", "Connections", "d", "Connections To Repository (Per Session)"), - Format("object_count", "Objects", "d", "Objects In Repository (Distinct URIs Per Session)"), - Format("objects_per_connection", "Objects/Connection", ".3f", "Objects In Repository / Connections To Repository"), - Format("seconds_per_object", "Seconds/Object", ".3f", "Seconds To Transfer / Object (Average Per Session)"), - Format("failure_rate_running", "Failure Rate", ".3f%%", "Sessions With Failed Connections Within Last %d Hours" % window_hours), - Format("average_connection_time", "Average Connection", ".3f", "Seconds / Connection (Average Per Session)"), - Format("hostname", "Hostname", "s")) - - format_dict = dict((fmt.attr, fmt) for fmt in format) - - def format_field(self, name): - result = self.format_dict[name](self) - return None if result is None else result.strip() - -class Session(dict): - """ - A session corresponds to one XML file. This is a dictionary of Host - objects, keyed by hostname. - """ - - def __init__(self, session_id, msg_key): - self.session_id = session_id - self.msg_key = msg_key - self.date = parse_utc(session_id) - self.calculated_failure_history = False - - @property - def hostnames(self): - return set(self.iterkeys()) - - def get_plot_row(self, name, hostnames): - return (self.session_id,) + tuple(self[h].format_field(name) if h in self else "" for h in hostnames) - - def add_rsync_history(self, h): - if h.hostname not in self: - self[h.hostname] = Host(h.hostname, self.session_id) - self[h.hostname].add_rsync_history(h) - - def add_uri(self, u): - h = urlparse.urlparse(u).hostname - if h and h in self: - self[h].add_uri(u) - - def finalize(self): - for h in self.itervalues(): - h.finalize() - - def calculate_failure_history(self, sessions): - start = self.date - datetime.timedelta(hours = window_hours) - sessions = tuple(s for s in sessions if s.date <= self.date and s.date > start) - for hostname, h in self.iteritems(): - i = n = 0 - for s in sessions: - if hostname in s: - i += s[hostname].failed - n += 1 - h.failure_rate_running = float(100 * i) / n - self.calculated_failure_history = True - -def plotter(f, hostnames, field, logscale = False): - plotlines = sorted(session.get_plot_row(field, hostnames) for session in sessions) - title = Host.format_dict[field].title - ylabel = Host.format_dict[field].ylabel - n = len(hostnames) + 1 - assert all(n == len(plotline) for plotline in plotlines) - if "%%" in Host.format_dict[field].fmt: - f.write('set format y "%.0f%%"\n') - else: - f.write('set format y\n') - if logscale: - f.write("set logscale y\n") - else: - f.write("unset logscale y\n") - f.write(""" - set xdata time - set timefmt '%Y-%m-%dT%H:%M:%SZ' - #set format x '%m/%d' - #set format x '%b%d' - #set format x '%Y-%m-%d' - set format x '%Y-%m' - #set title '""" + title + """' - set ylabel '""" + ylabel + """' - plot""" + ",".join(" '-' using 1:2 with linespoints pointinterval 500 title '%s'" % h for h in hostnames) + "\n") - for i in xrange(1, n): - for plotline in plotlines: - if plotline[i] is not None: - f.write("%s %s\n" % (plotline[0], plotline[i].rstrip("%"))) - f.write("e\n") - -def plot_hosts(hostnames, fields): - for field in fields: - for logscale in (False, True): - gnuplot = subprocess.Popen(("gnuplot",), stdin = subprocess.PIPE) - gnuplot.stdin.write("set terminal pdf\n") - gnuplot.stdin.write("set output '%s/%s-%s.pdf'\n" % (outdir, field, "log" if logscale else "linear")) - plotter(gnuplot.stdin, hostnames, field, logscale = logscale) - gnuplot.stdin.close() - gnuplot.wait() - -outdir = "images" - -if not os.path.exists(outdir): - os.makedirs(outdir) - -mb = mailbox.Maildir("/u/sra/rpki/rcynic-xml", factory = None, create = False) - -if sys.platform == "darwin": # Sigh - shelf = shelve.open("rcynic-xml", "c") -else: - shelf = shelve.open("rcynic-xml.db", "c") - -sessions = [] - -latest = None -parsed = 0 - -for i, key in enumerate(mb.iterkeys(), 1): - sys.stderr.write("\r%s %d/%d/%d..." % ("|\\-/"[i & 3], parsed, i, len(mb))) - - if key in shelf: - session = shelf[key] - - else: - assert not mb[key].is_multipart() - input = ElementTreeFromString(mb[key].get_payload()) - date = input.get("date") - sys.stderr.write("%s..." % date) - session = Session(date, key) - for elt in input.findall("rsync_history"): - session.add_rsync_history(Rsync_History(elt)) - for elt in input.findall("validation_status"): - if elt.get("generation") == "current": - session.add_uri(elt.text.strip()) - session.finalize() - shelf[key] = session - parsed += 1 - - sessions.append(session) - if latest is None or session.session_id > latest.session_id: - latest = session - -sys.stderr.write("\n") - -shelf.sync() - -for session in sessions: - if not getattr(session, "calculated_failure_history", False): - session.calculate_failure_history(sessions) - shelf[session.msg_key] = session - -if plot_all_hosts: - hostnames = sorted(reduce(lambda x, y: x | y, - (s.hostnames for s in sessions), - set())) - -else: - hostnames = ("rpki.apnic.net", "rpki.ripe.net", "localcert.ripe.net", - "repository.lacnic.net", "rpki.afrinic.net", "rpki.arin.net", - "arin.rpki.net", "repo0.rpki.net", "rgnet.rpki.net") - -plot_hosts(hostnames, [fmt.attr for fmt in Host.format if fmt.attr != "hostname"]) - -if latest is not None: - f = open("rcynic.xml", "wb") - f.write(mb[latest.msg_key].get_payload()) - f.close() - -shelf.close() diff --git a/scripts/analyze-transition.py b/scripts/analyze-transition.py deleted file mode 100644 index e2125dfb..00000000 --- a/scripts/analyze-transition.py +++ /dev/null @@ -1,88 +0,0 @@ -# $Id$ -# -# Copyright (C) 2012 Internet Systems Consortium, Inc. ("ISC") -# -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Compare rcynic.xml files, tell the user what became invalid, and why. -""" - -import sys - -try: - from lxml.etree import ElementTree -except ImportError: - from xml.etree.ElementTree import ElementTree - -class Object(object): - - def __init__(self, session, uri): - self.session = session - self.uri = uri - self.labels = [] - - def add(self, label): - self.labels.append(label) - - def __cmp__(self, other): - return cmp(self.labels, other.labels) - - @property - def accepted(self): - return "object_accepted" in self.labels - -class Session(dict): - - def __init__(self, name): - self.name = name - tree = ElementTree(file = name) - labels = tuple((elt.tag.strip(), elt.text.strip()) for elt in tree.find("labels")) - self.labels = tuple(pair[0] for pair in labels) - self.descrs = dict(labels) - self.date = tree.getroot().get("date") - for elt in tree.findall("validation_status"): - status = elt.get("status") - uri = elt.text.strip() - if status.startswith("rsync_transfer_") or elt.get("generation") != "current": - continue - if uri not in self: - self[uri] = Object(self, uri) - self[uri].add(status) - -skip_labels = frozenset(("object_accepted", "object_rejected")) - -old_db = new_db = None - -for arg in sys.argv[1:]: - - old_db = new_db - new_db = Session(arg) - if old_db is None: - continue - - old_uris = frozenset(old_db) - new_uris = frozenset(new_db) - - for uri in sorted(old_uris - new_uris): - print new_db.date, uri, "dropped" - - for uri in sorted(old_uris & new_uris): - old = old_db[uri] - new = new_db[uri] - if old.accepted and not new.accepted: - print new_db.date, uri, "invalid" - labels = frozenset(new.labels) - frozenset(old.labels) - skip_labels - for label in new.labels: - if label in labels: - print " ", new_db.descrs[label] diff --git a/scripts/apnic-to-csv.py b/scripts/apnic-to-csv.py deleted file mode 100644 index 62293a51..00000000 --- a/scripts/apnic-to-csv.py +++ /dev/null @@ -1,55 +0,0 @@ -# $Id$ -# -# Copyright (C) 2010-2012 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Parse APNIC "Extended Allocation and Assignment" reports and write -out (just) the RPKI-relevant fields in myrpki-format CSV syntax. -""" - -from rpki.csv_utils import csv_writer -from rpki.ipaddrs import v4addr - -asns = csv_writer("asns.csv") -prefixes = csv_writer("prefixes.csv") - -for line in open("delegated-apnic-extended-latest"): - - line = line.rstrip() - - if not line.startswith("apnic|") or line.endswith("|summary"): - continue - - try: - registry, cc, rectype, start, value, date, status, opaque_id = line.split("|") - except ValueError: - continue - - if not opaque_id: - continue - - assert registry == "apnic" - - if rectype == "asn": - asns.writerow((opaque_id, "%s-%s" % (start, int(start) + int(value) - 1))) - - elif rectype == "ipv4": - prefixes.writerow((opaque_id, "%s-%s" % (start, v4addr(v4addr(start) + long(value) - 1)))) - - elif rectype == "ipv6": - prefixes.writerow((opaque_id, "%s/%s" % (start, value))) - -asns.close() -prefixes.close() diff --git a/scripts/application-x-rpki-mailcap-handler.sh b/scripts/application-x-rpki-mailcap-handler.sh deleted file mode 100755 index 891987d9..00000000 --- a/scripts/application-x-rpki-mailcap-handler.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/sh - -# $Id$ -# -# Copyright (C) 2010 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -# Given the Maildir dump format, one can use Mutt as a viewer with two -# tweaks: -# -# 1) Add to ~/.muttrc -# -# auto_view application/x-rpki -# -# 2) Add to ~/.mailcap -# -# application/x-rpki; /path/to/this/script.sh ; copiousoutput -# -# "copiousoutput" is required by mutt to enable auto_view (inline -# display) behavior. -# -# This script could do fancier things (pretty XML formatting, -# verification checks of the CMS, etcetera) if anybody cared. -# For the moment the main use for this script is debugging. - -# We have to jump through some hoops to figure out where our OpenSSL -# binary is. If you have already installed an OpenSSL binary that -# understands CMS, feel free to use that instead. - -#exec 2>&1; set -x - -: ${AWK=/usr/bin/awk} -: ${OPENSSL=$(/usr/bin/dirname $0)/../openssl/openssl/apps/openssl} -: ${SPLITBASE64=$(/usr/bin/dirname $0)/splitbase64.xsl} -: ${XMLINDENT=/usr/local/bin/xmlindent} -: ${XMLLINT=/usr/local/bin/xmllint} -: ${XSLTPROC=/usr/local/bin/xsltproc} - -# This produces prettier output, but also hangs sometimes, apparently some xmlindent bug dealing with really long XML attributes -#OPENSSL_CONF=/dev/null $OPENSSL cms -verify -nosigs -noverify -inform DER 2>/dev/null | $XSLTPROC $SPLITBASE64 - | $XMLINDENT -i 2 | $AWK NF - -# So we do this instead -OPENSSL_CONF=/dev/null $OPENSSL cms -verify -nosigs -noverify -inform DER 2>/dev/null | $XSLTPROC $SPLITBASE64 - | $XMLLINT -format - diff --git a/scripts/arin-to-csv.py b/scripts/arin-to-csv.py deleted file mode 100644 index a4e7ffc3..00000000 --- a/scripts/arin-to-csv.py +++ /dev/null @@ -1,114 +0,0 @@ -# $Id$ -# -# Copyright (C) 2009-2012 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Parse an ARIN database research dump and write out (just) the -RPKI-relevant fields in myrpki-format CSV syntax. - -NB: The input data for this script comes from ARIN under an agreement -that allows research use but forbids redistribution, so if you think -you need a copy of the data, please talk to ARIN about it, not us. - -Input format used to be RPSL WHOIS dump, but ARIN recently went Java, -so we have to parse a 3.5GB XML "document". Credit to Liza Daly for -explaining the incantations needed to convince lxml to do this nicely, -see: http://www.ibm.com/developerworks/xml/library/x-hiperfparse/ -""" - -import sys -import lxml.etree - -from rpki.csv_utils import csv_writer - -def ns(tag): - return "{http://www.arin.net/bulkwhois/core/v1}" + tag - -tag_asn = ns("asn") -tag_net = ns("net") -tag_org = ns("org") -tag_poc = ns("poc") -tag_orgHandle = ns("orgHandle") -tag_netBlock = ns("netBlock") -tag_type = ns("type") -tag_startAddress = ns("startAddress") -tag_endAddress = ns("endAddress") -tag_startAsNumber = ns("startAsNumber") -tag_endAsNumber = ns("endAsNumber") - -def find(node, tag): - return node.findtext(tag).strip() - -def do_asn(node): - asns.writerow((find(node, tag_orgHandle), - "%s-%s" % (find(node, tag_startAsNumber), - find(node, tag_endAsNumber)))) - -erx_table = { - "AF" : "afrinic", - "AP" : "apnic", - "AR" : "arin", - "AV" : "arin", - "FX" : "afrinic", - "LN" : "lacnic", - "LX" : "lacnic", - "PV" : "apnic", - "PX" : "apnic", - "RN" : "ripe", - "RV" : "ripe", - "RX" : "ripe" } - -def do_net(node): - handle = find(node, tag_orgHandle) - for netblock in node.iter(tag_netBlock): - tag = find(netblock, tag_type) - startAddress = find(netblock, tag_startAddress) - endAddress = find(netblock, tag_endAddress) - if not startAddress.endswith(".000") and not startAddress.endswith(":0000"): - continue - if not endAddress.endswith(".255") and not endAddress.endswith(":FFFF"): - continue - if tag in ("DS", "DA", "IU"): - prefixes.writerow((handle, "%s-%s" % (startAddress, endAddress))) - elif tag in erx_table: - erx.writerow((erx_table[tag], "%s-%s" % (startAddress, endAddress))) - -dispatch = { tag_asn : do_asn, tag_net : do_net } - -asns = csv_writer("asns.csv") -prefixes = csv_writer("prefixes.csv") -erx = csv_writer("erx.csv") - -root = None - -for event, node in lxml.etree.iterparse(sys.stdin): - - if root is None: - root = node - while root.getparent() is not None: - root = root.getparent() - - if node.getparent() is root: - - if node.tag in dispatch: - dispatch[node.tag](node) - - node.clear() - while node.getprevious() is not None: - del node.getparent()[0] - -asns.close() -prefixes.close() -erx.close() diff --git a/scripts/cross_certify.py b/scripts/cross_certify.py deleted file mode 100644 index fab7743b..00000000 --- a/scripts/cross_certify.py +++ /dev/null @@ -1,74 +0,0 @@ -# $Id$ -# -# Copyright (C) 2014 Dragon Research Labs ("DRL") -# Portions copyright (C) 2009--2012 Internet Systems Consortium ("ISC") -# Portions copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notices and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND DRL, ISC, AND ARIN DISCLAIM ALL -# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL, -# ISC, OR ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION -# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -""" -Cross-certification tool to issue a new certificate based on an old -one that was issued by somebody else. The point of the exercise is to -end up with a valid certificate in our own BPKI which has the same -subject name and subject public key as the one we're replacing. -""" - -import os -import sys -import time -import argparse -import rpki.x509 -import rpki.sundial - -os.environ["TZ"] = "UTC" -time.tzset() - -parser = argparse.ArgumentParser(description = __doc__) -parser.add_argument("-i", "--in", required = True, dest = "input", - type = lambda s: rpki.x509.X509(Auto_file = s), - help = "input certificate") -parser.add_argument("-c", "--ca", required = True, - type = lambda s: rpki.x509.X509(Auto_file = s), - help = "issuing certificate") -parser.add_argument("-k", "--key", required = True, - type = lambda s: rpki.x509.RSA(Auto_file = s), - help = "private key of issuing certificate") -parser.add_argument("-s", "--serial", required = True, - help = "serial number file") -parser.add_argument("-o", "--out", - help = "output filename") -parser.add_argument("-l", "--lifetime", - type = rpki.sundial.timedelta, default = "30d", - help = "lifetime of generated certificate") -args = parser.parse_args() - -now = rpki.sundial.now() -notAfter = now + args.lifetime - -try: - with open(args.serial, "r") as f: - serial = int(f.read().splitlines()[0], 16) -except IOError: - serial = 1 - -cert = args.ca.cross_certify(args.key, args.input, serial, notAfter, now) - -with open(args.serial, "w") as f: - f.write("%02x\n" % (serial + 1)) - -if args.out is None: - sys.stdout.write(cert.get_PEM()) -else: - with open(args.out, "w") as f: - f.write(cert.get_PEM()) diff --git a/scripts/csvgrep.py b/scripts/csvgrep.py deleted file mode 100644 index 68bdd259..00000000 --- a/scripts/csvgrep.py +++ /dev/null @@ -1,72 +0,0 @@ -# $Id$ -# -# Copyright (C) 2010-2012 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Utility to simplify finding handles in one of the pseudo-RIR databases. - -Usage: python csvgrep.py datum [datum ...] - -where each datum is an ASN, IP address, or IP prefix. - -ASNs are recognized by being pure integers; IP addreses are recognized -by having dots (IPv4) or colons (IPv6). - -After eating all of the command line arguments, we search asns.csv for -any ASNs given, and prefixes.csv for any prefixes given. -""" - -import sys -from rpki.resource_set import resource_set_as, resource_set_ipv4, resource_set_ipv6 -from rpki.csv_utils import csv_reader - -asn = resource_set_as() -ipv4 = resource_set_ipv4() -ipv6 = resource_set_ipv6() - -for datum in sys.argv[1:]: - if datum.replace("-", "").isdigit(): - t = asn - else: - t = ipv6 if ":" in datum else ipv4 - if "-" not in datum and "/" not in datum: - datum = datum + "-" + datum - try: - t.append(t.parse_str(datum)) - except: - print "Error attempting to parse", datum - raise - -#print "Looking for: ASNs %s IPv4 %s IPv6 %s" % (asn, ipv4, ipv6) - -def matches(set1, datum): - set2 = set1.__class__(datum) - if set1.intersection(set2): - return set2 - else: - return False - -if asn: - for h, a in csv_reader("asns.csv", columns = 2): - m = matches(asn, a) - if m: - print h, m - -if ipv4 or ipv6: - for h, a in csv_reader("prefixes.csv", columns = 2): - t = ipv6 if ":" in a else ipv4 - m = t and matches(t, a) - if m: - print h, m diff --git a/scripts/expand-roa-prefixes.py b/scripts/expand-roa-prefixes.py deleted file mode 100644 index ae34ea0a..00000000 --- a/scripts/expand-roa-prefixes.py +++ /dev/null @@ -1,79 +0,0 @@ -# $Id$ -# -# Copyright (C) 2011 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -I got tired of trying to explain in English how the maxLength macro -hack works in ROAs, so this is an attempt to explain it as code. - -Given one or more ROA prefix sets on the command line, this script -prints out the expansion as a list of prefixes. -""" - -import sys -import rpki.resource_set -import rpki.ipaddrs - -class NotAPrefix(Exception): - """ - Address is not a proper prefix. - """ - -class address_range(object): - """ - Iterator for rpki.ipaddrs address objects. - """ - - def __init__(self, start, stop, step): - self.addr = start - self.stop = stop - self.step = step - self.type = type(start) - - def __iter__(self): - while self.addr < self.stop: - yield self.addr - self.addr = self.type(self.addr + self.step) - -def main(argv): - - prefix_sets = [] - for arg in argv: - if ":" in arg: - prefix_sets.extend(rpki.resource_set.roa_prefix_set_ipv6(arg)) - else: - prefix_sets.extend(rpki.resource_set.roa_prefix_set_ipv4(arg)) - - for prefix_set in prefix_sets: - sys.stdout.write("%s expands to:\n" % prefix_set) - - prefix_type = prefix_set.range_type.datum_type - prefix_min = prefix_set.prefix - prefix_max = prefix_set.prefix + (1L << (prefix_type.bits - prefix_set.prefixlen)) - - for prefixlen in xrange(prefix_set.prefixlen, prefix_set.max_prefixlen + 1): - - step = (1L << (prefix_type.bits - prefixlen)) - mask = step - 1 - - for addr in address_range(prefix_min, prefix_max, step): - if (addr & mask) != 0: - raise NotAPrefix, "%s is not a /%d prefix" % (addr, prefixlen) - sys.stdout.write(" %s/%d\n" % (addr, prefixlen)) - - sys.stdout.write("\n") - -if __name__ == "__main__": - main(sys.argv[1:] if len(sys.argv) > 1 else ["18.0.0.0/8-24"]) diff --git a/scripts/extract-key.py b/scripts/extract-key.py deleted file mode 100644 index b85c3d55..00000000 --- a/scripts/extract-key.py +++ /dev/null @@ -1,64 +0,0 @@ -# $Id$ - -# Copyright (C) 2014 Dragon Research Labs ("DRL") -# Portions copyright (C) 2008 American Registry for Internet Numbers ("ARIN") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notices and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND AND ARIN DISCLAIM ALL -# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR -# ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL -# DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA -# OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Extract a private key from rpkid's database. - -This is a debugging tool. rpkid goes to some trouble not to expose -private keys, which is correct for normal operation, but for debugging -it is occasionally useful to be able to extract the private key from -MySQL. This script is just a convenience, it doesn't enable anything -that couldn't be done via the mysql command line tool. - -While we're at this we also extract the corresponding certificate. -""" - -import os -import time -import argparse -import sys -import MySQLdb -import rpki.x509 - -os.environ["TZ"] = "UTC" -time.tzset() - -parser = argparse.ArgumentParser(description = __doc__) -parser.add_argument("-s", "--self", required = True, help = "self handle") -parser.add_argument("-b", "--bsc", required = True, help = "BSC handle") -parser.add_argument("-u", "--user", required = True, help = "MySQL user name") -parser.add_argument("-d", "--db", required = True, help = "MySQL database name") -parser.add_argument("-p", "--password", required = True, help = "MySQL password") -args = parser.parse_args() - -cur = MySQLdb.connect(user = args.user, db = args.db, passwd = args.password).cursor() - -cur.execute( - """ - SELECT bsc.private_key_id, bsc.signing_cert - FROM bsc, self - WHERE self.self_handle = %s AND self.self_id = bsc.self_id AND bsc_handle = %s - """, - (args.self, args.bsc)) - -key, cer = cur.fetchone() - -print rpki.x509.RSA(DER = key).get_PEM() - -if cer: - print rpki.x509.X509(DER = cer).get_PEM() diff --git a/scripts/fakerootd.py b/scripts/fakerootd.py deleted file mode 100644 index 6275a2a9..00000000 --- a/scripts/fakerootd.py +++ /dev/null @@ -1,50 +0,0 @@ -# $Id$ -# -# Copyright (C) 2011 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Hack to fake a catatonic rootd, for testing. -""" - -import sys -import socket -import datetime -import signal - -port = int(sys.argv[1]) if len(sys.argv) > 1 else 4405 -limit = int(sys.argv[2]) if len(sys.argv) > 2 else 5 - -print "Listening on port", port - -s4 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -s4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) -s4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) -s4.bind(('', port)) -s4.listen(limit) - -s6 = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) -s6.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) -s6.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) -s6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1) -s6.bind(('::1', port)) -s6.listen(limit) - -print "Going to sleep at", datetime.datetime.utcnow() - -try: - signal.pause() -except KeyboardInterrupt: - sys.exit(0) - diff --git a/scripts/find-roa-expiration.py b/scripts/find-roa-expiration.py deleted file mode 100644 index 1401dc42..00000000 --- a/scripts/find-roa-expiration.py +++ /dev/null @@ -1,61 +0,0 @@ -# $Id$ -# -# Copyright (C) 2012 Internet Systems Consortium, Inc. ("ISC") -# -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Look for ROAs for particular prefixes, like find_roa, then, for each -ROA we find, dig out the expiration times of all the certificates -involved in the authorization chain, all the way back to the root. -""" - -import sys -import subprocess -import rpki.POW - -def filename_to_uri(filename): - if not filename.startswith(sys.argv[1]): - raise ValueError - return "rsync://" + filename[len(sys.argv[1]):].lstrip("/") - -def uri_to_filename(uri): - if not uri.startswith("rsync://"): - raise ValueError - return sys.argv[1] + "/" + uri[len("rsync://"):] - -def get_aia(x): - for i in xrange(x.countExtensions()): - ext = x.getExtension(i) - if ext[0] == "authorityInfoAccess": - return ext[2][ext[2].index("rsync://"):] - return None - -for line in subprocess.check_output(["find_roa"] + sys.argv[1:]).splitlines(): - - words = line.split() - fn = words.pop() - del words[-1] - print " ".join(words) - - x = rpki.POW.CMS.derReadFile(fn).certs()[0] - uri = get_aia(x) - print x.getNotAfter(), filename_to_uri(fn) - - while uri: - fn = uri_to_filename(uri) - x = rpki.POW.X509.derReadFile(fn) - print x.getNotAfter(), uri - uri = get_aia(x) - - print diff --git a/scripts/format-application-x-rpki.py b/scripts/format-application-x-rpki.py deleted file mode 100644 index 184103f9..00000000 --- a/scripts/format-application-x-rpki.py +++ /dev/null @@ -1,132 +0,0 @@ -# $Id$ -# -# Copyright (C) 2014 Dragon Research Labs ("DRL") -# Portions copyright (C) 2010--2012 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notices and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL -# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR -# ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL -# DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA -# OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Take the basic application/x-rpki messages that rpkid and friends -log and translate them into a text version that's easier to search, -without losing any of the original data. We use MH for the output -format because nmh makes a handy viewer. -""" - -import email.mime -import email.mime.application -import email.mime.text -import email.mime.multipart -import email.utils -import email.encoders -import mailbox -import rpki.POW -import lxml.etree -import argparse -import sys -import base64 - -parser = argparse.ArgumentParser(description = __doc__) -parser.add_argument("-i", "--input", required = True, - help = "input Maildir") -parser.add_argument("-m", "--mark", action = "store_true", - help = "mark seen messages") -parser.add_argument("-k", "--kill", action = "store_true", - help = "kill seen messages") -parser.add_argument("-o", "--output", required = True, - help = "output MH folder") -parser.add_argument("-t", "--tag", - default = "{http://www.apnic.net/specs/rescerts/up-down/}message", - help = "XML namespace tag for an input message") -parser.add_argument("-u", "--unseen", action = "store_true", - help = "only process unseen messages") -args = parser.parse_args() - -def pprint_cert(b64): - return rpki.POW.X509.derRead(base64.b64decode(b64)).pprint() - -def up_down(): - msg["X-RPKI-Up-Down-Type"] = xml.get("type") - msg["X-RPKI-Up-Down-Sender"] = xml.get("sender") - msg["X-RPKI-Up-Down-Recipient"] = xml.get("recipient") - msg["Subject"] = "Up-down %s %s => %s" % (xml.get("type"), xml.get("sender"), xml.get("recipient")) - for x in xml: - if x.tag.endswith("class"): - for y in x: - if y.tag.endswith("certificate") or y.tag.endswith("issuer"): - msg.attach(email.mime.text.MIMEText(pprint_cert(y.text))) - -def left_right(): - msg["X-RPKI-Left-Right-Type"] = xml.get("type") - msg["Subject"] = "Left-right %s" % xml.get("type") - -def publication(): - msg["X-RPKI-Left-Right-Type"] = xml.get("type") - msg["Subject"] = "Publication %s" % xml.get("type") - -dispatch = { "{http://www.apnic.net/specs/rescerts/up-down/}message" : up_down, - "{http://www.hactrn.net/uris/rpki/left-right-spec/}msg" : left_right, - "{http://www.hactrn.net/uris/rpki/publication-spec/}msg" : publication } - -def fix_headers(): - if "X-RPKI-PID" in srcmsg or "X-RPKI-Object" in srcmsg: - msg["X-RPKI-PID"] = srcmsg["X-RPKI-PID"] - msg["X-RPKI-Object"] = srcmsg["X-RPKI-Object"] - else: - words = srcmsg["Subject"].split() - msg["X-RPKI-PID"] = words[1] - msg["X-RPKI-Object"] = " ".join(words[4:]) - -destination = None -source = None -try: - destination = mailbox.MH(args.output, factory = None, create = True) - source = mailbox.Maildir(args.input, factory = None) - - for srckey, srcmsg in source.iteritems(): - if args.unseen and "S" in srcmsg.get_flags(): - continue - assert not srcmsg.is_multipart() and srcmsg.get_content_type() == "application/x-rpki" - payload = srcmsg.get_payload(decode = True) - cms = rpki.POW.CMS.derRead(payload) - txt = cms.verify(rpki.POW.X509Store(), None, rpki.POW.CMS_NOCRL | rpki.POW.CMS_NO_SIGNER_CERT_VERIFY | rpki.POW.CMS_NO_ATTR_VERIFY | rpki.POW.CMS_NO_CONTENT_VERIFY) - xml = lxml.etree.fromstring(txt) - tag = xml.tag - if args.tag and tag != args.tag: - continue - msg = email.mime.multipart.MIMEMultipart("related") - msg["X-RPKI-Tag"] = tag - for i in ("Date", "Message-ID", "X-RPKI-Timestamp"): - msg[i] = srcmsg[i] - fix_headers() - if tag in dispatch: - dispatch[tag]() - if "Subject" not in msg: - msg["Subject"] = srcmsg["Subject"] - msg.attach(email.mime.text.MIMEText(txt)) - msg.attach(email.mime.application.MIMEApplication(payload, "x-rpki")) - msg.epilogue = "\n" # Force trailing newline - key = destination.add(msg) - print "Added", key - if args.kill: - del source[srckey] - elif args.mark: - srcmsg.set_subdir("cur") - srcmsg.add_flag("S") - source[srckey] = srcmsg - -finally: - if destination: - destination.close() - if source: - source.close() diff --git a/scripts/gc_summary.awk b/scripts/gc_summary.awk deleted file mode 100644 index b3b1bc6a..00000000 --- a/scripts/gc_summary.awk +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/awk -f - -# $Id$ -# -# Copyright (C) 2010 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -# Use gnuplot to graph interesting data from gc_summary lines in rpkid logs. - -BEGIN { - target = ENVIRON["TARGET"] ? ENVIRON["TARGET"] : "tuple"; - outtype = ENVIRON["OUTTYPE"] ? ENVIRON["OUTTYPE"] : "png"; - outname = ENVIRON["OUTNAME"] ? ENVIRON["OUTNAME"] : ""; - print "set xdata time"; - print "set timefmt '%Y-%m-%dT%H:%M:%S'"; - #print "set format x '%d%b'"; - print "set format x '%T'"; - print "set key right bottom"; - if (outname) { - print "set terminal", outtype; - print "set output '" outname "." outtype "'"; - print "set term png size 1024,1024"; - } - if (ARGC <= 2) { - print "plot '-' using 1:2 with linespoints title 'rpkid use of", target, "objects'"; - } else { - cmd = "plot '-' using 1:2 with linespoints title '" ARGV[1] "'"; - for (i = 2; i < ARGC; i++) - cmd = cmd ", '-' using 1:2 with linespoints title '" ARGV[i] "'"; - print cmd; - } -} - -FILENAME != filename && filename { - print "e"; -} - -FILENAME != filename { - print "#", FILENAME - filename = FILENAME; - proc = ""; -} - -$6 == target && proc != $3 && proc { - print ""; -} - -$6 == target && proc != $3 { - proc = $3; -} - -$6 == target { - print "#", $0; - print $1 "T" $2, $5; -} - -END { - print "e"; - if (!outname) - print "pause mouse any"; -} diff --git a/scripts/gc_summary.py b/scripts/gc_summary.py deleted file mode 100644 index 1f6987bf..00000000 --- a/scripts/gc_summary.py +++ /dev/null @@ -1,112 +0,0 @@ -# $Id$ -# -# Copyright (C) 2010 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -# Use gnuplot to graph interesting data from gc_summary lines in rpkid logs. - -import sys, os, time - -class datapoint(object): - - outtype = os.getenv("OUTTYPE", "png") - outname = os.getenv("OUTNAME", "") - timefmt = os.getenv("TIMEFMT", "%T") - pretend = os.getenv("PRETEND_EVERYTHING_CHANGED", False) - threshold = int(os.getenv("THRESHOLD", "100")) - - raw = [] - filenames = [] - - def __init__(self, filename, timestamp, process, count, typesig, line): - self.filename = filename - self.timestamp = timestamp - self.process = process - self.count = count - self.typesig = typesig - self.line = line - self.key = "%s %s" % (filename, typesig) - self.raw.append(self) - if filename not in self.filenames: - self.filenames.append(filename) - - def __cmp__(self, other): - c = cmp(self.key, other.key) - return c if c else cmp(self.timestamp, other.timestamp) - - @classmethod - def plot(cls): - - print "# [%s] Looking for interesting records" % time.strftime("%T") - changed = {} - for i in cls.raw: - if i.key not in changed: - changed[i.key] = set() - changed[i.key].add(i.count) - if cls.pretend: - changed = set(changed.iterkeys()) - else: - changed = set(k for k, v in changed.iteritems() if max(v) - min(v) > cls.threshold) - - if not changed: - print "# [%s] Apparently nothing worth reporting" % time.strftime("%T") - print "print 'Nothing to plot'" - return - - print "# [%s] Header" % time.strftime("%T") - print "set xdata time" - print "set timefmt '%Y-%m-%dT%H:%M:%S'" - print "set format x '%s'" % cls.timefmt - print "set key right bottom" - if cls.outname: - print "set terminal", cls.outtype - print "set output '%s.%s'" % (cls.outname, cls.outtype) - print "set term png size 1024,1024" - print "plot", ", ".join("'-' using 1:2 with linespoints title '%s'" % i for i in changed) - - print "# [%s] Sorting" % time.strftime("%T") - cls.raw.sort() - - key = None - proc = None - for i in cls.raw: - if i.key not in changed: - continue - if key is not None and i.key != key: - print "e" - elif proc is not None and i.process != proc: - print "" - key = i.key - proc = i.process - print "#", i.key, i.line - print i.timestamp, i.count - print "e" - if not cls.outname: - print "pause mouse any" - -for filename in sys.argv[1:]: - print "# [%s] Reading %s" % (time.strftime("%T"), filename) - for line in open(filename): - if "gc_summary:" in line: - word = line.split(None, 6) - if word[4].isdigit() and word[5].startswith("(") and word[5].endswith(")"): - datapoint(filename = filename, - timestamp = word[0] + "T" + word[1], - process = word[2], - count = int(word[4]), - typesig = word[5], - line = line.strip()) - -print "# [%s] Plotting" % time.strftime("%T") -datapoint.plot() diff --git a/scripts/generate-ripe-root-cert.py b/scripts/generate-ripe-root-cert.py deleted file mode 100644 index 3407bc51..00000000 --- a/scripts/generate-ripe-root-cert.py +++ /dev/null @@ -1,57 +0,0 @@ -# $Id$ -# -# Copyright (C) 2010-2012 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Parse IANA XML data and write out just what we need to generate a root -cert for Pseudo-RIPE. -""" - -import sys -import lxml.etree -from rpki.csv_utils import csv_writer - -def iterate_xml(filename, tag): - return lxml.etree.parse(filename).getroot().getiterator(tag) - -def ns(tag): - return "{http://www.iana.org/assignments}" + tag - -tag_description = ns("description") -tag_designation = ns("designation") -tag_record = ns("record") -tag_number = ns("number") -tag_prefix = ns("prefix") - -asns = csv_writer("asns.csv") -prefixes = csv_writer("prefixes.csv") - -for record in iterate_xml("as-numbers.xml", tag_record): - if record.findtext(tag_description) == "Assigned by RIPE NCC": - asns.writerow(("RIPE", record.findtext(tag_number))) - -for record in iterate_xml("ipv4-address-space.xml", tag_record): - if record.findtext(tag_designation) in ("RIPE NCC", "Administered by RIPE NCC"): - prefix = record.findtext(tag_prefix) - p, l = prefix.split("/") - assert l == "8", "Violated /8 assumption: %r" % prefix - prefixes.writerow(("RIPE", "%d.0.0.0/8" % int(p))) - -for record in iterate_xml("ipv6-unicast-address-assignments.xml", tag_record): - if record.findtext(tag_description) == "RIPE NCC": - prefixes.writerow(("RIPE", record.findtext(tag_prefix))) - -asns.close() -prefixes.close() diff --git a/scripts/gski.py b/scripts/gski.py deleted file mode 100644 index 083a59c8..00000000 --- a/scripts/gski.py +++ /dev/null @@ -1,21 +0,0 @@ -# $Id$ - -# Copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -import rpki.x509, sys - -for file in sys.argv[1:]: - cert = rpki.x509.X509(Auto_file = file) - print cert.gSKI(), cert.hSKI(), file diff --git a/scripts/guess-roas-from-routeviews.py b/scripts/guess-roas-from-routeviews.py deleted file mode 100644 index d8fb9c4c..00000000 --- a/scripts/guess-roas-from-routeviews.py +++ /dev/null @@ -1,63 +0,0 @@ -# $Id$ -# -# Copyright (C) 2009 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Pull RFC 3779 resources from a cert, attempt to mine routeviews (via -DNS, using the dnspython toolkit) for what the ROAs might look like -for the addresses found in the cert. - -This doesn't handle IPv6, because neither, apparently, does the -routeviews DNS interface. Oh well. - -NB: this is wild-assed guessing at best. Even if the routeviews data -were signed, which it is not, you have no particular reason to believe -it. Do not use output of this script production. Sanity check. -Beware of dog. If you issue ROAs using this script and your wallpaper -peels, your cat explodes, or your children turn blue, it's your own -fault for using this script. You have been warned. -""" - -import sys -import dns.resolver -import rpki.x509 -from rpki.ipaddrs import v4addr -from rpki.resource_set import roa_prefix_ipv4, resource_set_ipv4, resource_range_ipv4 - -roas = [] - -for filename in sys.argv[1:]: - resources = rpki.x509.X509(Auto_file = filename).get_3779resources().v4 - - while resources: - labels = str(resources[0].min).split(".") - labels.reverse() - - try: - for answer in dns.resolver.query(".".join(labels) + ".asn.routeviews.org", "txt"): - asn, prefix, prefixlen = answer.strings - roa_prefix = roa_prefix_ipv4(v4addr(prefix), long(prefixlen)) - roa = "%s\t%s\t%s" % (roa_prefix, long(asn), filename) - if roa not in roas: - roas.append(roa) - resources = resources.difference(resource_set_ipv4([roa_prefix.to_resource_range()])) - - except dns.resolver.NXDOMAIN: - resources = resources.difference(resource_set_ipv4([resource_range_ipv4(resources[0].min, v4addr(resources[0].min + 256))])) - -roas.sort() - -for roa in roas: - print roa diff --git a/scripts/iana-to-csv.py b/scripts/iana-to-csv.py deleted file mode 100644 index f803a21e..00000000 --- a/scripts/iana-to-csv.py +++ /dev/null @@ -1,85 +0,0 @@ -# $Id$ -# -# Copyright (C) 2010-2012 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Parse IANA XML data. -""" - -import sys -import lxml.etree -from rpki.csv_utils import csv_reader, csv_writer -from rpki.resource_set import resource_bag - -def iterate_xml(filename, tag): - return lxml.etree.parse(filename).getroot().getiterator(tag) - -def ns(tag): - return "{http://www.iana.org/assignments}" + tag - -tag_description = ns("description") -tag_designation = ns("designation") -tag_record = ns("record") -tag_number = ns("number") -tag_prefix = ns("prefix") -tag_status = ns("status") - -handles = {} -rirs = { "legacy" : resource_bag() } - -for rir in ("AfriNIC", "APNIC", "ARIN", "LACNIC", "RIPE NCC"): - handle = rir.split()[0].lower() - handles[rir] = handles["Assigned by %s" % rir] = handles["Administered by %s" % rir] = handle - rirs[handle] = resource_bag() - -asns = csv_writer("asns.csv") -prefixes = csv_writer("prefixes.csv") - -for record in iterate_xml("as-numbers.xml", tag_record): - description = record.findtext(tag_description) - if description in handles: - asns.writerow((handles[description], record.findtext(tag_number))) - -for record in iterate_xml("ipv4-address-space.xml", tag_record): - designation = record.findtext(tag_designation) - if record.findtext(tag_status) != "RESERVED": - prefix, prefixlen = [int(i) for i in record.findtext(tag_prefix).split("/")] - if prefixlen != 8: - raise ValueError("%s violated /8 assumption" % record.findtext(tag_prefix)) - rirs[handles.get(designation, "legacy")] |= resource_bag.from_str("%d.0.0.0/8" % prefix) - -for record in iterate_xml("ipv6-unicast-address-assignments.xml", tag_record): - description = record.findtext(tag_description) - if record.findtext(tag_description) in handles: - rirs[handles[description]] |= resource_bag.from_str(record.findtext(tag_prefix)) - -erx = list(csv_reader("erx.csv")) -assert all(r in rirs for r, p in erx) - -erx_overrides = resource_bag.from_str(",".join(p for r, p in erx), allow_overlap = True) - -for rir in rirs: - if rir != "legacy": - rirs[rir] -= erx_overrides - rirs[rir] |= resource_bag.from_str(",".join(p for r, p in erx if r == rir), allow_overlap = True) - -for rir, bag in rirs.iteritems(): - for p in bag.v4: - prefixes.writerow((rir, p)) - for p in bag.v6: - prefixes.writerow((rir, p)) - -asns.close() -prefixes.close() diff --git a/scripts/missing-oids.py b/scripts/missing-oids.py deleted file mode 100644 index 16316eac..00000000 --- a/scripts/missing-oids.py +++ /dev/null @@ -1,38 +0,0 @@ -# $Id$ -# -# Copyright (C) 2008 American Registry for Internet Numbers ("ARIN") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Figure out what OIDs from rpki.oids are missing from dumpasn1's database. -""" - -import rpki.POW.pkix, rpki.oids - -need_header = True - -for oid, name in rpki.oids.oid2name.items(): - try: - rpki.POW.pkix.oid2obj(oid) - except: - o = rpki.POW.pkix.Oid() - o.set(oid) - if need_header: - print - print "# Local additions" - need_header = False - print - print "OID =", " ".join(("%02X" % ord(c)) for c in o.toString()) - print "Comment = RPKI project" - print "Description =", name, "(" + " ".join((str(i) for i in oid)) + ")" diff --git a/scripts/object-dates.py b/scripts/object-dates.py deleted file mode 100644 index b99441d6..00000000 --- a/scripts/object-dates.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# $Id$ - -# Extract notBefore, notAfter, thisUpdate and nextUpdate dates from -# RPKI objects. - -# Copyright (C) 2013--2014 Dragon Research Labs ("DRL") -# -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -import sys -import os.path -import rpki.POW - -extract_flags = (rpki.POW.CMS_NOCRL | - rpki.POW.CMS_NO_SIGNER_CERT_VERIFY | - rpki.POW.CMS_NO_ATTR_VERIFY | - rpki.POW.CMS_NO_CONTENT_VERIFY) - -def get_mft(fn): - cms = rpki.POW.Manifest.derReadFile(fn) - cms.verify(rpki.POW.X509Store(), None, extract_flags) - return cms, cms.certs()[0] - -def get_roa(fn): - return None, rpki.POW.CMS.derReadFile(fn).certs()[0] - -def get_gbr(fn): - return None, rpki.POW.CMS.derReadFile(fn).certs()[0] - -def get_crl(fn): - return rpki.POW.CRL.derReadFile(fn), None - -def get_cer(fn): - return None, rpki.POW.X509.derReadFile(fn) - -dispatch = dict(mft = get_mft, - roa = get_roa, - gbr = get_gbr, - crl = get_crl, - cer = get_cer) - -for fn in sys.argv[1:]: - obj, cer = dispatch[os.path.splitext(fn)[1][1:]](fn) - print fn - if cer is not None: - print "notBefore: ", cer.getNotBefore() - if obj is not None: - print "thisUpdate:", obj.getThisUpdate() - print "nextUpdate:", obj.getNextUpdate() - if cer is not None: - print "notAfter: ", cer.getNotAfter() - print diff --git a/scripts/pcap-to-xml.sh b/scripts/pcap-to-xml.sh deleted file mode 100644 index 73c30880..00000000 --- a/scripts/pcap-to-xml.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh - -# $Id$ -# -# Copyright (C) 2011 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -# Zeroeth cut at a packet decoder for RPKI up-down / left-right / -# publication traffic captured off the wire. Needs work, not suitable -# for general use, depends on a bunch of external programs that I -# happen to have installed...but has been useful to me. - -for p in *.pcap -do - tcptrace -e $p - for i in *.dat - do - j=${i%_contents.dat} - sed '1,/^
$/d' $i >$j.der - openssl cms -verify -noverify -inform DER -in $j.der | xmlindent > $j.xml - k=$(dumpasn1 -a $j.der 2>/dev/null | awk 'BEGIN {FS = "[ \t/:]+"} /signingTime/ {nr = NR + 2} NR == nr {print $6 "-" $5 "-" $4 "T" $7 ":" $8 ":" $9 "Z"}') - mv $j.der $k.$j.der - mv $j.xml $k.$j.xml - rm $i - done -done diff --git a/scripts/print-profile.py b/scripts/print-profile.py deleted file mode 100644 index 081d2602..00000000 --- a/scripts/print-profile.py +++ /dev/null @@ -1,20 +0,0 @@ -# $Id$ -# -# Copyright (C) 2010 Internet Systems Consortium, Inc. ("ISC") -# -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -import pstats, glob - -for f in glob.iglob("*.prof"): - pstats.Stats(f).sort_stats("time").print_stats(50) diff --git a/scripts/rcynic-diff.py b/scripts/rcynic-diff.py deleted file mode 100644 index 327a7b71..00000000 --- a/scripts/rcynic-diff.py +++ /dev/null @@ -1,114 +0,0 @@ -# $Id$ -# -# Copyright (C) 2012 Internet Systems Consortium, Inc. ("ISC") -# -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Diff a series of rcynic.xml files, sort of. -""" - -import sys - -try: - from lxml.etree import ElementTree -except ImportError: - from xml.etree.ElementTree import ElementTree - -show_backup_generation = False -show_rsync_transfer = False - -class Object(object): - - def __init__(self, session, uri, generation): - self.session = session - self.uri = uri - self.generation = generation - self.labels = [] - - def add(self, label): - self.labels.append(label) - - def __cmp__(self, other): - return cmp(self.labels, other.labels) - -def show(old = None, new = None): - assert old is not None or new is not None - assert old is None or new is None or old.uri == new.uri - if old is None: - obj = new - labels = ["+" + label for label in new.labels] - elif new is None: - obj = old - labels = ["-" + label for label in old.labels] - else: - obj = new - labels = [] - for label in new.session.labels: - if label in new.labels and label in old.labels: - labels.append(label) - elif label in new.labels: - labels.append("+" + label) - elif label in old.labels: - labels.append("-" + label) - labels = " ".join(labels) - if show_backup_generation: - print " ", obj.uri, obj.generation, labels - else: - print " ", obj.uri, labels - -class Session(dict): - - def __init__(self, name): - self.name = name - tree = ElementTree(file = name) - self.labels = [elt.tag.strip() for elt in tree.find("labels")] - for elt in tree.findall("validation_status"): - generation = elt.get("generation") - status = elt.get("status") - uri = elt.text.strip() - if not show_rsync_transfer and status.startswith("rsync_transfer_"): - continue - if show_backup_generation: - key = (uri, generation) - elif generation == "backup": - continue - else: - key = uri - if key not in self: - self[key] = Object(self, uri, generation) - self[key].add(status) - -old_db = new_db = None - -for arg in sys.argv[1:]: - - old_db = new_db - new_db = Session(arg) - - if old_db is None: - continue - - only_old = set(old_db) - set(new_db) - only_new = set(new_db) - set(old_db) - changed = set(key for key in (set(old_db) & set(new_db)) if old_db[key] != new_db[key]) - - if only_old or changed or only_new: - print "Comparing", old_db.name, "with", new_db.name - for key in sorted(only_old): - show(old = old_db[key]) - for key in sorted(changed): - show(old = old_db[key], new = new_db[key]) - for key in sorted(only_new): - show(new = new_db[key]) - print diff --git a/scripts/rcynic-lta b/scripts/rcynic-lta deleted file mode 100755 index 4c55db92..00000000 --- a/scripts/rcynic-lta +++ /dev/null @@ -1,1055 +0,0 @@ -#!/usr/local/bin/python - -# $Id$ - -# Copyright (C) 2013 Dragon Research Labs ("DRL") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -######################################################################## -# -# DANGER WILL ROBINSON -# -# This is a PROTOTYPE of a local trust anchor mechanism. At the -# moment, it DOES NOT WORK by any sane standard of measurement. It -# produces output, but there is no particular reason to believe said -# output is useful, and fairly good reason to believe that it is not. -# -# With luck, this may eventually mutate into something useful. For -# now, just leave it alone unless you really know what you are doing, -# in which case, on your head be it. -# -# YOU HAVE BEEN WARNED -# -######################################################################## - -import os -import sys -import yaml -import glob -import time -import shutil -import base64 -import socket -import sqlite3 -import weakref -import rpki.POW -import rpki.x509 -import rpki.sundial -import rpki.resource_set - -# Teach SQLite3 about our data types. - -sqlite3.register_adapter(rpki.POW.IPAddress, - lambda x: buffer("_" + x.toBytes())) - -sqlite3.register_converter("RangeVal", - lambda s: long(s) if s.isdigit() else rpki.POW.IPAddress.fromBytes(s[1:])) - -sqlite3.register_adapter(rpki.x509.X501DN, str) - - -class main(object): - - tal_directory = None - constraints = None - rcynic_input = None - rcynic_output = None - tals = None - keyfile = None - - ltakey = None - ltacer = None - - ltauri = "rsync://localhost/lta" - ltasia = ltauri + "/" - ltaaia = ltauri + ".cer" - ltamft = ltauri + "/lta.mft" - ltacrl = ltauri + "/lta.crl" - - cer_delta = rpki.sundial.timedelta(days = 7) - crl_delta = rpki.sundial.timedelta(hours = 1) - - all_mentioned_resources = rpki.resource_set.resource_bag() - - - def __init__(self): - print "Parsing YAML" - self.parse_yaml() - print - print "Parsing TALs" - self.parse_tals() - print - print "Creating DB" - self.rpdb = RPDB(self.db_name) - print - print "Creating CA" - self.create_ca() - print - print "Loading DB" - self.rpdb.load(self.rcynic_input) - print - print "Processing adds and drops" - self.process_add_drop() - print - print "Processing deletions" - self.process_constraint_deletions() - print - print "Re-parenting TAs" - self.re_parent_tas() - print - print "Generating CRL and manifest" - self.generate_crl_and_manifest() - print - print "Committing final changes to DB" - self.rpdb.commit() - print - print "Dumping para-objects" - self.rpdb.dump_paras(self.rcynic_output) - print - print "Closing DB" - self.rpdb.close() - - - def create_ca(self): - self.serial = Serial() - self.ltakey = rpki.x509.RSA.generate(quiet = True) - cer = OutgoingX509.self_certify( - cn = "%s LTA Root Certificate" % socket.getfqdn(), - keypair = self.ltakey, - subject_key = self.ltakey.get_RSApublic(), - serial = self.serial(), - sia = (self.ltasia, self.ltamft, None), - notAfter = rpki.sundial.now() + self.cer_delta, - resources = rpki.resource_set.resource_bag.from_str("0-4294967295,0.0.0.0/0,::/0")) - subject_id = self.rpdb.find_keyname(cer.getSubject(), cer.get_SKI()) - self.rpdb.cur.execute("INSERT INTO outgoing (der, fn2, subject, issuer, uri, key) " - "VALUES (?, 'cer', ?, ?, ?, ?)", - (buffer(cer.get_DER()), subject_id, subject_id, self.ltaaia, - buffer(self.ltakey.get_DER()))) - self.ltacer = self.rpdb.find_outgoing_by_id(self.rpdb.cur.lastrowid) - - - def parse_yaml(self, fn = "rcynic-lta.yaml"): - y = yaml.safe_load(open(fn, "r")) - self.db_name = y["db-name"] - self.tal_directory = y["tal-directory"] - self.rcynic_input = y["rcynic-input"] - self.rcynic_output = y["rcynic-output"] - self.keyfile = y["keyfile"] - self.constraints = [Constraint(yc) for yc in y["constraints"]] - - - def parse_tals(self): - self.tals = {} - for fn in glob.iglob(os.path.join(self.tal_directory, "*.tal")): - with open(fn, "r") as f: - uri = f.readline().strip() - key = rpki.POW.Asymmetric.derReadPublic(base64.b64decode(f.read())) - self.tals[uri] = key - - - @staticmethod - def show_candidates(constraint, candidates): - print - print "Constraint:", repr(constraint) - print "Resources: ", constraint.mentioned_resources - for i, candidate in enumerate(candidates): - print " Candidate #%d id %d depth %d name %s uri %s" % ( - i, candidate.rowid, - candidate.depth, - candidate.subject_name, - candidate.uri) - if constraint.mentioned_resources <= candidate.resources: - print " Matched" - #print " Constraint resources:", constraint.mentioned_resources - #print " Candidate resources: ", candidate.resources - break - else: - print " No match" - - - def process_add_drop(self): - # - # We probably need to create the output root before running this, - # otherwise there's a chance that an "add" constraint will yield - # no viable candidate parent. Not likely to happen with current - # test setup where several of our roots claim 0/0. - # - for constraint in self.constraints: - candidates = self.rpdb.find_by_resource_bag(constraint.mentioned_resources) - candidates.sort(reverse = True, key = lambda candidate: candidate.depth) - #self.show_candidates(constraint, candidates) - constraint.drop(candidates) - constraint.add(candidates) - - - def process_constraint_deletions(self): - for obj in self.rpdb.find_by_resource_bag(self.all_mentioned_resources): - self.add_para(obj, obj.resources - self.all_mentioned_resources) - - - def re_parent_tas(self): - for uri, key in self.tals.iteritems(): - for ta in self.rpdb.find_by_ski_or_uri(key.calculateSKI(), uri): - if ta.para_obj is None: - self.add_para(ta, ta.resources - self.all_mentioned_resources) - - - def add_para(self, obj, resources): - return self.rpdb.add_para( - obj = obj, - resources = resources, - serial = self.serial, - ltacer = self.ltacer, - ltasia = self.ltasia, - ltaaia = self.ltaaia, - ltamft = self.ltamft, - ltacrl = self.ltacrl, - ltakey = self.ltakey) - - - def generate_crl_and_manifest(self): - thisUpdate = rpki.sundial.now() - nextUpdate = thisUpdate + self.crl_delta - serial = self.serial() - issuer = self.ltacer.getSubject() - aki = buffer(self.ltacer.get_SKI()) - - crl = OutgoingCRL.generate( - keypair = self.ltakey, - issuer = self.ltacer, - serial = serial, - thisUpdate = thisUpdate, - nextUpdate = nextUpdate, - revokedCertificates = ()) - - issuer_id = self.rpdb.find_keyname(issuer, aki) - - self.rpdb.cur.execute("INSERT INTO outgoing (der, fn2, subject, issuer, uri) " - "VALUES (?, 'crl', NULL, ?, ?)", - (buffer(crl.get_DER()), issuer_id, self.ltacrl)) - crl = self.rpdb.find_outgoing_by_id(self.rpdb.cur.lastrowid) - - key = rpki.x509.RSA.generate(quiet = True) - - cer = self.ltacer.issue( - keypair = self.ltakey, - subject_key = key.get_RSApublic(), - serial = serial, - sia = (None, None, self.ltamft), - aia = self.ltaaia, - crldp = self.ltacrl, - resources = rpki.resource_set.resource_bag.from_inheritance(), - notAfter = self.ltacer.getNotAfter(), - is_ca = False) - - # Temporary kludge, need more general solution but that requires - # more refactoring than I feel like doing this late in the day. - # - names_and_objs = [(uri, OutgoingObject.create(fn2 = fn2, der = der, uri = uri, - rpdb = None, rowid = None, - subject_id = None, issuer_id = None)) - for fn2, der, uri in - self.rpdb.cur.execute("SELECT fn2, der, uri FROM outgoing WHERE issuer = ?", - (self.ltacer.rowid,))] - - mft = OutgoingSignedManifest.build( - serial = serial, - thisUpdate = thisUpdate, - nextUpdate = nextUpdate, - names_and_objs = names_and_objs, - keypair = key, - certs = cer) - - subject_id = self.rpdb.find_keyname(cer.getSubject(), cer.get_SKI()) - - self.rpdb.cur.execute("INSERT INTO outgoing (der, fn2, subject, issuer, uri, key) " - "VALUES (?, 'mft', ?, ?, ?, ?)", - (buffer(mft.get_DER()), subject_id, issuer_id, self.ltamft, buffer(key.get_DER()))) - - - @staticmethod - def parse_xki(s): - """ - Parse text form of an SKI or AKI. We accept two encodings: - colon-delimited hexadecimal, and URL-safe Base64. The former is - what OpenSSL prints in its text representation of SKI and AKI - extensions; the latter is the g(SKI) value that some RPKI CA engines - (including rpkid) use when constructing filenames. - - In either case, we check that the decoded result contains the right - number of octets to be a SHA-1 hash. - """ - - if ":" in s: - b = "".join(chr(int(c, 16)) for c in s.split(":")) - else: - b = base64.urlsafe_b64decode(s + ("=" * (4 - len(s) % 4))) - if len(b) != 20: - raise RuntimeError("Bad length for SHA1 xKI value: %r" % s) - return b - - - -class Serial(object): - - def __init__(self): - self.value = long(time.time()) << 32 - - def __call__(self): - self.value += 1 - return self.value - - -class ConstrainedObject(object): - # I keep expecting the classes derived from this to have some common - # methods, but so far it hasn't happened. Clean up eventually if not. - pass - -class ConstrainedROA(ConstrainedObject): - - def __init__(self, constraint, y): - self.constraint = constraint - self.asn = long(y["asn"]) if y is not None else None - self.maxlen = long(y["maxlen"]) if y is not None and "maxlen" in y else None - - def drop(self, candidates): - for candidate in candidates: - if isinstance(candidate, IncomingROA) and \ - self.constraint.mentioned_resources == candidate.resources and \ - (self.asn is None or self.asn == candidate.get_POW().getASID()): - print "Dropping ROA %r" % candidate - candidate.disposition = "delete" - - def add(self, candidates): - assert self.asn is not None - for candidate in candidates: - if isinstance(candidate, IncomingX509) and self.constraint.mentioned_resources <= candidate.resources: - print "Should add ROA %s %s\nunder candidate %s (depth %s resources %s)" % ( - self.asn, self.constraint.prefixes, candidate.subject_name, candidate.depth, candidate.resources) - break - -class ConstrainedGBR(ConstrainedObject): - - def __init__(self, constraint, y): - self.constraint = constraint - self.vcard = y - - def drop(self, candidates): - for candidate in candidates: - if isinstance(candidate, IncomingX509) and self.constraint.mentioned_resources == candidate.resources: - print "Dropping GBRs directly under %r" % candidate - for gbr in candidate.find_children("gbr"): - print "Dropping GBR %r" % gbr - gbr.disposition = "delete" - - def add(self, candidates): - assert self.vcard is not None - for candidate in candidates: - if isinstance(candidate, IncomingX509) and self.constraint.mentioned_resources <= candidate.resources: - print "Should add GBR\n%s\nunder candidate %s (depth %s resources %s)" % ( - "\n".join((" " * 4) + line for line in self.vcard.splitlines()), - candidate.subject_name, candidate.depth, candidate.resources) - break - -class ConstrainedRTR(ConstrainedObject): - - def __init__(self, constraint, y): - self.constraint = constraint - self.key = y["key"] if y is not None else None - self.subject = y["subject"] if y is not None else None - - def add(self, candidates): - raise NotImplementedError - - def drop(self, candidates): - for candidate in candidates: - if isinstance(candidate, IncomingX509) and not candidate.is_ca and \ - self.constraint.mentioned_resources == candidate.resources and \ - (self.subject is None or candidate.getSubject() == self.subject): - print "Dropping RTR certificate %r" % candidate - candidate.disposition = "delete" - -class Constraint(object): - - dispatch = dict(roa = ConstrainedROA, - gbr = ConstrainedGBR, - rtr = ConstrainedRTR) - - def __init__(self, y): - self.y = y # Mostly for debugging. I think. - self.prefixes = rpki.resource_set.resource_bag.from_str(str(y.get("prefix", ""))) - self.asns = rpki.resource_set.resource_bag.from_str(str(y.get("asn", ""))) - self.init_drops(y.get("drop", ())) - self.init_adds( y.get("add", ())) - - def init_drops(self, drops): - if drops == "all": - self.drops = tuple(d(self, None) for d in self.dispatch.itervalues()) - else: - dd = [] - for d in (drops if isinstance(drops, (list, tuple)) else [drops]): - if isinstance(d, str): - dd.append(self.dispatch[d[:-1]](self, None)) - elif isinstance(d, dict) and len(d) == 1: - dd.append(self.dispatch[d.keys()[0]](self, d.values()[0])) - else: - raise ValueError("Unexpected drop clause " + repr(drops)) - self.drops = tuple(dd) - - def init_adds(self, adds): - if not all(isinstance(a, dict) and len(a) == 1 for a in adds): - raise ValueError("Expected list of single-entry mappings, got " + repr(adds)) - self.adds = tuple(self.dispatch[a.keys()[0]](self, a.values()[0]) for a in adds) - - def drop(self, candidates): - for d in self.drops: - d.drop(candidates) - - def add(self, candidates): - for a in self.adds: - a.add(candidates) - - def __repr__(self): - return "<%s:%s %r>" % (self.__class__.__module__, self.__class__.__name__, self.y) - - @property - def mentioned_resources(self): - return self.prefixes | self.asns - - -class BaseObject(object): - """ - Mixin to add some SQL-related methods to classes derived from - rpki.x509.DER_object. - """ - - _rpdb = None - _rowid = None - _fn2 = None - _fn2map = None - _uri = None - _subject_id = None - _issuer_id = None - - @property - def rowid(self): - return self._rowid - - @property - def para_resources(self): - return self.resources if self.para_obj is None else self.para_obj.resources - - @property - def fn2(self): - return self._fn2 - - @property - def uri(self): - return self._uri - - @classmethod - def setfn2map(cls, **map): - cls._fn2map = map - for k, v in map.iteritems(): - v._fn2 = k - - @classmethod - def create(cls, rpdb, rowid, fn2, der, uri, subject_id, issuer_id): - self = cls._fn2map[fn2]() - if der is not None: - self.set(DER = der) - self._rpdb = rpdb - self._rowid = rowid - self._uri = uri - self._subject_id = subject_id - self._issuer_id = issuer_id - return self - - @property - def subject_id(self): - return self._subject_id - - @property - def subject_name(self): - return self._rpdb.find_keyname_by_id(self._subject_id)[0] - - @property - def issuer_id(self): - return self._issuer_id - - @property - def issuer_name(self): - return self._rpdb.find_keyname_by_id(self._subject_id)[0] - - -class IncomingObject(BaseObject): - - _depth = None - _is_ca = False - _disposition = None - - @property - def para_obj(self): - if getattr(self, "_para_id", None) is None: - self._rpdb.cur.execute("SELECT replacement FROM incoming WHERE id = ?", (self.rowid,)) - self._para_id = self._rpdb.cur.fetchone()[0] - return self._rpdb.find_outgoing_by_id(self._para_id) - - @para_obj.setter - def para_obj(self, value): - if value is None: - self._rpdb.cur.execute("DELETE FROM outgoing WHERE id IN (SELECT replacement FROM incoming WHERE id = ?)", - (self.rowid,)) - try: - del self._para_id - except AttributeError: - pass - else: - assert isinstance(value.rowid, int) - self._rpdb.cur.execute("UPDATE incoming SET replacement = ? WHERE id = ?", (value.rowid, self.rowid)) - self._para_id = value.rowid - - @property - def disposition(self): - if self._disposition is None: - self._disposition = self._rpdb.cur.execute("SELECT disposition FROM incoming " - "WHERE id = ?", (self.rowid,)).fetchone()[0] - return self._disposition - - @disposition.setter - def disposition(self, value): - self._rpdb.cur.execute("UPDATE incoming SET disposition = ? WHERE id = ?", (value, self.rowid)) - self._disposition = value - - @classmethod - def fromFile(cls, fn): - return cls._fn2map[os.path.splitext(fn)[1][1:]](DER_file = fn) - - @classmethod - def create(cls, rpdb, rowid, fn2, der, uri, subject_id, issuer_id, depth = None, is_ca = False): - assert der is not None - self = super(IncomingObject, cls).create(rpdb, rowid, fn2, der, uri, subject_id, issuer_id) - self._depth = depth - self._is_ca = is_ca - return self - - @property - def depth(self): - return self._depth - - @property - def is_ca(self): - return self._is_ca - - @property - def issuer(self): - if self._issuer_id is None or self._issuer_id == self._subject_id: - return None - return self._rpdb.find_incoming_by_id(self._issuer_id) - - -class OutgoingObject(BaseObject): - - @property - def orig_obj(self): - if getattr(self, "_orig_id", None) is None: - self._rpdb.cur.execute("SELECT id FROM incoming WHERE replacement = ?", (self.rowid,)) - r = self._rpdb.cur.fetchone() - self._orig_id = None if r is None else r[0] - return self._rpdb.find_incoming_by_id(self._orig_id) - - -class BaseX509(rpki.x509.X509): - - @property - def resources(self): - r = self.get_3779resources() - r.valid_until = None - return r - - def find_children(self, fn2 = None): - return self._rpdb._find_results(fn2, "WHERE issuer = ?", [self.subject_id]) - - -class BaseCRL(rpki.x509.CRL): - - @property - def resources(self): - return None - - -class CommonCMS(object): - - @property - def resources(self): - r = rpki.x509.X509(POW = self.get_POW().certs()[0]).get_3779resources() - r.valid_until = None - return r - - -class BaseSignedManifest (rpki.x509.SignedManifest, CommonCMS): pass -class BaseROA (rpki.x509.ROA, CommonCMS): pass -class BaseGhostbuster (rpki.x509.Ghostbuster, CommonCMS): pass - -class IncomingX509 (BaseX509, IncomingObject): pass -class IncomingCRL (BaseCRL, IncomingObject): pass -class IncomingSignedManifest (BaseSignedManifest, IncomingObject): pass -class IncomingROA (BaseROA, IncomingObject): pass -class IncomingGhostbuster (BaseGhostbuster, IncomingObject): pass - -class OutgoingX509 (BaseX509, OutgoingObject): pass -class OutgoingCRL (BaseCRL, OutgoingObject): pass -class OutgoingSignedManifest (BaseSignedManifest, OutgoingObject): pass -class OutgoingROA (BaseROA, OutgoingObject): pass -class OutgoingGhostbuster (BaseGhostbuster, OutgoingObject): pass - -IncomingObject.setfn2map(cer = IncomingX509, - crl = IncomingCRL, - mft = IncomingSignedManifest, - roa = IncomingROA, - gbr = IncomingGhostbuster) - -OutgoingObject.setfn2map(cer = OutgoingX509, - crl = OutgoingCRL, - mft = OutgoingSignedManifest, - roa = OutgoingROA, - gbr = OutgoingGhostbuster) - - -class RPDB(object): - """ - Relying party database. - """ - - def __init__(self, db_name): - - try: - os.unlink(db_name) - except: - pass - - self.db = sqlite3.connect(db_name, detect_types = sqlite3.PARSE_DECLTYPES) - self.db.text_factory = str - self.cur = self.db.cursor() - - self.incoming_cache = weakref.WeakValueDictionary() - self.outgoing_cache = weakref.WeakValueDictionary() - - self.cur.executescript(''' - PRAGMA foreign_keys = on; - - CREATE TABLE keyname ( - id INTEGER PRIMARY KEY NOT NULL, - name TEXT NOT NULL, - keyid BLOB NOT NULL, - UNIQUE (name, keyid)); - - CREATE TABLE incoming ( - id INTEGER PRIMARY KEY NOT NULL, - der BLOB NOT NULL, - fn2 TEXT NOT NULL - CHECK (fn2 IN ('cer', 'crl', 'mft', 'roa', 'gbr')), - uri TEXT NOT NULL, - depth INTEGER, - is_ca BOOLEAN NOT NULL DEFAULT 0, - disposition TEXT NOT NULL - DEFAULT 'keep' - CHECK (disposition IN ('keep', 'delete', 'replace')), - subject INTEGER - REFERENCES keyname(id) - ON DELETE RESTRICT - ON UPDATE RESTRICT, - issuer INTEGER NOT NULL - REFERENCES keyname(id) - ON DELETE RESTRICT - ON UPDATE RESTRICT, - replacement INTEGER - REFERENCES outgoing(id) - ON DELETE SET NULL - ON UPDATE SET NULL, - UNIQUE (der), - UNIQUE (subject, issuer), - CHECK ((subject IS NULL) == (fn2 == 'crl'))); - - CREATE TABLE outgoing ( - id INTEGER PRIMARY KEY NOT NULL, - der BLOB, - key BLOB, - fn2 TEXT NOT NULL - CHECK (fn2 IN ('cer', 'crl', 'mft', 'roa', 'gbr')), - uri TEXT NOT NULL, - subject INTEGER - REFERENCES keyname(id) - ON DELETE RESTRICT - ON UPDATE RESTRICT, - issuer INTEGER NOT NULL - REFERENCES keyname(id) - ON DELETE RESTRICT - ON UPDATE RESTRICT, - UNIQUE (subject, issuer), - CHECK ((key IS NULL) == (fn2 == 'crl')), - CHECK ((subject IS NULL) == (fn2 == 'crl'))); - - CREATE TABLE range ( - id INTEGER NOT NULL - REFERENCES incoming(id) - ON DELETE CASCADE - ON UPDATE CASCADE, - min RangeVal NOT NULL, - max RangeVal NOT NULL, - UNIQUE (id, min, max)); - - ''') - - - def load(self, rcynic_input, spinner = 100): - - start = rpki.sundial.now() - nobj = 0 - - for root, dirs, files in os.walk(rcynic_input): - for fn in files: - fn = os.path.join(root, fn) - - try: - obj = IncomingObject.fromFile(fn) - except: - if spinner: - sys.stderr.write("\r") - sys.stderr.write("Couldn't read %s, skipping\n" % fn) - continue - - if spinner and nobj % spinner == 0: - sys.stderr.write("\r%s %d %s..." % ("|\\-/"[(nobj/spinner) & 3], nobj, rpki.sundial.now() - start)) - - nobj += 1 - - if obj.fn2 == "crl": - ski = None - aki = buffer(obj.get_AKI()) - cer = None - bag = None - issuer = obj.getIssuer() - subject = None - is_ca = False - - else: - if obj.fn2 == "cer": - cer = obj - else: - cer = rpki.x509.X509(POW = obj.get_POW().certs()[0]) - issuer = cer.getIssuer() - subject = cer.getSubject() - ski = buffer(cer.get_SKI()) - aki = cer.get_AKI() - if aki is None: - assert subject == issuer - aki = ski - else: - aki = buffer(aki) - bag = cer.get_3779resources() - is_ca = cer.is_CA() - - der = buffer(obj.get_DER()) - uri = "rsync://" + fn[len(rcynic_input) + 1:] - - self.cur.execute("SELECT id FROM incoming WHERE der = ?", (der,)) - r = self.cur.fetchone() - - if r is not None: - rowid = r[0] - - else: - subject_id = None if ski is None else self.find_keyname(subject, ski) - issuer_id = self.find_keyname(issuer, aki) - - self.cur.execute("INSERT INTO incoming (der, fn2, subject, issuer, uri, is_ca) " - "VALUES (?, ?, ?, ?, ?, ?)", - (der, obj.fn2, subject_id, issuer_id, uri, is_ca)) - rowid = self.cur.lastrowid - - if bag is not None: - for rset in (bag.asn, bag.v4, bag.v6): - if rset is not None: - self.cur.executemany("REPLACE INTO range (id, min, max) VALUES (?, ?, ?)", - ((rowid, i.min, i.max) for i in rset)) - - if spinner: - sys.stderr.write("\r= %d objects in %s.\n" % (nobj, rpki.sundial.now() - start)) - - self.cur.execute("UPDATE incoming SET depth = 0 WHERE subject = issuer") - - for depth in xrange(1, 500): - - self.cur.execute("SELECT COUNT(*) FROM incoming WHERE depth IS NULL") - if self.cur.fetchone()[0] == 0: - break - - if spinner: - sys.stderr.write("\rSetting depth %d..." % depth) - - self.cur.execute(""" - UPDATE incoming SET depth = ? - WHERE depth IS NULL - AND issuer IN (SELECT subject FROM incoming WHERE depth = ?) - """, - (depth, depth - 1)) - - else: - if spinner: - sys.stderr.write("\rSetting depth %d is absurd, giving up, " % depth) - - if spinner: - sys.stderr.write("\nCommitting...") - - self.db.commit() - - if spinner: - sys.stderr.write("done.\n") - - - def add_para(self, obj, resources, serial, ltacer, ltasia, ltaaia, ltamft, ltacrl, ltakey): - - assert isinstance(obj, IncomingX509) - - if obj.para_obj is not None: - resources &= obj.para_obj.resources - - obj.para_obj = None - - if not resources: - return - - pow = obj.get_POW() - - x = rpki.POW.X509() - - x.setVersion( pow.getVersion()) - x.setSubject( pow.getSubject()) - x.setNotBefore( pow.getNotBefore()) - x.setNotAfter( pow.getNotAfter()) - x.setPublicKey( pow.getPublicKey()) - x.setSKI( pow.getSKI()) - x.setBasicConstraints( pow.getBasicConstraints()) - x.setKeyUsage( pow.getKeyUsage()) - x.setCertificatePolicies( pow.getCertificatePolicies()) - x.setSIA( *pow.getSIA()) - - x.setIssuer( ltacer.get_POW().getIssuer()) - x.setAKI( ltacer.get_POW().getSKI()) - x.setAIA( (ltaaia,)) - x.setCRLDP( (ltacrl,)) - - x.setSerial( serial()) - x.setRFC3779( - asn = ((r.min, r.max) for r in resources.asn), - ipv4 = ((r.min, r.max) for r in resources.v4), - ipv6 = ((r.min, r.max) for r in resources.v6)) - - x.sign(ltakey.get_POW(), rpki.POW.SHA256_DIGEST) - cer = OutgoingX509(POW = x) - - ski = buffer(cer.get_SKI()) - aki = buffer(cer.get_AKI()) - bag = cer.get_3779resources() - issuer = cer.getIssuer() - subject = cer.getSubject() - der = buffer(cer.get_DER()) - uri = ltasia + cer.gSKI() + ".cer" - - # This will want to change when we start generating replacement keys for everything. - # This should really be a keypair, not just a public key, same comment. - # - key = buffer(pow.getPublicKey().derWritePublic()) - - subject_id = self.find_keyname(subject, ski) - issuer_id = self.find_keyname(issuer, aki) - - self.cur.execute("INSERT INTO outgoing (der, fn2, subject, issuer, uri, key) " - "VALUES (?, 'cer', ?, ?, ?, ?)", - (der, subject_id, issuer_id, uri, key)) - rowid = self.cur.lastrowid - self.cur.execute("UPDATE incoming SET replacement = ? WHERE id = ?", - (rowid, obj.rowid)) - - # Fix up _orig_id and _para_id here? Maybe later. - - #self.db.commit() - - - def dump_paras(self, rcynic_output): - shutil.rmtree(rcynic_output, ignore_errors = True) - rsync = "rsync://" - for der, uri in self.cur.execute("SELECT der, uri FROM outgoing"): - assert uri.startswith(rsync) - fn = os.path.join(rcynic_output, uri[len(rsync):]) - dn = os.path.dirname(fn) - if not os.path.exists(dn): - os.makedirs(dn) - with open(fn, "wb") as f: - #print ">> Writing", f.name - f.write(der) - - - def find_keyname(self, name, keyid): - keys = (name, buffer(keyid)) - self.cur.execute("SELECT id FROM keyname WHERE name = ? AND keyid = ?", keys) - result = self.cur.fetchone() - if result is None: - self.cur.execute("INSERT INTO keyname (name, keyid) VALUES (?, ?)", keys) - result = self.cur.lastrowid - else: - result = result[0] - return result - - - def find_keyname_by_id(self, rowid): - self.cur.execute("SELECT name, keyid FROM keyname WHERE id = ?", (rowid,)) - result = self.cur.fetchone() - return (None, None) if result is None else result - - - def find_incoming_by_id(self, rowid): - if rowid is None: - return None - if rowid in self.incoming_cache: - return self.incoming_cache[rowid] - r = self._find_results(None, "WHERE id = ?", [rowid]) - assert len(r) < 2 - return r[0] if r else None - - - def find_outgoing_by_id(self, rowid): - if rowid is None: - return None - if rowid in self.outgoing_cache: - return self.outgoing_cache[rowid] - self.cur.execute("SELECT fn2, der, key, uri, subject, issuer FROM outgoing WHERE id = ?", (rowid,)) - r = self.cur.fetchone() - if r is None: - return None - fn2, der, key, uri, subject_id, issuer_id = r - obj = OutgoingObject.create(rpdb = self, rowid = rowid, fn2 = fn2, der = der, uri = uri, - subject_id = subject_id, issuer_id = issuer_id) - self.outgoing_cache[rowid] = obj - return obj - - - def find_by_ski_or_uri(self, ski, uri): - if not ski and not uri: - return [] - j = "" - w = [] - a = [] - if ski: - j = "JOIN keyname ON incoming.subject = keyname.id" - w.append("keyname.keyid = ?") - a.append(buffer(ski)) - if uri: - w.append("incoming.uri = ?") - a.append(uri) - return self._find_results(None, "%s WHERE %s" % (j, " AND ".join(w)), a) - - - # It's easiest to understand overlap conditions by understanding - # non-overlap then inverting and and applying De Morgan's law. - # Ranges A and B do not overlap if: A.min > B.max or B.min > A.max; - # therefore A and B do overlap if: A.min <= B.max and B.min <= A.max. - - def find_by_range(self, range_min, range_max = None, fn2 = None): - if range_max is None: - range_max = range_min - if isinstance(range_min, (str, unicode)): - range_min = long(range_min) if range_min.isdigit() else rpki.POW.IPAddress(range_min) - if isinstance(range_max, (str, unicode)): - range_max = long(range_max) if range_max.isdigit() else rpki.POW.IPAddress(range_max) - assert isinstance(range_min, (int, long, rpki.POW.IPAddress)) - assert isinstance(range_max, (int, long, rpki.POW.IPAddress)) - return self._find_results(fn2, - "JOIN range ON incoming.id = range.id " - "WHERE ? <= range.max AND ? >= range.min", - [range_min, range_max]) - - - def find_by_resource_bag(self, bag, fn2 = None): - assert bag.asn or bag.v4 or bag.v6 - qset = [] - aset = [] - for rset in (bag.asn, bag.v4, bag.v6): - if rset: - for r in rset: - qset.append("(? <= max AND ? >= min)") - aset.append(r.min) - aset.append(r.max) - return self._find_results( - fn2, - """ - JOIN range ON incoming.id = range.id - WHERE - """ + (" OR ".join(qset)), - aset) - - - def _find_results(self, fn2, query, args = None): - if args is None: - args = [] - if fn2 is not None: - query += " AND fn2 = ?" - args.append(fn2) - results = [] - for rowid, fn2, der, uri, subject_id, issuer_id, depth, is_ca in self.cur.execute( - ''' - SELECT DISTINCT - incoming.id, incoming.fn2, - incoming.der, incoming.uri, - incoming.subject, incoming.issuer, - incoming.depth, incoming.is_ca - FROM incoming - ''' + query, args): - if rowid in self.incoming_cache: - obj = self.incoming_cache[rowid] - assert obj.rowid == rowid - else: - obj = IncomingObject.create(rpdb = self, rowid = rowid, fn2 = fn2, der = der, uri = uri, - subject_id = subject_id, issuer_id = issuer_id, depth = depth, - is_ca = is_ca) - self.incoming_cache[rowid] = obj - results.append(obj) - return results - - - def commit(self): - self.db.commit() - - - def close(self): - self.commit() - self.cur.close() - self.db.close() - -if __name__ == "__main__": - #profile = None - profile = "rcynic-lta.prof" - if profile: - import cProfile - prof = cProfile.Profile() - try: - prof.runcall(main) - finally: - prof.dump_stats(profile) - sys.stderr.write("Dumped profile data to %s\n" % profile) - else: - main() - diff --git a/scripts/rcynic-lta.yaml b/scripts/rcynic-lta.yaml deleted file mode 100644 index ab17a56c..00000000 --- a/scripts/rcynic-lta.yaml +++ /dev/null @@ -1,69 +0,0 @@ -db-name: - /u/sra/rpki/subvert-rpki.hactrn.net/trunk/scripts/rcynic-lta.db - -rcynic-input: - /u/sra/rpki/subvert-rpki.hactrn.net/trunk/rcynic/rcynic-data/authenticated - -rcynic-output: - /u/sra/rpki/subvert-rpki.hactrn.net/trunk/rcynic/rcynic-data/lta-unauthenticated - -tal-directory: - /u/sra/rpki/subvert-rpki.hactrn.net/trunk/rcynic/sample-trust-anchors - -keyfile: - /u/sra/rpki/subvert-rpki.hactrn.net/trunk/scripts/rcynic-lta.key - -common: - - - &VCARD-1 | - BEGIN:VCARD - VERSION:4.0 - FN:R0 - ORG:Organizational Entity - ADR;TYPE=WORK:;;42 Twisty Passage;Deep Cavern;WA;98666;U.S.A. - TEL;TYPE=VOICE,TEXT,WORK;VALUE=uri:tel:+1-666-555-1212 - TEL;TYPE=FAX,WORK;VALUE=uri:tel:+1-666-555-1213 - EMAIL:human@example.com - END:VCARD - - &GBR-1 { gbr: *VCARD-1 } - - - &VCARD-2 | - BEGIN:VCARD - VERSION:4.0 - ORG:Epilogue Technology Corporation - EMAIL:sra@epilogue.com - END:VCARD - - &GBR-2 { gbr: *VCARD-2 } - - - &ROA-666 { roa: { asn: 666 } } - -constraints: - - # Need something for a drop test, sorry Randy - - prefix: 147.28.224.0/19 - drop: roas - - - prefix: 10.0.0.0/8 - add: [ { roa: { asn: 666, maxlen: 16 }}, *GBR-1 ] - - - prefix: 192.168.0.0/16 - drop: all - - - asn: 666 - add: [ *GBR-1 ] - - - prefix: 128.224.0.0/16 - drop: all - add: [ *GBR-2 ] - - - prefix: 128.224.1.0/24 - add: [ *GBR-2, *ROA-666 ] - - - prefix: 128.224.2.0/24 - add: [ *GBR-2, *ROA-666 ] - - - prefix: 149.20.0.0/16 - add: [ *ROA-666 ] - - - prefix: 2001:4f8:3:d::/64 - add: [ *ROA-666 ] diff --git a/scripts/repo0-testbed-daily b/scripts/repo0-testbed-daily deleted file mode 100755 index 576464c4..00000000 --- a/scripts/repo0-testbed-daily +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -# $Id: daily 602 2013-06-02 18:00:25Z sra $ - -home=/home/sra/rpki.testbed - -exec >>$home/logs/daily.log 2>&1 - -set -x - -date - -cd $home - -/usr/local/sbin/rpkic update_bpki - -/usr/local/bin/svn update -/usr/local/bin/svn add --force . -/usr/local/bin/svn ci --message 'Daily auto update' -/usr/local/bin/svn update diff --git a/scripts/repo0-testbed-monthly b/scripts/repo0-testbed-monthly deleted file mode 100755 index b7ada110..00000000 --- a/scripts/repo0-testbed-monthly +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh - -# $Id: monthly 602 2013-06-02 18:00:25Z sra $ - -home=/home/sra/rpki.testbed - -exec >>$home/logs/monthly.log 2>&1 - -set -x - -date - -cd $home - -for identity in iana afrinic apnic arin lacnic legacy ripe -do - /usr/local/sbin/rpkic -i $identity renew_all_children -done - -/usr/local/bin/svn update -/usr/local/bin/svn add --force . -/usr/local/bin/svn ci --message 'Monthly auto update' -/usr/local/bin/svn update diff --git a/scripts/repo0-testbed-weekly b/scripts/repo0-testbed-weekly deleted file mode 100755 index 6f1f8ead..00000000 --- a/scripts/repo0-testbed-weekly +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/sh - -# $Id: weekly 756 2013-11-21 22:54:28Z sra $ -# -# Run weekly periodic IR back-end tasks. - -home=/home/sra/rpki.testbed - -top=/home/sra/subvert-rpki.hactrn.net/trunk - -exec >>$home/logs/weekly.log 2>&1 -set -x -date - -export OPENSSL_CONF=/dev/null -for openssl in $top/openssl/openssl/apps/openssl /usr/local/bin/openssl -do - test -x $openssl && break -done - -## Download all input files. See the fetch script for the current -## list of files, but for a long time now it's been: -## -## http://www.iana.org/assignments/as-numbers/as-numbers.xml -## http://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.xml -## http://www.iana.org/assignments/ipv6-unicast-address-assignments/ipv6-unicast-address-assignments.xml -## ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz -## ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz -## ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz -## ftp://ftp.ripe.net/pub/stats/ripencc/membership/alloclist.txt -## ftp://ftp.apnic.net/public/stats/apnic/delegated-apnic-extended-latest -## -## Along with an ARIN bulkwhois dump which we get under a research NDA -## and retrieve via a mechanism that I'm not going to describe here. - -/bin/sh -x $home/scripts/fetch - -## Process ARIN data first -- we need erx.csv, which comes from ARIN. - -cd $home/arin -/usr/local/bin/unzip -p arin_db.zip arin_db.xml | -/usr/local/bin/python $top/scripts/arin-to-csv.py -/usr/local/bin/python $top/scripts/translate-handles.py asns.csv prefixes.csv - -## Process IANA data, now that we have erx.csv. - -cd $home/iana -/usr/local/bin/python $top/scripts/iana-to-csv.py - -## Process APNIC data. - -cd $home/apnic -/usr/local/bin/python $top/scripts/apnic-to-csv.py -/usr/local/bin/python $top/scripts/translate-handles.py asns.csv prefixes.csv - -## Process RIPE data. RIPE's database is a horror, the less said -## about it the better. -## -## Somewhere along the line we seem to have stopped even trying to -## generate the ASN database for RIPE, not sure why. I've restored it -## here, guess we'll find out if there was a reason why we disabled it. - -cd $home/ripe -/usr/local/bin/python $top/scripts/ripe-asns-to-csv.py -/usr/bin/awk -f $top/scripts/ripe-prefixes-to-csv.awk alloclist.txt | -/bin/cat extra-prefixes.csv - | -/usr/bin/sort -uo prefixes.csv -/usr/local/bin/python $top/scripts/translate-handles.py asns.csv prefixes.csv -/usr/bin/sort -uo prefixes.csv prefixes.csv - -## Not yet doing anything for AfriNIC, LACNIC, or Legacy. - -## Generate root certificate. This is a lot simpler now that we're pretending to be the One True Root. - -cd $home/root -$openssl req -new -x509 -days 90 -set_serial $(/bin/date -u +%s) -config root.conf -out root.cer -key root.key -outform DER -/bin/cp -fp root.cer root.cer.dup && -/bin/mv -f root.cer.dup /home/pubd/publication/root.cer - -## Whack all the files into subversion. - -cd $home -/usr/local/bin/svn update -/usr/local/bin/svn add --force . -/usr/local/bin/svn ci --message 'Weekly auto update' -/usr/local/bin/svn update - -## Feed all the new data into the IRDB. - -for entity in iana afrinic apnic arin lacnic legacy ripe -do - for resource in asns prefixes - do - /bin/test -r $entity/$resource.csv && - /usr/local/sbin/rpkic --identity $entity load_$resource $entity/$resource.csv - done -done diff --git a/scripts/ripe-asns-to-csv.py b/scripts/ripe-asns-to-csv.py deleted file mode 100644 index 50251ce8..00000000 --- a/scripts/ripe-asns-to-csv.py +++ /dev/null @@ -1,108 +0,0 @@ -# $Id$ -# -# Copyright (C) 2009-2012 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Parse a WHOIS research dump and write out (just) the RPKI-relevant -fields in myrpki-format CSV syntax. - -NB: The input data for this script is publicly available via FTP, but -you'll have to fetch the data from RIPE yourself, and be sure to see -the terms and conditions referenced by the data file header comments. -""" - -import gzip -from rpki.csv_utils import csv_writer - -class Handle(dict): - - want_tags = () - - debug = False - - def set(self, tag, val): - if tag in self.want_tags: - self[tag] = "".join(val.split(" ")) - - def check(self): - for tag in self.want_tags: - if not tag in self: - return False - if self.debug: - self.log() - return True - - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, - " ".join("%s:%s" % (tag, self.get(tag, "?")) - for tag in self.want_tags)) - - def log(self): - print repr(self) - - def finish(self, ctx): - self.check() - -class aut_num(Handle): - want_tags = ("aut-num", "mnt-by", "as-name") - - def set(self, tag, val): - if tag == "aut-num" and val.startswith("AS"): - val = val[2:] - Handle.set(self, tag, val) - - def finish(self, ctx): - if self.check(): - ctx.asns.writerow((self["mnt-by"], self["aut-num"])) - -class main(object): - - types = dict((x.want_tags[0], x) for x in (aut_num,)) - - - def finish_statement(self, done): - if self.statement: - tag, sep, val = self.statement.partition(":") - assert sep, "Couldn't find separator in %r" % self.statement - tag = tag.strip().lower() - val = val.strip().upper() - if self.cur is None: - self.cur = self.types[tag]() if tag in self.types else False - if self.cur is not False: - self.cur.set(tag, val) - if done and self.cur: - self.cur.finish(self) - self.cur = None - - filenames = ("ripe.db.aut-num.gz",) - - def __init__(self): - self.asns = csv_writer("asns.csv") - for fn in self.filenames: - f = gzip.open(fn) - self.statement = "" - self.cur = None - for line in f: - line = line.expandtabs().partition("#")[0].rstrip("\n") - if line and not line[0].isalpha(): - self.statement += line[1:] if line[0] == "+" else line - else: - self.finish_statement(not line) - self.statement = line - self.finish_statement(True) - f.close() - self.asns.close() - -main() diff --git a/scripts/ripe-prefixes-to-csv.awk b/scripts/ripe-prefixes-to-csv.awk deleted file mode 100644 index 37327484..00000000 --- a/scripts/ripe-prefixes-to-csv.awk +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/awk -f -# $Id$ - -# ftp -pa ftp://ftp.ripe.net/pub/stats/ripencc/membership/alloclist.txt - -function done() { - for (i = 1; i <= n_allocs; i++) - print handle "\t" alloc[i]; - n_allocs = 0; -} - -/^[a-z]/ { - done(); - handle = $0; - nr = NR; -} - -NR == nr + 1 { - name = $0; -} - -NR > nr + 2 && NF > 1 && $2 !~ /:/ { - split($2, a, "/"); - len = a[2]; - split(a[1], a, /[.]/); - for (i = length(a); i < 4; i++) - a[i+1] = 0; - alloc[++n_allocs] = sprintf("%d.%d.%d.%d/%d", a[1], a[2], a[3], a[4], len); -} - -NR > nr + 2 && NF > 1 && $2 ~ /:/ { - alloc[++n_allocs] = $2; -} - -END { - done(); -} diff --git a/scripts/ripe-to-csv.awk b/scripts/ripe-to-csv.awk deleted file mode 100644 index 5325574f..00000000 --- a/scripts/ripe-to-csv.awk +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/awk -f - -# Parse a WHOIS research dump and write out (just) the RPKI-relevant -# fields in myrpki-format CSV syntax. -# -# Unfortunately, unlike the ARIN and APNIC databases, the RIPE database -# doesn't really have any useful concept of an organizational handle. -# More precisely, while it has handles out the wazoo, none of them are -# useful as a reliable grouping mechanism for tracking which set of -# resources are held by a particular organization. So, instead of being -# able to track all of an organization's resources with a single handle -# as we can in the ARIN and APNIC databases, the best we can do with the -# RIPE database is to track individual resources, each with its own -# resource handle. Well, for prefixes -- ASN entries behave more like -# in the ARIN and APNIC databases. -# -# This is an AWK script rather than a Python script because it is a -# fairly simple stream parser that has to process a ridiculous amount -# of text. AWK turns out to be significantly faster for this. -# -# NB: The input data for this script is publicly available via FTP, but -# you'll have to fetch the data from RIPE yourself, and be sure to see -# the terms and conditions referenced by the data file header comments. -# -# $Id$ -# -# Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -# On input, ":" is the most useful delimiter -# On output, we want tab-delimited text. -BEGIN { - FS = "[ \t]*:"; - OFS = "\t"; -} - -# Clean up trailing whitespace. -{ - sub(/[ \t]+$/, ""); -} - -# Continuation line: strip comment, if any, then append value, if any, -# to what we had from previous line(s). -/^[^A-Z]/ { - sub(/[ \t]*#.*$/, ""); - if (NF) - val = val $0; - next; -} - -# Anything other than line continuation terminates the previous line, -# so if we were working on a line, we're done with it now, process it. -key { - do_line(); -} - -# Non-empty line and we have no tag, this must be start of a new block. -NF && !tag { - tag = $1; -} - -# One of the tags we care about, clean up and save the data. -/^(AS-NAME|AUT-NUM|INET6NUM|INETNUM|MNT-BY|NETNAME|STATUS):/ { - key = $1; - sub(/^[^ \t]+:/, ""); - sub(/[ \t]*#.*$/, ""); - val = $0; -} - -# Blank line and we have something, process it. -!NF && tag { - do_block(); -} - -# End of file, process final data, if any. -END { - do_line(); - do_block(); -} - -# Handle one line, after line icky RPSL continuation. -function do_line() { - gsub(/[ \t]/, "", val); - if (key && val) - tags[key] = val; - key = ""; - val = ""; -} - -# Dispatch to handle known block types, then clean up so we can start -# a new block. -function do_block() { - if (tag == "INETNUM" || tag == "INET6NUM") - do_prefix(); - else if (tag == "AUT-NUM") - do_asn(); - delete tags; - tag = ""; -} - -# Handle an AUT-NUM block: extract the ASN, use MNT-BY as the handle. -function do_asn() { - sub(/^AS/, "", tags[tag]); - if (tags["MNT-BY"] && tags[tag]) - print tags["MNT-BY"], tags[tag] >"asns.csv"; -} - -# Handle an INETNUM or INET6NUM block: check for the status values we -# care about, use NETNAME as the handle. -function do_prefix() { - if (tags["STATUS"] ~ /^ASSIGNED(P[AI])$/ && tags["NETNAME"] && tags[tag]) - print tags["NETNAME"], tags[tag] >"prefixes.csv"; -} diff --git a/scripts/ripe-to-csv.py b/scripts/ripe-to-csv.py deleted file mode 100644 index b864345b..00000000 --- a/scripts/ripe-to-csv.py +++ /dev/null @@ -1,138 +0,0 @@ -# $Id$ -# -# Copyright (C) 2009-2012 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Parse a WHOIS research dump and write out (just) the RPKI-relevant -fields in myrpki-format CSV syntax. - -Unfortunately, unlike the ARIN and APNIC databases, the RIPE database -doesn't really have any useful concept of an organizational handle. -More precisely, while it has handles out the wazoo, none of them are -useful as a reliable grouping mechanism for tracking which set of -resources are held by a particular organization. So, instead of being -able to track all of an organization's resources with a single handle -as we can in the ARIN and APNIC databases, the best we can do with the -RIPE database is to track individual resources, each with its own -resource handle. Well, for prefixes -- ASN entries behave more like -in the ARIN and APNIC databases. - -Feh. - -NB: The input data for this script is publicly available via FTP, but -you'll have to fetch the data from RIPE yourself, and be sure to see -the terms and conditions referenced by the data file header comments. -""" - -import gzip -from rpki.csv_utils import csv_writer - -class Handle(dict): - - want_tags = () - - want_status = ("ASSIGNED", "ASSIGNEDPA", "ASSIGNEDPI") - - debug = False - - def set(self, tag, val): - if tag in self.want_tags: - self[tag] = "".join(val.split(" ")) - - def check(self): - for tag in self.want_tags: - if not tag in self: - return False - if self.debug: - self.log() - return True - - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, - " ".join("%s:%s" % (tag, self.get(tag, "?")) - for tag in self.want_tags)) - - def log(self): - print repr(self) - - def finish(self, ctx): - self.check() - -class aut_num(Handle): - want_tags = ("aut-num", "mnt-by") # "as-name" - - def set(self, tag, val): - if tag == "aut-num" and val.startswith("AS"): - val = val[2:] - Handle.set(self, tag, val) - - def finish(self, ctx): - if self.check(): - ctx.asns.writerow((self["mnt-by"], self["aut-num"])) - -class inetnum(Handle): - want_tags = ("inetnum", "netname", "status") # "mnt-by" - - def finish(self, ctx): - if self.check() and self["status"] in self.want_status: - ctx.prefixes.writerow((self["netname"], self["inetnum"])) - -class inet6num(Handle): - want_tags = ("inet6num", "netname", "status") # "mnt-by" - - def finish(self, ctx): - if self.check() and self["status"] in self.want_status: - ctx.prefixes.writerow((self["netname"], self["inet6num"])) - -class main(object): - - types = dict((x.want_tags[0], x) for x in (aut_num, inetnum, inet6num)) - - def finish_statement(self, done): - if self.statement: - tag, sep, val = self.statement.partition(":") - assert sep, "Couldn't find separator in %r" % self.statement - tag = tag.strip().lower() - val = val.strip().upper() - if self.cur is None: - self.cur = self.types[tag]() if tag in self.types else False - if self.cur is not False: - self.cur.set(tag, val) - if done and self.cur: - self.cur.finish(self) - self.cur = None - - filenames = ("ripe.db.aut-num.gz", "ripe.db.inet6num.gz", "ripe.db.inetnum.gz") - - def __init__(self): - self.asns = csv_writer("asns.csv") - self.prefixes = csv_writer("prefixes.csv") - for fn in self.filenames: - f = gzip.open(fn) - self.statement = "" - self.cur = None - for line in f: - line = line.expandtabs().partition("#")[0].rstrip("\n") - if line and not line[0].isalpha(): - self.statement += line[1:] if line[0] == "+" else line - else: - self.finish_statement(not line) - self.statement = line - self.finish_statement(True) - f.close() - self.asns.close() - self.prefixes.close() - -main() diff --git a/scripts/roa-to-irr.py b/scripts/roa-to-irr.py deleted file mode 100644 index 01b2aac8..00000000 --- a/scripts/roa-to-irr.py +++ /dev/null @@ -1,159 +0,0 @@ -# $Id$ -# -# Copyright (C) 2014 Dragon Research Labs ("DRL") -# Portions copyright (C) 2010--2012 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notices and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL -# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR -# ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL -# DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA -# OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Generate IRR route and route6 objects from ROAs. -""" - -import os -import socket -import sys -import argparse -import errno -import time -import rpki.x509 - -args = None - -class route(object): - """ - Interesting parts of a route object. - """ - - def __init__(self, label, uri, asnum, date, prefix, prefixlen, max_prefixlen): - self.label = label - self.uri = uri - self.asn = asnum - self.date = date - self.prefix = prefix - self.prefixlen = prefixlen - self.max_prefixlen = self.prefixlen if max_prefixlen is None else max_prefixlen - - def __cmp__(self, other): - result = cmp(self.asn, other.asn) - if result == 0: - result = cmp(self.prefix, other.prefix) - if result == 0: - result = cmp(self.prefixlen, other.prefixlen) - if result == 0: - result = cmp(self.max_prefixlen, other.max_prefixlen) - if result == 0: - result = cmp(self.date, other.date) - return result - - def __str__(self): - lines = ( - "%-14s%s/%s" % (self.label, self.prefix, self.prefixlen), - "descr: %s/%s-%s" % (self.prefix, self.prefixlen, self.max_prefixlen), - "origin: AS%d" % self.asn, - "notify: %s" % args.notify, - "mnt-by: %s" % args.mnt_by, - "changed: %s %s" % (args.changed_by, self.date), - "source: %s" % args.source, - "override: %s" % args.password if args.password is not None else None, - "") - return "\n".join(line for line in lines if line is not None) - - def write(self, output_directory): - name = "%s-%s-%s-AS%d-%s" % (self.prefix, self.prefixlen, self.max_prefixlen, self.asn, self.date) - with open(os.path.join(output_directory, name), "w") as f: - f.write(str(self)) - - -class route_list(list): - """ - A list of route objects. - """ - - def __init__(self, rcynic_dir): - for root, dirs, files in os.walk(rcynic_dir): - for f in files: - if f.endswith(".roa"): - path = os.path.join(root, f) - uri = "rsync://" + path[len(rcynic_dir):].lstrip("/") - roa = rpki.x509.ROA(DER_file = path) - roa.extract() - assert roa.get_POW().getVersion() == 0, "ROA version is %d, expected 0" % roa.get_POW().getVersion() - asnum = roa.get_POW().getASID() - notBefore = roa.get_POW().certs()[0].getNotBefore().strftime("%Y%m%d") - v4, v6 = roa.get_POW().getPrefixes() - if v4 is not None: - for prefix, prefixlen, max_prefixlen in v4: - self.append(route("route:", uri, asnum, notBefore, prefix, prefixlen, max_prefixlen)) - if v6 is not None: - for prefix, prefixlen, max_prefixlen in v6: - self.append(route("route6:", uri, asnum, notBefore, prefix, prefixlen, max_prefixlen)) - self.sort() - for i in xrange(len(self) - 2, -1, -1): - if self[i] == self[i + 1]: - del self[i + 1] - -def email_header(f): - if args.email: - f.write("\n".join(( - "From %s" % args.email_from, - "Date: %s" % time.strftime("%d %b %Y %T %z"), - "From: %s" % args.email_from, - "Subject: Fake email header to make irr_rpsl_submit happy", - "Message-Id: <%s.%s@%s>" % (os.getpid(), time.time(), socket.gethostname()), - "", ""))) - -def main(): - - global args - whoami = "%s@%s" % (os.getlogin(), socket.gethostname()) - - parser = argparse.ArgumentParser(description = __doc__) - parser.add_argument("-c", "--changed_by", default = whoami, - help = "override \"changed:\" value") - parser.add_argument("-f", "--from", dest = "email_from", default = whoami, - help = "override \"from:\" header when using --email") - parser.add_argument("-m", "--mnt_by", default = "MAINT-RPKI", - help = "override \"mnt-by:\" value") - parser.add_argument("-n", "--notify", default = whoami, - help = "override \"notify:\" value") - parser.add_argument("-p", "--password", - help = "specify \"override:\" password") - parser.add_argument("-s", "--source", default = "RPKI", - help = "override \"source:\" value") - group = parser.add_mutually_exclusive_group() - group.add_argument("-e", "--email", action = "store_true", - help = "generate fake RFC 822 header suitable for piping to irr_rpsl_submit") - group.add_argument("-o", "--output", - help = "write route and route6 objects to directory OUTPUT, one object per file") - parser.add_argument("authenticated_directory", - help = "directory tree containing authenticated rcynic output") - args = parser.parse_args() - - if not os.path.isdir(args.authenticated_directory): - sys.exit("\"%s\" is not a directory" % args.authenticated_directory) - - routes = route_list(args.authenticated_directory) - - if args.output: - if not os.path.isdir(args.output): - os.makedirs(args.output) - for r in routes: - r.write(args.output) - else: - email_header(sys.stdout) - for r in routes: - sys.stdout.write("%s\n" % r) - -if __name__ == "__main__": - main() diff --git a/scripts/rp-sqlite b/scripts/rp-sqlite deleted file mode 100755 index ee43096d..00000000 --- a/scripts/rp-sqlite +++ /dev/null @@ -1,425 +0,0 @@ -#!/usr/local/bin/python - -# $Id$ - -# Copyright (C) 2013 Dragon Research Labs ("DRL") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -# Preliminary script to work out what's involved in building an -# SQLite3 database of RP objects. We haven't bothered with this until -# now in rcynic, because we mostly just walk the filesystem tree, but -# LTA and some of the ideas Tim is playing with require a lot of -# lookups based on things that are not the URIs we use as filenames, -# so some kind of indexing may become necessary. Given the complexity -# of building any kind of real index over RFC 3779 resources, -# otherwise fine lightweight tools like the Python shelve library -# probably won't cut it here, and I don't want to add a dependency on -# MySQL on the RP side (yet?), so let's see what we can do with SQLite3. - -import os -import sys -import yaml -import base64 -import sqlite3 -import weakref -import rpki.POW -import rpki.x509 -import rpki.resource_set - -sqlite3.register_adapter(rpki.POW.IPAddress, - lambda x: buffer("_" + x.toBytes())) - -sqlite3.register_converter("RangeVal", - lambda s: long(s) if s.isdigit() else rpki.POW.IPAddress.fromBytes(s[1:])) - - -def main(): - rpdb = RPDB() - rpdb.load() - test(rpdb) - rpdb.close() - -def test(rpdb): - fn2s = [None] + rpdb.fn2map.keys() - - print - print "Testing YAML parsing" - parse_yaml(rpdb) - - print - print "Looking for certificates without AKI" - for r in rpdb.find_by_aki(None, "cer"): - print r, r.uris - print - print "Testing range functions" - for fn2 in fn2s: - if fn2 is not None: - print - print "Restricting search to type", fn2 - print - print "Looking for range that should include adrilankha and psg again" - for r in rpdb.find_by_range("147.28.0.19", "147.28.0.62", fn2): - print r, r.uris - print - print "Looking for range that should include adrilankha" - for r in rpdb.find_by_range("147.28.0.19", "147.28.0.19", fn2): - print r, r.uris - print - print "Looking for range that should include ASN 3130" - for r in rpdb.find_by_range(3130, 3130, fn2): - print r, r.uris - print - print "Moving on to resource sets" - for fn2 in fn2s: - if fn2 is not None: - print - print "Restricting search to type", fn2 - for expr in ("147.28.0.19-147.28.0.62", - "3130", - "2001:418:1::19/128", - "147.28.0.19-147.28.0.62,198.180.150.50/32", - "3130,147.28.0.19-147.28.0.62,198.180.150.50/32", - "2001:418:1::62/128,198.180.150.50/32,2001:418:8006::50/128", - "147.28.0.19-147.28.0.62,2001:418:1::19/128,2001:418:1::62/128,198.180.150.50/32,2001:418:8006::50/128"): - print - print "Trying", expr - for r in rpdb.find_by_resource_bag(rpki.resource_set.resource_bag.from_str(expr), fn2): - print r, r.uris - - -def parse_xki(s): - """ - Parse text form of an SKI or AKI. We accept two encodings: - colon-delimited hexadecimal, and URL-safe Base64. The former is - what OpenSSL prints in its text representation of SKI and AKI - extensions; the latter is the g(SKI) value that some RPKI CA engines - (including rpkid) use when constructing filenames. - - In either case, we check that the decoded result contains the right - number of octets to be a SHA-1 hash. - """ - - if ":" in s: - b = "".join(chr(int(c, 16)) for c in s.split(":")) - else: - b = base64.urlsafe_b64decode(s + ("=" * (4 - len(s) % 4))) - if len(b) != 20: - raise RuntimeError("Bad length for SHA1 xKI value: %r" % s) - return b - - -def parse_yaml(rpdb, fn = "rp-sqlite.yaml"): - yy = yaml.safe_load(open(fn, "r")) - for y in yy: - - ski = None - uri = None - obj = set() - - print - - if "ski" in y: - ski = parse_xki(y["ski"]) - obj.update(rpdb.find_by_ski(ski)) - if "uri" in y: - uri = y["uri"] - obj.update(rpdb.find_by_uri(uri)) - if len(obj) == 1: - obj = obj.pop() - else: - raise RuntimeError("Constraint entry must name a unique object using SKI, URI, or both (%r, %r, %r)" % ( - ski, uri, obj)) - - print "URI:", uri - print "SKI:", " ".join("%02X" % ord(c) for c in ski), "(" + y["ski"] + ")" - - new_resources = old_resources = obj.get_3779resources() - - if "set" in y: - new_resources = rpki.resource_set.resource_bag.from_str(y["set"]) - - if "add" in y: - new_resources = new_resources | rpki.resource_set.resource_bag.from_str(y["add"]) - - if "sub" in y: - new_resources = new_resources - rpki.resource_set.resource_bag.from_str(y["sub"]) - - if new_resources == old_resources: - print "No resource change, skipping" - continue - - print "Old:", old_resources - print "New:", new_resources - print "Add:", new_resources - old_resources - print "Sub:", old_resources - new_resources - - # See draft-ietf-sidr-ltamgmt-08.txt for real processing details, but overview: - # - # - Process constraints file as above to determine list of target - # certificates (2.1). May need to add more fields to YAML hash - # for things like CP, CRLDP, etc, although I'm not entirely sure - # yet which of those it really makes sense to tweak via - # constraints. - # - # - Use resources from selected target certificates to determine - # which additional certificates we need to reissue to remove those - # resources (2.2, "perforation"). In theory we already have SQL - # that will just locate all of these for us. - # - # - Figure out which trust anchors to process (2.3, TA - # re-parenting); we can look in SQL for NULL AKI, but that's just - # a hint, we either have to verify that rcynic accepted those TAs - # or we have to look at the TALs. Looking at TALs is probably - # easier. - # - # At some point we probably need to parse the constraints file into - # Constraints objects or something like that, except that we may - # really need something more general that will accomodate - # perforation and TA reparenting as well. Figure out and refactor - # as we go along, most likely. - - -class RPDB(object): - """ - Relying party database. - - For now just wire in the database name and rcynic root, fix this - later if overall approach seems usable. Might even end up just - being an in-memory SQL database, who knows? - """ - - fn2map = dict(cer = rpki.x509.X509, - crl = rpki.x509.CRL, - mft = rpki.x509.SignedManifest, - roa = rpki.x509.ROA, - gbr = rpki.x509.Ghostbuster) - - def __init__(self, db_name = "rp-sqlite.db", delete_old_db = True): - - if delete_old_db: - try: - os.unlink(db_name) - except: - pass - - exists = os.path.exists(db_name) - - self.db = sqlite3.connect(db_name, detect_types = sqlite3.PARSE_DECLTYPES) - self.db.text_factory = str - self.cur = self.db.cursor() - - self.cache = weakref.WeakValueDictionary() - - if exists: - return - - self.cur.executescript(''' - PRAGMA foreign_keys = on; - - CREATE TABLE object ( - id INTEGER PRIMARY KEY NOT NULL, - der BLOB NOT NULL, - fn2 TEXT NOT NULL, - ski BLOB, - aki BLOB, - inherits BOOLEAN NOT NULL, - UNIQUE (der)); - - CREATE TABLE uri ( - id INTEGER NOT NULL, - uri TEXT NOT NULL, - UNIQUE (uri), - FOREIGN KEY (id) REFERENCES object(id) - ON DELETE CASCADE - ON UPDATE CASCADE); - - CREATE INDEX uri_index ON uri(id); - - CREATE TABLE range ( - id INTEGER NOT NULL, - min RangeVal NOT NULL, - max RangeVal NOT NULL, - UNIQUE (id, min, max), - FOREIGN KEY (id) REFERENCES object(id) - ON DELETE CASCADE - ON UPDATE CASCADE); - - CREATE INDEX range_index ON range(min, max); - ''') - - def load(self, - rcynic_root = os.path.expanduser("~/rpki/subvert-rpki.hactrn.net/trunk/" - "rcynic/rcynic-data/unauthenticated"), - spinner = 100): - - nobj = 0 - - for root, dirs, files in os.walk(rcynic_root): - for fn in files: - fn = os.path.join(root, fn) - fn2 = os.path.splitext(fn)[1][1:] - - try: - obj = self.fn2map[fn2](DER_file = fn) - except: - continue - - if spinner and nobj % spinner == 0: - sys.stderr.write("\r%s %d..." % ("|\\-/"[(nobj/spinner) & 3], nobj)) - - nobj += 1 - - inherits = False - - if fn2 == "crl": - ski = None - aki = buffer(obj.get_AKI()) - cer = None - bag = None - - else: - if fn2 == "cer": - cer = obj - else: - cer = rpki.x509.X509(POW = obj.get_POW().certs()[0]) - ski = buffer(cer.get_SKI()) - try: - aki = buffer(cer.get_AKI()) - except: - aki = None - bag = cer.get_3779resources() - inherits = bag.asn.inherit or bag.v4.inherit or bag.v6.inherit - - der = buffer(obj.get_DER()) - uri = "rsync://" + fn[len(rcynic_root) + 1:] - - try: - self.cur.execute("INSERT INTO object (der, fn2, ski, aki, inherits) VALUES (?, ?, ?, ?, ?)", - (der, fn2, ski, aki, inherits)) - rowid = self.cur.lastrowid - - except sqlite3.IntegrityError: - self.cur.execute("SELECT id FROM object WHERE der = ? AND fn2 = ?", (der, fn2)) - rows = self.cur.fetchall() - rowid = rows[0][0] - assert len(rows) == 1 - - else: - if bag is not None: - for rset in (bag.asn, bag.v4, bag.v6): - if rset is not None: - self.cur.executemany("REPLACE INTO range (id, min, max) VALUES (?, ?, ?)", - ((rowid, i.min, i.max) for i in rset)) - - self.cur.execute("INSERT INTO uri (id, uri) VALUES (?, ?)", - (rowid, uri)) - - if spinner: - sys.stderr.write("\r= %d objects, committing..." % nobj) - - self.db.commit() - - if spinner: - sys.stderr.write("done.\n") - - - def find_by_ski(self, ski, fn2 = None): - if ski is None: - return self._find_results(fn2, "SELECT id, fn2, der FROM object WHERE ski IS NULL") - else: - return self._find_results(fn2, "SELECT id, fn2, der FROM object WHERE ski = ?", [buffer(ski)]) - - def find_by_aki(self, aki, fn2 = None): - if aki is None: - return self._find_results(fn2, "SELECT id, fn2, der FROM object WHERE aki IS NULL") - else: - return self._find_results(fn2, "SELECT id, fn2, der FROM object WHERE aki = ?", [buffer(aki)]) - - def find_by_uri(self, uri): - return self._find_results(None, "SELECT object.id, fn2, der FROM object, uri WHERE uri.uri = ? AND object.id = uri.id", [uri]) - - - # It's easiest to understand overlap conditions by understanding - # non-overlap then inverting and and applying De Morgan's law. Ranges - # A and B do not overlap if either A.min > B.max or A.max < B.min; - # therefore they do overlap if A.min <= B.max and A.max >= B.min. - - def find_by_range(self, range_min, range_max = None, fn2 = None): - if range_max is None: - range_max = range_min - if isinstance(range_min, (str, unicode)): - range_min = long(range_min) if range_min.isdigit() else rpki.POW.IPAddress(range_min) - if isinstance(range_max, (str, unicode)): - range_max = long(range_max) if range_max.isdigit() else rpki.POW.IPAddress(range_max) - assert isinstance(range_min, (int, long, rpki.POW.IPAddress)) - assert isinstance(range_max, (int, long, rpki.POW.IPAddress)) - return self._find_results( - fn2, - """ - SELECT object.id, fn2, der FROM object, range - WHERE ? <= max AND ? >= min AND object.id = range.id - """, - [range_min, range_max]) - - - def find_by_resource_bag(self, bag, fn2 = None): - assert bag.asn or bag.v4 or bag.v6 - qset = [] - aset = [] - for rset in (bag.asn, bag.v4, bag.v6): - if rset: - for r in rset: - qset.append("(? <= max AND ? >= min)") - aset.append(r.min) - aset.append(r.max) - return self._find_results( - fn2, - """ - SELECT object.id, fn2, der FROM object, range - WHERE object.id = range.id AND (%s) - """ % (" OR ".join(qset)), - aset) - - - def _find_results(self, fn2, query, args = None): - if args is None: - args = [] - if fn2 is not None: - assert fn2 in self.fn2map - query += " AND fn2 = ?" - args.append(fn2) - query += " GROUP BY object.id" - results = [] - self.cur.execute(query, args) - selections = self.cur.fetchall() - for rowid, fn2, der in selections: - if rowid in self.cache: - obj = self.cache[rowid] - else: - obj = self.fn2map[fn2](DER = der) - self.cur.execute("SELECT uri FROM uri WHERE id = ?", (rowid,)) - obj.uris = [u[0] for u in self.cur.fetchall()] - obj.uri = obj.uris[0] if len(obj.uris) == 1 else None - self.cache[rowid] = obj - results.append(obj) - return results - - - def close(self): - self.cur.close() - self.db.close() - - -if __name__ == "__main__": - main() diff --git a/scripts/rp-sqlite.yaml b/scripts/rp-sqlite.yaml deleted file mode 100644 index 4c0a0b8d..00000000 --- a/scripts/rp-sqlite.yaml +++ /dev/null @@ -1,53 +0,0 @@ -- ski: B8:14:5D:13:53:7D:AE:6E:E2:E3:95:84:A8:99:EB:7D:1A:7D:E5:DF - uri: rsync://repo0.rpki.net/rpki/root.cer - add: 10.0.0.44/32 - -- ski: A2:B3:2A:99:20:07:7A:E9:A4:9F:3F:02:F2:32:F9:3D:54:F8:7E:78 - uri: rsync://repo0.rpki.net/rpki/root/iana.cer - sub: 10.0.0.0/8 - -- ski: 3NYgwt_HYV91MeCGLWdUL4uq65M - uri: rsync://repo0.rpki.net/rpki/root/iana/5/3NYgwt_HYV91MeCGLWdUL4uq65M.cer - add: 10.0.0.0/8 - -- ski: 3YFTaQOUkPQfhckX_ikYzy0mR9s - uri: rsync://repo0.rpki.net/rpki/root/iana/5/3YFTaQOUkPQfhckX_ikYzy0mR9s.cer - -- ski: 7ew1d5WFCSfhd8lnpmjwOohS_DQ - uri: rsync://repo0.rpki.net/rpki/root/iana/5/7ew1d5WFCSfhd8lnpmjwOohS_DQ.cer - -- ski: PWxDsvUgDdeloE3eQfceV8vbIyg - uri: rsync://repo0.rpki.net/rpki/root/iana/5/PWxDsvUgDdeloE3eQfceV8vbIyg.cer - -- ski: WnOFfpqobEKxzmvddJue3PXsEIQ - uri: rsync://repo0.rpki.net/rpki/root/iana/5/WnOFfpqobEKxzmvddJue3PXsEIQ.cer - -- ski: m6TQTKwKYyVva-Yq__I-Zz25Vns - uri: rsync://repo0.rpki.net/rpki/root/iana/5/m6TQTKwKYyVva-Yq__I-Zz25Vns.cer - -- ski: 8U5wQ47aZuuOXYPGX5BPvlcTfNg - uri: rsync://repo0.rpki.net/rpki/root/iana/ripe/8/8U5wQ47aZuuOXYPGX5BPvlcTfNg.cer - -- ski: bccxGl4Xl4ur3nd1fwQ-1QIwtNA - uri: rsync://repo0.rpki.net/rpki/root/iana/ripe/8/bccxGl4Xl4ur3nd1fwQ-1QIwtNA.cer - -- ski: zbXifbEkZNmOVhYZTCZFfLPxhjM - uri: rsync://repo0.rpki.net/rpki/root/iana/ripe/8/zbXifbEkZNmOVhYZTCZFfLPxhjM.cer - -- ski: XYjTToHopYme-hlwgUyUyYRD_VQ - uri: rsync://repo0.rpki.net/rpki/root/iana/arin/6/XYjTToHopYme-hlwgUyUyYRD_VQ.cer - -- ski: _3I3i3uVmnliCinb2STR2MaxuU8 - uri: rsync://repo0.rpki.net/rpki/root/iana/arin/6/_3I3i3uVmnliCinb2STR2MaxuU8.cer - -- ski: qdtoqOMCNSOdYuNNC7ya3dUaPl4 - uri: rsync://repo0.rpki.net/rpki/root/iana/arin/6/qdtoqOMCNSOdYuNNC7ya3dUaPl4.cer - -- ski: yq4x9MteoM0DQYTh6NLbbmf--QY - uri: rsync://repo0.rpki.net/rpki/root/iana/arin/6/yq4x9MteoM0DQYTh6NLbbmf--QY.cer - -- ski: j2TDGclJnZ7mXpZCQJS0cfkOL34 - uri: rsync://repo0.rpki.net/rpki/root/iana/apnic/7/j2TDGclJnZ7mXpZCQJS0cfkOL34.cer - -- ski: yRk89OOx4yyJHJ6Z1JLLnk0_oDc - uri: rsync://repo0.rpki.net/rpki/root/iana/apnic/7/yRk89OOx4yyJHJ6Z1JLLnk0_oDc.cer diff --git a/scripts/rpki b/scripts/rpki deleted file mode 120000 index 168548eb..00000000 --- a/scripts/rpki +++ /dev/null @@ -1 +0,0 @@ -../rpkid/rpki
\ No newline at end of file diff --git a/scripts/rpkidemo b/scripts/rpkidemo deleted file mode 100755 index fdb4e1bb..00000000 --- a/scripts/rpkidemo +++ /dev/null @@ -1,495 +0,0 @@ -#!/usr/bin/env python - -""" -Hosted GUI client startup script, for workshops, etc. - -As of when this is run, we assume that the tarball (contents TBD and -perhaps changing from one workshop to another) have been unpacked, -that we are on some Unix-like machine, and that we are executing in -a Python interpreter. We have to check anything else we care about. - -In what we hope is the most common case, this script should be run -with no options. - -$Id$ - -Copyright (C) 2010 Internet Systems Consortium ("ISC") - -Permission to use, copy, modify, and distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THIS SOFTWARE. -""" - -# Check Python version before doing anything else - -import sys - -python_version = sys.version_info[:2] - -have_ssl_module = python_version >= (2, 6) - -if python_version == (2, 5): - print """ - WARNING WARNING WARNING - - You are running Python version 2.5, which does not include - real SSL support. This means that sessions created by this - script will be vulnerable to monkey-in-the-middle attacks. - - Python 2.6 does not have this problem. - """ - while True: - answer = raw_input("Are you SURE you want to proceed? (yes/NO) ").strip().lower() - if answer in ("", "n", "no"): - sys.exit("You have chosen wisely") - elif answer in ("y", "yes"): - print "You have been warned" - break - else: - print 'Please answer "yes" or "no"' - -elif have_ssl_module: - try: - import ssl - except ImportError: - sys.exit("You're running Python 2.6+, but I can't find the ssl module, so you have no SSL support at all, argh!") - -else: - sys.exit("Sorry, this script requires Python 2.6+, I seem to be running in %s" % sys.version) - -# Ok, it's safe to import the other stuff we need now - -import os, subprocess, webbrowser, urllib2, getpass, re, errno, time, email.utils, httplib, socket, getopt, urllib, cookielib -import tempfile -from xml.etree.ElementTree import fromstring as ElementFromString - -def save(filename, data): - """ - Save data to a file. - """ - - tempname = "%s.%d.tmp" % (filename, os.getpid()) - f = open(tempname, "w") - f.write(data) - f.close() - os.rename(tempname, filename) - -def save_error(err): - """ - Save the data from the file-like object "f" into a temporary file - and open a web browser to view the result. - """ - - with tempfile.NamedTemporaryFile(prefix = "rpkidemo-error", suffix = ".html", delete = False) as tmpf: - tmpf.write(err.read()) - - # Save filename for use outside the with statement. This ensures - # the file is properly flushed prior to invoking the web browser. - fname = tmpf.name - - sys.stderr.write("errors saved in %s\n" % fname) - webbrowser.open("file://" + fname) - -class CSV_File(object): - """ - A CSV file that's being maintained by the GUI but being monitored, - downloaded, and used here. - """ - - def __init__(self, filename, url): - self.filename = filename - self.url = url - try: - self.timestamp = os.stat(filename).st_mtime - except: - self.store(0, "") - - def last_modified(self): - """ - Return CSV file timestamp formatted for use with HTTP. - """ - return email.utils.formatdate(self.timestamp, False, True) - - def store(self, timestamp, data): - """ - Save CSV file, and record new timestamp. - """ - save(self.filename, data) - self.timestamp = timestamp - os.utime(self.filename, (time.time(), timestamp)) - - -class AbstractHTTPSConnection(httplib.HTTPSConnection): - """ - Customization of httplib.HTTPSConnection to enable certificate - validation. - - This is an abstract class; subclass must set trust_anchor to the - filename of a anchor file in the format that the ssl module - expects. - """ - - trust_anchor = None - - def connect(self): - assert self.trust_anchor is not None - sock = socket.create_connection((self.host, self.port), self.timeout) - if getattr(self, "_tunnel_host", None): - self.sock = sock - self._tunnel() - self.sock = ssl.wrap_socket(sock, - keyfile = self.key_file, - certfile = self.cert_file, - cert_reqs = ssl.CERT_REQUIRED, - ssl_version = ssl.PROTOCOL_TLSv1, - ca_certs = self.trust_anchor) - - -class main(object): - """ - Main program. - """ - - # Environmental parameters - - top = os.path.realpath(os.path.join((sys.path[0] or "."), "..")) - cwd = os.getcwd() - - # Parameters that we might want to get from a config file. - # Just wire them all in for the moment. - - base_url = "https://demo.rpki.net/" - myrpki_url = base_url + "rpki/" - auth_url = myrpki_url + "demo/login" - example_myrpki_cfg = "%s/rpkid/examples/rpki.conf" % top - working_dir = "%s/rpkidemo-data" % cwd - myrpki_py = "%s/rpkid/myrpki.py" % top - user_agent = "RPKIDemo" - delay = 15 - trust_anchor = "%s/scripts/rpkidemo.pem" % top - - openssl = None - - def setup_openssl(self): - """ - Find a usable version of OpenSSL, or build one if we must. - """ - - def scrape(*args): - return subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT).communicate()[0] - - def usable_openssl(f): - return f is not None and os.path.exists(f) and "-ss_cert" in scrape(f, "ca", "-?") and "Usage cms" in scrape(f, "cms", "-?") - - for d in os.environ["PATH"].split(":"): - f = os.path.join(d, "openssl") - if usable_openssl(f): - self.openssl = f - break - - if self.openssl is None: - print "Couldn't find usable openssl on path, attempting to build one" - subprocess.check_call(("./configure",), cwd = self.top) - subprocess.check_call(("make",), cwd = os.path.join(self.top, "openssl")) - self.openssl = os.path.join(self.top, "openssl", "openssl", "apps", "openssl") - print "Done building openssl" - print - - if usable_openssl(self.openssl): - print "Using", self.openssl - else: - sys.exit("Could not find or build usable version of openssl, giving up") - - @staticmethod - def setup_utc(): - """ - This script thinks in UTC. - """ - - os.environ["TZ"] = "UTC" - time.tzset() - - def setup_username(self): - """ - Get username and password for web interface, construct urllib2 - "opener" tailored for our use, perform an initial GET (ignoring - result, other than exceptions) to test the username and password. - """ - - print "I need to know your username and password on the Django GUI server to proceed" - - while True: - - try: - self.username = raw_input("Username: ") - self.password = getpass.getpass() - - handlers = [] - - self.cookiejar = cookielib.CookieJar() - handlers.append(urllib2.HTTPCookieProcessor(self.cookiejar)) - - if have_ssl_module: - - class HTTPSConnection(AbstractHTTPSConnection): - trust_anchor = self.trust_anchor - - class HTTPSHandler(urllib2.HTTPSHandler): - def https_open(self, req): - return self.do_open(HTTPSConnection, req) - - handlers.append(HTTPSHandler) - - self.opener = urllib2.build_opener(*handlers) - - # Test login credentials - resp = self.opener.open(self.auth_url) # GET - - r = self.opener.open(urllib2.Request( - url = self.auth_url, - data = urllib.urlencode({ "username" : self.username, - "password" : self.password, - "csrfmiddlewaretoken" : self.csrftoken() }), - headers = { "Referer" : self.auth_url, - "User-Agent" : self.user_agent})) # POST - return - - except urllib2.URLError, e: - print "Could not log in to server: %s" % e - print "Please try again" - save_error(e) - - def csrftoken(self): - """ - Pull Django's CSFR token from cookie database. - - Django's login form requires the "csrfmiddlewaretoken." It turns out - this is the same value as the "csrftoken" cookie, so we don't need - to bother parsing the form. - """ - - return [c.value for c in self.cookiejar if c.name == "csrftoken"][0] - - def setup_working_directory(self): - """ - Create working directory and move to it. - """ - - try: - print "Creating", self.working_dir - os.mkdir(self.working_dir) - except OSError, e: - if e.errno != errno.EEXIST: - raise - print self.working_dir, "already exists, reusing it" - os.chdir(self.working_dir) - - def setup_config_file(self): - """ - Generate rpki.conf - """ - - if os.path.exists("rpki.conf"): - print "You already have a rpki.conf file, so I will use it" - return - - print "Generating rpki.conf" - section_regexp = re.compile("\s*\[\s*(.+?)\s*\]\s*$") - variable_regexp = re.compile("\s*([-a-zA-Z0-9_]+)\s*=\s*(.+?)\s*$") - f = open("rpki.conf", "w") - f.write("# Automatically generated, do not edit\n") - section = None - for line in open(self.example_myrpki_cfg): - m = section_regexp.match(line) - if m: - section = m.group(1) - m = variable_regexp.match(line) - option = m.group(1) if m and section == "myrpki" else None - value = m.group(2) if option else None - if option == "handle": - line = "handle = %s\n" % self.username - if option == "openssl": - line = "openssl = %s\n" % self.openssl - if option in ("run_rpkid", "run_pubd", "run_rootd") and value != "false": - line = "%s = false\n" % option - f.write(line) - f.close() - - def myrpki(self, *cmd): - """ - Run a myrpki command. - """ - return subprocess.check_call((sys.executable, self.myrpki_py) + cmd) - - def upload(self, url, filename): - """ - Upload filename to URL, return result. - """ - - url = "%s%s/%s" % (self.myrpki_url, url, self.username) - data = open(filename).read() - print "Uploading", filename, "to", url - post_data = urllib.urlencode({ - "content" : data, - "csrfmiddlewaretoken" : self.csrftoken() }) # POST - try: - return self.opener.open(urllib2.Request(url, post_data, { - "User-Agent" : self.user_agent, - "Referer" : url})) - except urllib2.HTTPError, e: - sys.stderr.write("Problem uploading to URL %s\n" % url) - save_error(e) - raise - - def update(self): - """ - Run configure_resources, upload result, download updated result. - """ - - self.myrpki("configure_resources") - r = self.upload("demo/myrpki-xml", "myrpki.xml") - save("myrpki.xml", r.read()) - - def setup_csv_files(self): - """ - Create CSV file objects and synchronize timestamps. - """ - - self.csv_files = [ - CSV_File("asns.csv", "demo/down/asns/%s" % self.username), - CSV_File("prefixes.csv", "demo/down/prefixes/%s" % self.username), - CSV_File("roas.csv", "demo/down/roas/%s" % self.username) ] - - def upload_for_response(self, url, path): - """ - Upload an XML file to the requested URL and wait for for the server - to signal that a response is ready. - """ - - self.upload(url, path) - - print """ - Waiting for response to upload. This may require action by a human - being on the server side, so it may take a while, please be patient. - """ - - while True: - try: - return self.opener.open(urllib2.Request( - "%s%s/%s" % (self.myrpki_url, url, self.username), - None, - { "User-Agent" : self.user_agent })) - except urllib2.HTTPError, e: - # Portal GUI uses response code 503 to signal "not ready" - if e.code != 503: - sys.stderr.write("Problem getting response from %s: %s\n" % (url, e)) - save_error(e) - raise - time.sleep(self.delay) - - def setup_parent(self): - """ - Upload the user's identity.xml and wait for the portal gui to send - back the parent.xml response. - """ - - r = self.upload_for_response("demo/parent-request", "entitydb/identity.xml") - parent_data = r.read() - save("parent.xml", parent_data) - self.myrpki("configure_parent", "parent.xml") - - # Extract the parent_handle from the xml response and save it for use by - # setup_repository() - self.parent_handle = ElementFromString(parent_data).get("parent_handle") - - def setup_repository(self): - """ - Upload the repository referral to the portal-gui and wait the - response from the repository operator. - """ - - r = self.upload_for_response("demo/repository-request", "entitydb/repositories/%s.xml" % self.parent_handle) - save("repository.xml", r.read()) - self.myrpki("configure_repository", "repository.xml") - - def poll(self, csv_file): - """ - Poll for new version of a CSV file, save if changed, return - boolean indicating whether file has changed. - """ - - try: - url = self.myrpki_url + csv_file.url - r = self.opener.open(urllib2.Request(url, None, { - "If-Modified-Since" : csv_file.last_modified(), - "User-Agent" : self.user_agent })) - timestamp = time.mktime(r.info().getdate("Last-Modified")) - csv_file.store(timestamp, r.read()) - return True - except urllib2.HTTPError, e: - if e.code == 304: # 304 == "Not Modified" - return False - else: - sys.stderr.write("Problem polling URL %s\n" % url) - save_error(e) - raise - - def poll_loop(self): - """ - Loop forever, polling for updates. - """ - - while True: - changed = False - for csv_file in self.csv_files: - if self.poll(csv_file): - changed = True - if changed: - self.update() - time.sleep(self.delay) - - def getopt(self): - """ - Parse options. - """ - opts, argv = getopt.getopt(sys.argv[1:], "hi?", ["help"]) - for o, a in opts: - if o in ("-h", "--help", "-?"): - print __doc__ - sys.exit(0) - if argv: - sys.exit("Unexpected arguments %r" % (argv,)) - - def __init__(self): - self.getopt() - self.setup_utc() - self.setup_openssl() - self.setup_username() - self.setup_working_directory() - self.setup_config_file() - self.setup_csv_files() - self.myrpki("initialize") - self.setup_parent() - self.setup_repository() - self.update() - self.update() - - webbrowser.open(self.myrpki_url) - - self.poll_loop() - -main() - -# Local Variables: -# mode:python -# End: - -# vim:sw=2 ts=8 expandtab diff --git a/scripts/rpkidemo.pem b/scripts/rpkidemo.pem deleted file mode 100644 index f96fdb9e..00000000 --- a/scripts/rpkidemo.pem +++ /dev/null @@ -1,23 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDxjCCAy+gAwIBAgIJALc3/mkoVFOMMA0GCSqGSIb3DQEBBQUAMIGaMR4wHAYD -VQQDExVSR25ldC9QU0duZXQgU2VjdXJpdHkxCzAJBgNVBAYTAlVTMQswCQYDVQQI -EwJXQTEaMBgGA1UEBxMRQmFpbmJyaWRnZSBJc2xhbmQxEzARBgNVBAoTClJHbmV0 -LCBMTEMxDzANBgNVBAsTBlBTR25ldDEcMBoGCSqGSIb3DQEJARYNcmFuZHlAcHNn -LmNvbTAeFw0xMTAyMjcwNDMxMzhaFw0yMTAyMjQwNDMxMzhaMIGaMR4wHAYDVQQD -ExVSR25ldC9QU0duZXQgU2VjdXJpdHkxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJX -QTEaMBgGA1UEBxMRQmFpbmJyaWRnZSBJc2xhbmQxEzARBgNVBAoTClJHbmV0LCBM -TEMxDzANBgNVBAsTBlBTR25ldDEcMBoGCSqGSIb3DQEJARYNcmFuZHlAcHNnLmNv -bTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAw/lD46076mS4vDHI943dlNPh -KN1EY0wnx7nDga7jh4x8U5gt+MdvdH+kRABR7KVjkaUYKcd+DlAQeNLYXXkXOECz -yN90PgADWucPij6GJn7o9xwNsh2yAhICzqc19RgiKPVJL9V5xWBwKgzkpGG7dcUm -97TjZwhj8B8xcHjVbJ8CAwEAAaOCARAwggEMMAwGA1UdEwQFMAMBAf8wCwYDVR0P -BAQDAgEGMB0GA1UdDgQWBBRUmkatFo7oAUl5SJqUCfAC0LpkgDCBzwYDVR0jBIHH -MIHEgBRUmkatFo7oAUl5SJqUCfAC0LpkgKGBoKSBnTCBmjEeMBwGA1UEAxMVUkdu -ZXQvUFNHbmV0IFNlY3VyaXR5MQswCQYDVQQGEwJVUzELMAkGA1UECBMCV0ExGjAY -BgNVBAcTEUJhaW5icmlkZ2UgSXNsYW5kMRMwEQYDVQQKEwpSR25ldCwgTExDMQ8w -DQYDVQQLEwZQU0duZXQxHDAaBgkqhkiG9w0BCQEWDXJhbmR5QHBzZy5jb22CCQC3 -N/5pKFRTjDANBgkqhkiG9w0BAQUFAAOBgQBHBN06mk++v2fb3DnDiwt0Xqna4oNH -LNN69VaKLHhQ8J0shO4386E9ejLTutWO5VCdRim3a7WuneYSIncFBY76ddipWmuK -chitDDRUn/ccx4pkPoiHBGqthMqSbNGVsamAMOAJzzPyGXdur46wpzz6DtWObsQg -2/Q6evShgNYmtg== ------END CERTIFICATE----- diff --git a/scripts/rrd-rcynic-history.py b/scripts/rrd-rcynic-history.py deleted file mode 100644 index 8a0d50a8..00000000 --- a/scripts/rrd-rcynic-history.py +++ /dev/null @@ -1,201 +0,0 @@ -# $Id$ -# -# Copyright (C) 2011-2012 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Parse traffic data out of rynic XML output, whack it a bit, and stuff -it into one or more RRDs (Round Robin Databases -- see rrdtool). - -Haven't decided yet whether to draw the resulting pictures here or -elsewhere. - -This is an initial adaptation of analyze-rcynic-history.py, which uses -gnuplot and a shelve database. It's also my first attempt to do -anything with rrdtool, so no doubt I'll get half of it wrong. -""" - -import mailbox -import sys -import urlparse -import os -import time - -from xml.etree.cElementTree import (ElementTree as ElementTree, - fromstring as ElementTreeFromString) - -os.putenv("TZ", "UTC") -time.tzset() - -def parse_utc(s): - return int(time.mktime(time.strptime(s, "%Y-%m-%dT%H:%M:%SZ"))) - -class Host(object): - """ - A host object represents all the data collected for one host for a given session. - """ - - def __init__(self, hostname, timestamp): - self.hostname = hostname - self.timestamp = timestamp - self.elapsed = 0 - self.connections = 0 - self.failures = 0 - self.uris = set() - - def add_connection(self, elt): - self.elapsed += parse_utc(elt.get("finished")) - parse_utc(elt.get("started")) - self.connections += 1 - if elt.get("error") is not None: - self.failures += 1 - - def add_object_uri(self, u): - self.uris.add(u) - - @property - def failed(self): - return 1 if self.failures > 0 else 0 - - @property - def objects(self): - return len(self.uris) - - field_table = (("timestamp", None, None, None), - ("connections", "GAUGE", "Connections", "FF0000"), - ("objects", "GAUGE", "Objects", "00FF00"), - ("elapsed", "GAUGE", "Fetch Time", "0000FF"), - ("failed", "ABSOLUTE", "Failed", "00FFFF")) - - @property - def field_values(self): - return tuple(str(getattr(self, field[0])) for field in self.field_table) - - @classmethod - def field_ds_specifiers(cls, heartbeat = 24 * 60 * 60, minimum = 0, maximum = "U"): - return ["DS:%s:%s:%s:%s:%s" % (field[0], field[1], heartbeat, minimum, maximum) - for field in cls.field_table if field[1] is not None] - - @classmethod - def field_graph_specifiers(cls, hostname): - result = [] - for field in cls.field_table: - if field[1] is not None: - result.append("DEF:%s=%s.rrd:%s:AVERAGE" % (field[0], hostname, field[0])) - result.append("'LINE1:%s#%s:%s'" % (field[0], field[3], field[2])) - return result - - def save(self, rrdtable): - rrdtable.add(self.hostname, self.field_values) - -class Session(dict): - """ - A session corresponds to one XML file. This is a dictionary of Host - objects, keyed by hostname. - """ - - def __init__(self, timestamp): - dict.__init__(self) - self.timestamp = timestamp - - @property - def hostnames(self): - return set(self.iterkeys()) - - def add_connection(self, elt): - hostname = urlparse.urlparse(elt.text.strip()).hostname - if hostname not in self: - self[hostname] = Host(hostname, self.timestamp) - self[hostname].add_connection(elt) - - def add_object_uri(self, u): - h = urlparse.urlparse(u).hostname - if h and h in self: - self[h].add_object_uri(u) - - def save(self, rrdtable): - for h in self.itervalues(): - h.save(rrdtable) - -class RRDTable(dict): - """ - Final data we're going to be sending to rrdtool. We need to buffer - it until we're done because we have to sort it. Might be easier - just to sort the maildir, then again it might be easier to get rid - of the maildir too once we're dealing with current data. We'll see. - """ - - def __init__(self, rrdtool = sys.stdout): - dict.__init__(self) - self.rrdtool = rrdtool - - def add(self, hostname, data): - if hostname not in self: - self[hostname] = [] - self[hostname].append(data) - - def sort(self): - for data in self.itervalues(): - data.sort() - - @property - def oldest(self): - return min(min(datum[0] for datum in data) for data in self.itervalues()) - - rras = tuple("RRA:AVERAGE:0.5:%s:9600" % steps for steps in (1, 4, 24)) - - def create(self): - start = self.oldest - ds_list = Host.field_ds_specifiers() - ds_list.extend(self.rras) - for hostname in self: - if not os.path.exists("%s.rrd" % hostname): - self.rrdtool("create %s.rrd --start %s --step 3600 %s\n" % (hostname, start, " ".join(ds_list))) - - def update(self): - for hostname, data in self.iteritems(): - for datum in data: - self.rrdtool("update %s.rrd %s\n" % (hostname, ":".join(str(d) for d in datum))) - - def graph(self): - for hostname in self: - self.rrdtool("graph %s.png --start -90d %s\n" % (hostname, " ".join(Host.field_graph_specifiers(hostname)))) - -mb = mailbox.Maildir("/u/sra/rpki/rcynic-xml", factory = None, create = False) - -rrdtable = RRDTable() - -for i, key in enumerate(mb.iterkeys(), 1): - sys.stderr.write("\r%s %d/%d..." % ("|\\-/"[i & 3], i, len(mb))) - - assert not mb[key].is_multipart() - input = ElementTreeFromString(mb[key].get_payload()) - date = input.get("date") - sys.stderr.write("%s..." % date) - session = Session(parse_utc(date)) - for elt in input.findall("rsync_history"): - session.add_connection(elt) - for elt in input.findall("validation_status"): - if elt.get("generation") == "current": - session.add_object_uri(elt.text.strip()) - session.save(rrdtable) - - # XXX - #if i > 4: break - -sys.stderr.write("\n") - -rrdtable.create() -rrdtable.sort() -rrdtable.update() -rrdtable.graph() diff --git a/scripts/setup-rootd.sh b/scripts/setup-rootd.sh deleted file mode 100644 index 41a271b8..00000000 --- a/scripts/setup-rootd.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh - -# -# $Id$ -# -# Copyright (C) 2010 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -# Setting up rootd requires cross-certifying rpkid's resource-holding -# BPKI trust anchor under the BPKI trust anchor that rootd uses. This -# script handles that, albiet in a very ugly way. -# -# Filenames are wired in, you might need to change these if you've -# done something more complicated. - -export RANDFILE=.OpenSSL.whines.unless.I.set.this -export BPKI_DIRECTORY=`pwd`/bpki/servers - -openssl=../openssl/openssl/apps/openssl - -$openssl ca -notext -batch -config rpki.conf \ - -ss_cert bpki/resources/ca.cer \ - -out $BPKI_DIRECTORY/child.cer \ - -extensions ca_x509_ext_xcert0 - -$openssl x509 -noout -text -in $BPKI_DIRECTORY/child.cer diff --git a/scripts/show-cms-ee-certs.sh b/scripts/show-cms-ee-certs.sh deleted file mode 100644 index 4f5168c7..00000000 --- a/scripts/show-cms-ee-certs.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -# -# $Id$ -# -# Copyright (C) 2010 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -openssl=../openssl/openssl/apps/openssl - -for i in "$@" -do - $openssl cms -verify -noverify -out /dev/null -inform DER -certsout /dev/stdout -in $1 | - $openssl x509 -noout -text -certopt no_sigdump,no_pubkey -done diff --git a/scripts/show-tracking-data.py b/scripts/show-tracking-data.py deleted file mode 100644 index 07e0a144..00000000 --- a/scripts/show-tracking-data.py +++ /dev/null @@ -1,39 +0,0 @@ -# $Id$ -# -# Copyright (C) 2012 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Show tracking data for a bunch of objects retrieved by rcynic. - -This script takes one required argument, which is the name of a -directory tree containing the validated outpt of an rcynic run. If -you follow the default naming scheme this will be -/some/where/rcynic-data/authenticated. -""" - -import os -import sys -import rpki.x509 -import rpki.sundial - -rcynic_dir = sys.argv[1] - -for root, dirs, files in os.walk(rcynic_dir): - for f in files: - path = os.path.join(root, f) - date = rpki.sundial.datetime.utcfromtimestamp(os.stat(path).st_mtime) - uri = "rsync://" + path[len(rcynic_dir):].lstrip("/") - obj = rpki.x509.uri_dispatch(uri)(DER_file = path) - print date, obj.tracking_data(uri) diff --git a/scripts/signed-object-dates.py b/scripts/signed-object-dates.py deleted file mode 100644 index fefd9448..00000000 --- a/scripts/signed-object-dates.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -# $Id$ - -# Extract notBefore, and notAfter values from an RPKI signed object; -# if the object is a manifest, also extract thisUpdate and nextUpdate. - -# Copyright (C) 2013 Dragon Research Labs ("DRL") -# -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -import sys -import rpki.POW - -extract_flags = (rpki.POW.CMS_NOCRL | - rpki.POW.CMS_NO_SIGNER_CERT_VERIFY | - rpki.POW.CMS_NO_ATTR_VERIFY | - rpki.POW.CMS_NO_CONTENT_VERIFY) - -for fn in sys.argv[1:]: - cls = rpki.POW.Manifest if fn.endswith(".mft") else rpki.POW.CMS - cms = cls.derReadFile(fn) - cer = cms.certs()[0] - print fn - print " notBefore: ", cer.getNotBefore() - if fn.endswith(".mft"): - cms.verify(rpki.POW.X509Store(), None, extract_flags) - print " thisUpdate:", cms.getThisUpdate() - print " nextUpdate:", cms.getNextUpdate() - print " notAfter: ", cer.getNotAfter() - print diff --git a/scripts/splitbase64.xsl b/scripts/splitbase64.xsl deleted file mode 100644 index 2b2d2fb1..00000000 --- a/scripts/splitbase64.xsl +++ /dev/null @@ -1,66 +0,0 @@ -<!-- -*- SGML -*- - - $Id$ - - - - Copyright (C) 2009 Internet Systems Consortium ("ISC") - - - - Permission to use, copy, modify, and distribute this software for any - - purpose with or without fee is hereby granted, provided that the above - - copyright notice and this permission notice appear in all copies. - - - - THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH - - REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - - AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, - - INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM - - LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE - - OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR - - PERFORMANCE OF THIS SOFTWARE. - --> - -<!-- - - Split Base64 XML text elements into reasonable length chunks, to - - make the result more readable, allow halfway-sane comparisions of - - XML using diff, etc. Makes no attempt to distinguish Base64 from - - other text, so not suitable for use on XML with text elements that - - are -not- Base64. Piping output of this transform into xmlindent - - produces something halfway readable. YMMV. - --> - -<xsl:transform xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0"> - - <xsl:output method="xml"/> - - <xsl:param name="width" select="64"/> - - <xsl:template match="text()"> - <xsl:text> </xsl:text> - <xsl:call-template name="wrap"> - <xsl:with-param name="input" select="translate(normalize-space(), ' ', '')"/> - </xsl:call-template> - </xsl:template> - - <xsl:template match="node()|@*"> - <xsl:copy> - <xsl:copy-of select="@*"/> - <xsl:apply-templates/> - </xsl:copy> - </xsl:template> - - <xsl:template name="wrap"> - <xsl:param name="input"/> - <xsl:text> </xsl:text> - <xsl:choose> - <xsl:when test="string-length($input) > $width"> - <xsl:value-of select="substring($input, 1, $width)"/> - <xsl:text> </xsl:text> - <xsl:call-template name="wrap"> - <xsl:with-param name="input" select="substring($input, $width+1)"/> - </xsl:call-template> - </xsl:when> - <xsl:otherwise> - <xsl:value-of select="$input"/> - <xsl:text> </xsl:text> - </xsl:otherwise> - </xsl:choose> - </xsl:template> - -</xsl:transform> diff --git a/scripts/testbed-rootcert.py b/scripts/testbed-rootcert.py deleted file mode 100644 index 0716be2f..00000000 --- a/scripts/testbed-rootcert.py +++ /dev/null @@ -1,66 +0,0 @@ -# $Id$ -# -# Copyright (C) 2009-2012 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Generate config for a test RPKI root certificate for resources -specified in asns.csv and prefixes.csv. - -This script is separate from arin-to-csv.py so that we can convert on -the fly rather than having to pull the entire database into memory. -""" - -import sys -from rpki.csv_utils import csv_reader - -if len(sys.argv) not in (2, 4): - sys.exit("Usage: %s holder [asns.csv prefixes.csv]" % sys.argv[0]) - -print '''\ -[req] -default_bits = 2048 -default_md = sha256 -distinguished_name = req_dn -prompt = no -encrypt_key = no - -[req_dn] -CN = Pseudo-%(HOLDER)s testbed root RPKI certificate - -[x509v3_extensions] -basicConstraints = critical,CA:true -subjectKeyIdentifier = hash -keyUsage = critical,keyCertSign,cRLSign -subjectInfoAccess = 1.3.6.1.5.5.7.48.5;URI:rsync://%(holder)s.rpki.net/rpki/%(holder)s/,1.3.6.1.5.5.7.48.10;URI:rsync://%(holder)s.rpki.net/rpki/%(holder)s/root.mft -certificatePolicies = critical,1.3.6.1.5.5.7.14.2 -sbgp-autonomousSysNum = critical,@rfc3779_asns -sbgp-ipAddrBlock = critical,@rfc3997_addrs - -[rfc3779_asns] -''' % { "holder" : sys.argv[1].lower(), - "HOLDER" : sys.argv[1].upper() } - -for i, asn in enumerate(asn for handle, asn in csv_reader(sys.argv[2] if len(sys.argv) > 2 else "asns.csv", columns = 2)): - print "AS.%d = %s" % (i, asn) - -print '''\ - -[rfc3997_addrs] - -''' - -for i, prefix in enumerate(prefix for handle, prefix in csv_reader(sys.argv[3] if len(sys.argv) > 2 else "prefixes.csv", columns = 2)): - v = 6 if ":" in prefix else 4 - print "IPv%d.%d = %s" % (v, i, prefix) diff --git a/scripts/translate-handles.py b/scripts/translate-handles.py deleted file mode 100644 index 49848277..00000000 --- a/scripts/translate-handles.py +++ /dev/null @@ -1,49 +0,0 @@ -# $Id$ -# -# Copyright (C) 2010-2012 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Translate handles from the ones provided in a database dump into the -ones we use in our testbed. This has been broken out into a separate -program for two reasons: - -- Conversion of some of the RIR data is a very slow process, and it's - both annoying and unnecessary to run it every time we add a new - participant to the testbed. - -- This handle translation business now has fingers into half a dozen - scripts, so it needs refactoring in any case, either as a common - library function or as a separate script. - -This program takes a list of .CSV files on its command line, and -rewrites them as needed after performing the translation. -""" - -import os -import sys -from rpki.csv_utils import csv_reader, csv_writer - -translations = dict((src, dst) for src, dst in csv_reader("translations.csv", columns = 2)) - -for filename in sys.argv[1:]: - - f = csv_writer(filename) - - for cols in csv_reader(filename): - if cols[0] in translations: - cols[0] = translations[cols[0]] - f.writerow(cols) - - f.close() diff --git a/scripts/upgrade-add-ghostbusters.py b/scripts/upgrade-add-ghostbusters.py deleted file mode 100644 index a8c8a92b..00000000 --- a/scripts/upgrade-add-ghostbusters.py +++ /dev/null @@ -1,73 +0,0 @@ -# $Id$ -# -# Copyright (C) 2009--2011 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Add SQL tables needed for Ghostbusters support. -Most of the code here lifted from rpki-sql-setup.py -""" - -import getopt, sys, rpki.config, warnings - -from rpki.mysql_import import MySQLdb - -def fix(name, *statements): - db = MySQLdb.connect(db = cfg.get("sql-database", section = name), - user = cfg.get("sql-username", section = name), - passwd = cfg.get("sql-password", section = name)) - cur = db.cursor() - for statement in statements: - cur.execute(statement) - db.commit() - db.close() - -cfg_file = None - -opts, argv = getopt.getopt(sys.argv[1:], "c:h?", ["config=", "help"]) -for o, a in opts: - if o in ("-h", "--help", "-?"): - print __doc__ - sys.exit(0) - if o in ("-c", "--config"): - cfg_file = a - -cfg = rpki.config.parser(cfg_file, "myrpki") - -fix("irdbd", """ - CREATE TABLE ghostbuster_request ( - ghostbuster_request_id SERIAL NOT NULL, - self_handle VARCHAR(40) NOT NULL, - parent_handle VARCHAR(40), - vcard LONGBLOB NOT NULL, - PRIMARY KEY (ghostbuster_request_id) - ) ENGINE=InnoDB; -""") - -fix("rpkid", """ - CREATE TABLE ghostbuster ( - ghostbuster_id SERIAL NOT NULL, - vcard LONGBLOB NOT NULL, - cert LONGBLOB NOT NULL, - ghostbuster LONGBLOB NOT NULL, - published DATETIME, - self_id BIGINT UNSIGNED NOT NULL, - ca_detail_id BIGINT UNSIGNED NOT NULL, - PRIMARY KEY (ghostbuster_id), - CONSTRAINT ghostbuster_self_id - FOREIGN KEY (self_id) REFERENCES self (self_id) ON DELETE CASCADE, - CONSTRAINT ghostbuster_ca_detail_id - FOREIGN KEY (ca_detail_id) REFERENCES ca_detail (ca_detail_id) ON DELETE CASCADE - ) ENGINE=InnoDB; -""") diff --git a/scripts/verify-bpki.sh b/scripts/verify-bpki.sh deleted file mode 100755 index 0e36d796..00000000 --- a/scripts/verify-bpki.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh - -# $Id$ -# -# Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -# Tests of generated BPKI certificates. Kind of cheesy, but does test -# the basic stuff. - -exec 2>&1 - -for bpki in bpki/* -do - crls=$(find $bpki -name '*.crl') - - # Check that CRLs verify properly - for crl in $crls - do - echo -n "$crl: " - openssl crl -CAfile $bpki/ca.cer -noout -in $crl - done - - # Check that issued certificates verify properly - cat $bpki/ca.cer $crls | openssl verify -crl_check -CAfile /dev/stdin $(find $bpki -name '*.cer' ! -name 'ca.cer' ! -name '*.cacert.cer') - -done - -# Check that cross-certified BSC certificates verify properly -if test -d bpki/servers -then - cat bpki/servers/xcert.*.cer | openssl verify -verbose -CAfile bpki/servers/ca.cer -untrusted /dev/stdin bpki/resources/bsc.*.cer -fi diff --git a/scripts/whack-ripe-asns.py b/scripts/whack-ripe-asns.py deleted file mode 100644 index 9c702271..00000000 --- a/scripts/whack-ripe-asns.py +++ /dev/null @@ -1,83 +0,0 @@ -# $Id$ -# -# Copyright (C) 2010 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Fix problems in asns.csv generated from RIPE's database. - -RIPE's database contains inconsistancies, overlaps, and format errors -that make it impossible to feed the output of ripe-to-csv.awk directly -into testbed-rootcert.py without OpenSSL rejecting the resulting -root.conf. This script takes a brute force approach to fixing this: -it converts all ASNs to range form, runs the resulting file through -the unix sort program to put the data into canonical order, then reads -it back, merging overlaps, and writing the result in a form acceptable -to testbed-rootcert.py. - -Since we're doing all this anyway, the script also merges adjacent -blocks. - -Ordinarily, it would be dangerous to have the same program act as both -the source and sink of a pipeline, particularly for such a large data -set, as the chance of deadlock would approach 100%, but in this case -we know that the sort program must consume and buffer (somehow) all of -its input before writing a single line of output, so a single script -can safely act as a filter both before and after sort. -""" - -import sys, subprocess - -sorter = subprocess.Popen(("sort", "-T.", "-n"), - stdin = subprocess.PIPE, - stdout = subprocess.PIPE) - -for line in sys.stdin: - handle, asn = line.split() - - if "-" in asn: - range_min, range_max = asn.split("-") - else: - range_min, range_max = asn, asn - - sorter.stdin.write("%d %d\n" % (long(range_min), long(range_max))) - -sorter.stdin.close() - -prev_min = None -prev_max = None - -def show(): - if prev_min and prev_max: - sys.stdout.write("x\t%s-%s\n" % (prev_min, prev_max)) - -for line in sorter.stdout: - this_min, this_max = line.split() - this_min = long(this_min) - this_max = long(this_max) - - if prev_min and prev_max and prev_max + 1 >= this_min: - prev_min = min(prev_min, this_min) - prev_max = max(prev_max, this_max) - - else: - show() - prev_min = this_min - prev_max = this_max - -show() - -sorter.stdout.close() - -sys.exit(sorter.wait()) diff --git a/scripts/whack-ripe-prefixes.py b/scripts/whack-ripe-prefixes.py deleted file mode 100644 index 52ea3f18..00000000 --- a/scripts/whack-ripe-prefixes.py +++ /dev/null @@ -1,101 +0,0 @@ -# $Id$ -# -# Copyright (C) 2010 Internet Systems Consortium ("ISC") -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THIS SOFTWARE. - -""" -Fix problems in prefixes.csv generated from RIPE's database. - -RIPE's database contains inconsistancies, overlaps, and format errors -that make it impossible to feed the output of ripe-to-csv.awk directly -into testbed-rootcert.py without OpenSSL rejecting the resulting -root.conf. This script takes a brute force approach to fixing this: -it converts all prefixes and address ranges into pairs of unsigned -decimal integers representing range min and range max, runs the -resulting 3+ million entry file through the unix sort program to put -the data into canonical order, then reads it back, merging overlaps -and converting everything back to ranges of IP addresses, and writing -the result in a form acceptable to testbed-rootcert.py. - -Since we're doing all this anyway, the script also merges adjacent -address blocks, not because OpenSSL can't handle them (it can) but -because doing so cuts out a lot of unnecessary I/O. - -Ordinarily, it would be dangerous to have the same program act as both -the source and sink of a pipeline, particularly for such a large data -set, as the chance of deadlock would approach 100%, but in this case -we know that the sort program must consume and buffer (somehow) all of -its input before writing a single line of output, so a single script -can safely act as a filter both before and after sort. -""" - -import sys -import subprocess -import rpki.ipaddrs - -sorter = subprocess.Popen(("sort", "-T.", "-n"), - stdin = subprocess.PIPE, - stdout = subprocess.PIPE) - -for line in sys.stdin: - handle, prefix = line.split() - - if "-" in prefix: - range_min, range_max = prefix.split("-") - range_min = rpki.ipaddrs.parse(range_min) - range_max = rpki.ipaddrs.parse(range_max) - - else: - address, length = prefix.split("/") - address = rpki.ipaddrs.parse(address) - mask = (1L << (address.bits - int(length))) - 1 - range_min = address & ~mask - range_max = address | mask - - sorter.stdin.write("%d %d\n" % (long(range_min), long(range_max))) - -sorter.stdin.close() - -prev_min = None -prev_max = None - -def address(number): - if number > 0xffffffff: - return rpki.ipaddrs.v6addr(number) - else: - return rpki.ipaddrs.v4addr(number) - -def show(): - if prev_min and prev_max: - sys.stdout.write("x\t%s-%s\n" % (address(prev_min), address(prev_max))) - -for line in sorter.stdout: - this_min, this_max = line.split() - this_min = long(this_min) - this_max = long(this_max) - - if prev_min and prev_max and prev_max + 1 >= this_min: - prev_min = min(prev_min, this_min) - prev_max = max(prev_max, this_max) - - else: - show() - prev_min = this_min - prev_max = this_max - -show() - -sorter.stdout.close() - -sys.exit(sorter.wait()) diff --git a/scripts/x509-dot.py b/scripts/x509-dot.py deleted file mode 100644 index 42e1543a..00000000 --- a/scripts/x509-dot.py +++ /dev/null @@ -1,170 +0,0 @@ -# $Id$ - -""" -Generate .dot description of a certificate tree. - -Copyright (C) 2009-2012 Internet Systems Consortium ("ISC") - -Permission to use, copy, modify, and distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH -REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, -INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THIS SOFTWARE. - -Portions copyright (C) 2008 American Registry for Internet Numbers ("ARIN") - -Permission to use, copy, modify, and distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH -REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, -INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THIS SOFTWARE. -""" - -import rpki.POW, sys, glob, os - -class x509(object): - - ski = None - aki = None - - show_file = False - show_ski = False - show_aki = False - show_issuer = True - show_subject = True - - cn_only = True - - subjects = {} - - def __init__(self, filename): - - while filename.startswith("./"): - filename = filename[2:] - - self.filename = filename - - f = open(filename, "rb") - text = f.read() - f.close() - - if "-----BEGIN" in text: - self.pow = rpki.POW.X509.pemRead(text) - else: - self.pow = rpki.POW.X509.derRead(text) - - - try: - self.ski = ":".join(["%02X" % ord(i) for i in self.pow.getSKI()]) - except: - pass - - try: - self.aki = ":".join(["%02X" % ord(i) for i in self.pow.getAKI()]) - except: - pass - - self.subject = self.canonize(self.pow.getSubject()) - self.issuer = self.canonize(self.pow.getIssuer()) - - if self.subject in self.subjects: - self.subjects[self.subject].append(self) - else: - self.subjects[self.subject] = [self] - - def canonize(self, name): - - # Probably should just use rpki.x509.X501DN class here. - - try: - if self.cn_only and name[0][0][0] == "2.5.4.3": - return name[0][0][1] - except: - pass - - return name - - def set_node(self, node): - - self.node = node - - def dot(self): - - label = [] - - if self.show_issuer: - label.append(("Issuer", self.issuer)) - - if self.show_subject: - label.append(("Subject", self.subject)) - - if self.show_file: - label.append(("File", self.filename)) - - if self.show_aki: - label.append(("AKI", self.aki)) - - if self.show_ski: - label.append(("SKI", self.ski)) - - print "#", repr(label) - - if len(label) > 1: - print '%s [shape = record, label = "{%s}"];' % (self.node, "|".join("{%s|%s}" % (x, y) for x, y in label if y is not None)) - else: - print '%s [label = "%s"];' % (self.node, label[0][1]) - - for issuer in self.subjects.get(self.issuer, ()): - - if issuer is self: - print "# Issuer is self" - issuer = None - - if issuer is not None and self.aki is not None and self.ski is not None and self.aki == self.ski: - print "# Self-signed" - issuer = None - - if issuer is not None and self.aki is not None and issuer.ski is not None and self.aki != issuer.ski: - print "# AKI does not match issuer SKI" - issuer = None - - if issuer is not None: - print "%s -> %s;" % (issuer.node, self.node) - - print - -certs = [] - -for topdir in sys.argv[1:] or ["."]: - for dirpath, dirnames, filenames in os.walk(topdir): - certs += [x509(dirpath + "/" + filename) for filename in filenames if filename.endswith(".cer")] - -for i, cert in enumerate(certs): - cert.set_node("cert_%d" % i) - -print """\ -digraph certificates { - -rotate = 90; -#size = "11,8.5"; -splines = true; -ratio = fill; - -""" - -for cert in certs: - cert.dot() - -print "}" |