diff options
| -rw-r--r-- | myrpki/ripe-to-csv.py | 111 |
1 files changed, 111 insertions, 0 deletions
diff --git a/myrpki/ripe-to-csv.py b/myrpki/ripe-to-csv.py new file mode 100644 index 00000000..e700e669 --- /dev/null +++ b/myrpki/ripe-to-csv.py @@ -0,0 +1,111 @@ +""" +Parse a WHOIS research dump and write out (just) the RPKI-relevant +fields in myrpki-format CSV syntax. + +NB: The input data for this script is publicly available via FTP, but +you'll have to fetch the data from RIPE yourself, and be sure to see +the terms and conditions referenced by the data file header comments. + +$Id$ + +Copyright (C) 2009 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import gzip, csv, myrpki + +class Handle(dict): + + want_tags = () + + debug = True + + def set(self, tag, val): + if tag in self.want_tags: + self[tag] = "".join(val.split(" ")) + + def check(self): + for tag in self.want_tags: + if not tag in self: + return False + if self.debug: + self.log() + return True + + def __repr__(self): + return "<%s %s>" % (self.__class__.__name__, + " ".join("%s:%s" % (tag, self.get(tag, "?")) + for tag in self.want_tags)) + + def log(self): + print repr(self) + + def finish(self, ctx): + self.check() + +class as_block(Handle): + # This one is less useful than I had hoped, no useful links to owners + want_tags = ("as-block", "mnt-by", "org", "mnt-lower") + +class as_set(Handle): + # This is probably useless + want_tags = ("as-set", "mnt-by", "members") + +class aut_num(Handle): + want_tags = ("aut-num", "mnt-by", "as-name") + +class inetnum(Handle): + want_tags = ("inetnum", "mnt-by", "netname") + +class inet6num(Handle): + want_tags = ("inet6num", "mnt-by", "netname") + +class main(object): + + types = dict((x.want_tags[0], x) for x in (as_block, aut_num, as_set, inetnum, inet6num)) + + @staticmethod + def csvout(fn): + return csv.writer(open(fn, "w"), dialect = myrpki.csv_dialect) + + def finish_statement(self, done): + if self.statement: + tag, sep, val = self.statement.partition(":") + assert sep, "Couldn't find separator in %r" % self.statement + tag = tag.strip() + val = val.strip() + if self.cur is None: + self.cur = self.types[tag]() if tag in self.types else False + if self.cur is not False: + self.cur.set(tag, val) + if done and self.cur: + self.cur.finish(self) + self.cur = None + + def __init__(self): + self.asns = self.csvout("asns.csv") + self.prefixes = self.csvout("prefixes.csv") + f = gzip.open("ripe.db.gz") + self.statement = "" + self.cur = None + for line in f: + line = line.expandtabs().partition("#")[0].rstrip("\n") + if line and not line[0].isalpha(): + self.statement += line[1:] if line[0] == "+" else line + else: + self.finish_statement(not line) + self.statement = line + self.finish_statement(True) + +main() |