diff options
Diffstat (limited to 'rpkid/portal-gui/scripts/rpkigui-import-routes.py')
| -rw-r--r-- | rpkid/portal-gui/scripts/rpkigui-import-routes.py | 211 |
1 files changed, 11 insertions, 200 deletions
diff --git a/rpkid/portal-gui/scripts/rpkigui-import-routes.py b/rpkid/portal-gui/scripts/rpkigui-import-routes.py index 9a3748d3..439274ce 100644 --- a/rpkid/portal-gui/scripts/rpkigui-import-routes.py +++ b/rpkid/portal-gui/scripts/rpkigui-import-routes.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 SPARTA, Inc. a Parsons Company +# Copyright (C) 2012, 2013 SPARTA, Inc. a Parsons Company # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above @@ -14,165 +14,19 @@ __version__ = '$Id$' -import itertools -import _mysql_exceptions import optparse -import os.path -import re -import sys -import struct -import subprocess -import time import logging +from rpki.gui.routeview.util import import_routeviews_dump -from django.db import transaction, connection -from rpki.resource_set import resource_range_ipv4, resource_range_ipv6 -from rpki.exceptions import BadIPResource -import rpki.gui.app.timestamp - -# globals -BGPDUMP = 'bgpdump' -logger = logging.getLogger(__name__) - - -def parse_text(f): - ip_re = re.compile(r'^[0-9a-fA-F:.]+/\d{1,3}$') - last_prefix = None - cursor = connection.cursor() - range_class = resource_range_ipv4 - table = 'routeview_routeorigin' - sql = "INSERT INTO %s_new SET asn=%%s, prefix_min=%%s, prefix_max=%%s" % table - - try: - logger.info('Dropping existing staging table...') - cursor.execute('DROP TABLE IF EXISTS %s_new' % table) - except _mysql_exceptions.Warning: - pass - - logger.info('Creating staging table...') - cursor.execute('CREATE TABLE %(table)s_new LIKE %(table)s' % {'table': table}) - - logger.info('Disabling autocommit...') - cursor.execute('SET autocommit=0') - - logger.info('Adding rows to table...') - for row in itertools.islice(f, 5, None): - cols = row.split() - - # index -1 is i/e/? for igp/egp - origin_as = cols[-2] - # FIXME: skip AS_SETs - if origin_as[0] == '{': - continue - - prefix = cols[1] - - # validate the prefix since the "sh ip bgp" output is sometimes - # corrupt by no space between the prefix and the next hop IP - # address. - net, bits = prefix.split('/') - if len(bits) > 2: - s = ['mask for %s looks fishy...' % prefix] - prefix = '%s/%s' % (net, bits[0:2]) - s.append('assuming it should be %s' % prefix) - logger.warning(' '.join(s)) - - # the output may contain multiple paths to the same origin. - # if this is the same prefix as the last entry, we don't need - # to validate it again. - # - # prefixes are sorted, but the origin_as is not, so we keep a set to - # avoid duplicates, and insert into the db once we've seen all the - # origin_as values for a given prefix - if prefix != last_prefix: - # output routes for previous prefix - if last_prefix is not None: - try: - rng = range_class.parse_str(last_prefix) - rmin = long(rng.min) - rmax = long(rng.max) - cursor.executemany(sql, [(asn, rmin, rmax) for asn in asns]) - except BadIPResource: - logger.warning('skipping bad prefix: ' + last_prefix) - - asns = set() - last_prefix = prefix - - asns.add(int(origin_as)) - - logger.info('Committing...') - cursor.execute('COMMIT') - - try: - logger.info('Dropping old table...') - cursor.execute('DROP TABLE IF EXISTS %s_old' % table) - except _mysql_exceptions.Warning: - pass - - logger.info('Swapping staging table with live table...') - cursor.execute('RENAME TABLE %(table)s TO %(table)s_old, %(table)s_new TO %(table)s' % {'table': table}) - - transaction.commit_unless_managed() - - logger.info('Updating timestamp metadata...') - rpki.gui.app.timestamp.update('bgp_v4_import') - - -def parse_mrt(f): - # filter input through bgpdump - pipe = subprocess.Popen([BGPDUMP, '-m', '-v', '-'], stdin=f, - stdout=subprocess.PIPE) - - last_prefix = None - last_as = None - for e in pipe.stdout.readlines(): - a = e.split('|') - prefix = a[5] - try: - origin_as = int(a[6].split()[-1]) - except ValueError: - # skip AS_SETs - continue - - if prefix != last_prefix: - last_prefix = prefix - elif last_as == origin_as: - continue - last_as = origin_as - - asns = PREFIXES.get(prefix) - if not asns: - asns = set() - PREFIXES[prefix] = asns - asns.add(origin_as) - - pipe.wait() - if pipe.returncode: - raise ProgException('bgpdump exited with code %d' % pipe.returncode) - - -class ProgException(Exception): - pass - - -class BadArgument(ProgException): - pass - - -class UnknownInputType(ProgException): - pass - - -class PipeFailed(ProgException): +class BadArgument(Exception): pass if __name__ == '__main__': - start_time = time.time() - - parser = optparse.OptionParser(usage='%prog [options] PATH', - description="""This tool is used to import the IPv4/6 BGP table dumps + parser = optparse.OptionParser( + usage='%prog [options] PATH', + description="""This tool is used to import the IPv4/6 BGP table dumps from routeviews.org into the RPKI Web Portal database. If the input file is a bzip2 compressed file, it will be decompressed automatically.""") @@ -188,9 +42,8 @@ automatically.""") options, args = parser.parse_args() v = getattr(logging, options.log_level.upper()) - logger.setLevel(v) - logging.basicConfig() - logger.info('logging level set to ' + logging.getLevelName(v)) + logging.basicConfig(level=v) + logging.info('logging level set to ' + logging.getLevelName(v)) if options.bgpdump: BGPDUMP = os.path.expanduser(options.bgpdump) @@ -199,52 +52,10 @@ automatically.""") if len(args) != 1: raise BadArgument('no filename specified, or more than one filename specified') filename = args[0] + import_routeviews_dump(filename) - if options.filetype == 'auto': - # try to determine input type from filename, based on the default - # filenames from archive.routeviews.org - bname = os.path.basename(filename) - if bname.startswith('oix-full-snapshot-latest'): - filetype = 'text' - elif bname.startswith('rib.'): - filetype = 'mrt' - else: - raise UnknownInputType('unable to automatically determine input file type') - logging.info('Detected import format as "%s"' % filetype) - else: - filetype = options.filetype - - pipe = None - if filename.endswith('.bz2'): - bunzip = 'bunzip2' if not options.bunzip else os.path.expanduser(options.bunzip) - logging.info('Decompressing input file on the fly...') - pipe = subprocess.Popen([bunzip, '--stdout', filename], - stdout=subprocess.PIPE) - input_file = pipe.stdout - else: - input_file = open(filename) - - try: - dispatch = {'text': parse_text, 'mrt': parse_mrt} - dispatch[filetype](input_file) - except KeyError: - raise UnknownInputType('"%s" is an unknown input file type' % filetype) - - if pipe: - logging.debug('Waiting for child to exit...') - pipe.wait() - if pipe.returncode: - raise PipeFailed('Child exited code %d' % pipe.returncode) - pipe = None - else: - input_file.close() - - logger.info('Elapsed time %d secs' % (time.time() - start_time)) - rc = 0 - - except ProgException, e: - logger.exception(e) + except Exception as e: + logging.exception(e) rc = 1 logging.shutdown() - sys.exit(rc) |