diff options
Diffstat (limited to 'rpkid/portal-gui/scripts/rpkigui-import-routes.py')
| -rw-r--r-- | rpkid/portal-gui/scripts/rpkigui-import-routes.py | 250 |
1 files changed, 250 insertions, 0 deletions
diff --git a/rpkid/portal-gui/scripts/rpkigui-import-routes.py b/rpkid/portal-gui/scripts/rpkigui-import-routes.py new file mode 100644 index 00000000..9a3748d3 --- /dev/null +++ b/rpkid/portal-gui/scripts/rpkigui-import-routes.py @@ -0,0 +1,250 @@ +# Copyright (C) 2012 SPARTA, Inc. a Parsons Company +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND SPARTA DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL SPARTA BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +__version__ = '$Id$' + +import itertools +import _mysql_exceptions +import optparse +import os.path +import re +import sys +import struct +import subprocess +import time +import logging + +from django.db import transaction, connection + +from rpki.resource_set import resource_range_ipv4, resource_range_ipv6 +from rpki.exceptions import BadIPResource +import rpki.gui.app.timestamp + +# globals +BGPDUMP = 'bgpdump' +logger = logging.getLogger(__name__) + + +def parse_text(f): + ip_re = re.compile(r'^[0-9a-fA-F:.]+/\d{1,3}$') + last_prefix = None + cursor = connection.cursor() + range_class = resource_range_ipv4 + table = 'routeview_routeorigin' + sql = "INSERT INTO %s_new SET asn=%%s, prefix_min=%%s, prefix_max=%%s" % table + + try: + logger.info('Dropping existing staging table...') + cursor.execute('DROP TABLE IF EXISTS %s_new' % table) + except _mysql_exceptions.Warning: + pass + + logger.info('Creating staging table...') + cursor.execute('CREATE TABLE %(table)s_new LIKE %(table)s' % {'table': table}) + + logger.info('Disabling autocommit...') + cursor.execute('SET autocommit=0') + + logger.info('Adding rows to table...') + for row in itertools.islice(f, 5, None): + cols = row.split() + + # index -1 is i/e/? for igp/egp + origin_as = cols[-2] + # FIXME: skip AS_SETs + if origin_as[0] == '{': + continue + + prefix = cols[1] + + # validate the prefix since the "sh ip bgp" output is sometimes + # corrupt by no space between the prefix and the next hop IP + # address. + net, bits = prefix.split('/') + if len(bits) > 2: + s = ['mask for %s looks fishy...' % prefix] + prefix = '%s/%s' % (net, bits[0:2]) + s.append('assuming it should be %s' % prefix) + logger.warning(' '.join(s)) + + # the output may contain multiple paths to the same origin. + # if this is the same prefix as the last entry, we don't need + # to validate it again. + # + # prefixes are sorted, but the origin_as is not, so we keep a set to + # avoid duplicates, and insert into the db once we've seen all the + # origin_as values for a given prefix + if prefix != last_prefix: + # output routes for previous prefix + if last_prefix is not None: + try: + rng = range_class.parse_str(last_prefix) + rmin = long(rng.min) + rmax = long(rng.max) + cursor.executemany(sql, [(asn, rmin, rmax) for asn in asns]) + except BadIPResource: + logger.warning('skipping bad prefix: ' + last_prefix) + + asns = set() + last_prefix = prefix + + asns.add(int(origin_as)) + + logger.info('Committing...') + cursor.execute('COMMIT') + + try: + logger.info('Dropping old table...') + cursor.execute('DROP TABLE IF EXISTS %s_old' % table) + except _mysql_exceptions.Warning: + pass + + logger.info('Swapping staging table with live table...') + cursor.execute('RENAME TABLE %(table)s TO %(table)s_old, %(table)s_new TO %(table)s' % {'table': table}) + + transaction.commit_unless_managed() + + logger.info('Updating timestamp metadata...') + rpki.gui.app.timestamp.update('bgp_v4_import') + + +def parse_mrt(f): + # filter input through bgpdump + pipe = subprocess.Popen([BGPDUMP, '-m', '-v', '-'], stdin=f, + stdout=subprocess.PIPE) + + last_prefix = None + last_as = None + for e in pipe.stdout.readlines(): + a = e.split('|') + prefix = a[5] + try: + origin_as = int(a[6].split()[-1]) + except ValueError: + # skip AS_SETs + continue + + if prefix != last_prefix: + last_prefix = prefix + elif last_as == origin_as: + continue + last_as = origin_as + + asns = PREFIXES.get(prefix) + if not asns: + asns = set() + PREFIXES[prefix] = asns + asns.add(origin_as) + + pipe.wait() + if pipe.returncode: + raise ProgException('bgpdump exited with code %d' % pipe.returncode) + + +class ProgException(Exception): + pass + + +class BadArgument(ProgException): + pass + + +class UnknownInputType(ProgException): + pass + + +class PipeFailed(ProgException): + pass + + +if __name__ == '__main__': + start_time = time.time() + + parser = optparse.OptionParser(usage='%prog [options] PATH', + description="""This tool is used to import the IPv4/6 BGP table dumps +from routeviews.org into the RPKI Web Portal database. If the +input file is a bzip2 compressed file, it will be decompressed +automatically.""") + parser.add_option('-t', '--type', dest='filetype', metavar='TYPE', + help='Specify the input file type (auto, text, mrt) [Default: %default]') + parser.add_option('-l', '--level', dest='log_level', default='INFO', + help='Set logging level [Default: %default]') + parser.add_option('-u', '--bunzip2', dest='bunzip', metavar='PROG', + help='Specify bunzip2 program to use') + parser.add_option('-b', '--bgpdump', dest='bgpdump', metavar='PROG', + help='Specify path to bgdump binary') + parser.set_defaults(debug=False, verbose=False, filetype='auto') + options, args = parser.parse_args() + + v = getattr(logging, options.log_level.upper()) + logger.setLevel(v) + logging.basicConfig() + logger.info('logging level set to ' + logging.getLevelName(v)) + + if options.bgpdump: + BGPDUMP = os.path.expanduser(options.bgpdump) + + try: + if len(args) != 1: + raise BadArgument('no filename specified, or more than one filename specified') + filename = args[0] + + if options.filetype == 'auto': + # try to determine input type from filename, based on the default + # filenames from archive.routeviews.org + bname = os.path.basename(filename) + if bname.startswith('oix-full-snapshot-latest'): + filetype = 'text' + elif bname.startswith('rib.'): + filetype = 'mrt' + else: + raise UnknownInputType('unable to automatically determine input file type') + logging.info('Detected import format as "%s"' % filetype) + else: + filetype = options.filetype + + pipe = None + if filename.endswith('.bz2'): + bunzip = 'bunzip2' if not options.bunzip else os.path.expanduser(options.bunzip) + logging.info('Decompressing input file on the fly...') + pipe = subprocess.Popen([bunzip, '--stdout', filename], + stdout=subprocess.PIPE) + input_file = pipe.stdout + else: + input_file = open(filename) + + try: + dispatch = {'text': parse_text, 'mrt': parse_mrt} + dispatch[filetype](input_file) + except KeyError: + raise UnknownInputType('"%s" is an unknown input file type' % filetype) + + if pipe: + logging.debug('Waiting for child to exit...') + pipe.wait() + if pipe.returncode: + raise PipeFailed('Child exited code %d' % pipe.returncode) + pipe = None + else: + input_file.close() + + logger.info('Elapsed time %d secs' % (time.time() - start_time)) + rc = 0 + + except ProgException, e: + logger.exception(e) + rc = 1 + + logging.shutdown() + sys.exit(rc) |