diff options
| -rwxr-xr-x | ca/rpkigui-import-routes | 92 | ||||
| -rw-r--r-- | rp/config/rpki-confgen.xml | 94 | ||||
| -rw-r--r-- | rpki/django_settings/gui_script.py | 47 | ||||
| -rw-r--r-- | rpki/gui/routeview/util.py | 17 | ||||
| -rw-r--r-- | rpki/gui/script_util.py | 2 |
5 files changed, 192 insertions, 60 deletions
diff --git a/ca/rpkigui-import-routes b/ca/rpkigui-import-routes index fb8e381e..aeecea6e 100755 --- a/ca/rpkigui-import-routes +++ b/ca/rpkigui-import-routes @@ -14,6 +14,11 @@ # OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR # PERFORMANCE OF THIS SOFTWARE. +"""This tool is used to import the IPv4/6 BGP table dumps +from routeviews.org into the RPKI Web Portal database. If the +input file is a bzip2 compressed file, it will be decompressed +automatically.""" + __version__ = '$Id$' import optparse @@ -27,12 +32,15 @@ import fcntl import sys import os +import rpki.config + # configure django ORM from rpki.gui.script_util import setup setup() from rpki.gui.routeview.util import import_routeviews_dump +default_url = 'http://archive.routeviews.org/oix-route-views/oix-full-snapshot-latest.dat.bz2' class BadArgument(Exception): pass @@ -44,67 +52,47 @@ def timed_out(*ignored): if __name__ == '__main__': - parser = optparse.OptionParser( - usage='%prog [options] [PATH]', - description="""This tool is used to import the IPv4/6 BGP table dumps -from routeviews.org into the RPKI Web Portal database. If the -input file is a bzip2 compressed file, it will be decompressed -automatically.""") - parser.add_option('-t', '--type', dest='filetype', metavar='TYPE', - help='Specify the input file type (text, mrt) [Default: %default]') - parser.add_option('-l', '--level', dest='log_level', default='ERROR', - help='Set logging level [Default: %default]') - parser.add_option('-b', '--bgpdump', dest='bgpdump', metavar='PROG', - help='Specify path to bgdump binary') - parser.add_option('-j', '--jitter', dest='jitter', type='int', - help='Specify upper bound of startup delay, in seconds [Default: %default]') - parser.add_option('--lockfile', dest='lockfile', - help='Set name of lock file; empty string disables locking [Default: %default]') - parser.add_option('--timeout', dest='timeout', type='int', - help='Specify timeout for download and import, in seconds [Default: %default]') - parser.set_defaults(debug=False, verbose=False, filetype='text', jitter=0, - lockfile='/tmp/rpkigui-import-routes.lock', timeout=90*60) - options, args = parser.parse_args() - - v = getattr(logging, options.log_level.upper()) - logging.basicConfig(level=v) - atexit.register(logging.shutdown) - logging.info('logging level set to ' + logging.getLevelName(v)) - - if options.bgpdump: - BGPDUMP = os.path.expanduser(options.bgpdump) - - if options.jitter > 0: + myname = 'rpkigui-import-routes' + + cfg = rpki.config.argparser(section=myname, doc=__doc__) + cfg.add_logging_arguments() + args = cfg.argparser.parse_args() + cfg.configure_logging(ident=myname, args=args) + + jitter = cfg.getint('jitter', section=myname, default=600) + if jitter > 0: try: - delay = random.SystemRandom().randint(0, options.jitter) + delay = random.SystemRandom().randint(0, jitter) except NotImplementedError: - delay = random.randint(0, options.jitter) + delay = random.randint(0, jitter) logging.info('jitter active, delaying startup for %d seconds', delay) time.sleep(delay) - if options.lockfile: - try: - lock = os.open(options.lockfile, os.O_RDONLY | os.O_CREAT | os.O_NONBLOCK, 0666) - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - except (IOError, OSError), e: - if e.errno == errno.EAGAIN: - logging.info('lock held by another process') - sys.exit(0) - else: - logging.exception(e) - sys.exit(1) - + lockfile = cfg.get('lockfile', section=myname, default='/tmp/rpkigui-import-routes.lock') try: - if len(args) > 1: - raise BadArgument('more than one filename specified') - - if options.timeout > 0: + lock = os.open(lockfile, os.O_RDONLY | os.O_CREAT | os.O_NONBLOCK, 0666) + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except (IOError, OSError), e: + if e.errno == errno.EAGAIN: + logging.info('lock held by another process') + sys.exit(0) + else: + logging.exception(e) + sys.exit(1) + + filename = cfg.get('filename', section=myname, default=default_url) + filetype = cfg.get('filetype', section=myname, default='text') + download_dir = cfg.get('download-directory', section=myname, default='/var/tmp') + + timeout = cfg.getint('timeout', section=myname, default=5400) + try: + if timeout > 0: signal.signal(signal.SIGALRM, timed_out) - signal.setitimer(signal.ITIMER_REAL, options.timeout) + signal.setitimer(signal.ITIMER_REAL, timeout) - import_routeviews_dump(*args, filetype=options.filetype) + import_routeviews_dump(filename=filename, filetype=filetype, download_dir=download_dir) - if options.timeout > 0: + if timeout > 0: signal.setitimer(signal.ITIMER_REAL, 0) except Exception as e: diff --git a/rp/config/rpki-confgen.xml b/rp/config/rpki-confgen.xml index bc1458ea..a8533815 100644 --- a/rp/config/rpki-confgen.xml +++ b/rp/config/rpki-confgen.xml @@ -1028,6 +1028,100 @@ </section> + <section name = "rpkigui-import-routes"> + + <doc> + Configuration for the rpkigui-import-routes auxillary script. + </doc> + + <option name = "type" + value = "text"> + <doc> + Select the input format. Valid input types are 'text' for 'sh ip bgp' format, and 'mrt' + for MRT. + </doc> + </option> + + <option name = "bgpdump" + value = "bgpdump"> + <doc> + Path to the bgpdump command line utility, used for importing MRT format files. + </doc> + </option> + + <option name = "jitter" + value = "0"> + <doc> + Sleep for a random amount of seconds between 0 and this value before starting the download. + </doc> + </option> + + <option name = "lockfile" + value = "/tmp/rpkigui-import-routes.lock"> + <doc> + Path to use for the lockfile to prevent multiple executions. + </doc> + </option> + + <option name = "timeout" + value = "5400"> + <doc> + How long in seconds to wait for the download to complete before aborting. + </doc> + </option> + + <option name = "filename" + value = "http://archive.routeviews.org/oix-route-views/oix-full-snapshot-latest.dat.bz2"> + <doc> + Specify where to fetch the route dump. May be a filename or URL. + </doc> + </option> + + <option name = "download-directory" + value = "/var/tmp"> + <doc> + Where to store local copy of route dump when 'filename' is a URL. + </doc> + </option> + + <option name = "log-destination" + value = "${myrpki::log-destination}"> + <doc> + Logging mechanism, can be "file", "syslog", "stderr", or "stdout". + </doc> + </option> + + <option name = "log-filename" + value = "${myrpki::log-directory}/rpkgui-import-routes.log"> + <doc> + Where to write log file when logging to a file. + </doc> + </option> + + <option name = "log-level" + value = "error"> + <doc> + Default logging level. + </doc> + </option> + + <option name = "log-time-limit" + value = "${myrpki::log-time-limit}"> + <doc> + Interval between log file rotations, in hours. + Set to zero to disable automatic rotations. + </doc> + </option> + + <option name = "log-count" + value = "${myrpki::log-count}"> + <doc> + How many old logs to keep before deleting. + </doc> + </option> + + </section> + <section name = "autoconf"> <doc> diff --git a/rpki/django_settings/gui_script.py b/rpki/django_settings/gui_script.py new file mode 100644 index 00000000..4aa18e2c --- /dev/null +++ b/rpki/django_settings/gui_script.py @@ -0,0 +1,47 @@ +# $Id: gui.py 6427 2016-05-07 04:14:02Z sra $ + +# Copyright (C) 2015-2016 Parsons Government Services ("PARSONS") +# Portions copyright (C) 2014 Dragon Research Labs ("DRL") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notices and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND PARSONS AND DRL DISCLAIM ALL +# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL +# PARSONS OR DRL BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION +# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +""" +This module contains settings for the GUI's auxillary scripts which just need +enough configuration to use the ORM. +""" + +# Pull in the irdb configuration, which in turn pulls in the common configuration. + +from .irdb import * # pylint: disable=W0401,W0614 + +__version__ = "$Id: gui.py 6427 2016-05-07 04:14:02Z sra $" + +INSTALLED_APPS.extend(( + #"django.contrib.contenttypes", # not sure if required for aux scripts? + "rpki.gui.app", + "rpki.gui.gui_rpki_cache", + "rpki.gui.routeview", + "rpki.rcynicdb" +)) + +LOGGING_CONFIG = None # we handle logging configuration ourself via rpki.conf + +# Allow local site to override any setting above -- but if there's +# anything that local sites routinely need to modify, please consider +# putting that configuration into rpki.conf and just adding code here +# to read that configuration. +try: + from local_settings import * # pylint: disable=W0401,F0401 +except ImportError: + pass diff --git a/rpki/gui/routeview/util.py b/rpki/gui/routeview/util.py index 14ac3cf9..1e2a9906 100644 --- a/rpki/gui/routeview/util.py +++ b/rpki/gui/routeview/util.py @@ -25,7 +25,6 @@ import bz2 from urllib import urlretrieve, unquote from django.db import transaction -from django.conf import settings from rpki.resource_set import resource_range_ipv4, resource_range_ipv6 from rpki.exceptions import BadIPResource @@ -35,9 +34,6 @@ from rpki.gui.routeview.models import RouteOrigin # globals logger = logging.getLogger(__name__) -# Eventually this can be retrived from rpki.conf -DEFAULT_URL = 'http://archive.routeviews.org/oix-route-views/oix-full-snapshot-latest.dat.bz2' - class ParseError(Exception): pass class RouteDumpParser(object): @@ -63,6 +59,9 @@ class RouteDumpParser(object): logger.warning('error while parsing line: {} ({})'.format(line, str(e))) continue + if prefix is None: # used when encountering AS sets that we skip over + continue + # the output may contain multiple paths to the same origin. # if this is the same prefix as the last entry, we don't need # to validate it again. @@ -121,6 +120,9 @@ class TextDumpParser(RouteDumpParser): # index -1 is i/e/? for igp/egp try: + if cols[-2][0] == '{' and cols[-2][-1] == '}': + # skip AS sets + return None, None origin_as = int(cols[-2]) except IndexError: raise ParseError('unexpected format') @@ -129,7 +131,8 @@ class TextDumpParser(RouteDumpParser): # FIXME Django doesn't have a field for positive integers up to 2^32-1 if origin_as < 0 or origin_as > 2147483647: - raise ParseError('AS value out of supported database range') + logger.debug('AS value out of range: %d', origin_as) + return None, None prefix = cols[1] @@ -186,7 +189,7 @@ class PipeFailed(ProgException): pass -def import_routeviews_dump(filename=DEFAULT_URL, filetype='text'): +def import_routeviews_dump(filename, filetype='text', download_dir='/var/tmp'): """Load the oix-full-snapshot-latest.bz2 from routeview.org into the rpki.gui.routeview database. @@ -205,7 +208,7 @@ def import_routeviews_dump(filename=DEFAULT_URL, filetype='text'): #get filename from the basename of the URL u = urlparse.urlparse(filename) bname = os.path.basename(unquote(u.path)) - tmpname = os.path.join(settings.DOWNLOAD_DIRECTORY, bname) + tmpname = os.path.join(download_dir, bname) logger.info("Downloading %s to %s", filename, tmpname) if os.path.exists(tmpname): diff --git a/rpki/gui/script_util.py b/rpki/gui/script_util.py index 289dbbb7..d0159840 100644 --- a/rpki/gui/script_util.py +++ b/rpki/gui/script_util.py @@ -26,7 +26,7 @@ def setup(): import os - os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings.gui") + os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings.gui_script") # Initialize Django. import django |