Make rpkigui-import-routes use logging settings from /etc/rpki.conf instead of rpki.django_settings.gui

Create new rpki.django_settings.gui_script Django settings file with a minimal subset required to use the ORM, to be used in auxillary scripts

Add [rpkigui-import-routes] section to rpki.conf

Change log level of warning about AS value errors in routeviews dumps to DEBUG


svn path=/branches/tk705/; revision=6439

3 files changed, 58 insertions, 8 deletions

diff --git a/rpki/django_settings/gui_script.py b/rpki/django_settings/gui_script.py
new file mode 100644
index 00000000..4aa18e2c
--- /dev/null
+++ b/rpki/django_settings/gui_script.py
@@ -0,0 +1,47 @@
+# $Id: gui.py 6427 2016-05-07 04:14:02Z sra $
+
+# Copyright (C) 2015-2016  Parsons Government Services ("PARSONS")
+# Portions copyright (C) 2014  Dragon Research Labs ("DRL")
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notices and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND PARSONS AND DRL DISCLAIM ALL
+# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS.  IN NO EVENT SHALL
+# PARSONS OR DRL BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""
+This module contains settings for the GUI's auxillary scripts which just need
+enough configuration to use the ORM.
+"""
+
+# Pull in the irdb configuration, which in turn pulls in the common configuration.
+
+from .irdb import *                     # pylint: disable=W0401,W0614
+
+__version__ = "$Id: gui.py 6427 2016-05-07 04:14:02Z sra $"
+
+INSTALLED_APPS.extend((
+    #"django.contrib.contenttypes", # not sure if required for aux scripts?
+    "rpki.gui.app",
+    "rpki.gui.gui_rpki_cache",
+    "rpki.gui.routeview",
+    "rpki.rcynicdb"
+))
+
+LOGGING_CONFIG = None # we handle logging configuration ourself via rpki.conf
+
+# Allow local site to override any setting above -- but if there's
+# anything that local sites routinely need to modify, please consider
+# putting that configuration into rpki.conf and just adding code here
+# to read that configuration.
+try:
+    from local_settings import *        # pylint: disable=W0401,F0401
+except ImportError:
+    pass
diff --git a/rpki/gui/routeview/util.py b/rpki/gui/routeview/util.py
index 14ac3cf9..1e2a9906 100644
--- a/rpki/gui/routeview/util.py
+++ b/rpki/gui/routeview/util.py
@@ -25,7 +25,6 @@ import bz2
 from urllib import urlretrieve, unquote
 
 from django.db import transaction
-from django.conf import settings
 
 from rpki.resource_set import resource_range_ipv4, resource_range_ipv6
 from rpki.exceptions import BadIPResource
@@ -35,9 +34,6 @@ from rpki.gui.routeview.models import RouteOrigin
 # globals
 logger = logging.getLogger(__name__)
 
-# Eventually this can be retrived from rpki.conf
-DEFAULT_URL = 'http://archive.routeviews.org/oix-route-views/oix-full-snapshot-latest.dat.bz2'
-
 class ParseError(Exception): pass
 
 class RouteDumpParser(object):
@@ -63,6 +59,9 @@ class RouteDumpParser(object):
                 logger.warning('error while parsing line: {} ({})'.format(line, str(e)))
                 continue
 
+	    if prefix is None: # used when encountering AS sets that we skip over
+		continue
+
             # the output may contain multiple paths to the same origin.
             # if this is the same prefix as the last entry, we don't need
             # to validate it again.
@@ -121,6 +120,9 @@ class TextDumpParser(RouteDumpParser):
 
         # index -1 is i/e/? for igp/egp
         try:
+	    if cols[-2][0] == '{' and cols[-2][-1] == '}':
+		# skip AS sets
+		return None, None
             origin_as = int(cols[-2])
         except IndexError:
             raise ParseError('unexpected format')
@@ -129,7 +131,8 @@ class TextDumpParser(RouteDumpParser):
 
         # FIXME Django doesn't have a field for positive integers up to 2^32-1
 	if origin_as < 0 or origin_as > 2147483647:
-	    raise ParseError('AS value out of supported database range')
+            logger.debug('AS value out of range: %d', origin_as)
+            return None, None
 
         prefix = cols[1]
 
@@ -186,7 +189,7 @@ class PipeFailed(ProgException):
     pass
 
 
-def import_routeviews_dump(filename=DEFAULT_URL, filetype='text'):
+def import_routeviews_dump(filename, filetype='text', download_dir='/var/tmp'):
     """Load the oix-full-snapshot-latest.bz2 from routeview.org into the
     rpki.gui.routeview database.
 
@@ -205,7 +208,7 @@ def import_routeviews_dump(filename=DEFAULT_URL, filetype='text'):
             #get filename from the basename of the URL
             u = urlparse.urlparse(filename)
             bname = os.path.basename(unquote(u.path))
-            tmpname = os.path.join(settings.DOWNLOAD_DIRECTORY, bname)
+            tmpname = os.path.join(download_dir, bname)
 
             logger.info("Downloading %s to %s", filename, tmpname)
             if os.path.exists(tmpname):
diff --git a/rpki/gui/script_util.py b/rpki/gui/script_util.py
index 289dbbb7..d0159840 100644
--- a/rpki/gui/script_util.py
+++ b/rpki/gui/script_util.py
@@ -26,7 +26,7 @@ def setup():
 
     import os
 
-    os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings.gui")
+    os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings.gui_script")
 
     # Initialize Django.
     import django