Source tree reorg, phase 1. Almost everything moved, no file contents changed.

svn path=/branches/tk685/; revision=5757

4 files changed, 386 insertions, 0 deletions

diff --git a/rpki/gui/routeview/__init__.py b/rpki/gui/routeview/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/rpki/gui/routeview/__init__.py
diff --git a/rpki/gui/routeview/api.py b/rpki/gui/routeview/api.py
new file mode 100644
index 00000000..cf699c9a
--- /dev/null
+++ b/rpki/gui/routeview/api.py
@@ -0,0 +1,69 @@
+# Copyright (C) 2012  SPARTA, Inc. a Parsons Company
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND SPARTA DISCLAIMS ALL WARRANTIES WITH
+# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS.  IN NO EVENT SHALL SPARTA BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+
+__version__ = '$Id$'
+
+import json
+from django import http
+from rpki.gui.routeview.models import RouteOrigin, RouteOriginV6
+from rpki import resource_set
+import rpki.exceptions
+
+def route_list(request):
+    """Implements the REST query against the route models to allow the client
+    to search for routes.
+
+    The only search currently supported is returning all the routes covered by
+    the prefix given in the 'prefix__in=' query string parameter.
+
+    By default, only returns up to 10 matching routes, but the client may
+    request a different limit with the 'count=' query string parameter.
+
+    """
+    hard_limit = 100
+
+    if request.method == 'GET' and 'prefix__in' in request.GET:
+        # find all routers covered by this prefix
+        match_prefix = request.GET.get('prefix__in')
+        # max number of items to return
+        limit = request.GET.get('count', 10)
+        if limit < 1 or limit > hard_limit:
+            return http.HttpResponseBadRequest('invalid value for count parameter')
+
+        try:
+            if ':' in match_prefix:
+                # v6
+                pfx = resource_set.resource_range_ipv6.parse_str(match_prefix)
+                manager = RouteOriginV6
+            else:
+                # v4
+                pfx = resource_set.resource_range_ipv4.parse_str(match_prefix)
+                manager = RouteOrigin
+        except (AssertionError, rpki.exceptions.BadIPResource), e:
+            return http.HttpResponseBadRequest(e)
+
+        try:
+            qs = manager.objects.filter(prefix_min__gte=pfx.min,
+                                        prefix_max__lte=pfx.max)[:limit]
+            # FIXME - a REST API should really return the url of the resource,
+            # but since we are combining two separate tables, the .pk is not a
+            # unique identifier.
+            matches = [{'prefix': str(x.as_resource_range()), 'asn': x.asn} for x in qs]
+        except IndexError:
+            # no matches
+            matches = []
+
+        return http.HttpResponse(json.dumps(matches), content_type='text/javascript')
+
+    return http.HttpResponseBadRequest()
diff --git a/rpki/gui/routeview/models.py b/rpki/gui/routeview/models.py
new file mode 100644
index 00000000..052860c4
--- /dev/null
+++ b/rpki/gui/routeview/models.py
@@ -0,0 +1,81 @@
+# Copyright (C) 2010, 2011  SPARTA, Inc. dba Cobham Analytic Solutions
+# Copyright (C) 2012  SPARTA, Inc. a Parsons Company
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND SPARTA DISCLAIMS ALL WARRANTIES WITH
+# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS.  IN NO EVENT SHALL SPARTA BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+
+__version__ = '$Id$'
+
+from django.db.models import PositiveIntegerField, permalink
+import rpki.gui.models
+
+
+class RouteOrigin(rpki.gui.models.PrefixV4):
+    "Represents an IPv4 BGP routing table entry."
+
+    asn = PositiveIntegerField(help_text='origin AS', null=False)
+
+    def __unicode__(self):
+        return u"AS%d's route origin for %s" % (self.asn,
+                                                self.get_prefix_display())
+
+    @property
+    def roas(self):
+        "Return a queryset of ROAs which cover this route."
+        return rpki.gui.cacheview.models.ROA.objects.filter(
+            prefixes__prefix_min__lte=self.prefix_min,
+            prefixes__prefix_max__gte=self.prefix_max
+        )
+
+    @property
+    def roa_prefixes(self):
+        "Return a queryset of ROA prefixes which cover this route."
+        return rpki.gui.cacheview.models.ROAPrefixV4.objects.filter(
+            prefix_min__lte=self.prefix_min,
+            prefix_max__gte=self.prefix_max
+        )
+
+    @property
+    def status(self):
+        "Returns the validation status of this route origin object."
+        roas = self.roas
+        # subselect exact match
+        if self.asn != 0 and roas.filter(asid=self.asn, prefixes__max_length__gte=self.prefixlen).exists():
+            return 'valid'
+        elif roas.exists():
+            return 'invalid'
+        return 'unknown'
+
+    @permalink
+    def get_absolute_url(self):
+        return ('rpki.gui.app.views.route_detail', [str(self.pk)])
+
+    class Meta:
+        # sort by increasing mask length (/16 before /24)
+        ordering = ('prefix_min', '-prefix_max')
+
+
+class RouteOriginV6(rpki.gui.models.PrefixV6):
+    "Represents an IPv6 BGP routing table entry."
+
+    asn = PositiveIntegerField(help_text='origin AS', null=False)
+
+    def __unicode__(self):
+        return u"AS%d's route origin for %s" % (self.asn,
+                                                self.get_prefix_display())
+
+    class Meta:
+        ordering = ('prefix_min', '-prefix_max')
+
+
+# this goes at the end of the file to avoid problems with circular imports
+import rpki.gui.cacheview.models
diff --git a/rpki/gui/routeview/util.py b/rpki/gui/routeview/util.py
new file mode 100644
index 00000000..7884224c
--- /dev/null
+++ b/rpki/gui/routeview/util.py
@@ -0,0 +1,236 @@
+# Copyright (C) 2012, 2013  SPARTA, Inc. a Parsons Company
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND SPARTA DISCLAIMS ALL WARRANTIES WITH
+# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS.  IN NO EVENT SHALL SPARTA BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+
+__version__ = '$Id$'
+__all__ = ('import_routeviews_dump')
+
+import itertools
+import _mysql_exceptions
+import os.path
+import subprocess
+import time
+import logging
+import urlparse
+from urllib import urlretrieve, unquote
+
+from django.db import transaction, connection
+
+from rpki.resource_set import resource_range_ipv4, resource_range_ipv6
+from rpki.exceptions import BadIPResource
+import rpki.gui.app.timestamp
+
+# globals
+logger = logging.getLogger(__name__)
+
+# Eventually this can be retrived from rpki.conf
+DEFAULT_URL = 'http://archive.routeviews.org/oix-route-views/oix-full-snapshot-latest.dat.bz2'
+
+def parse_text(f):
+    last_prefix = None
+    cursor = connection.cursor()
+    range_class = resource_range_ipv4
+    table = 'routeview_routeorigin'
+    sql = "INSERT INTO %s_new SET asn=%%s, prefix_min=%%s, prefix_max=%%s" % table
+
+    try:
+        logger.info('Dropping existing staging table...')
+        cursor.execute('DROP TABLE IF EXISTS %s_new' % table)
+    except _mysql_exceptions.Warning:
+        pass
+
+    logger.info('Creating staging table...')
+    cursor.execute('CREATE TABLE %(table)s_new LIKE %(table)s' % {'table': table})
+
+    logger.info('Disabling autocommit...')
+    cursor.execute('SET autocommit=0')
+
+    logger.info('Adding rows to table...')
+    for row in itertools.islice(f, 5, None):
+        cols = row.split()
+
+        # index -1 is i/e/? for igp/egp
+        origin_as = cols[-2]
+        # FIXME: skip AS_SETs
+        if origin_as[0] == '{':
+            continue
+
+        prefix = cols[1]
+
+        # validate the prefix since the "sh ip bgp" output is sometimes
+        # corrupt by no space between the prefix and the next hop IP
+        # address.
+        net, bits = prefix.split('/')
+        if len(bits) > 2:
+            s = ['mask for %s looks fishy...' % prefix]
+            prefix = '%s/%s' % (net, bits[0:2])
+            s.append('assuming it should be %s' % prefix)
+            logger.warning(' '.join(s))
+
+        # the output may contain multiple paths to the same origin.
+        # if this is the same prefix as the last entry, we don't need
+        # to validate it again.
+        #
+        # prefixes are sorted, but the origin_as is not, so we keep a set to
+        # avoid duplicates, and insert into the db once we've seen all the
+        # origin_as values for a given prefix
+        if prefix != last_prefix:
+            # output routes for previous prefix
+            if last_prefix is not None:
+                try:
+                    rng = range_class.parse_str(last_prefix)
+                    rmin = long(rng.min)
+                    rmax = long(rng.max)
+                    cursor.executemany(sql, [(asn, rmin, rmax) for asn in asns])
+                except BadIPResource:
+                    logger.warning('skipping bad prefix: ' + last_prefix)
+
+            asns = set()
+            last_prefix = prefix
+
+        try:
+            asns.add(int(origin_as))
+        except ValueError as err:
+            logger.warning('\n'.join(
+                ['unable to parse origin AS: ' + origin_as],
+                ['ValueError: ' + str(err)]
+                ['route entry was: ' + row],
+            ))
+
+    logger.info('Committing...')
+    cursor.execute('COMMIT')
+
+    try:
+        logger.info('Dropping old table...')
+        cursor.execute('DROP TABLE IF EXISTS %s_old' % table)
+    except _mysql_exceptions.Warning:
+        pass
+
+    logger.info('Swapping staging table with live table...')
+    cursor.execute('RENAME TABLE %(table)s TO %(table)s_old, %(table)s_new TO %(table)s' % {'table': table})
+
+    transaction.commit_unless_managed()
+
+    logger.info('Updating timestamp metadata...')
+    rpki.gui.app.timestamp.update('bgp_v4_import')
+
+
+def parse_mrt(f):
+    # filter input through bgpdump
+    pipe = subprocess.Popen(['bgpdump', '-m', '-v', '-'], stdin=f,
+                            stdout=subprocess.PIPE)
+
+    last_prefix = None
+    last_as = None
+    for e in pipe.stdout.readlines():
+        a = e.split('|')
+        prefix = a[5]
+        try:
+            origin_as = int(a[6].split()[-1])
+        except ValueError:
+            # skip AS_SETs
+            continue
+
+        if prefix != last_prefix:
+            last_prefix = prefix
+        elif last_as == origin_as:
+            continue
+        last_as = origin_as
+
+        asns = PREFIXES.get(prefix)
+        if not asns:
+            asns = set()
+            PREFIXES[prefix] = asns
+        asns.add(origin_as)
+
+    pipe.wait()
+    if pipe.returncode:
+        raise ProgException('bgpdump exited with code %d' % pipe.returncode)
+
+
+class ProgException(Exception):
+    pass
+
+
+class UnknownInputType(ProgException):
+    pass
+
+
+class PipeFailed(ProgException):
+    pass
+
+
+def import_routeviews_dump(filename=DEFAULT_URL, filetype='auto'):
+    """Load the oix-full-snapshot-latest.bz2 from routeview.org into the
+    rpki.gui.routeview database.
+
+    Arguments:
+
+        filename [optional]: the full path to the downloaded file to parse
+
+        filetype [optional]: 'text' or 'mrt'
+
+    """
+    start_time = time.time()
+
+    if filename.startswith('http://'):
+        #get filename from the basename of the URL
+        u = urlparse.urlparse(filename)
+        bname = os.path.basename(unquote(u.path))
+        tmpname = os.path.join('/tmp', bname)
+
+        logger.info("Downloading %s to %s" % (filename, tmpname))
+        if os.path.exists(tmpname):
+            os.remove(tmpname)
+        # filename is replaced with a local filename containing cached copy of
+        # URL
+        filename, headers = urlretrieve(filename, tmpname)
+
+    if filetype == 'auto':
+        # try to determine input type from filename, based on the default
+        # filenames from archive.routeviews.org
+        bname = os.path.basename(filename)
+        if bname.startswith('oix-full-snapshot-latest'):
+            filetype = 'text'
+        elif bname.startswith('rib.'):
+            filetype = 'mrt'
+        else:
+            raise UnknownInputType('unable to automatically determine input file type')
+        logging.info('Detected import format as "%s"' % filetype)
+
+    pipe = None
+    if filename.endswith('.bz2'):
+        bunzip = 'bunzip2'
+        logging.info('Decompressing input file on the fly...')
+        pipe = subprocess.Popen([bunzip, '--stdout', filename],
+                                stdout=subprocess.PIPE)
+        input_file = pipe.stdout
+    else:
+        input_file = open(filename)
+
+    try:
+        dispatch = {'text': parse_text, 'mrt': parse_mrt}
+        dispatch[filetype](input_file)
+    except KeyError:
+        raise UnknownInputType('"%s" is an unknown input file type' % filetype)
+
+    if pipe:
+        logging.debug('Waiting for child to exit...')
+        pipe.wait()
+        if pipe.returncode:
+            raise PipeFailed('Child exited code %d' % pipe.returncode)
+        pipe = None
+    else:
+        input_file.close()
+
+    logger.info('Elapsed time %d secs' % (time.time() - start_time))