diff options
| -rw-r--r-- | ca/rpki-confgen.xml | 8 | ||||
| -rwxr-xr-x | ca/rpkigui-apache-conf-gen | 20 | ||||
| -rwxr-xr-x | ca/rpkigui-import-routes | 10 | ||||
| -rw-r--r-- | ca/tests/testpoke.py | 3 | ||||
| -rw-r--r-- | rp/rcynic/rcynic.c | 6 | ||||
| -rw-r--r-- | rpki/gui/app/forms.py | 2 | ||||
| -rw-r--r-- | rpki/gui/app/models.py | 37 | ||||
| -rw-r--r-- | rpki/gui/app/templates/app/roarequest_confirm_multi_form.html | 37 | ||||
| -rw-r--r-- | rpki/gui/app/templates/app/roarequest_multi_form.html | 11 | ||||
| -rw-r--r-- | rpki/gui/app/templates/app/route_detail.html | 8 | ||||
| -rw-r--r-- | rpki/gui/app/urls.py | 10 | ||||
| -rw-r--r-- | rpki/gui/app/views.py | 87 | ||||
| -rw-r--r-- | rpki/gui/decorators.py | 24 | ||||
| -rw-r--r-- | rpki/gui/routeview/util.py | 286 | ||||
| -rw-r--r-- | rpki/gui/script_util.py | 1 |
15 files changed, 334 insertions, 216 deletions
diff --git a/ca/rpki-confgen.xml b/ca/rpki-confgen.xml index b3e50823..14f160ab 100644 --- a/ca/rpki-confgen.xml +++ b/ca/rpki-confgen.xml @@ -895,6 +895,14 @@ </doc> </option> + <option name = "download-directory" + value = "/var/tmp"> + <doc> + A directory large enough to hold the RouteViews.org routing table dump + fetched by the rpkigui-import-routes script. + </doc> + </option> + </section> <section name = "autoconf"> diff --git a/ca/rpkigui-apache-conf-gen b/ca/rpkigui-apache-conf-gen index 1270ad15..0658254f 100755 --- a/ca/rpkigui-apache-conf-gen +++ b/ca/rpkigui-apache-conf-gen @@ -141,6 +141,26 @@ Alias /rrdp %(datarootdir)s/rpki/rrdp-publication/ SSLCertificateKeyFile %(sysconfdir)s/rpki/apache.key # + # Recommended settings based on + # https://wiki.mozilla.org/Security/Server_Side_TLS + # (Currently using the Intermediate cipher suite) + # + SSLProtocol all -SSLv2 -SSLv3 + SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA + SSLHonorCipherOrder on + SSLCompression off + + # OCSP Stapling, only in httpd 2.3.3 and later + #SSLUseStapling on + #SSLStaplingResponderTimeout 5 + #SSLStaplingReturnResponderErrors off + # On Apache 2.4+, SSLStaplingCache must be set *outside* of the VirtualHost + #SSLStaplingCache shmcb:/var/run/ocsp(128000) + + # Enable this if your want HSTS (recommended) + # Header add Strict-Transport-Security "max-age=15768000" + + # # Take pity on users running Internet Exploder # BrowserMatch "MSIE [2-6]" ssl-unclean-shutdown nokeepalive downgrade-1.0 force-response-1.0 diff --git a/ca/rpkigui-import-routes b/ca/rpkigui-import-routes index 3dce26b3..0fbe0126 100755 --- a/ca/rpkigui-import-routes +++ b/ca/rpkigui-import-routes @@ -39,7 +39,7 @@ class BadArgument(Exception): def timed_out(*ignored): - logging.info('timed out') + logging.error('timed out') sys.exit(1) @@ -51,11 +51,9 @@ from routeviews.org into the RPKI Web Portal database. If the input file is a bzip2 compressed file, it will be decompressed automatically.""") parser.add_option('-t', '--type', dest='filetype', metavar='TYPE', - help='Specify the input file type (auto, text, mrt) [Default: %default]') + help='Specify the input file type (text, mrt) [Default: %default]') parser.add_option('-l', '--level', dest='log_level', default='ERROR', help='Set logging level [Default: %default]') - parser.add_option('-u', '--bunzip2', dest='bunzip', metavar='PROG', - help='Specify bunzip2 program to use') parser.add_option('-b', '--bgpdump', dest='bgpdump', metavar='PROG', help='Specify path to bgdump binary') parser.add_option('-j', '--jitter', dest='jitter', type='int', @@ -64,7 +62,7 @@ automatically.""") help='Set name of lock file; empty string disables locking [Default: %default]') parser.add_option('--timeout', dest='timeout', type='int', help='Specify timeout for download and import, in seconds [Default: %default]') - parser.set_defaults(debug=False, verbose=False, filetype='auto', jitter=0, + parser.set_defaults(debug=False, verbose=False, filetype='text', jitter=0, lockfile='/tmp/rpkigui-import-routes.lock', timeout=90*60) options, args = parser.parse_args() @@ -104,7 +102,7 @@ automatically.""") signal.signal(signal.SIGALRM, timed_out) signal.setitimer(signal.ITIMER_REAL, options.timeout) - import_routeviews_dump(*args) + import_routeviews_dump(*args, filetype=options.filetype) if options.timeout > 0: signal.setitimer(signal.ITIMER_REAL, 0) diff --git a/ca/tests/testpoke.py b/ca/tests/testpoke.py index 8a443e0d..9cd7b8fd 100644 --- a/ca/tests/testpoke.py +++ b/ca/tests/testpoke.py @@ -105,7 +105,8 @@ def query_up_down(q_pdu): msg = q_der, url = yaml_data["posturl"], callback = done, - errback = fail) + errback = fail, + content_type = rpki.up_down.content_type) def do_list(): query_up_down(rpki.up_down.list_pdu()) diff --git a/rp/rcynic/rcynic.c b/rp/rcynic/rcynic.c index a6f80890..36c1950f 100644 --- a/rp/rcynic/rcynic.c +++ b/rp/rcynic/rcynic.c @@ -2061,8 +2061,10 @@ static int walk_ctx_loop_this(const rcynic_ctx_t *rc, } if (name == NULL) { - logmsg(rc, log_sys_err, "Can't find a URI in walk context, this shouldn't happen: state %d, manifest_iteration %d, filename_iteration %d", - (int) w->state, w->manifest_iteration, w->filename_iteration); + logmsg(rc, log_sys_err, + "Can't find a URI in walk context, this shouldn't happen: " + "state %d, manifest_iteration %d, filename_iteration %d, manifest URI %s", + (int) w->state, w->manifest_iteration, w->filename_iteration, w->certinfo.manifest.s); return 0; } diff --git a/rpki/gui/app/forms.py b/rpki/gui/app/forms.py index f173c15d..306b8dce 100644 --- a/rpki/gui/app/forms.py +++ b/rpki/gui/app/forms.py @@ -193,7 +193,7 @@ def ROARequestFormFactory(conf): 'class': 'span1' }) ) - confirmed = forms.BooleanField(widget=forms.HiddenInput, required=False) + protect_children = forms.BooleanField(required=False) def __init__(self, *args, **kwargs): kwargs['auto_id'] = False diff --git a/rpki/gui/app/models.py b/rpki/gui/app/models.py index ed32e9d2..40bdbe2c 100644 --- a/rpki/gui/app/models.py +++ b/rpki/gui/app/models.py @@ -18,6 +18,7 @@ __version__ = '$Id$' from django.db import models from django.contrib.auth.models import User from django.core.mail import send_mail +from django.db.models import Q import rpki.resource_set import rpki.exceptions @@ -65,6 +66,15 @@ class Child(rpki.irdb.models.Child): proxy = True verbose_name_plural = 'children' + @property + def routes(self): + "Return a list of RouteOrigin objects (potentially) originated by this child." + query = Q() + for r in self.address_ranges.filter(version='IPv4'): + rng = r.as_resource_range() + query |= Q(prefix_min__gte=rng.min, prefix_max__lte=rng.max) + return RouteOrigin.objects.filter(query) + class ChildASN(rpki.irdb.models.ChildASN): """Proxy model for irdb ChildASN.""" @@ -121,17 +131,30 @@ class Conf(rpki.irdb.models.ResourceHolderCA): def parents(self): """Simulates irdb.models.Parent.objects, but returns app.models.Parent proxy objects. - """ + """ return Parent.objects.filter(issuer=self) @property def children(self): """Simulates irdb.models.Child.objects, but returns app.models.Child proxy objects. + + When running rootd, we need to exclude the Child object for self. + """ + return Child.objects.filter(issuer=self).exclude(handle=self.handle) - return Child.objects.filter(issuer=self) + @property + def child_routes(self): + """Return currently announced routes for prefixes covered by child + sub-allocations. + """ + query = Q() + for pfx in ChildNet.objects.filter(child__issuer=self, version='IPv4'): + rng = pfx.as_resource_range() + query |= Q(prefix_min__gte=rng.min, prefix_max__lte=rng.max) + return RouteOrigin.objects.filter(query) @property def ghostbusters(self): @@ -149,8 +172,8 @@ class Conf(rpki.irdb.models.ResourceHolderCA): def routes(self): """Return all IPv4 routes covered by RPKI certs issued to this resource holder. - """ + """ # build a Q filter to select all RouteOrigin objects covered by # prefixes in the resource holder's certificates prefixes = ResourceRangeAddressV4.objects.filter(cert__conf=self) @@ -167,8 +190,8 @@ class Conf(rpki.irdb.models.ResourceHolderCA): def routes_v6(self): """Return all IPv6 routes covered by RPKI certs issued to this resource holder. - """ + """ # build a Q filter to select all RouteOrigin objects covered by # prefixes in the resource holder's certificates prefixes = ResourceRangeAddressV6.objects.filter(cert__conf=self) @@ -183,7 +206,6 @@ class Conf(rpki.irdb.models.ResourceHolderCA): def send_alert(self, subject, message, from_email, severity=Alert.INFO): """Store an alert for this resource holder.""" - self.alerts.create(subject=subject, text=message, severity=severity) send_mail( @@ -199,8 +221,8 @@ class Conf(rpki.irdb.models.ResourceHolderCA): Contact emails are extract from any ghostbuster requests, and any linked user accounts. - """ + """ notify_emails = [gbr.email_address for gbr in self.ghostbusters if gbr.email_address] notify_emails.extend( [acl.user.email for acl in ConfACL.objects.filter(conf=self) if acl.user.email] @@ -225,6 +247,7 @@ class ResourceCert(models.Model): """Represents a resource certificate. This model is used to cache the output of <list_received_resources/>. + """ # Handle to which this cert was issued @@ -252,7 +275,6 @@ class ResourceCert(models.Model): def get_cert_chain(self): """Return a list containing the complete certificate chain for this certificate.""" - cert = self x = [cert] while cert.issuer: @@ -426,6 +448,7 @@ class RouteOriginV6(rpki.gui.routeview.models.RouteOriginV6): class ConfACL(models.Model): """Stores access control for which users are allowed to manage a given resource handle. + """ conf = models.ForeignKey(Conf) diff --git a/rpki/gui/app/templates/app/roarequest_confirm_multi_form.html b/rpki/gui/app/templates/app/roarequest_confirm_multi_form.html index 4a06a4aa..d1d8171f 100644 --- a/rpki/gui/app/templates/app/roarequest_confirm_multi_form.html +++ b/rpki/gui/app/templates/app/roarequest_confirm_multi_form.html @@ -1,5 +1,4 @@ {% extends "app/app_base.html" %} -{% load url from future %} {% load app_extras %} {% block content %} @@ -9,17 +8,20 @@ <div class='row-fluid'> <div class='span6'> - <div class='alert alert-block-message alert-warning'> - <p><strong>Please confirm</strong> that you would like to create the following ROA(s). + <div class='alert'> + <strong>Please confirm</strong> that you would like to create the following ROA(s). The accompanying table indicates how the validation status may change as a result. </div> <table class='table table-condensed table-striped'> - <tr> - <th>Prefix</th> - <th>Max Length</th> - <th>AS</th> - </tr> + <thead> + <tr> + <th>Prefix</th> + <th>Max Length</th> + <th>AS</th> + </tr> + </thead> + <tbody> {% for roa in roas %} <tr> <td>{{ roa.prefix }}</td> @@ -27,6 +29,7 @@ <td>{{ roa.asn }}</td> </tr> {% endfor %} + </tbody> </table> <form method='POST' action='{% url "rpki.gui.app.views.roa_create_multi_confirm" %}'> @@ -47,18 +50,22 @@ <h2>Matched Routes</h2> <table class='table table-striped table-condensed'> - <tr> - <th>Prefix</th> - <th>Origin AS</th> - <th>Validation Status</th> - </tr> + <thead> + <tr> + <th>Prefix</th> + <th>Origin AS</th> + <th>Validation Status</th> + </tr> + </thead> + <tbody> {% for r in routes %} <tr> - <td>{{ r.get_prefix_display }}</td> - <td>{{ r.asn }}</td> + <td>{{ r.get_prefix_display }}</td> + <td>{{ r.asn }}</td> <td>{% validity_label r.newstatus %}</td> </tr> {% endfor %} + </tbody> </table> </div> diff --git a/rpki/gui/app/templates/app/roarequest_multi_form.html b/rpki/gui/app/templates/app/roarequest_multi_form.html index 0fbc49ae..ed2135ed 100644 --- a/rpki/gui/app/templates/app/roarequest_multi_form.html +++ b/rpki/gui/app/templates/app/roarequest_multi_form.html @@ -1,27 +1,28 @@ {% extends "app/app_base.html" %} -{% load url from future %} {% block content %} <div class='page-title'> <h1>Create ROAs</h1> </div> -<form method='POST' action='{{ request.get_full_path }}'> +<form class='form-inline' method='POST' action='{{ request.get_full_path }}'> {% csrf_token %} {{ formset.management_form }} {% for form in formset %} - <div class="controls controls-row"> + {{ form.prefix }} {{ form.max_prefixlen }} {{ form.asn }} + <label class="checkbox" title='create additional ROAs for child routes'>{{ form.protect_children }} Protect children</label> + {# <label class="checkbox inline span1">{{ form.DELETE }} Delete</label> #} {% if form.errors %}<span class="help-inline">{{ form.errors }}</span>{% endif %} {% if form.non_field_errors %}<span class="help-inline">{{ form.non_field_errors }}</span>{% endif %} - </div> + {% endfor %} <div class="form-actions"> - <input class="btn" type="submit" value="Preview"> + <button class='btn btn-primary' type='submit'>Preview</button> <a class="btn" href="{% url "rpki.gui.app.views.dashboard" %}">Cancel</a> </div> </form> diff --git a/rpki/gui/app/templates/app/route_detail.html b/rpki/gui/app/templates/app/route_detail.html index 84add4a8..8bd744df 100644 --- a/rpki/gui/app/templates/app/route_detail.html +++ b/rpki/gui/app/templates/app/route_detail.html @@ -42,13 +42,15 @@ </thead> <tbody> {% for pfx in roa_prefixes %} + {% for roa in pfx.roas.all %} <tr> <td>{{ pfx.as_resource_range }}</td> <td>{{ pfx.max_length }}</td> - <td>{{ pfx.roas.all.0.asid }}</td> - <td>{{ pfx.roas.all.0.not_after }}</td> - <td>{{ pfx.roas.all.0.repo.uri }}</td> + <td>{{ roa.asid }}</td> + <td>{{ roa.not_after }}</td> + <td>{{ roa.repo.uri }}</td> </tr> + {% endfor %} {% endfor %} </tbody> </table> diff --git a/rpki/gui/app/urls.py b/rpki/gui/app/urls.py index f595ea8f..81c9b127 100644 --- a/rpki/gui/app/urls.py +++ b/rpki/gui/app/urls.py @@ -88,12 +88,14 @@ urlpatterns = patterns( #{'post_reset_redirect' : '/user/password/reset/done/'}, {'extra_context': {'form_title': 'Password Reset'}}, name="password_reset"), - (r'^user/password/reset/done/$', - 'django.contrib.auth.views.password_reset_done'), + url(r'^user/password/reset/done/$', + 'django.contrib.auth.views.password_reset_done', + name='password_reset_done'), url(r'^user/password/reset/(?P<uidb36>[0-9A-Za-z]+)-(?P<token>.+)/$', 'django.contrib.auth.views.password_reset_confirm', #{'post_reset_redirect' : '/user/password/done/'}, name="password_reset_confirm"), - (r'^user/password/done/$', - 'django.contrib.auth.views.password_reset_complete'), + url(r'^user/password/done/$', + 'django.contrib.auth.views.password_reset_complete', + name='password_reset_complete'), ) diff --git a/rpki/gui/app/views.py b/rpki/gui/app/views.py index dfd36dbb..bf152f8e 100644 --- a/rpki/gui/app/views.py +++ b/rpki/gui/app/views.py @@ -27,6 +27,7 @@ from tempfile import NamedTemporaryFile import cStringIO import csv import logging +import lxml.etree from django.utils.decorators import method_decorator from django.contrib.auth.decorators import login_required @@ -38,8 +39,8 @@ from django.contrib.auth.models import User from django.views.generic import DetailView, ListView, DeleteView, FormView from django.core.paginator import Paginator, InvalidPage from django.forms.formsets import formset_factory, BaseFormSet -import django.db.models from django.contrib import messages +from django.db.models import Q from rpki.irdb import Zookeeper, ChildASN, ChildNet, ROARequestPrefix from rpki.gui.app import models, forms, glue, range_list @@ -146,19 +147,28 @@ def generic_import(request, queryset, configure, form_class=None, # expects it. if handle == '': handle = None - # configure_repository returns None, so can't use tuple expansion - # here. Unpack the tuple below if post_import_redirect is None. - r = configure(z, tmpf.name, handle) - # force rpkid run now - z.synchronize_ca(poke=True) - os.remove(tmpf.name) - if post_import_redirect: - url = post_import_redirect + try: + # configure_repository returns None, so can't use tuple expansion + # here. Unpack the tuple below if post_import_redirect is None. + r = configure(z, tmpf.name, handle) + except lxml.etree.XMLSyntaxError as e: + logger.exception('caught XMLSyntaxError while parsing uploaded file') + messages.error( + request, + 'The uploaded file has an invalid XML syntax' + ) else: - _, handle = r - url = queryset.get(issuer=conf, - handle=handle).get_absolute_url() - return http.HttpResponseRedirect(url) + # force rpkid run now + z.synchronize_ca(poke=True) + if post_import_redirect: + url = post_import_redirect + else: + _, handle = r + url = queryset.get(issuer=conf, + handle=handle).get_absolute_url() + return http.HttpResponseRedirect(url) + finally: + os.remove(tmpf.name) else: form = form_class() @@ -622,7 +632,6 @@ def get_covered_routes(rng, max_prefixlen, asn): return routes - @handle_required def roa_create(request): """Present the user with a form to create a ROA. @@ -708,20 +717,58 @@ def roa_create_multi(request): formset = formset_factory(forms.ROARequestFormFactory(conf), extra=extra)(initial=init) elif request.method == 'POST': formset = formset_factory(forms.ROARequestFormFactory(conf), extra=0)(request.POST, request.FILES) - if formset.is_valid(): + # We need to check .has_changed() because .is_valid() will return true + # if the user clicks the Preview button without filling in the blanks + # in the ROA form, leaving the form invalid from this view's POV. + if formset.has_changed() and formset.is_valid(): routes = [] v = [] + query = Q() # for matching routes + roas = [] for form in formset: asn = form.cleaned_data['asn'] rng = resource_range_ip.parse_str(form.cleaned_data['prefix']) max_prefixlen = int(form.cleaned_data['max_prefixlen']) - # FIXME: This won't do the right thing in the event that a - # route is covered by multiple ROAs created in the form. - # You will see duplicate entries, each with a potentially - # different validation status. - routes.extend(get_covered_routes(rng, max_prefixlen, asn)) + protect_children = form.cleaned_data['protect_children'] + + roas.append((rng, max_prefixlen, asn, protect_children)) v.append({'prefix': str(rng), 'max_prefixlen': max_prefixlen, 'asn': asn}) + + query |= Q(prefix_min__gte=rng.min, prefix_max__lte=rng.max) + + for rt in RouteOrigin.objects.filter(query): + status = rt.status # cache the value + newstatus = status + if status == 'unknown': + # possible change to valid or invalid + for rng, max_prefixlen, asn, protect in roas: + if rng.min <= rt.prefix_min and rng.max >= rt.prefix_max: + # this route is covered + if asn == rt.asn and rt.prefixlen <= max_prefixlen: + newstatus = 'valid' + break # no need to continue for this route + else: + newstatus = 'invalid' + elif status == 'invalid': + # possible change to valid + for rng, max_prefixlen, asn, protect in roas: + if rng.min <= rt.prefix_min and rng.max >= rt.prefix_max: + # this route is covered + if asn == rt.asn and rt.prefixlen <= max_prefixlen: + newstatus = 'valid' + break # no need to continue for this route + + if status != newstatus: + if protect_children and newstatus == 'invalid' and conf.child_routes.filter(pk=rt.pk).exists(): + rng = rt.as_resource_range() + v.append({'prefix': str(rng), + 'max_prefixlen': rng.prefixlen, + 'asn': rt.asn}) + newstatus = 'valid' + rt.newstatus = newstatus # I"M A MUHNKAY!!! + routes.append(rt) + # if there were no rows, skip the confirmation step if v: formset = formset_factory(forms.ROARequestConfirm, extra=0)(initial=v) diff --git a/rpki/gui/decorators.py b/rpki/gui/decorators.py index 75efeae0..b5c52afb 100644 --- a/rpki/gui/decorators.py +++ b/rpki/gui/decorators.py @@ -15,24 +15,18 @@ __version__ = '$Id$' from django import http -from os import getenv - - -# Don't set this in production, ever. Really. You have been warned. -# -_allow_plain_http_for_testing = getenv("ALLOW_PLAIN_HTTP_FOR_TESTING") == "I solemnly swear that I am not running this in production" +from django.conf import settings def tls_required(f): - """ - Decorator which returns a 500 error if the connection is not - secured with TLS (https). - """ + """Decorator which returns a 500 error if the connection is not secured + with TLS (https). + """ def _tls_required(request, *args, **kwargs): - if not request.is_secure() and not _allow_plain_http_for_testing: - return http.HttpResponseServerError( - 'This resource may only be accessed securely via https', - content_type='text/plain') - return f(request, *args, **kwargs) + if settings.DEBUG or request.is_secure(): + return f(request, *args, **kwargs) + return http.HttpResponseServerError( + 'This resource may only be accessed securely via https', + content_type='text/plain') return _tls_required diff --git a/rpki/gui/routeview/util.py b/rpki/gui/routeview/util.py index a2b515c8..77ff04c7 100644 --- a/rpki/gui/routeview/util.py +++ b/rpki/gui/routeview/util.py @@ -22,9 +22,11 @@ import subprocess import time import logging import urlparse +import bz2 from urllib import urlretrieve, unquote from django.db import transaction, connection +from django.conf import settings from rpki.resource_set import resource_range_ipv4, resource_range_ipv6 from rpki.exceptions import BadIPResource @@ -36,34 +38,118 @@ logger = logging.getLogger(__name__) # Eventually this can be retrived from rpki.conf DEFAULT_URL = 'http://archive.routeviews.org/oix-route-views/oix-full-snapshot-latest.dat.bz2' -def parse_text(f): - last_prefix = None - cursor = connection.cursor() - range_class = resource_range_ipv4 +class ParseError(Exception): pass + +class RouteDumpParser(object): + """Base class for parsing various route dump formats.""" + table = 'routeview_routeorigin' sql = "INSERT INTO %s_new SET asn=%%s, prefix_min=%%s, prefix_max=%%s" % table + range_class = resource_range_ipv4 - try: - logger.info('Dropping existing staging table...') - cursor.execute('DROP TABLE IF EXISTS %s_new' % table) - except _mysql_exceptions.Warning: - pass + def __init__(self, path, *args, **kwargs): + self.path = path + self.cursor = connection.cursor() + self.last_prefix = None + self.asns = set() + + def parse(self): + try: + logger.info('Dropping existing staging table...') + self.cursor.execute('DROP TABLE IF EXISTS %s_new' % self.table) + except _mysql_exceptions.Warning: + pass + + logger.info('Creating staging table...') + self.cursor.execute('CREATE TABLE %(table)s_new LIKE %(table)s' % {'table': self.table}) + + logger.info('Disabling autocommit...') + self.cursor.execute('SET autocommit=0') + + logger.info('Adding rows to table...') + for line in self.input: + try: + prefix, origin_as = self.parse_line(line) + except ParseError as e: + logger.warning('error while parsing line: {} ({})'.format(line, str(e))) + continue + + # the output may contain multiple paths to the same origin. + # if this is the same prefix as the last entry, we don't need + # to validate it again. + # + # prefixes are sorted, but the origin_as is not, so we keep a set to + # avoid duplicates, and insert into the db once we've seen all the + # origin_as values for a given prefix + if prefix != self.last_prefix: + self.ins_routes() + self.last_prefix = prefix + self.asns.add(origin_as) + + self.ins_routes() # process data from last line + + logger.info('Committing...') + self.cursor.execute('COMMIT') - logger.info('Creating staging table...') - cursor.execute('CREATE TABLE %(table)s_new LIKE %(table)s' % {'table': table}) + try: + logger.info('Dropping old table...') + self.cursor.execute('DROP TABLE IF EXISTS %s_old' % self.table) + except _mysql_exceptions.Warning: + pass + + logger.info('Swapping staging table with live table...') + self.cursor.execute('RENAME TABLE %(table)s TO %(table)s_old, %(table)s_new TO %(table)s' % {'table': self.table}) + + self.cleanup() # allow cleanup function to throw prior to COMMIT + + transaction.commit_unless_managed() + + logger.info('Updating timestamp metadata...') + rpki.gui.app.timestamp.update('bgp_v4_import') + + def parse_line(self, row): + "Parse one line of input. Return a (prefix, origin_as) tuple." + return None + + def cleanup(self): + pass - logger.info('Disabling autocommit...') - cursor.execute('SET autocommit=0') + def ins_routes(self): + # output routes for previous prefix + if self.last_prefix is not None: + try: + rng = self.range_class.parse_str(self.last_prefix) + rmin = long(rng.min) + rmax = long(rng.max) + self.cursor.executemany(self.sql, [(asn, rmin, rmax) for asn in self.asns]) + except BadIPResource: + logger.warning('skipping bad prefix: ' + self.last_prefix) + self.asns = set() # reset + + +class TextDumpParser(RouteDumpParser): + """Parses the RouteViews.org text dump.""" + + def __init__(self, *args, **kwargs): + super(TextDumpParser, self).__init__(*args, **kwargs) + if self.path.endswith('.bz2'): + logger.info('decompressing bz2 file') + self.file = bz2.BZ2File(self.path, buffering=4096) + else: + self.file = open(self.path, buffering=-1) + self.input = itertools.islice(self.file, 5, None) # skip first 5 lines - logger.info('Adding rows to table...') - for row in itertools.islice(f, 5, None): + def parse_line(self, row): + "Parse one line of input" cols = row.split() # index -1 is i/e/? for igp/egp - origin_as = cols[-2] - # FIXME: skip AS_SETs - if origin_as[0] == '{': - continue + try: + origin_as = int(cols[-2]) + except IndexError: + raise ParseError('unexpected format') + except ValueError: + raise ParseError('bad AS value') prefix = cols[1] @@ -77,85 +163,35 @@ def parse_text(f): s.append('assuming it should be %s' % prefix) logger.warning(' '.join(s)) - # the output may contain multiple paths to the same origin. - # if this is the same prefix as the last entry, we don't need - # to validate it again. - # - # prefixes are sorted, but the origin_as is not, so we keep a set to - # avoid duplicates, and insert into the db once we've seen all the - # origin_as values for a given prefix - if prefix != last_prefix: - # output routes for previous prefix - if last_prefix is not None: - try: - rng = range_class.parse_str(last_prefix) - rmin = long(rng.min) - rmax = long(rng.max) - cursor.executemany(sql, [(asn, rmin, rmax) for asn in asns]) - except BadIPResource: - logger.warning('skipping bad prefix: ' + last_prefix) - - asns = set() - last_prefix = prefix - - try: - asns.add(int(origin_as)) - except ValueError as err: - logger.warning('\n'.join( - ['unable to parse origin AS: ' + origin_as], - ['ValueError: ' + str(err)] - ['route entry was: ' + row], - )) - - logger.info('Committing...') - cursor.execute('COMMIT') - - try: - logger.info('Dropping old table...') - cursor.execute('DROP TABLE IF EXISTS %s_old' % table) - except _mysql_exceptions.Warning: - pass - - logger.info('Swapping staging table with live table...') - cursor.execute('RENAME TABLE %(table)s TO %(table)s_old, %(table)s_new TO %(table)s' % {'table': table}) + return prefix, origin_as - transaction.commit_unless_managed() + def cleanup(self): + self.file.close() - logger.info('Updating timestamp metadata...') - rpki.gui.app.timestamp.update('bgp_v4_import') +class MrtDumpParser(RouteDumpParser): + def __init__(self, *args, **kwargs): + super(MrtDumpParser, self).__init__(*args, **kwargs) + # filter input through bgpdump + # bgpdump can decompress bz2 files directly, no need to do it here + self.pipe = subprocess.Popen(['bgpdump', '-m', '-v', self.path], stdout=subprocess.PIPE, bufsize=-1) + self.input = self.pipe.stdout -def parse_mrt(f): - # filter input through bgpdump - pipe = subprocess.Popen(['bgpdump', '-m', '-v', '-'], stdin=f, - stdout=subprocess.PIPE) - - last_prefix = None - last_as = None - for e in pipe.stdout.readlines(): - a = e.split('|') + def parse_line(self, row): + a = row.split('|') prefix = a[5] try: origin_as = int(a[6].split()[-1]) except ValueError: - # skip AS_SETs - continue - - if prefix != last_prefix: - last_prefix = prefix - elif last_as == origin_as: - continue - last_as = origin_as + raise ParseError('bad AS value') - asns = PREFIXES.get(prefix) - if not asns: - asns = set() - PREFIXES[prefix] = asns - asns.add(origin_as) + return prefix, origin_as - pipe.wait() - if pipe.returncode: - raise ProgException('bgpdump exited with code %d' % pipe.returncode) + def cleanup(self): + logger.info('waiting for child process to terminate') + self.pipe.wait() + if self.pipe.returncode: + raise PipeFailed('bgpdump exited with code %d' % self.pipe.returncode) class ProgException(Exception): @@ -170,7 +206,7 @@ class PipeFailed(ProgException): pass -def import_routeviews_dump(filename=DEFAULT_URL, filetype='auto'): +def import_routeviews_dump(filename=DEFAULT_URL, filetype='text'): """Load the oix-full-snapshot-latest.bz2 from routeview.org into the rpki.gui.routeview database. @@ -182,55 +218,31 @@ def import_routeviews_dump(filename=DEFAULT_URL, filetype='auto'): """ start_time = time.time() - - if filename.startswith('http://'): - #get filename from the basename of the URL - u = urlparse.urlparse(filename) - bname = os.path.basename(unquote(u.path)) - tmpname = os.path.join('/tmp', bname) - - logger.info("Downloading %s to %s", filename, tmpname) - if os.path.exists(tmpname): - os.remove(tmpname) - # filename is replaced with a local filename containing cached copy of - # URL - filename, headers = urlretrieve(filename, tmpname) - - if filetype == 'auto': - # try to determine input type from filename, based on the default - # filenames from archive.routeviews.org - bname = os.path.basename(filename) - if bname.startswith('oix-full-snapshot-latest'): - filetype = 'text' - elif bname.startswith('rib.'): - filetype = 'mrt' - else: - raise UnknownInputType('unable to automatically determine input file type') - logging.info('Detected import format as "%s"', filetype) - - pipe = None - if filename.endswith('.bz2'): - bunzip = 'bunzip2' - logging.info('Decompressing input file on the fly...') - pipe = subprocess.Popen([bunzip, '--stdout', filename], - stdout=subprocess.PIPE) - input_file = pipe.stdout - else: - input_file = open(filename) + tmpname = None try: - dispatch = {'text': parse_text, 'mrt': parse_mrt} - dispatch[filetype](input_file) - except KeyError: - raise UnknownInputType('"%s" is an unknown input file type' % filetype) - - if pipe: - logging.debug('Waiting for child to exit...') - pipe.wait() - if pipe.returncode: - raise PipeFailed('Child exited code %d' % pipe.returncode) - pipe = None - else: - input_file.close() + if filename.startswith('http://'): + #get filename from the basename of the URL + u = urlparse.urlparse(filename) + bname = os.path.basename(unquote(u.path)) + tmpname = os.path.join(settings.DOWNLOAD_DIRECTORY, bname) + + logger.info("Downloading %s to %s", filename, tmpname) + if os.path.exists(tmpname): + os.remove(tmpname) + # filename is replaced with a local filename containing cached copy of + # URL + filename, headers = urlretrieve(filename, tmpname) + + try: + dispatch = {'text': TextDumpParser, 'mrt': MrtDumpParser} + dispatch[filetype](filename).parse() + except KeyError: + raise UnknownInputType('"%s" is an unknown input file type' % filetype) + + finally: + # make sure to always clean up the temp download file + if tmpname is not None: + os.unlink(tmpname) logger.info('Elapsed time %d secs', (time.time() - start_time)) diff --git a/rpki/gui/script_util.py b/rpki/gui/script_util.py index 1941f6f7..24b6d313 100644 --- a/rpki/gui/script_util.py +++ b/rpki/gui/script_util.py @@ -51,6 +51,7 @@ def setup(): }, }, MIDDLEWARE_CLASSES = (), + DOWNLOAD_DIRECTORY = cfg.get('download-directory', '/var/tmp'), ) # Can't populate apps if we don't know what they are. If this # explodes with an AppRegistryNotReady exception, the above comment |