More relatively straightforward merges.

svn path=/branches/tk705/; revision=6083

15 files changed, 334 insertions, 216 deletions

diff --git a/ca/rpki-confgen.xml b/ca/rpki-confgen.xml
index b3e50823..14f160ab 100644
--- a/ca/rpki-confgen.xml
+++ b/ca/rpki-confgen.xml
@@ -895,6 +895,14 @@
       </doc>
     </option>
 
+    <option name = "download-directory"
+            value = "/var/tmp">
+      <doc>
+	A directory large enough to hold the RouteViews.org routing table dump
+        fetched by the rpkigui-import-routes script.
+      </doc>
+    </option>
+
   </section>
 
   <section name = "autoconf">
diff --git a/ca/rpkigui-apache-conf-gen b/ca/rpkigui-apache-conf-gen
index 1270ad15..0658254f 100755
--- a/ca/rpkigui-apache-conf-gen
+++ b/ca/rpkigui-apache-conf-gen
@@ -141,6 +141,26 @@ Alias   /rrdp           %(datarootdir)s/rpki/rrdp-publication/
   SSLCertificateKeyFile  %(sysconfdir)s/rpki/apache.key
 
   #
+  # Recommended settings based on
+  # https://wiki.mozilla.org/Security/Server_Side_TLS
+  # (Currently using the Intermediate cipher suite)
+  #
+  SSLProtocol             all -SSLv2 -SSLv3
+  SSLCipherSuite          ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
+  SSLHonorCipherOrder     on
+  SSLCompression          off
+
+  # OCSP Stapling, only in httpd 2.3.3 and later
+  #SSLUseStapling          on
+  #SSLStaplingResponderTimeout 5
+  #SSLStaplingReturnResponderErrors off
+  # On Apache 2.4+, SSLStaplingCache must be set *outside* of the VirtualHost
+  #SSLStaplingCache        shmcb:/var/run/ocsp(128000)
+
+  # Enable this if your want HSTS (recommended)
+  # Header add Strict-Transport-Security "max-age=15768000"
+
+  #
   # Take pity on users running Internet Exploder
   #
   BrowserMatch "MSIE [2-6]"  ssl-unclean-shutdown nokeepalive downgrade-1.0 force-response-1.0
diff --git a/ca/rpkigui-import-routes b/ca/rpkigui-import-routes
index 3dce26b3..0fbe0126 100755
--- a/ca/rpkigui-import-routes
+++ b/ca/rpkigui-import-routes
@@ -39,7 +39,7 @@ class BadArgument(Exception):
 
 
 def timed_out(*ignored):
-    logging.info('timed out')
+    logging.error('timed out')
     sys.exit(1)
 
 
@@ -51,11 +51,9 @@ from routeviews.org into the RPKI Web Portal database.  If the
 input file is a bzip2 compressed file, it will be decompressed
 automatically.""")
     parser.add_option('-t', '--type', dest='filetype', metavar='TYPE',
-                      help='Specify the input file type (auto, text, mrt) [Default: %default]')
+                      help='Specify the input file type (text, mrt) [Default: %default]')
     parser.add_option('-l', '--level', dest='log_level', default='ERROR',
                       help='Set logging level [Default: %default]')
-    parser.add_option('-u', '--bunzip2', dest='bunzip', metavar='PROG',
-                      help='Specify bunzip2 program to use')
     parser.add_option('-b', '--bgpdump', dest='bgpdump', metavar='PROG',
                       help='Specify path to bgdump binary')
     parser.add_option('-j', '--jitter', dest='jitter', type='int',
@@ -64,7 +62,7 @@ automatically.""")
                       help='Set name of lock file; empty string disables locking [Default: %default]')
     parser.add_option('--timeout', dest='timeout', type='int',
                       help='Specify timeout for download and import, in seconds [Default: %default]')
-    parser.set_defaults(debug=False, verbose=False, filetype='auto', jitter=0,
+    parser.set_defaults(debug=False, verbose=False, filetype='text', jitter=0,
                         lockfile='/tmp/rpkigui-import-routes.lock', timeout=90*60)
     options, args = parser.parse_args()
 
@@ -104,7 +102,7 @@ automatically.""")
             signal.signal(signal.SIGALRM, timed_out)
             signal.setitimer(signal.ITIMER_REAL, options.timeout)
 
-        import_routeviews_dump(*args)
+        import_routeviews_dump(*args, filetype=options.filetype)
 
         if options.timeout > 0:
             signal.setitimer(signal.ITIMER_REAL, 0)
diff --git a/ca/tests/testpoke.py b/ca/tests/testpoke.py
index 8a443e0d..9cd7b8fd 100644
--- a/ca/tests/testpoke.py
+++ b/ca/tests/testpoke.py
@@ -105,7 +105,8 @@ def query_up_down(q_pdu):
     msg          = q_der,
     url          = yaml_data["posturl"],
     callback     = done,
-    errback      = fail)
+    errback      = fail,
+    content_type = rpki.up_down.content_type)
 
 def do_list():
   query_up_down(rpki.up_down.list_pdu())
diff --git a/rp/rcynic/rcynic.c b/rp/rcynic/rcynic.c
index a6f80890..36c1950f 100644
--- a/rp/rcynic/rcynic.c
+++ b/rp/rcynic/rcynic.c
@@ -2061,8 +2061,10 @@ static int walk_ctx_loop_this(const rcynic_ctx_t *rc,
   }
 
   if (name == NULL) {
-    logmsg(rc, log_sys_err, "Can't find a URI in walk context, this shouldn't happen: state %d, manifest_iteration %d, filename_iteration %d",
-	   (int) w->state, w->manifest_iteration, w->filename_iteration);
+    logmsg(rc, log_sys_err, 
+	   "Can't find a URI in walk context, this shouldn't happen: "
+	   "state %d, manifest_iteration %d, filename_iteration %d, manifest URI %s",
+	   (int) w->state, w->manifest_iteration, w->filename_iteration, w->certinfo.manifest.s);
     return 0;
   }
 
diff --git a/rpki/gui/app/forms.py b/rpki/gui/app/forms.py
index f173c15d..306b8dce 100644
--- a/rpki/gui/app/forms.py
+++ b/rpki/gui/app/forms.py
@@ -193,7 +193,7 @@ def ROARequestFormFactory(conf):
 		'class': 'span1'
 	    })
 	)
-	confirmed = forms.BooleanField(widget=forms.HiddenInput, required=False)
+	protect_children = forms.BooleanField(required=False)
 
 	def __init__(self, *args, **kwargs):
 	    kwargs['auto_id'] = False
diff --git a/rpki/gui/app/models.py b/rpki/gui/app/models.py
index ed32e9d2..40bdbe2c 100644
--- a/rpki/gui/app/models.py
+++ b/rpki/gui/app/models.py
@@ -18,6 +18,7 @@ __version__ = '$Id$'
 from django.db import models
 from django.contrib.auth.models import User
 from django.core.mail import send_mail
+from django.db.models import Q
 
 import rpki.resource_set
 import rpki.exceptions
@@ -65,6 +66,15 @@ class Child(rpki.irdb.models.Child):
         proxy = True
         verbose_name_plural = 'children'
 
+    @property
+    def routes(self):
+        "Return a list of RouteOrigin objects (potentially) originated by this child."
+        query = Q()
+        for r in self.address_ranges.filter(version='IPv4'):
+            rng = r.as_resource_range()
+            query |= Q(prefix_min__gte=rng.min, prefix_max__lte=rng.max)
+        return RouteOrigin.objects.filter(query)
+
 
 class ChildASN(rpki.irdb.models.ChildASN):
     """Proxy model for irdb ChildASN."""
@@ -121,17 +131,30 @@ class Conf(rpki.irdb.models.ResourceHolderCA):
     def parents(self):
         """Simulates irdb.models.Parent.objects, but returns app.models.Parent
         proxy objects.
-        """
 
+        """
         return Parent.objects.filter(issuer=self)
 
     @property
     def children(self):
         """Simulates irdb.models.Child.objects, but returns app.models.Child
         proxy objects.
+
+        When running rootd, we need to exclude the Child object for self.
+
         """
+        return Child.objects.filter(issuer=self).exclude(handle=self.handle)
 
-        return Child.objects.filter(issuer=self)
+    @property
+    def child_routes(self):
+        """Return currently announced routes for prefixes covered by child
+        sub-allocations.
+        """
+        query = Q()
+        for pfx in ChildNet.objects.filter(child__issuer=self, version='IPv4'):
+            rng = pfx.as_resource_range()
+            query |= Q(prefix_min__gte=rng.min, prefix_max__lte=rng.max)
+        return RouteOrigin.objects.filter(query)
 
     @property
     def ghostbusters(self):
@@ -149,8 +172,8 @@ class Conf(rpki.irdb.models.ResourceHolderCA):
     def routes(self):
         """Return all IPv4 routes covered by RPKI certs issued to this resource
         holder.
-        """
 
+        """
         # build a Q filter to select all RouteOrigin objects covered by
         # prefixes in the resource holder's certificates
         prefixes = ResourceRangeAddressV4.objects.filter(cert__conf=self)
@@ -167,8 +190,8 @@ class Conf(rpki.irdb.models.ResourceHolderCA):
     def routes_v6(self):
         """Return all IPv6 routes covered by RPKI certs issued to this resource
         holder.
-        """
 
+        """
         # build a Q filter to select all RouteOrigin objects covered by
         # prefixes in the resource holder's certificates
         prefixes = ResourceRangeAddressV6.objects.filter(cert__conf=self)
@@ -183,7 +206,6 @@ class Conf(rpki.irdb.models.ResourceHolderCA):
 
     def send_alert(self, subject, message, from_email, severity=Alert.INFO):
         """Store an alert for this resource holder."""
-
         self.alerts.create(subject=subject, text=message, severity=severity)
 
         send_mail(
@@ -199,8 +221,8 @@ class Conf(rpki.irdb.models.ResourceHolderCA):
 
         Contact emails are extract from any ghostbuster requests, and any
         linked user accounts.
-        """
 
+        """
         notify_emails = [gbr.email_address for gbr in self.ghostbusters if gbr.email_address]
         notify_emails.extend(
             [acl.user.email for acl in ConfACL.objects.filter(conf=self) if acl.user.email]
@@ -225,6 +247,7 @@ class ResourceCert(models.Model):
     """Represents a resource certificate.
 
     This model is used to cache the output of <list_received_resources/>.
+
     """
 
     # Handle to which this cert was issued
@@ -252,7 +275,6 @@ class ResourceCert(models.Model):
     def get_cert_chain(self):
         """Return a list containing the complete certificate chain for this
         certificate."""
-
         cert = self
         x = [cert]
         while cert.issuer:
@@ -426,6 +448,7 @@ class RouteOriginV6(rpki.gui.routeview.models.RouteOriginV6):
 class ConfACL(models.Model):
     """Stores access control for which users are allowed to manage a given
     resource handle.
+
     """
 
     conf = models.ForeignKey(Conf)
diff --git a/rpki/gui/app/templates/app/roarequest_confirm_multi_form.html b/rpki/gui/app/templates/app/roarequest_confirm_multi_form.html
index 4a06a4aa..d1d8171f 100644
--- a/rpki/gui/app/templates/app/roarequest_confirm_multi_form.html
+++ b/rpki/gui/app/templates/app/roarequest_confirm_multi_form.html
@@ -1,5 +1,4 @@
 {% extends "app/app_base.html" %}
-{% load url from future %}
 {% load app_extras %}
 
 {% block content %}
@@ -9,17 +8,20 @@
 
 <div class='row-fluid'>
   <div class='span6'>
-    <div class='alert alert-block-message alert-warning'>
-      <p><strong>Please confirm</strong> that you would like to create the following ROA(s).
+    <div class='alert'>
+      <strong>Please confirm</strong> that you would like to create the following ROA(s).
       The accompanying table indicates how the validation status may change as a result.
     </div>
 
     <table class='table table-condensed table-striped'>
-      <tr>
-	<th>Prefix</th>
-	<th>Max Length</th>
-	<th>AS</th>
-      </tr>
+      <thead>
+	<tr>
+	  <th>Prefix</th>
+	  <th>Max Length</th>
+	  <th>AS</th>
+	</tr>
+      </thead>
+      <tbody>
       {% for roa in roas %}
       <tr>
 	<td>{{ roa.prefix }}</td>
@@ -27,6 +29,7 @@
 	<td>{{ roa.asn }}</td>
       </tr>
       {% endfor %}
+      </tbody>
     </table>
 
     <form method='POST' action='{% url "rpki.gui.app.views.roa_create_multi_confirm" %}'>
@@ -47,18 +50,22 @@
     <h2>Matched Routes</h2>
 
     <table class='table table-striped table-condensed'>
-      <tr>
-        <th>Prefix</th>
-        <th>Origin AS</th>
-        <th>Validation Status</th>
-      </tr>
+      <thead>
+	<tr>
+	  <th>Prefix</th>
+	  <th>Origin AS</th>
+	  <th>Validation Status</th>
+	</tr>
+      </thead>
+      <tbody>
       {% for r in routes %}
       <tr>
-        <td>{{ r.get_prefix_display }}</td>
-        <td>{{ r.asn }}</td>
+	<td>{{ r.get_prefix_display }}</td>
+	<td>{{ r.asn }}</td>
 	<td>{% validity_label r.newstatus %}</td>
       </tr>
       {% endfor %}
+      </tbody>
     </table>
   </div>
 
diff --git a/rpki/gui/app/templates/app/roarequest_multi_form.html b/rpki/gui/app/templates/app/roarequest_multi_form.html
index 0fbc49ae..ed2135ed 100644
--- a/rpki/gui/app/templates/app/roarequest_multi_form.html
+++ b/rpki/gui/app/templates/app/roarequest_multi_form.html
@@ -1,27 +1,28 @@
 {% extends "app/app_base.html" %}
-{% load url from future %}
 
 {% block content %}
 <div class='page-title'>
   <h1>Create ROAs</h1>
 </div>
 
-<form method='POST' action='{{ request.get_full_path }}'>
+<form class='form-inline' method='POST' action='{{ request.get_full_path }}'>
   {% csrf_token %}
   {{ formset.management_form }}
   {% for form in formset %}
-  <div class="controls controls-row">
+
     {{ form.prefix }}
     {{ form.max_prefixlen }}
     {{ form.asn }}
+    <label class="checkbox" title='create additional ROAs for child routes'>{{ form.protect_children }} Protect children</label>
+
     {# <label class="checkbox inline span1">{{ form.DELETE }} Delete</label> #}
     {% if form.errors %}<span class="help-inline">{{ form.errors }}</span>{% endif %}
     {% if form.non_field_errors %}<span class="help-inline">{{ form.non_field_errors }}</span>{% endif %}
-  </div>
+
   {% endfor %}
 
   <div class="form-actions">
-    <input class="btn" type="submit" value="Preview">
+    <button class='btn btn-primary' type='submit'>Preview</button>
     <a class="btn" href="{% url "rpki.gui.app.views.dashboard" %}">Cancel</a>
   </div>
 </form>
diff --git a/rpki/gui/app/templates/app/route_detail.html b/rpki/gui/app/templates/app/route_detail.html
index 84add4a8..8bd744df 100644
--- a/rpki/gui/app/templates/app/route_detail.html
+++ b/rpki/gui/app/templates/app/route_detail.html
@@ -42,13 +42,15 @@
       </thead>
       <tbody>
 	{% for pfx in roa_prefixes %}
+          {% for roa in pfx.roas.all %}
 	<tr>
 	  <td>{{ pfx.as_resource_range }}</td>
 	  <td>{{ pfx.max_length }}</td>
-	  <td>{{ pfx.roas.all.0.asid }}</td>
-	  <td>{{ pfx.roas.all.0.not_after }}</td>
-	  <td>{{ pfx.roas.all.0.repo.uri }}</td>
+	  <td>{{ roa.asid }}</td>
+	  <td>{{ roa.not_after }}</td>
+	  <td>{{ roa.repo.uri }}</td>
 	</tr>
+          {% endfor %}
 	{% endfor %}
       </tbody>
     </table>
diff --git a/rpki/gui/app/urls.py b/rpki/gui/app/urls.py
index f595ea8f..81c9b127 100644
--- a/rpki/gui/app/urls.py
+++ b/rpki/gui/app/urls.py
@@ -88,12 +88,14 @@ urlpatterns = patterns(
         #{'post_reset_redirect' : '/user/password/reset/done/'},
         {'extra_context': {'form_title': 'Password Reset'}},
         name="password_reset"),
-    (r'^user/password/reset/done/$',
-        'django.contrib.auth.views.password_reset_done'),
+    url(r'^user/password/reset/done/$',
+        'django.contrib.auth.views.password_reset_done',
+        name='password_reset_done'),
     url(r'^user/password/reset/(?P<uidb36>[0-9A-Za-z]+)-(?P<token>.+)/$',
         'django.contrib.auth.views.password_reset_confirm',
         #{'post_reset_redirect' : '/user/password/done/'},
         name="password_reset_confirm"),
-    (r'^user/password/done/$',
-        'django.contrib.auth.views.password_reset_complete'),
+    url(r'^user/password/done/$',
+        'django.contrib.auth.views.password_reset_complete',
+        name='password_reset_complete'),
 )
diff --git a/rpki/gui/app/views.py b/rpki/gui/app/views.py
index dfd36dbb..bf152f8e 100644
--- a/rpki/gui/app/views.py
+++ b/rpki/gui/app/views.py
@@ -27,6 +27,7 @@ from tempfile import NamedTemporaryFile
 import cStringIO
 import csv
 import logging
+import lxml.etree
 
 from django.utils.decorators import method_decorator
 from django.contrib.auth.decorators import login_required
@@ -38,8 +39,8 @@ from django.contrib.auth.models import User
 from django.views.generic import DetailView, ListView, DeleteView, FormView
 from django.core.paginator import Paginator, InvalidPage
 from django.forms.formsets import formset_factory, BaseFormSet
-import django.db.models
 from django.contrib import messages
+from django.db.models import Q
 
 from rpki.irdb import Zookeeper, ChildASN, ChildNet, ROARequestPrefix
 from rpki.gui.app import models, forms, glue, range_list
@@ -146,19 +147,28 @@ def generic_import(request, queryset, configure, form_class=None,
             # expects it.
             if handle == '':
                 handle = None
-            # configure_repository returns None, so can't use tuple expansion
-            # here.  Unpack the tuple below if post_import_redirect is None.
-            r = configure(z, tmpf.name, handle)
-            # force rpkid run now
-            z.synchronize_ca(poke=True)
-            os.remove(tmpf.name)
-            if post_import_redirect:
-                url = post_import_redirect
+            try:
+		# configure_repository returns None, so can't use tuple expansion
+		# here.  Unpack the tuple below if post_import_redirect is None.
+		r = configure(z, tmpf.name, handle)
+            except lxml.etree.XMLSyntaxError as e:
+		logger.exception('caught XMLSyntaxError while parsing uploaded file')
+                messages.error(
+                    request,
+                    'The uploaded file has an invalid XML syntax'
+                )
             else:
-                _, handle = r
-                url = queryset.get(issuer=conf,
-                                   handle=handle).get_absolute_url()
-            return http.HttpResponseRedirect(url)
+		# force rpkid run now
+		z.synchronize_ca(poke=True)
+		if post_import_redirect:
+		    url = post_import_redirect
+		else:
+		    _, handle = r
+		    url = queryset.get(issuer=conf,
+				       handle=handle).get_absolute_url()
+		return http.HttpResponseRedirect(url)
+            finally:
+		os.remove(tmpf.name)
     else:
         form = form_class()
 
@@ -622,7 +632,6 @@ def get_covered_routes(rng, max_prefixlen, asn):
 
     return routes
 
-
 @handle_required
 def roa_create(request):
     """Present the user with a form to create a ROA.
@@ -708,20 +717,58 @@ def roa_create_multi(request):
         formset = formset_factory(forms.ROARequestFormFactory(conf), extra=extra)(initial=init)
     elif request.method == 'POST':
         formset = formset_factory(forms.ROARequestFormFactory(conf), extra=0)(request.POST, request.FILES)
-        if formset.is_valid():
+	# We need to check .has_changed() because .is_valid() will return true
+	# if the user clicks the Preview button without filling in the blanks
+	# in the ROA form, leaving the form invalid from this view's POV.
+        if formset.has_changed() and formset.is_valid():
             routes = []
             v = []
+            query = Q()  # for matching routes
+            roas = []
             for form in formset:
 		asn = form.cleaned_data['asn']
 		rng = resource_range_ip.parse_str(form.cleaned_data['prefix'])
 		max_prefixlen = int(form.cleaned_data['max_prefixlen'])
-		# FIXME: This won't do the right thing in the event that a
-		# route is covered by multiple ROAs created in the form.
-		# You will see duplicate entries, each with a potentially
-		# different validation status.
-		routes.extend(get_covered_routes(rng, max_prefixlen, asn))
+                protect_children = form.cleaned_data['protect_children']
+
+                roas.append((rng, max_prefixlen, asn, protect_children))
 		v.append({'prefix': str(rng), 'max_prefixlen': max_prefixlen,
 			  'asn': asn})
+
+                query |= Q(prefix_min__gte=rng.min, prefix_max__lte=rng.max)
+
+            for rt in RouteOrigin.objects.filter(query):
+                status = rt.status  # cache the value
+                newstatus = status
+                if status == 'unknown':
+                    # possible change to valid or invalid
+                    for rng, max_prefixlen, asn, protect in roas:
+                        if rng.min <= rt.prefix_min and rng.max >= rt.prefix_max:
+                            # this route is covered
+                            if asn == rt.asn and rt.prefixlen <= max_prefixlen:
+                                newstatus = 'valid'
+                                break  # no need to continue for this route
+                            else:
+                                newstatus = 'invalid'
+                elif status == 'invalid':
+                    # possible change to valid
+                    for rng, max_prefixlen, asn, protect in roas:
+                        if rng.min <= rt.prefix_min and rng.max >= rt.prefix_max:
+                            # this route is covered
+                            if asn == rt.asn and rt.prefixlen <= max_prefixlen:
+                                newstatus = 'valid'
+                                break  # no need to continue for this route
+
+                if status != newstatus:
+                    if protect_children and newstatus == 'invalid' and conf.child_routes.filter(pk=rt.pk).exists():
+                        rng = rt.as_resource_range()
+                        v.append({'prefix': str(rng),
+                                    'max_prefixlen': rng.prefixlen,
+                                    'asn': rt.asn})
+                        newstatus = 'valid'
+                    rt.newstatus = newstatus  # I"M A MUHNKAY!!!
+                    routes.append(rt)
+
             # if there were no rows, skip the confirmation step
             if v:
                 formset = formset_factory(forms.ROARequestConfirm, extra=0)(initial=v)
diff --git a/rpki/gui/decorators.py b/rpki/gui/decorators.py
index 75efeae0..b5c52afb 100644
--- a/rpki/gui/decorators.py
+++ b/rpki/gui/decorators.py
@@ -15,24 +15,18 @@
 __version__ = '$Id$'
 
 from django import http
-from os import getenv
-
-
-# Don't set this in production, ever.  Really.  You have been warned.
-#
-_allow_plain_http_for_testing = getenv("ALLOW_PLAIN_HTTP_FOR_TESTING") == "I solemnly swear that I am not running this in production"
+from django.conf import settings
 
 
 def tls_required(f):
-    """
-    Decorator which returns a 500 error if the connection is not
-    secured with TLS (https).
-    """
+    """Decorator which returns a 500 error if the connection is not secured
+    with TLS (https).
 
+    """
     def _tls_required(request, *args, **kwargs):
-        if not request.is_secure() and not _allow_plain_http_for_testing:
-            return http.HttpResponseServerError(
-                'This resource may only be accessed securely via https',
-                content_type='text/plain')
-        return f(request, *args, **kwargs)
+        if settings.DEBUG or request.is_secure():
+            return f(request, *args, **kwargs)
+        return http.HttpResponseServerError(
+            'This resource may only be accessed securely via https',
+            content_type='text/plain')
     return _tls_required
diff --git a/rpki/gui/routeview/util.py b/rpki/gui/routeview/util.py
index a2b515c8..77ff04c7 100644
--- a/rpki/gui/routeview/util.py
+++ b/rpki/gui/routeview/util.py
@@ -22,9 +22,11 @@ import subprocess
 import time
 import logging
 import urlparse
+import bz2
 from urllib import urlretrieve, unquote
 
 from django.db import transaction, connection
+from django.conf import settings
 
 from rpki.resource_set import resource_range_ipv4, resource_range_ipv6
 from rpki.exceptions import BadIPResource
@@ -36,34 +38,118 @@ logger = logging.getLogger(__name__)
 # Eventually this can be retrived from rpki.conf
 DEFAULT_URL = 'http://archive.routeviews.org/oix-route-views/oix-full-snapshot-latest.dat.bz2'
 
-def parse_text(f):
-    last_prefix = None
-    cursor = connection.cursor()
-    range_class = resource_range_ipv4
+class ParseError(Exception): pass
+
+class RouteDumpParser(object):
+    """Base class for parsing various route dump formats."""
+
     table = 'routeview_routeorigin'
     sql = "INSERT INTO %s_new SET asn=%%s, prefix_min=%%s, prefix_max=%%s" % table
+    range_class = resource_range_ipv4
 
-    try:
-        logger.info('Dropping existing staging table...')
-        cursor.execute('DROP TABLE IF EXISTS %s_new' % table)
-    except _mysql_exceptions.Warning:
-        pass
+    def __init__(self, path, *args, **kwargs):
+        self.path = path
+        self.cursor = connection.cursor()
+        self.last_prefix = None
+        self.asns = set()
+
+    def parse(self):
+        try:
+            logger.info('Dropping existing staging table...')
+            self.cursor.execute('DROP TABLE IF EXISTS %s_new' % self.table)
+        except _mysql_exceptions.Warning:
+            pass
+
+        logger.info('Creating staging table...')
+        self.cursor.execute('CREATE TABLE %(table)s_new LIKE %(table)s' % {'table': self.table})
+
+        logger.info('Disabling autocommit...')
+        self.cursor.execute('SET autocommit=0')
+
+        logger.info('Adding rows to table...')
+        for line in self.input:
+            try:
+                prefix, origin_as = self.parse_line(line)
+            except ParseError as e:
+                logger.warning('error while parsing line: {} ({})'.format(line, str(e)))
+                continue
+
+            # the output may contain multiple paths to the same origin.
+            # if this is the same prefix as the last entry, we don't need
+            # to validate it again.
+            #
+            # prefixes are sorted, but the origin_as is not, so we keep a set to
+            # avoid duplicates, and insert into the db once we've seen all the
+            # origin_as values for a given prefix
+            if prefix != self.last_prefix:
+                self.ins_routes()
+                self.last_prefix = prefix
+            self.asns.add(origin_as)
+
+        self.ins_routes() # process data from last line
+
+        logger.info('Committing...')
+        self.cursor.execute('COMMIT')
 
-    logger.info('Creating staging table...')
-    cursor.execute('CREATE TABLE %(table)s_new LIKE %(table)s' % {'table': table})
+        try:
+            logger.info('Dropping old table...')
+            self.cursor.execute('DROP TABLE IF EXISTS %s_old' % self.table)
+        except _mysql_exceptions.Warning:
+            pass
+
+        logger.info('Swapping staging table with live table...')
+        self.cursor.execute('RENAME TABLE %(table)s TO %(table)s_old, %(table)s_new TO %(table)s' % {'table': self.table})
+
+        self.cleanup()  # allow cleanup function to throw prior to COMMIT
+
+        transaction.commit_unless_managed()
+
+        logger.info('Updating timestamp metadata...')
+        rpki.gui.app.timestamp.update('bgp_v4_import')
+
+    def parse_line(self, row):
+        "Parse one line of input. Return a (prefix, origin_as) tuple."
+        return None
+
+    def cleanup(self):
+        pass
 
-    logger.info('Disabling autocommit...')
-    cursor.execute('SET autocommit=0')
+    def ins_routes(self):
+        # output routes for previous prefix
+        if self.last_prefix is not None:
+            try:
+                rng = self.range_class.parse_str(self.last_prefix)
+                rmin = long(rng.min)
+                rmax = long(rng.max)
+                self.cursor.executemany(self.sql, [(asn, rmin, rmax) for asn in self.asns])
+            except BadIPResource:
+                logger.warning('skipping bad prefix: ' + self.last_prefix)
+            self.asns = set() # reset
+
+
+class TextDumpParser(RouteDumpParser):
+    """Parses the RouteViews.org text dump."""
+
+    def __init__(self, *args, **kwargs):
+        super(TextDumpParser, self).__init__(*args, **kwargs)
+        if self.path.endswith('.bz2'):
+            logger.info('decompressing bz2 file')
+            self.file = bz2.BZ2File(self.path, buffering=4096)
+        else:
+            self.file = open(self.path, buffering=-1)
+        self.input = itertools.islice(self.file, 5, None)  # skip first 5 lines
 
-    logger.info('Adding rows to table...')
-    for row in itertools.islice(f, 5, None):
+    def parse_line(self, row):
+        "Parse one line of input"
         cols = row.split()
 
         # index -1 is i/e/? for igp/egp
-        origin_as = cols[-2]
-        # FIXME: skip AS_SETs
-        if origin_as[0] == '{':
-            continue
+        try:
+            origin_as = int(cols[-2])
+        except IndexError:
+            raise ParseError('unexpected format')
+        except ValueError:
+            raise ParseError('bad AS value')
 
         prefix = cols[1]
 
@@ -77,85 +163,35 @@ def parse_text(f):
             s.append('assuming it should be %s' % prefix)
             logger.warning(' '.join(s))
 
-        # the output may contain multiple paths to the same origin.
-        # if this is the same prefix as the last entry, we don't need
-        # to validate it again.
-        #
-        # prefixes are sorted, but the origin_as is not, so we keep a set to
-        # avoid duplicates, and insert into the db once we've seen all the
-        # origin_as values for a given prefix
-        if prefix != last_prefix:
-            # output routes for previous prefix
-            if last_prefix is not None:
-                try:
-                    rng = range_class.parse_str(last_prefix)
-                    rmin = long(rng.min)
-                    rmax = long(rng.max)
-                    cursor.executemany(sql, [(asn, rmin, rmax) for asn in asns])
-                except BadIPResource:
-                    logger.warning('skipping bad prefix: ' + last_prefix)
-
-            asns = set()
-            last_prefix = prefix
-
-        try:
-            asns.add(int(origin_as))
-        except ValueError as err:
-            logger.warning('\n'.join(
-                ['unable to parse origin AS: ' + origin_as],
-                ['ValueError: ' + str(err)]
-                ['route entry was: ' + row],
-            ))
-
-    logger.info('Committing...')
-    cursor.execute('COMMIT')
-
-    try:
-        logger.info('Dropping old table...')
-        cursor.execute('DROP TABLE IF EXISTS %s_old' % table)
-    except _mysql_exceptions.Warning:
-        pass
-
-    logger.info('Swapping staging table with live table...')
-    cursor.execute('RENAME TABLE %(table)s TO %(table)s_old, %(table)s_new TO %(table)s' % {'table': table})
+        return prefix, origin_as
 
-    transaction.commit_unless_managed()
+    def cleanup(self):
+        self.file.close()
 
-    logger.info('Updating timestamp metadata...')
-    rpki.gui.app.timestamp.update('bgp_v4_import')
 
+class MrtDumpParser(RouteDumpParser):
+    def __init__(self, *args, **kwargs):
+        super(MrtDumpParser, self).__init__(*args, **kwargs)
+        # filter input through bgpdump
+        # bgpdump can decompress bz2 files directly, no need to do it here
+        self.pipe = subprocess.Popen(['bgpdump', '-m', '-v', self.path], stdout=subprocess.PIPE, bufsize=-1)
+        self.input = self.pipe.stdout
 
-def parse_mrt(f):
-    # filter input through bgpdump
-    pipe = subprocess.Popen(['bgpdump', '-m', '-v', '-'], stdin=f,
-                            stdout=subprocess.PIPE)
-
-    last_prefix = None
-    last_as = None
-    for e in pipe.stdout.readlines():
-        a = e.split('|')
+    def parse_line(self, row):
+        a = row.split('|')
         prefix = a[5]
         try:
             origin_as = int(a[6].split()[-1])
         except ValueError:
-            # skip AS_SETs
-            continue
-
-        if prefix != last_prefix:
-            last_prefix = prefix
-        elif last_as == origin_as:
-            continue
-        last_as = origin_as
+            raise ParseError('bad AS value')
 
-        asns = PREFIXES.get(prefix)
-        if not asns:
-            asns = set()
-            PREFIXES[prefix] = asns
-        asns.add(origin_as)
+        return prefix, origin_as
 
-    pipe.wait()
-    if pipe.returncode:
-        raise ProgException('bgpdump exited with code %d' % pipe.returncode)
+    def cleanup(self):
+        logger.info('waiting for child process to terminate')
+        self.pipe.wait()
+        if self.pipe.returncode:
+            raise PipeFailed('bgpdump exited with code %d' % self.pipe.returncode)
 
 
 class ProgException(Exception):
@@ -170,7 +206,7 @@ class PipeFailed(ProgException):
     pass
 
 
-def import_routeviews_dump(filename=DEFAULT_URL, filetype='auto'):
+def import_routeviews_dump(filename=DEFAULT_URL, filetype='text'):
     """Load the oix-full-snapshot-latest.bz2 from routeview.org into the
     rpki.gui.routeview database.
 
@@ -182,55 +218,31 @@ def import_routeviews_dump(filename=DEFAULT_URL, filetype='auto'):
     """
 
     start_time = time.time()
-
-    if filename.startswith('http://'):
-        #get filename from the basename of the URL
-        u = urlparse.urlparse(filename)
-        bname = os.path.basename(unquote(u.path))
-        tmpname = os.path.join('/tmp', bname)
-
-        logger.info("Downloading %s to %s", filename, tmpname)
-        if os.path.exists(tmpname):
-            os.remove(tmpname)
-        # filename is replaced with a local filename containing cached copy of
-        # URL
-        filename, headers = urlretrieve(filename, tmpname)
-
-    if filetype == 'auto':
-        # try to determine input type from filename, based on the default
-        # filenames from archive.routeviews.org
-        bname = os.path.basename(filename)
-        if bname.startswith('oix-full-snapshot-latest'):
-            filetype = 'text'
-        elif bname.startswith('rib.'):
-            filetype = 'mrt'
-        else:
-            raise UnknownInputType('unable to automatically determine input file type')
-        logging.info('Detected import format as "%s"', filetype)
-
-    pipe = None
-    if filename.endswith('.bz2'):
-        bunzip = 'bunzip2'
-        logging.info('Decompressing input file on the fly...')
-        pipe = subprocess.Popen([bunzip, '--stdout', filename],
-                                stdout=subprocess.PIPE)
-        input_file = pipe.stdout
-    else:
-        input_file = open(filename)
+    tmpname = None
 
     try:
-        dispatch = {'text': parse_text, 'mrt': parse_mrt}
-        dispatch[filetype](input_file)
-    except KeyError:
-        raise UnknownInputType('"%s" is an unknown input file type' % filetype)
-
-    if pipe:
-        logging.debug('Waiting for child to exit...')
-        pipe.wait()
-        if pipe.returncode:
-            raise PipeFailed('Child exited code %d' % pipe.returncode)
-        pipe = None
-    else:
-        input_file.close()
+        if filename.startswith('http://'):
+            #get filename from the basename of the URL
+            u = urlparse.urlparse(filename)
+            bname = os.path.basename(unquote(u.path))
+            tmpname = os.path.join(settings.DOWNLOAD_DIRECTORY, bname)
+
+            logger.info("Downloading %s to %s", filename, tmpname)
+            if os.path.exists(tmpname):
+                os.remove(tmpname)
+            # filename is replaced with a local filename containing cached copy of
+            # URL
+            filename, headers = urlretrieve(filename, tmpname)
+
+        try:
+            dispatch = {'text': TextDumpParser, 'mrt': MrtDumpParser}
+            dispatch[filetype](filename).parse()
+        except KeyError:
+            raise UnknownInputType('"%s" is an unknown input file type' % filetype)
+
+    finally:
+        # make sure to always clean up the temp download file
+        if tmpname is not None:
+            os.unlink(tmpname)
 
     logger.info('Elapsed time %d secs', (time.time() - start_time))
diff --git a/rpki/gui/script_util.py b/rpki/gui/script_util.py
index 1941f6f7..24b6d313 100644
--- a/rpki/gui/script_util.py
+++ b/rpki/gui/script_util.py
@@ -51,6 +51,7 @@ def setup():
 		},
 	    },
 	    MIDDLEWARE_CLASSES = (),
+	    DOWNLOAD_DIRECTORY = cfg.get('download-directory', '/var/tmp'),
 	)
 	# Can't populate apps if we don't know what they are.  If this
 	# explodes with an AppRegistryNotReady exception, the above comment