Convert pubd to use Django ORM and lxml.etree.

smoketest temporarily broken as it doesn't know anything about Django.

svn path=/branches/tk705/; revision=5961

4 files changed, 449 insertions, 1 deletions

diff --git a/rpki/pubdb/__init__.py b/rpki/pubdb/__init__.py
index 5e25c7e3..2c83051f 100644
--- a/rpki/pubdb/__init__.py
+++ b/rpki/pubdb/__init__.py
@@ -1,3 +1,21 @@
 # $Id$
 #
-# Placeholder for pubdb Django models not yet written.
+# Copyright (C) 2014 Dragon Research Labs ("DRL")
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH
+# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS.  IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+
+"""
+Package for Django ORM models relating to pubd.
+"""
+
+from rpki.pubdb.models import *
diff --git a/rpki/pubdb/migrations/0001_initial.py b/rpki/pubdb/migrations/0001_initial.py
new file mode 100644
index 00000000..c796d020
--- /dev/null
+++ b/rpki/pubdb/migrations/0001_initial.py
@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import dbs
+from south.v2 import SchemaMigration
+from django.db import models
+
+db = dbs["pubdb"]
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'Client'
+        db.create_table(u'pubdb_client', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('client_handle', self.gf('django.db.models.fields.CharField')(unique=True, max_length=255)),
+            ('base_uri', self.gf('django.db.models.fields.TextField')()),
+            ('bpki_cert', self.gf('rpki.fields.BlobField')(default=None, blank=True)),
+            ('bpki_glue', self.gf('rpki.fields.BlobField')(default=None, null=True, blank=True)),
+            ('last_cms_timestamp', self.gf('rpki.fields.SundialField')(null=True, blank=True)),
+        ))
+        db.send_create_signal(u'pubdb', ['Client'])
+
+        # Adding model 'Session'
+        db.create_table(u'pubdb_session', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('uuid', self.gf('django.db.models.fields.CharField')(unique=True, max_length=36)),
+            ('serial', self.gf('django.db.models.fields.BigIntegerField')()),
+            ('snapshot', self.gf('django.db.models.fields.TextField')(blank=True)),
+            ('hash', self.gf('django.db.models.fields.CharField')(max_length=64, blank=True)),
+        ))
+        db.send_create_signal(u'pubdb', ['Session'])
+
+        # Adding model 'Delta'
+        db.create_table(u'pubdb_delta', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('serial', self.gf('django.db.models.fields.BigIntegerField')()),
+            ('xml', self.gf('django.db.models.fields.TextField')()),
+            ('hash', self.gf('django.db.models.fields.CharField')(max_length=64)),
+            ('expires', self.gf('rpki.fields.SundialField')()),
+            ('session', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['pubdb.Session'])),
+        ))
+        db.send_create_signal(u'pubdb', ['Delta'])
+
+        # Adding model 'PublishedObject'
+        db.create_table(u'pubdb_publishedobject', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('uri', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('der', self.gf('rpki.fields.BlobField')(default=None, blank=True)),
+            ('hash', self.gf('django.db.models.fields.CharField')(max_length=64)),
+            ('client', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['pubdb.Client'])),
+            ('session', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['pubdb.Session'])),
+        ))
+        db.send_create_signal(u'pubdb', ['PublishedObject'])
+
+        # Adding unique constraint on 'PublishedObject', fields ['session', 'hash']
+        db.create_unique(u'pubdb_publishedobject', ['session_id', 'hash'])
+
+        # Adding unique constraint on 'PublishedObject', fields ['session', 'uri']
+        db.create_unique(u'pubdb_publishedobject', ['session_id', 'uri'])
+
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'PublishedObject', fields ['session', 'uri']
+        db.delete_unique(u'pubdb_publishedobject', ['session_id', 'uri'])
+
+        # Removing unique constraint on 'PublishedObject', fields ['session', 'hash']
+        db.delete_unique(u'pubdb_publishedobject', ['session_id', 'hash'])
+
+        # Deleting model 'Client'
+        db.delete_table(u'pubdb_client')
+
+        # Deleting model 'Session'
+        db.delete_table(u'pubdb_session')
+
+        # Deleting model 'Delta'
+        db.delete_table(u'pubdb_delta')
+
+        # Deleting model 'PublishedObject'
+        db.delete_table(u'pubdb_publishedobject')
+
+
+    models = {
+        u'pubdb.client': {
+            'Meta': {'object_name': 'Client'},
+            'base_uri': ('django.db.models.fields.TextField', [], {}),
+            'bpki_cert': ('rpki.fields.BlobField', [], {'default': 'None', 'blank': 'True'}),
+            'bpki_glue': ('rpki.fields.BlobField', [], {'default': 'None', 'null': 'True', 'blank': 'True'}),
+            'client_handle': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_cms_timestamp': ('rpki.fields.SundialField', [], {'null': 'True', 'blank': 'True'})
+        },
+        u'pubdb.delta': {
+            'Meta': {'object_name': 'Delta'},
+            'expires': ('rpki.fields.SundialField', [], {}),
+            'hash': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'serial': ('django.db.models.fields.BigIntegerField', [], {}),
+            'session': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['pubdb.Session']"}),
+            'xml': ('django.db.models.fields.TextField', [], {})
+        },
+        u'pubdb.publishedobject': {
+            'Meta': {'unique_together': "((u'session', u'hash'), (u'session', u'uri'))", 'object_name': 'PublishedObject'},
+            'client': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['pubdb.Client']"}),
+            'der': ('rpki.fields.BlobField', [], {'default': 'None', 'blank': 'True'}),
+            'hash': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'session': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['pubdb.Session']"}),
+            'uri': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+        },
+        u'pubdb.session': {
+            'Meta': {'object_name': 'Session'},
+            'hash': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'serial': ('django.db.models.fields.BigIntegerField', [], {}),
+            'snapshot': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '36'})
+        }
+    }
+
+    complete_apps = ['pubdb']
diff --git a/rpki/pubdb/migrations/__init__.py b/rpki/pubdb/migrations/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/rpki/pubdb/migrations/__init__.py
diff --git a/rpki/pubdb/models.py b/rpki/pubdb/models.py
new file mode 100644
index 00000000..f7edfd4a
--- /dev/null
+++ b/rpki/pubdb/models.py
@@ -0,0 +1,310 @@
+# $Id$
+#
+# Copyright (C) 2014 Dragon Research Labs ("DRL")
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND DRL DISCLAIMS ALL WARRANTIES WITH
+# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS.  IN NO EVENT SHALL DRL BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+
+"""
+Django ORM models for pubd.
+"""
+
+from __future__ import unicode_literals
+from django.db import models
+from rpki.fields import BlobField, CertificateField, SundialField
+from lxml.etree import Element, SubElement, tostring as ElementToString
+
+import os
+import logging
+import rpki.exceptions
+import rpki.relaxng
+
+logger = logging.getLogger(__name__)
+
+
+# Some of this probably ought to move into a rpki.rrdp module.
+
+rrdp_xmlns   = rpki.relaxng.rrdp.xmlns
+rrdp_nsmap   = rpki.relaxng.rrdp.nsmap
+rrdp_version = "1"
+
+rrdp_tag_delta        = rrdp_xmlns + "delta"
+rrdp_tag_deltas       = rrdp_xmlns + "deltas"
+rrdp_tag_notification = rrdp_xmlns + "notification"
+rrdp_tag_publish      = rrdp_xmlns + "publish"
+rrdp_tag_snapshot     = rrdp_xmlns + "snapshot"
+rrdp_tag_withdraw     = rrdp_xmlns + "withdraw"
+
+
+# This would probably be useful to more than just this module, not
+# sure quite where to put it at the moment.
+
+def DERSubElement(elt, name, der, attrib = None, **kwargs):
+  """
+  Convenience wrapper around SubElement for use with Base64 text.
+  """
+
+  se = SubElement(elt, name, attrib, **kwargs)
+  se.text = rpki.x509.base64_with_linebreaks(der)
+  se.tail = "\n"
+  return se
+
+
+
+class Client(models.Model):
+  client_handle = models.CharField(unique = True, max_length = 255)
+  base_uri = models.TextField()
+  bpki_cert = CertificateField()
+  bpki_glue = CertificateField(null = True)
+  last_cms_timestamp = SundialField(blank = True, null = True)
+
+
+  def check_allowed_uri(self, uri):
+    """
+    Make sure that a target URI is within this client's allowed URI space.
+    """
+
+    if not uri.startswith(self.base_uri):
+      raise rpki.exceptions.ForbiddenURI
+
+
+class Session(models.Model):
+  uuid = models.CharField(unique = True, max_length=36)
+  serial = models.BigIntegerField()
+  snapshot = models.TextField(blank = True)
+  hash = models.CharField(max_length = 64, blank = True)
+
+
+  def new_delta(self, expires):
+    """
+    Construct a new delta associated with this session.
+    """
+
+    delta = Delta(session = self,
+                  serial = self.serial + 1,
+                  expires = expires)
+    delta.deltas = Element(rrdp_tag_deltas,
+                           nsmap = rrdp_nsmap,
+                           version = rrdp_version,
+                           session_id = self.uuid)
+    delta.deltas.set("to",   str(delta.serial))
+    delta.deltas.set("from", str(delta.serial - 1))
+    SubElement(delta.deltas, rrdp_tag_delta, serial = str(delta.serial)).text = "\n"
+    return delta
+
+
+  def expire_deltas(self):
+    """
+    Delete deltas whose expiration date has passed.
+    """
+
+    self.delta_set.filter(expires__lt = rpki.sundial.now()).delete()
+  
+
+  def generate_snapshot(self):
+    """
+    Generate an XML snapshot of this session.
+    """
+
+    xml = Element(rrdp_tag_snapshot, nsmap = rrdp_nsmap,
+                  version = rrdp_version,
+                  session_id = self.uuid,
+                  serial = str(self.serial))
+    xml.text = "\n"
+    for obj in self.publishedobject_set.all():
+      DERSubElement(xml, rrdp_tag_publish,
+                    der = obj.der,
+                    uri = obj.uri)
+    rpki.relaxng.rrdp.assertValid(xml)
+    self.snapshot = ElementToString(xml, pretty_print = True)
+    self.hash = rpki.x509.sha256(self.snapshot).encode("hex")
+    self.save()
+
+
+  @property
+  def snapshot_fn(self):
+    return "%s/snapshot/%s.xml" % (self.uuid, self.serial)
+
+
+  @property
+  def notification_fn(self):
+    return "updates.xml"
+
+
+  @staticmethod
+  def _write_rrdp_file(fn, text, rrdp_publication_base, overwrite = False):
+    if overwrite or not os.path.exists(os.path.join(rrdp_publication_base, fn)):
+      tn = os.path.join(rrdp_publication_base, fn + ".%s.tmp" % os.getpid())
+      if not os.path.isdir(os.path.dirname(tn)):
+        os.makedirs(os.path.dirname(tn))
+      with open(tn, "w") as f:
+        f.write(text)
+      os.rename(tn, os.path.join(rrdp_publication_base, fn))
+
+
+  @staticmethod
+  def _rrdp_filename_to_uri(fn, rrdp_uri_base):
+    return "%s/%s" % (rrdp_uri_base.rstrip("/"), fn)
+
+
+  def _generate_update_xml(self, rrdp_uri_base):
+    xml = Element(rrdp_tag_notification, nsmap = rrdp_nsmap,
+                  version = rrdp_version,
+                  session_id = self.uuid,
+                  serial = str(self.serial))
+    SubElement(xml, rrdp_tag_snapshot,
+               uri = self._rrdp_filename_to_uri(self.snapshot_fn, rrdp_uri_base),
+               hash = self.hash)
+    for delta in self.delta_set.all():
+      se = SubElement(xml, rrdp_tag_delta,
+                      uri = self._rrdp_filename_to_uri(delta.fn, rrdp_uri_base),
+                      hash =  delta.hash)
+      se.set("to",   str(delta.serial))
+      se.set("from", str(delta.serial - 1))
+    rpki.relaxng.rrdp.assertValid(xml)
+    return ElementToString(xml, pretty_print = True)
+
+
+  def synchronize_rrdp_files(self, rrdp_publication_base, rrdp_uri_base):
+    """
+    Write current RRDP files to disk, clean up old files and directories.
+    """
+
+    current_filenames = set()
+
+    for delta in self.delta_set.all():
+      self._write_rrdp_file(delta.fn, delta.xml, rrdp_publication_base)
+      current_filenames.add(delta.fn)
+
+    self._write_rrdp_file(self.snapshot_fn, self.snapshot, rrdp_publication_base)
+    current_filenames.add(self.snapshot_fn)
+
+    self._write_rrdp_file(self.notification_fn, self._generate_update_xml(rrdp_uri_base),
+                          rrdp_publication_base, overwrite = True)
+    current_filenames.add(self.notification_fn)
+
+    for root, dirs, files in os.walk(rrdp_publication_base, topdown = False):
+      for fn in files:
+        fn = os.path.join(root, fn)
+        if fn[len(rrdp_publication_base):].lstrip("/") not in current_filenames:
+          os.remove(fn)
+      for dn in dirs:
+        try:
+          os.rmdir(os.path.join(root, dn))
+        except OSError:
+          pass
+
+
+class Delta(models.Model):
+  serial = models.BigIntegerField()
+  xml = models.TextField()
+  hash = models.CharField(max_length = 64)
+  expires = SundialField()
+  session = models.ForeignKey(Session)
+
+
+  @staticmethod
+  def _uri_to_filename(uri, publication_base):
+    if not uri.startswith("rsync://"):
+      raise rpki.exceptions.BadURISyntax(uri)
+    path = uri.split("/")[4:]
+    path.insert(0, publication_base.rstrip("/"))
+    filename = "/".join(path)
+    if "/../" in filename or filename.endswith("/.."):
+      raise rpki.exceptions.BadURISyntax(filename)
+    return filename
+
+
+  @property
+  def fn(self):
+    return "%s/deltas/%s-%s.xml" % (self.session.uuid, self.serial - 1, self.serial)
+
+
+  def activate(self):
+    rpki.relaxng.rrdp.assertValid(self.deltas)
+    self.xml = ElementToString(self.deltas, pretty_print = True)
+    self.hash = rpki.x509.sha256(self.xml).encode("hex")
+    self.save()
+    self.session.serial += 1
+    self.session.save()
+
+
+  def publish(self, client, der, uri, hash):
+    try:
+      obj = client.publishedobject_set.get(session = self.session, uri = uri)
+      if obj.hash == hash:
+        obj.delete()
+      elif hash is None:
+        raise rpki.exceptions.ExistingObjectAtURI("Object already published at %s" % uri)
+      else:
+        raise rpki.exceptions.DifferentObjectAtURI("Found different object at %s (old %s, new %s)" % (uri, obj.hash, hash))
+    except rpki.pubdb.PublishedObject.DoesNotExist:
+      pass
+    logger.debug("Publishing %s", uri)
+    PublishedObject.objects.create(session = self.session, client = client, der = der, uri = uri,
+                                   hash = rpki.x509.sha256(der).encode("hex"))
+    se = DERSubElement(self.deltas[0], rrdp_tag_publish, der = der, uri = uri)
+    if hash is not None:
+      se.set("hash", hash)
+    rpki.relaxng.rrdp.assertValid(self.deltas)
+
+
+  def withdraw(self, client, uri, hash):
+    obj = client.publishedobject_set.get(session = self.session, uri = uri)
+    if obj.hash != hash:
+      raise rpki.exceptions.DifferentObjectAtURI("Found different object at %s (old %s, new %s)" % (uri, obj.hash, hash))
+    logger.debug("Withdrawing %s", uri)
+    obj.delete()
+    SubElement(self.deltas[0], rrdp_tag_withdraw, uri = uri, hash = hash).tail = "\n"
+    rpki.relaxng.rrdp.assertValid(self.deltas)
+
+
+  def update_rsync_files(self, publication_base):
+    min_path_len = len(publication_base.rstrip("/"))
+    for pdu in self.deltas[0]:
+      assert pdu.tag in (rrdp_tag_publish, rrdp_tag_withdraw)
+      fn = self._uri_to_filename(pdu.get("uri"), publication_base)
+      if pdu.tag == rrdp_tag_publish:
+        tn = fn + ".tmp"
+        dn = os.path.dirname(fn)
+        if not os.path.isdir(dn):
+          os.makedirs(dn)
+        with open(tn, "wb") as f:
+          f.write(pdu.text.decode("base64"))
+        os.rename(tn, fn)
+      else:
+        try:
+          os.remove(fn)
+        except OSError, e:
+          if e.errno != errno.ENOENT:
+            raise
+        dn = os.path.dirname(fn)
+        while len(dn) > min_path_len:
+          try:
+            os.rmdir(dn)
+          except OSError:
+            break
+          else:
+            dn = os.path.dirname(dn)
+    del self.deltas
+
+
+class PublishedObject(models.Model):
+  uri = models.CharField(max_length = 255)
+  der = BlobField()
+  hash = models.CharField(max_length = 64)
+  client = models.ForeignKey(Client)
+  session = models.ForeignKey(Session)
+
+  class Meta:
+    unique_together = (("session", "hash"),
+                       ("session", "uri"))