Simplify RRDP schema again: one delta file now contains exactly one

delta rather than a range.

svn path=/branches/tk705/; revision=6010

7 files changed, 104 insertions, 142 deletions

diff --git a/ca/tests/rrdp-samples.xml b/ca/tests/rrdp-samples.xml
index 0318b169..966d9887 100644
--- a/ca/tests/rrdp-samples.xml
+++ b/ca/tests/rrdp-samples.xml
@@ -29,12 +29,11 @@
     
   <!-- Notification file: lists current snapshots and deltas -->
 
-   <notification version="1" xmlns="http://www.ripe.net/rpki/rrdp" session_id="d9f6dc91-0394-40b9-9663-66aef4bb623a" serial="203">
+   <notification xmlns="http://www.ripe.net/rpki/rrdp" version="1" session_id="d9f6dc91-0394-40b9-9663-66aef4bb623a" serial="185">
      <snapshot uri="http://host.example/d9f6dc91-0394-40b9-9663-66aeb623a/snapshot/202.xml" hash="279b79fd8389e20585f26735ee70e0e4d4b8af23bb2e2e611c70e92d2433edea"/>
-     <delta from="156" to="183" uri="http://host.example/d9f6c91-0394-40b9-9663-66aeb623a/deltas/156/183.xml" hash="a2d56ec180f2dde2a46bf90565932e25829b852a0b43107d5de6e41394c29100"/>
-     <delta from="183" to="184" uri="http://host.example/d9f6c91-0394-40b9-9663-66aeb623a/deltas/183/184.xml" hash="a2d56ec180f2dde2a46b2e0565932e25829b852a0b43107d5de6e41394c29200"/>
-     <delta from="184" to="197" uri="http://host.example/d9f6c91-0394-40b9-9663-66aeb623a/deltas/184/197.xml" hash="a2d56ec180f2dde2a46b2e0565932e25829b852a0b43107d5de6e41394c29201"/>
-     <delta from="197" to="203" uri="http://host.example/d9f6c91-0394-40b9-9663-66aeb623a/deltas/197/203.xml" hash="a2d56ec180f2dde2a4f92e0565932e25829b852a0b43107d5de6e41394c29300"/>
+     <delta serial="183" uri="http://host.example/d9f6c91-0394-40b9-9663-66aeb623a/deltas/183.xml" hash="a2d56ec180f2dde2a46bf90565932e25829b852a0b43107d5de6e41394c29100"/>
+     <delta serial="184" uri="http://host.example/d9f6c91-0394-40b9-9663-66aeb623a/deltas/184.xml" hash="a2d56ec180f2dde2a46b2e0565932e25829b852a0b43107d5de6e41394c29200"/>
+     <delta serial="185" uri="http://host.example/d9f6c91-0394-40b9-9663-66aeb623a/deltas/185.xml" hash="a2d56ec180f2dde2a46b2e0565932e25829b852a0b43107d5de6e41394c29201"/>
    </notification>
 
    <!-- Snapshot segment: think DNS AXFR -->
@@ -59,30 +58,24 @@
 
    <!-- Delta segment: think DNS IXFR -->
 
-   <deltas version="1" xmlns="http://www.ripe.net/rpki/rrdp" session_id="d9f6dc91-0394-40b9-9663-66aef4bb623a" from="0" to="3">
-     <delta serial="1">
-       <publish uri="http://host.example/foo/bar/cer1.cer">
-         MIIE+jCCA+KgAwIBAgIBDTANBgkqhkiG9w0BAQsFADAzMTEw
-         jRBODAxN0U2NkE5RTkxNzJFNDYxMkQ4Q0Y0QzgzRjIzOERFM
-         h8zeHoFVu6ghRPy5dbOA4akX/KG6b8XIx0iwPYdLiDbdWFbt
-       </publish>
-     </delta>
-     <delta serial="2">
-       <withdraw uri="http://host.example/foo/bar/cer1.cer" hash="deadf00d"/>
-       <publish uri="http://host.example/foo/bar/cer2.cer">
-         MIIE+jCCA+KgAwIBAgIBDTANBgkqhkiG9w0BAQsFADAzMTEw
-         h8zeHoFVu6ghRPy5dbOA4akX/KG6b8XIx0iwPYdLiDbdWFbt
-         jRBODAxN0U2NkE5RTkxNzJFNDYxMkQ4Q0Y0QzgzRjIzOERFM
-       </publish>
-       <publish uri="http://host.example/foo/bar/cer3.cer" hash="deadf00d">
-         MIIE+jCCA+KgAwIBAgIBDTANBgkqhkiG9w0BAQsFADAzMTEw
-         h8zeHoFVu6ghRPy5dbOA4akX/KG6b8XIx0iwPYdLiDbdWFbt
-         jRBODAxN0U2NkE5RTkxNzJFNDYxMkQ4Q0Y0QzgzRjIzOERFM
-       </publish>
-     </delta>
-     <delta serial="3">
-       <withdraw uri="http://host.example/foo/bar/cer2.cer" hash="deadf00d"/>
-     </delta>
-   </deltas>
+   <delta version="1" xmlns="http://www.ripe.net/rpki/rrdp" session_id="d9f6dc91-0394-40b9-9663-66aef4bb623a" serial="3">
+     <publish uri="http://host.example/foo/bar/cer1.cer">
+       MIIE+jCCA+KgAwIBAgIBDTANBgkqhkiG9w0BAQsFADAzMTEw
+       jRBODAxN0U2NkE5RTkxNzJFNDYxMkQ4Q0Y0QzgzRjIzOERFM
+       h8zeHoFVu6ghRPy5dbOA4akX/KG6b8XIx0iwPYdLiDbdWFbt
+     </publish>
+     <withdraw uri="http://host.example/foo/bar/cer1.cer" hash="deadf00d"/>
+     <publish uri="http://host.example/foo/bar/cer2.cer">
+       MIIE+jCCA+KgAwIBAgIBDTANBgkqhkiG9w0BAQsFADAzMTEw
+       h8zeHoFVu6ghRPy5dbOA4akX/KG6b8XIx0iwPYdLiDbdWFbt
+       jRBODAxN0U2NkE5RTkxNzJFNDYxMkQ4Q0Y0QzgzRjIzOERFM
+     </publish>
+     <publish uri="http://host.example/foo/bar/cer3.cer" hash="deadf00d">
+       MIIE+jCCA+KgAwIBAgIBDTANBgkqhkiG9w0BAQsFADAzMTEw
+       h8zeHoFVu6ghRPy5dbOA4akX/KG6b8XIx0iwPYdLiDbdWFbt
+       jRBODAxN0U2NkE5RTkxNzJFNDYxMkQ4Q0Y0QzgzRjIzOERFM
+     </publish>
+     <withdraw uri="http://host.example/foo/bar/cer4.cer" hash="deadf00d"/>
+   </delta>
 
 </completely_gratuitous_wrapper_element_to_let_me_run_this_through_xmllint>
diff --git a/ca/tests/test-rrdp.py b/ca/tests/test-rrdp.py
index db626a35..9b7f207c 100755
--- a/ca/tests/test-rrdp.py
+++ b/ca/tests/test-rrdp.py
@@ -34,6 +34,7 @@ parser.add_argument("--yaml-file", default = "smoketest.2.yaml")
 parser.add_argument("--delay", type = int, default = 30)
 parser.add_argument("--exhaustive", action = "store_true")
 parser.add_argument("--skip-daemons", action = "store_true")
+parser.add_argument("--dry-run", action = "store_true")
 args = parser.parse_args()
 
 def log(msg):
@@ -42,7 +43,8 @@ def log(msg):
 
 def run(*argv):
   log("Running: " + " ".join(argv))
-  subprocess.check_call(argv)
+  if not args.dry_run:
+    subprocess.check_call(argv)
 
 def dataglob(pattern):
   return glob.iglob(os.path.join(("smoketest.dir" if args.use_smoketest else "yamltest.dir/RIR"), pattern))
@@ -51,7 +53,7 @@ def snapshot_to_serial(fn):
   return int(os.path.splitext(os.path.basename(fn))[0])
 
 def delta_to_serial(fn):
-  return int(os.path.splitext(os.path.basename(fn))[0].split("-")[1])
+  return int(os.path.splitext(os.path.basename(fn))[0])
 
 top = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "..", ".."))
 
diff --git a/potpourri/rrdp-test-tool b/potpourri/rrdp-test-tool
index b4dc65da..8ea90f17 100755
--- a/potpourri/rrdp-test-tool
+++ b/potpourri/rrdp-test-tool
@@ -33,7 +33,7 @@ class Tags(object):
     for tag in tags:
       setattr(self, tag, rpki.relaxng.rrdp.xmlns + tag)
 
-tags = Tags("notification", "deltas", "delta", "snapshot", "publish", "withdraw")
+tags = Tags("notification", "delta", "snapshot", "publish", "withdraw")
 
 class main(object):
 
@@ -51,7 +51,7 @@ class main(object):
     for rrdp_file in self.args.rrdp_file:
       xml = lxml.etree.ElementTree(file = rrdp_file).getroot()
       rpki.relaxng.rrdp.assertValid(xml)
-      getattr(self, xml.tag[len(rpki.relaxng.rrdp.xmlns):])(xml)
+      getattr(self, "handle_" + xml.tag[len(rpki.relaxng.rrdp.xmlns):])(xml)
 
   @property
   def serial_filename(self):
@@ -65,15 +65,15 @@ class main(object):
     with open(self.serial_filename, "w") as f:
       f.write("%s\n" % value)
 
-  def notification(self, xml):
+  def handle_notification(self, xml):
     print "Notification version %s session %s serial %s" % (
       xml.get("version"), xml.get("session_id"), xml.get("serial"))
     assert xml[0].tag == tags.snapshot
     print " Snapshot URI %s hash %s" % (
       xml[0].get("uri"), xml[0].get("hash"))
     for i, elt in enumerate(xml.iterchildren(tags.delta)):
-      print " Delta %3d from %6s to %6s URI %s hash %s" % (
-        i, elt.get("from"), elt.get("to"), elt.get("uri"), elt.get("hash"))
+      print " Delta %3d serial %6s URI %s hash %s" % (
+        i, elt.get("serial"), elt.get("uri"), elt.get("hash"))
 
   def uri_to_filename(self, uri):
     assert uri.startswith("rsync://")
@@ -102,7 +102,7 @@ class main(object):
       else:
         dn = os.path.dirname(dn)
 
-  def snapshot(self, xml):
+  def handle_snapshot(self, xml):
     print "Unpacking snapshot version %s session %s serial %6s" % (
       xml.get("version"), xml.get("session_id"), xml.get("serial"))
     for elt in xml.iterchildren(tags.publish):
@@ -110,33 +110,26 @@ class main(object):
       self.add_obj(elt.get("uri"), elt.text.decode("base64"))
     self.set_serial(xml.get("serial"))
 
-  def deltas(self, xml):
-    cur = int(self.get_serial())
-    old = int(xml.get("from"))
-    new = int(xml.get("to"))
-    print "Unpacking deltas version %s session %s from %s to %s" % (
-      xml.get("version"), xml.get("session_id"), old, new)
-    if cur != old:
-      raise RuntimeError("Can't apply deltas: current %s old %s new %s" % (cur, old, new))
-    for i, delta in enumerate(xml.iterchildren(tags.delta)):
-      serial = int(delta.get("serial"))
-      print " Delta %3d serial %d" % (i, serial)
-      if cur != serial - 1:
-        raise RuntimeError("Can't apply delta: current %s delta serial %s" % (cur, serial))
-      for j, elt in enumerate(delta.iterchildren(tags.withdraw)):
-        uri = elt.get("uri")
-        hash = elt.get("hash")
-        print "  %3d withdraw URI %s hash %s" % (j, uri, hash)
+  def handle_delta(self, xml):
+    old_serial = int(self.get_serial())
+    new_serial = int(xml.get("serial"))
+    print "Unpacking deltas version %s session %s serial %s" % (
+      xml.get("version"), xml.get("session_id"), new_serial)
+    if old_serial != new_serial - 1:
+      raise RuntimeError("Can't apply deltas: old serial %s new serial %s" % (old_serial, new_serial))
+    for i, elt in enumerate(xml.iterchildren(tags.withdraw)):
+      uri = elt.get("uri")
+      hash = elt.get("hash")
+      print "  %3d withdraw URI %s hash %s" % (i, uri, hash)
+      self.del_obj(uri, hash)
+    for i, elt in enumerate(xml.iterchildren(tags.publish)):
+      uri = elt.get("uri")
+      hash = elt.get("hash", None)
+      print "  %3d publish  URI %s hash %s" % (i, uri, hash)
+      if hash is not None:
         self.del_obj(uri, hash)
-      for j, elt in enumerate(delta.iterchildren(tags.publish)):
-        uri = elt.get("uri")
-        hash = elt.get("hash", None)
-        print "  %3d publish  URI %s hash %s" % (j, uri, hash)
-        if hash is not None:
-          self.del_obj(uri, hash)
-        self.add_obj(elt.get("uri"), elt.text.decode("base64"))
-      cur += 1
-      self.set_serial(cur)
+      self.add_obj(elt.get("uri"), elt.text.decode("base64"))
+    self.set_serial(new_serial)
 
 if __name__ == "__main__":
   main()
diff --git a/rpki/pubdb/models.py b/rpki/pubdb/models.py
index f7edfd4a..5241ea45 100644
--- a/rpki/pubdb/models.py
+++ b/rpki/pubdb/models.py
@@ -38,7 +38,6 @@ rrdp_nsmap   = rpki.relaxng.rrdp.nsmap
 rrdp_version = "1"
 
 rrdp_tag_delta        = rrdp_xmlns + "delta"
-rrdp_tag_deltas       = rrdp_xmlns + "deltas"
 rrdp_tag_notification = rrdp_xmlns + "notification"
 rrdp_tag_publish      = rrdp_xmlns + "publish"
 rrdp_tag_snapshot     = rrdp_xmlns + "snapshot"
@@ -83,6 +82,11 @@ class Session(models.Model):
   snapshot = models.TextField(blank = True)
   hash = models.CharField(max_length = 64, blank = True)
 
+  ## @var keep_all_rrdp_files
+  # Debugging flag to prevent expiration of old RRDP files.
+  # This simplifies debugging delta code.  Need for this
+  # may go away once RRDP is fully integrated into rcynic.
+  keep_all_rrdp_files = False
 
   def new_delta(self, expires):
     """
@@ -92,13 +96,11 @@ class Session(models.Model):
     delta = Delta(session = self,
                   serial = self.serial + 1,
                   expires = expires)
-    delta.deltas = Element(rrdp_tag_deltas,
-                           nsmap = rrdp_nsmap,
-                           version = rrdp_version,
-                           session_id = self.uuid)
-    delta.deltas.set("to",   str(delta.serial))
-    delta.deltas.set("from", str(delta.serial - 1))
-    SubElement(delta.deltas, rrdp_tag_delta, serial = str(delta.serial)).text = "\n"
+    delta.elt = Element(rrdp_tag_delta,
+                        nsmap = rrdp_nsmap,
+                        version = rrdp_version,
+                        session_id = self.uuid,
+                        serial = str(delta.serial))
     return delta
 
 
@@ -165,11 +167,10 @@ class Session(models.Model):
                uri = self._rrdp_filename_to_uri(self.snapshot_fn, rrdp_uri_base),
                hash = self.hash)
     for delta in self.delta_set.all():
-      se = SubElement(xml, rrdp_tag_delta,
-                      uri = self._rrdp_filename_to_uri(delta.fn, rrdp_uri_base),
-                      hash =  delta.hash)
-      se.set("to",   str(delta.serial))
-      se.set("from", str(delta.serial - 1))
+      SubElement(xml, rrdp_tag_delta,
+                 uri = self._rrdp_filename_to_uri(delta.fn, rrdp_uri_base),
+                 hash =  delta.hash,
+                 serial = str(delta.serial))
     rpki.relaxng.rrdp.assertValid(xml)
     return ElementToString(xml, pretty_print = True)
 
@@ -192,16 +193,17 @@ class Session(models.Model):
                           rrdp_publication_base, overwrite = True)
     current_filenames.add(self.notification_fn)
 
-    for root, dirs, files in os.walk(rrdp_publication_base, topdown = False):
-      for fn in files:
-        fn = os.path.join(root, fn)
-        if fn[len(rrdp_publication_base):].lstrip("/") not in current_filenames:
-          os.remove(fn)
-      for dn in dirs:
-        try:
-          os.rmdir(os.path.join(root, dn))
-        except OSError:
-          pass
+    if not self.keep_all_rrdp_files:
+      for root, dirs, files in os.walk(rrdp_publication_base, topdown = False):
+        for fn in files:
+          fn = os.path.join(root, fn)
+          if fn[len(rrdp_publication_base):].lstrip("/") not in current_filenames:
+            os.remove(fn)
+        for dn in dirs:
+          try:
+            os.rmdir(os.path.join(root, dn))
+          except OSError:
+            pass
 
 
 class Delta(models.Model):
@@ -226,12 +228,12 @@ class Delta(models.Model):
 
   @property
   def fn(self):
-    return "%s/deltas/%s-%s.xml" % (self.session.uuid, self.serial - 1, self.serial)
+    return "%s/deltas/%s.xml" % (self.session.uuid, self.serial)
 
 
   def activate(self):
-    rpki.relaxng.rrdp.assertValid(self.deltas)
-    self.xml = ElementToString(self.deltas, pretty_print = True)
+    rpki.relaxng.rrdp.assertValid(self.elt)
+    self.xml = ElementToString(self.elt, pretty_print = True)
     self.hash = rpki.x509.sha256(self.xml).encode("hex")
     self.save()
     self.session.serial += 1
@@ -252,10 +254,10 @@ class Delta(models.Model):
     logger.debug("Publishing %s", uri)
     PublishedObject.objects.create(session = self.session, client = client, der = der, uri = uri,
                                    hash = rpki.x509.sha256(der).encode("hex"))
-    se = DERSubElement(self.deltas[0], rrdp_tag_publish, der = der, uri = uri)
+    se = DERSubElement(self.elt, rrdp_tag_publish, der = der, uri = uri)
     if hash is not None:
       se.set("hash", hash)
-    rpki.relaxng.rrdp.assertValid(self.deltas)
+    rpki.relaxng.rrdp.assertValid(self.elt)
 
 
   def withdraw(self, client, uri, hash):
@@ -264,13 +266,13 @@ class Delta(models.Model):
       raise rpki.exceptions.DifferentObjectAtURI("Found different object at %s (old %s, new %s)" % (uri, obj.hash, hash))
     logger.debug("Withdrawing %s", uri)
     obj.delete()
-    SubElement(self.deltas[0], rrdp_tag_withdraw, uri = uri, hash = hash).tail = "\n"
-    rpki.relaxng.rrdp.assertValid(self.deltas)
+    SubElement(self.elt, rrdp_tag_withdraw, uri = uri, hash = hash).tail = "\n"
+    rpki.relaxng.rrdp.assertValid(self.elt)
 
 
   def update_rsync_files(self, publication_base):
     min_path_len = len(publication_base.rstrip("/"))
-    for pdu in self.deltas[0]:
+    for pdu in self.elt:
       assert pdu.tag in (rrdp_tag_publish, rrdp_tag_withdraw)
       fn = self._uri_to_filename(pdu.get("uri"), publication_base)
       if pdu.tag == rrdp_tag_publish:
@@ -295,7 +297,7 @@ class Delta(models.Model):
             break
           else:
             dn = os.path.dirname(dn)
-    del self.deltas
+    del self.elt
 
 
 class PublishedObject(models.Model):
diff --git a/rpki/relaxng.py b/rpki/relaxng.py
index 394c0218..4906e019 100644
--- a/rpki/relaxng.py
+++ b/rpki/relaxng.py
@@ -2139,10 +2139,7 @@ rrdp = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?>
       </element>
       <zeroOrMore>
         <element name="delta">
-          <attribute name="from">
-            <ref name="serial"/>
-          </attribute>
-          <attribute name="to">
+          <attribute name="serial">
             <ref name="serial"/>
           </attribute>
           <attribute name="uri">
@@ -2179,28 +2176,18 @@ rrdp = RelaxNGParser(r'''<?xml version="1.0" encoding="UTF-8"?>
   </start>
   <!-- Delta segment: think DNS IXFR. -->
   <start combine="choice">
-    <element name="deltas">
+    <element name="delta">
       <attribute name="version">
         <ref name="version"/>
       </attribute>
       <attribute name="session_id">
         <ref name="uuid"/>
       </attribute>
-      <attribute name="from">
-        <ref name="serial"/>
-      </attribute>
-      <attribute name="to">
+      <attribute name="serial">
         <ref name="serial"/>
       </attribute>
       <oneOrMore>
-        <element name="delta">
-          <attribute name="serial">
-            <ref name="serial"/>
-          </attribute>
-          <oneOrMore>
-            <ref name="delta_element"/>
-          </oneOrMore>
-        </element>
+        <ref name="delta_element"/>
       </oneOrMore>
     </element>
   </start>
diff --git a/schemas/relaxng/rrdp.rnc b/schemas/relaxng/rrdp.rnc
index 2829605d..7809abdd 100644
--- a/schemas/relaxng/rrdp.rnc
+++ b/schemas/relaxng/rrdp.rnc
@@ -36,10 +36,9 @@ start |= element notification {
     attribute hash { hash }
   },
   element delta {
-    attribute from { serial },
-    attribute to   { serial },
-    attribute uri  { uri },
-    attribute hash { hash }
+    attribute serial { serial },
+    attribute uri    { uri },
+    attribute hash   { hash }
   }*
 }
 
@@ -49,20 +48,19 @@ start |= element snapshot {
   attribute version    { version },
   attribute session_id { uuid },
   attribute serial     { serial },
-  element publish      { attribute uri { uri }, base64 }*
+  element publish      {
+    attribute uri { uri },
+    base64
+  }*
 }
 
 # Delta segment: think DNS IXFR.
 
-start |= element deltas {
+start |= element delta {
   attribute version    { version },
   attribute session_id { uuid },
-  attribute from       { serial },
-  attribute to         { serial },
-  element delta {
-    attribute serial   { serial },
-    delta_element+
-  }+
+  attribute serial     { serial },
+  delta_element+
 }
 
 delta_element |= element publish  {
diff --git a/schemas/relaxng/rrdp.rng b/schemas/relaxng/rrdp.rng
index 9bd3a207..9ff2ff1e 100644
--- a/schemas/relaxng/rrdp.rng
+++ b/schemas/relaxng/rrdp.rng
@@ -65,10 +65,7 @@
       </element>
       <zeroOrMore>
         <element name="delta">
-          <attribute name="from">
-            <ref name="serial"/>
-          </attribute>
-          <attribute name="to">
+          <attribute name="serial">
             <ref name="serial"/>
           </attribute>
           <attribute name="uri">
@@ -105,28 +102,18 @@
   </start>
   <!-- Delta segment: think DNS IXFR. -->
   <start combine="choice">
-    <element name="deltas">
+    <element name="delta">
       <attribute name="version">
         <ref name="version"/>
       </attribute>
       <attribute name="session_id">
         <ref name="uuid"/>
       </attribute>
-      <attribute name="from">
-        <ref name="serial"/>
-      </attribute>
-      <attribute name="to">
+      <attribute name="serial">
         <ref name="serial"/>
       </attribute>
       <oneOrMore>
-        <element name="delta">
-          <attribute name="serial">
-            <ref name="serial"/>
-          </attribute>
-          <oneOrMore>
-            <ref name="delta_element"/>
-          </oneOrMore>
-        </element>
+        <ref name="delta_element"/>
       </oneOrMore>
     </element>
   </start>