Move root.cer to separate rsync module when building test

configurations, to avoid warnings about it being tainted.  This is
really a conflict between needing to support rootd and wanting to make
the configuration clean when not running rootd: either we add an extra
directory level to the publication structure which we don't use when
not running rootd, or we move root.cer somewhere else.  Since the
latter is simpler except for test configurations which are already
generated for us by programs, we do the latter.

svn path=/branches/tk274/; revision=4667

6 files changed, 134 insertions, 91 deletions

diff --git a/rpkid/examples/rpki.conf b/rpkid/examples/rpki.conf
index 880758ee..4fbfca0d 100644
--- a/rpkid/examples/rpki.conf
+++ b/rpkid/examples/rpki.conf
@@ -83,6 +83,7 @@ rootd_server_port		= 4401
 # relying parties can find and verify rpkid's published outputs.
 
 publication_base_directory     	= publication
+publication_root_cert_directory	= ${myrpki::publication_base_directory}.root
 
 # rsyncd module name corresponding to publication_base_directory.
 # This has to match the module you configured into rsyncd.conf.
@@ -90,6 +91,12 @@ publication_base_directory     	= publication
 
 publication_rsync_module	= rpki
 
+# rsyncd module name corresponding to publication_root_cert_directory.
+# This has to match the module you configured into rsyncd.conf.
+# Leave this alone unless you have some need to change it.
+
+publication_root_module		= root
+
 # Hostname and optional port number for rsync:// URIs.  In most cases
 # this should just be the same value as pubd_server_host.
 
@@ -304,7 +311,7 @@ rpki-base-uri           	= rsync://${myrpki::publication_rsync_server}/${myrpki:
 
 # rsync URI for rootd's root (self-signed) RPKI certificate
 
-rpki-root-cert-uri      	= rsync://${myrpki::publication_rsync_server}/${myrpki::publication_rsync_module}/root.cer
+rpki-root-cert-uri      	= rsync://${myrpki::publication_rsync_server}/${myrpki::publication_root_module}/root.cer
 
 # Private key corresponding to rootd's root RPKI certificate
 
@@ -312,7 +319,7 @@ rpki-root-key           	= ${myrpki::bpki_servers_directory}/root.key
 
 # Filename (as opposed to rsync URI) of rootd's root RPKI certificate
 
-rpki-root-cert          	= ${myrpki::publication_base_directory}/root.cer
+rpki-root-cert          	= ${myrpki::publication_root_cert_directory}/root.cer
 
 # Where rootd should stash a copy of the PKCS #10 request it gets from
 # its one (and only) child
diff --git a/rpkid/examples/rsyncd.conf b/rpkid/examples/rsyncd.conf
index 1bb60324..faf1dd0d 100644
--- a/rpkid/examples/rsyncd.conf
+++ b/rpkid/examples/rsyncd.conf
@@ -43,3 +43,11 @@ gid		= nobody
     transfer logging	= yes
     path		= /some/where/publication
     comment		= RPKI Testbed
+
+[root]
+    # This one is only relevant if you're running rootd.
+    use chroot		= no
+    read only		= yes
+    transfer logging	= yes
+    path		= /some/where/publication.root
+    comment		= RPKI Testbed Root
diff --git a/rpkid/rpki/csv_utils.py b/rpkid/rpki/csv_utils.py
index f7eed414..352aebd9 100644
--- a/rpkid/rpki/csv_utils.py
+++ b/rpkid/rpki/csv_utils.py
@@ -68,6 +68,12 @@ class csv_reader(object):
         fields += tuple(None for i in xrange(self.columns - len(fields)))
       yield fields
 
+  def __enter__(self):
+    return self
+
+  def __exit__(self, type, value, traceback):
+    self.file.close()
+
 class csv_writer(object):
   """
   Writer object for tab delimited text.  We just use the stock CSV
@@ -83,6 +89,12 @@ class csv_writer(object):
     self.file = open(self.renmwo, "w")
     self.writer = csv.writer(self.file, dialect = csv.get_dialect("excel-tab"))
 
+  def __enter__(self):
+    return self
+
+  def __exit__(self, type, value, traceback):
+    self.close()
+
   def close(self):
     """
     Close this writer.
diff --git a/rpkid/rpki/rootd.py b/rpkid/rpki/rootd.py
index 75257a80..45b0d32d 100644
--- a/rpkid/rpki/rootd.py
+++ b/rpkid/rpki/rootd.py
@@ -372,12 +372,12 @@ class main(object):
 
     self.rpki_root_key           = rpki.x509.RSA(Auto_update = self.cfg.get("rpki-root-key"))
     self.rpki_root_cert_file     = self.cfg.get("rpki-root-cert")
-    self.rpki_root_cert_uri      = self.cfg.get("rpki-root-cert-uri", self.rpki_base_uri + "Root.cer")
+    self.rpki_root_cert_uri      = self.cfg.get("rpki-root-cert-uri", self.rpki_base_uri + "root.cer")
 
-    self.rpki_root_manifest      = self.cfg.get("rpki-root-manifest", "Root.mft")
-    self.rpki_root_crl           = self.cfg.get("rpki-root-crl",      "Root.crl")
-    self.rpki_subject_cert       = self.cfg.get("rpki-subject-cert",  "Child.cer")
-    self.rpki_subject_pkcs10     = self.cfg.get("rpki-subject-pkcs10", "Child.pkcs10")
+    self.rpki_root_manifest      = self.cfg.get("rpki-root-manifest", "root.mft")
+    self.rpki_root_crl           = self.cfg.get("rpki-root-crl",      "root.crl")
+    self.rpki_subject_cert       = self.cfg.get("rpki-subject-cert",  "child.cer")
+    self.rpki_subject_pkcs10     = self.cfg.get("rpki-subject-pkcs10", "child.pkcs10")
 
     self.rpki_subject_lifetime   = rpki.sundial.timedelta.parse(self.cfg.get("rpki-subject-lifetime", "30d"))
     self.rpki_subject_regen      = rpki.sundial.timedelta.parse(self.cfg.get("rpki-subject-regen", self.rpki_subject_lifetime.convert_to_seconds() / 2))
diff --git a/rpkid/tests/yamlconf.py b/rpkid/tests/yamlconf.py
index 21bcf302..5a5f78f2 100644
--- a/rpkid/tests/yamlconf.py
+++ b/rpkid/tests/yamlconf.py
@@ -65,7 +65,8 @@ only_one_pubd = True
 yaml_file = None
 loopback = False
 dns_suffix = None
-mysql_rootpw = None
+mysql_rootuser = None
+mysql_rootpass = None
 
 # The SQL username mismatch between rpkid/examples/rpki.conf and
 # rpkid/tests/smoketest.setup.sql is completely stupid and really
@@ -289,23 +290,20 @@ class allocation(object):
                                            self.name)
 
   def dump_asns(self, fn):
-    f = self.csvout(fn)
-    for k in self.kids:    
-      f.writerows((k.name, a) for a in k.resources.asn)
-    f.close()
+    with self.csvout(fn) as f:
+      for k in self.kids:    
+        f.writerows((k.name, a) for a in k.resources.asn)
 
   def dump_prefixes(self, fn):
-    f = self.csvout(fn)
-    for k in self.kids:
-      f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6))
-    f.close()
+    with self.csvout(fn) as f:
+      for k in self.kids:
+        f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6))
 
   def dump_roas(self, fn):
-    f = self.csvout(fn)
-    for g1, r in enumerate(self.roa_requests):
-      f.writerows((p, r.asn, "G%08d%08d" % (g1, g2))
-                  for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ())))
-    f.close()
+    with self.csvout(fn) as f:
+      for g1, r in enumerate(self.roa_requests):
+        f.writerows((p, r.asn, "G%08d%08d" % (g1, g2))
+                    for g2, p in enumerate((r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ())))
 
   @property
   def pubd(self):
@@ -331,60 +329,68 @@ class allocation(object):
 
   def dump_conf(self):
 
-    r = { "handle"              : self.name,
-          "run_rpkid"           : str(not self.is_hosted),
-          "run_pubd"            : str(self.runs_pubd),
-          "run_rootd"           : str(self.is_root),
-          "irdbd_sql_database"  : self.irdb_name,
-          "irdbd_sql_username"  : "irdb",
-          "rpkid_sql_database"  : "rpki%d" % self.engine,
-          "rpkid_sql_username"  : "rpki",
-          "rpkid_server_host"   : self.hostname,
-          "rpkid_server_port"   : str(self.rpkid_port),
-          "irdbd_server_host"   : "localhost",
-          "irdbd_server_port"   : str(self.irdbd_port),
-          "rootd_server_port"   : str(self.rootd_port),
-          "pubd_sql_database"   : "pubd%d" % self.engine,
-          "pubd_sql_username"   : "pubd",
-          "pubd_server_host"    : self.pubd.hostname,
-          "pubd_server_port"    : str(self.pubd.pubd_port),
-          "publication_rsync_server" : self.rsync_server,
-          "bpki_servers_directory" : self.path() }
+    r = dict(
+      handle                    = self.name,
+      run_rpkid                 = str(not self.is_hosted),
+      run_pubd                  = str(self.runs_pubd),
+      run_rootd                 = str(self.is_root),
+      irdbd_sql_username        = "irdb",
+      rpkid_sql_username        = "rpki",
+      rpkid_server_host         = self.hostname,
+      rpkid_server_port         = str(self.rpkid_port),
+      irdbd_server_host         = "localhost",
+      irdbd_server_port         = str(self.irdbd_port),
+      rootd_server_port         = str(self.rootd_port),
+      pubd_sql_username         = "pubd",
+      pubd_server_host          = self.pubd.hostname,
+      pubd_server_port          = str(self.pubd.pubd_port),
+      publication_rsync_server  = self.rsync_server,
+      publication_base_directory = self.path("publication"),
+      bpki_servers_directory    = self.path())
     
-    r.update(config_overrides)
+    if loopback:
+      r.update(
+        irdbd_sql_database      = self.irdb_name,
+        rpkid_sql_database      = "rpki%d" % self.engine,
+        pubd_sql_database       = "pubd%d" % self.engine)
 
-    f = open(self.path("rpki.conf"), "w")
-    f.write("# Automatically generated, do not edit\n")
-    print "Writing", f.name
+    r.update(config_overrides)
 
-    section = None
-    for line in open(rpki_conf):
-      m = section_regexp.match(line)
-      if m:
-        section = m.group(1)
-      m = variable_regexp.match(line)
-      option = m.group(1) if m and section == "myrpki" else None
-      if option and option in r:
-        line = "%s = %s\n" % (option, r[option])
-      f.write(line)
+    with open(self.path("rpki.conf"), "w") as f:
+      f.write("# Automatically generated, do not edit\n")
+      print "Writing", f.name
 
-    f.close()
+      section = None
+      for line in open(rpki_conf):
+        m = section_regexp.match(line)
+        if m:
+          section = m.group(1)
+        m = variable_regexp.match(line)
+        option = m.group(1) if m and section == "myrpki" else None
+        if option and option in r:
+          line = "%s = %s\n" % (option, r[option])
+        f.write(line)
 
   def dump_rsyncd(self):
     if self.runs_pubd:
-      f = open(self.path("rsyncd.conf"), "w")
-      print "Writing", f.name
-      f.writelines(s + "\n" for s in
-                   ("# Automatically generated, do not edit",
-                    "port         = %d"           % self.rsync_port,
-                    "address      = %s"           % self.hostname,
-                    "[rpki]",
-                    "log file     = rsyncd.log",
-                    "read only    = yes",
-                    "use chroot   = no",
-                    "path         = %s"           % self.path("publication"),
-                    "comment      = RPKI test"))
-      f.close()
+      with open(self.path("rsyncd.conf"), "w") as f:
+        print "Writing", f.name
+        f.writelines(s + "\n" for s in
+                     ("# Automatically generated, do not edit",
+                      "port         = %d"           % self.rsync_port,
+                      "address      = %s"           % self.hostname,
+                      "[rpki]",
+                      "log file     = rsyncd.log",
+                      "read only    = yes",
+                      "use chroot   = no",
+                      "path         = %s"           % self.path("publication"),
+                      "comment      = RPKI test",
+                      "[root]",
+                      "log file     = rsyncd_root.log",
+                      "read only    = yes",
+                      "use chroot   = no",
+                      "path         = %s"           % self.path("publication.root"),
+                      "comment      = RPKI test root"))
 
   @property
   def irdb_name(self):
@@ -441,18 +447,15 @@ class allocation(object):
       notAfter    = rpki.sundial.now() + rpki.sundial.timedelta(days = 365),
       resources   = root_resources)
 
-    f = open(self.path("publication/root.cer"), "wb")
-    f.write(root_cert.get_DER())
-    f.close()
+    with open(self.path("publication.root/root.cer"), "wb") as f:
+      f.write(root_cert.get_DER())
 
-    f = open(self.path("root.key"), "wb")
-    f.write(root_key.get_DER())
-    f.close()
+    with open(self.path("root.key"), "wb") as f:
+      f.write(root_key.get_DER())
 
-    f = open(os.path.join(test_dir, "root.tal"), "w")
-    f.write(root_uri + "root.cer\n")
-    f.write(root_key.get_RSApublic().get_Base64())
-    f.close()
+    with open(os.path.join(test_dir, "root.tal"), "w") as f:
+      f.write("rsync://%s/root/root.cer\n\n%s" % (
+        self.rsync_server, root_key.get_RSApublic().get_Base64()))
 
   def mkdir(self, *path):
     path = self.path(*path)
@@ -489,11 +492,11 @@ def pre_django_sql_setup(needed):
   # databases as necessary, all we need to do here is provide empty
   # databases for the Django code to fill in.
 
-  if mysql_rootpw is not None:
-    if mysql_rootpw:
-      db = MySQLdb.connect(user = "root", passwd = mysql_rootpw)
+  if mysql_rootpass is not None:
+    if mysql_rootpass:
+      db = MySQLdb.connect(user = mysql_rootuser, passwd = mysql_rootpass)
     else:
-      db = MySQLdb.connect(user = "root")  
+      db = MySQLdb.connect(user = mysql_rootuser)  
     cur = db.cursor()
     for database in needed:
       try:
@@ -549,7 +552,8 @@ def main():
   global only_one_pubd
   global loopback
   global dns_suffix
-  global mysql_rootpw
+  global mysql_rootuser
+  global mysql_rootpass
   global yaml_file
 
   os.environ["TZ"] = "UTC"
@@ -597,9 +601,10 @@ def main():
     pass
 
   only_one_pubd = cfg.getboolean("only_one_pubd", True)
+  mysql_rootuser = cfg.get("mysql_rootuser", "root")
 
   try:
-    mysql_rootpw = cfg.get("mysql_rootpw", None)
+    mysql_rootpass = cfg.get("mysql_rootpass", None)
   except:
     pass
 
@@ -683,6 +688,8 @@ def body():
       d.mkdir()
     if d.runs_pubd:
       d.mkdir("publication")
+    if d.is_root:
+      d.mkdir("publication.root")
 
     if not d.is_hosted:
       d.dump_conf()
@@ -720,9 +727,10 @@ def body():
 
     ts()
 
-  print
-  for d in db:
-    d.dump_sql()
+  if not loopback:
+    print
+    for d in db:
+      d.dump_sql()
 
 if __name__ == "__main__":
   main()
diff --git a/rpkid/tests/yamltest.py b/rpkid/tests/yamltest.py
index 89a0c63d..2ae23f36 100644
--- a/rpkid/tests/yamltest.py
+++ b/rpkid/tests/yamltest.py
@@ -420,7 +420,13 @@ class allocation(object):
                     "read only    = yes",
                     "use chroot   = no",
                     "path         = %s"           % self.path("publication"),
-                    "comment      = RPKI test"))
+                    "comment      = RPKI test",
+                    "[root]",
+                    "log file     = rsyncd_root.log",
+                    "read only    = yes",
+                    "use chroot   = no",
+                    "path         = %s"           % self.path("publication.root"),
+                    "comment      = RPKI test root"))
       f.close()
 
   @classmethod
@@ -603,8 +609,10 @@ try:
     # Create publication directories.
 
     for d in db:
-      if d.is_root or d.runs_pubd:
+      if d.runs_pubd:
         os.makedirs(d.path("publication"))
+      if d.is_root:
+        os.makedirs(d.path("publication.root"))
 
     # Create RPKI root certificate.
 
@@ -630,7 +638,7 @@ try:
       notAfter    = rpki.sundial.now() + rpki.sundial.timedelta(days = 365),
       resources   = root_resources)
 
-    f = open(db.root.path("publication/root.cer"), "wb")
+    f = open(db.root.path("publication.root/root.cer"), "wb")
     f.write(root_cert.get_DER())
     f.close()
 
@@ -639,7 +647,7 @@ try:
     f.close()
 
     f = open(os.path.join(test_dir, "root.tal"), "w")
-    f.write(root_uri + "root.cer\n")
+    f.write("rsync://localhost:%d/root/root.cer\n\n" % db.root.pubd.rsync_port)
     f.write(root_key.get_RSApublic().get_Base64())
     f.close()