Start backing out all the old settings.configure() calls, which were

indeed masking the new migration stuff.  yamltest now runs migrations
as part of setting up test CAs; still need to decide whether running
migrations in production is something that should be handled
explicitly via rpki-manage or should be bundled into rpki-sql-setup.

Old settings.configure() code still present as a trail of breadcrumbs
to follow when backing out the rest of the tortuous startup sequence
required by the old way of doing things.

svn path=/branches/tk713/; revision=5950

6 files changed, 141 insertions, 56 deletions

diff --git a/ca/irbe_cli b/ca/irbe_cli
index c38cf93b..cd9c2165 100755
--- a/ca/irbe_cli
+++ b/ca/irbe_cli
@@ -322,19 +322,22 @@ while argv:
   argv = q_pdu.client_getopt(argv[1:])
   q_msg.append(q_pdu)
 
-from django.conf import settings
-
-settings.configure(
-  DATABASES = { "default" : {
-    "ENGINE"   : "django.db.backends.mysql",
-    "NAME"     : cfg.get("sql-database", section = "irdbd"),
-    "USER"     : cfg.get("sql-username", section = "irdbd"),
-    "PASSWORD" : cfg.get("sql-password", section = "irdbd"),
-    "HOST"     : "",
-    "PORT"     : "",
-    "OPTIONS"  : { "init_command": "SET storage_engine=INNODB" }}},
-  INSTALLED_APPS = ("rpki.irdb",),
-)
+if True:
+  os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings")
+
+else:
+  from django.conf import settings
+  settings.configure(
+    DATABASES = { "default" : {
+      "ENGINE"   : "django.db.backends.mysql",
+      "NAME"     : cfg.get("sql-database", section = "irdbd"),
+      "USER"     : cfg.get("sql-username", section = "irdbd"),
+      "PASSWORD" : cfg.get("sql-password", section = "irdbd"),
+      "HOST"     : "",
+      "PORT"     : "",
+      "OPTIONS"  : { "init_command": "SET storage_engine=INNODB" }}},
+    INSTALLED_APPS = ("rpki.irdb",),
+  )
 
 import rpki.irdb
 
diff --git a/ca/tests/yamlconf.py b/ca/tests/yamlconf.py
index 0f1467f7..efe161d6 100644
--- a/ca/tests/yamlconf.py
+++ b/ca/tests/yamlconf.py
@@ -759,6 +759,19 @@ def body():
   # https://docs.djangoproject.com/en/1.4/topics/db/multi-db/
   # https://docs.djangoproject.com/en/1.4/topics/db/sql/
 
+  # This will probably need hacking to work with rpki.django_settings,
+  # rpki.db_router, and all the related new code, but let's get the
+  # kinks ironed out of the normal stuff before messing with this.
+  #
+  # I suspect the correct strategy here will be to import
+  # rpki.django_settings and modify its DATABASES and DATABASE_ROUTERS
+  # settings, but we'll see.
+  #
+  # We will almost certainly have to run syncdb and migrate.  We may want
+  # our own private south database so we don't mess up anything else; that
+  # database might even be one of our outputs (ie, yamltest may need that
+  # too, and may end up generating its own when run without this).
+
   database_template = {
     "ENGINE"   : "django.db.backends.mysql",
     "USER"     : config_overrides["irdbd_sql_username"],
diff --git a/ca/tests/yamltest.py b/ca/tests/yamltest.py
index 62b1252b..e265fd46 100644
--- a/ca/tests/yamltest.py
+++ b/ca/tests/yamltest.py
@@ -73,13 +73,14 @@ def cleanpath(*names):
 
 this_dir  = os.getcwd()
 test_dir  = cleanpath(this_dir, "yamltest.dir")
-rpkid_dir = cleanpath(this_dir, "..")
+ca_dir    = cleanpath(this_dir, "..")
 
-prog_rpkic   = cleanpath(rpkid_dir, "rpkic")
-prog_rpkid   = cleanpath(rpkid_dir, "rpkid")
-prog_irdbd   = cleanpath(rpkid_dir, "irdbd")
-prog_pubd    = cleanpath(rpkid_dir, "pubd")
-prog_rootd   = cleanpath(rpkid_dir, "rootd")
+prog_rpkic = cleanpath(ca_dir, "rpkic")
+prog_rpkid = cleanpath(ca_dir, "rpkid")
+prog_irdbd = cleanpath(ca_dir, "irdbd")
+prog_pubd  = cleanpath(ca_dir, "pubd")
+prog_rootd = cleanpath(ca_dir, "rootd")
+prog_rpki_manage  = cleanpath(ca_dir, "rpki-manage")
 
 class roa_request(object):
   """
@@ -129,7 +130,7 @@ class router_cert(object):
   def __init__(self, asn, router_id):
     self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split()))
     self.router_id = router_id
-    self.keypair = rpki.x509.ECDSA.generate(self.ecparams())
+    self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True)
     self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair)
     self.gski = self.pkcs10.gSKI()
 
@@ -492,7 +493,7 @@ class allocation(object):
     print "Writing", f.name
 
     section = None
-    for line in open(cleanpath(rpkid_dir, "examples/rpki.conf")):
+    for line in open(cleanpath(ca_dir, "examples/rpki.conf")):
       m = section_regexp.match(line)
       if m:
         section = m.group(1)
@@ -539,6 +540,7 @@ class allocation(object):
     """
     Run rpkic for this entity.
     """
+
     cmd = [prog_rpkic, "-i", self.name]
     if args.profile:
       cmd.append("--profile")
@@ -550,11 +552,48 @@ class allocation(object):
                RPKI_CONF = self.path("rpki.conf"))
     subprocess.check_call(cmd, cwd = self.host.path(), env = env)
 
+  def syncdb(self):
+    """
+    Run whatever Django ORM commands are necessary to set up the
+    database this week.
+
+    This may end up moving back into rpkic as an explicit command, but
+    for the moment I'm assuming that production use handle this via
+    rpki-sql-setup and that we therefore must do it ourselves for
+    testing.  We'll see.
+    """
+
+    verbosity = 0                       # Set to 3 for copious output
+    use_management_api = False          # Set to use internal management API
+
+    # These are equivalent.  We probably want to use rpki-manage so we
+    # can show the user what's happening, but preserve both forms as a
+    # reference in case we want the internal form in rpki-sql-setup,
+    # in which case we might want the internal form here too after
+    # all.  Decisions, decisions.
+
+    if use_management_api:
+      if not os.fork():
+        os.environ["RPKI_CONF"] = self.path("rpki.conf")
+        import django.core.management
+        django.core.management.call_command("syncdb", verbosity = verbosity, load_initial_data = False, migrate = True)
+        sys.exit(0)
+      if os.wait()[1]:
+        raise RuntimeError("syncdb failed for %s" % self.name)
+
+    else:
+      cmd = (prog_rpki_manage, "syncdb", "--noinput", "--no-initial-data", "--migrate", "--verbosity", str(verbosity))
+      env = os.environ.copy()
+      env.update(RPKI_CONF = self.path("rpki.conf"))
+      print 'Running "%s"' % " ".join(cmd)
+      subprocess.check_call(cmd, cwd = self.host.path(), env = env)
+
   def run_python_daemon(self, prog):
     """
     Start a Python daemon and return a subprocess.Popen object
     representing the running daemon.
     """
+
     basename = os.path.splitext(os.path.basename(prog))[0]
     cmd = [prog, "--foreground", "--log-level", "debug",
            "--log-file", self.path(basename + ".log")]
@@ -637,8 +676,8 @@ def create_root_certificate(db_root):
   f.close()
 
 
-
-os.environ["TZ"] = "UTC"
+os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings",
+                  TZ = "UTC")
 time.tzset()
 
 parser = argparse.ArgumentParser(description = __doc__)
@@ -715,6 +754,7 @@ try:
 
     for d in db:
       if not d.is_hosted:
+        print "Initializing", d.name
         os.makedirs(d.path())
         d.dump_conf()
         if d.runs_pubd:
@@ -722,7 +762,9 @@ try:
           d.dump_rsyncd()
         if d.is_root:
           os.makedirs(d.path("publication.root"))
+        d.syncdb()
         d.run_rpkic("initialize_server_bpki")
+        print
 
     # Initialize resource holding BPKI and generate self-descriptor
     # for each entity.
diff --git a/rpki/gui/script_util.py b/rpki/gui/script_util.py
index c3a864fd..678ddae6 100644
--- a/rpki/gui/script_util.py
+++ b/rpki/gui/script_util.py
@@ -16,11 +16,6 @@
 This module contains utility functions for use in standalone scripts.
 """
 
-from django.conf import settings
-
-from rpki import config
-from rpki import autoconf
-
 __version__ = '$Id$'
 
 
@@ -28,16 +23,29 @@ def setup():
     """
     Configure Django enough to use the ORM.
     """
-    cfg = config.parser(section='web_portal')
-    # INSTALLED_APPS doesn't seem necessary so long as you are only accessing
-    # existing tables.
-    settings.configure(
-        DATABASES={
-            'default': {
-                'ENGINE': 'django.db.backends.mysql',
-                'NAME': cfg.get('sql-database'),
-                'USER': cfg.get('sql-username'),
-                'PASSWORD': cfg.get('sql-password'),
-            }
-        },
-    )
+
+    # In theory we no longer need to call settings.configure, which
+    # probably means this whole module can go away soon, but leave
+    # breadcrumbs for now.
+
+    if True:
+	os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings")
+
+    else:
+	from rpki import config
+	from rpki import autoconf
+	from django.conf import settings
+
+	cfg = config.parser(section='web_portal')
+	# INSTALLED_APPS doesn't seem necessary so long as you are only accessing
+	# existing tables.
+	settings.configure(
+	    DATABASES={
+		'default': {
+		    'ENGINE': 'django.db.backends.mysql',
+		    'NAME': cfg.get('sql-database'),
+		    'USER': cfg.get('sql-username'),
+		    'PASSWORD': cfg.get('sql-password'),
+		}
+	    },
+	)
diff --git a/rpki/irdb/models.py b/rpki/irdb/models.py
index 2db86ba8..f89cebd4 100644
--- a/rpki/irdb/models.py
+++ b/rpki/irdb/models.py
@@ -81,6 +81,11 @@ class SignedReferralField(DERField):
   description   = "CMS signed object containing XML"
   rpki_type     = rpki.x509.SignedReferral
 
+# Alias to keep old rpki.gui migrations happy.  Would generating a new
+# schema migration for rpki.gui remove the need, or do we have to
+# preserve every old field class we've ever used forever?  Dunno.
+
+RSAKeyField = KeyField
 
 # Introspection rules for Django South
 
diff --git a/rpki/rpkic.py b/rpki/rpkic.py
index 62921308..fdde6056 100644
--- a/rpki/rpkic.py
+++ b/rpki/rpkic.py
@@ -132,19 +132,29 @@ class main(Cmd):
     self.histfile = cfg.get("history_file", os.path.expanduser("~/.rpkic_history"))
     self.autosync = cfg.getboolean("autosync", True, section = "rpkic")
 
-    from django.conf import settings
-
-    settings.configure(
-      DATABASES = { "default" : {
-        "ENGINE"   : "django.db.backends.mysql",
-        "NAME"     : cfg.get("sql-database", section = "irdbd"),
-        "USER"     : cfg.get("sql-username", section = "irdbd"),
-        "PASSWORD" : cfg.get("sql-password", section = "irdbd"),
-        "HOST"     : "",
-        "PORT"     : "",
-        "OPTIONS"  : { "init_command": "SET storage_engine=INNODB" }}},
-      INSTALLED_APPS = ("rpki.irdb",),
-    )
+    # This should go away now that we have rpki.django_settings, but
+    # let's get a verbose log with it present first to see what
+    # changes.
+
+    use_south = True
+    setup_db  = False
+
+    if use_south:
+      os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings")
+
+    else:
+      from django.conf import settings
+      settings.configure(
+        DATABASES = { "default" : {
+          "ENGINE"   : "django.db.backends.mysql",
+          "NAME"     : cfg.get("sql-database", section = "irdbd"),
+          "USER"     : cfg.get("sql-username", section = "irdbd"),
+          "PASSWORD" : cfg.get("sql-password", section = "irdbd"),
+          "HOST"     : "",
+          "PORT"     : "",
+          "OPTIONS"  : { "init_command": "SET storage_engine=INNODB" }}},
+        INSTALLED_APPS = ["rpki.irdb"])
+
 
     import rpki.irdb                    # pylint: disable=W0621
 
@@ -166,8 +176,12 @@ class main(Cmd):
     except rpki.config.ConfigParser.Error:
       pass
 
-    import django.core.management
-    django.core.management.call_command("syncdb", verbosity = 0, load_initial_data = False)
+    if setup_db:
+      import django.core.management
+      django.core.management.call_command("syncdb", verbosity = 3, load_initial_data = False)
+
+    if setup_db and use_south:
+        django.core.management.call_command("migrate", verbosity = 3)
 
     self.zoo = rpki.irdb.Zookeeper(cfg = cfg, handle = self.handle, logstream = sys.stdout)