From 429adae788694f109174e35467c49d13b9533fe2 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Sat, 13 Sep 2014 03:56:54 +0000 Subject: Groundwork for Django ORM world conquest: sort out settings.py mess. svn path=/branches/tk713/; revision=5948 --- ca/tests/yamltest.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'ca/tests/yamltest.py') diff --git a/ca/tests/yamltest.py b/ca/tests/yamltest.py index 2b65dbd2..62b1252b 100644 --- a/ca/tests/yamltest.py +++ b/ca/tests/yamltest.py @@ -539,14 +539,15 @@ class allocation(object): """ Run rpkic for this entity. """ - cmd = [prog_rpkic, "-i", self.name, "-c", self.path("rpki.conf")] + cmd = [prog_rpkic, "-i", self.name] if args.profile: cmd.append("--profile") cmd.append(self.path("rpkic.%s.prof" % rpki.sundial.now())) cmd.extend(str(a) for a in argv if a is not None) print 'Running "%s"' % " ".join(cmd) env = os.environ.copy() - env["YAMLTEST_RPKIC_COUNTER"] = self.next_rpkic_counter() + env.update(YAMLTEST_RPKIC_COUNTER = self.next_rpkic_counter(), + RPKI_CONF = self.path("rpki.conf")) subprocess.check_call(cmd, cwd = self.host.path(), env = env) def run_python_daemon(self, prog): @@ -556,12 +557,13 @@ class allocation(object): """ basename = os.path.splitext(os.path.basename(prog))[0] cmd = [prog, "--foreground", "--log-level", "debug", - "--log-file", self.path(basename + ".log"), - "--config", self.path("rpki.conf")] + "--log-file", self.path(basename + ".log")] if args.profile and basename != "rootd": cmd.extend(( "--profile", self.path(basename + ".prof"))) - p = subprocess.Popen(cmd, cwd = self.path()) + env = os.environ.copy() + env.update(RPKI_CONF = self.path("rpki.conf")) + p = subprocess.Popen(cmd, cwd = self.path(), env = env) print 'Running %s for %s: pid %d process %r' % (" ".join(cmd), self.name, p.pid, p) return p @@ -672,7 +674,7 @@ try: # passwords: this is mostly so that I can show a complete working # example without publishing my own server's passwords. - cfg = rpki.config.parser(args.config, "yamltest", allow_missing = True) + cfg = rpki.config.parser(set_filename = args.config, section = "yamltest", allow_missing = True) only_one_pubd = cfg.getboolean("only_one_pubd", True) allocation.base_port = cfg.getint("base_port", 4400) -- cgit v1.2.3 From 1ca764a9456512a3dbdcbf83e4a337cdfc982dbe Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Sun, 14 Sep 2014 00:28:56 +0000 Subject: Start backing out all the old settings.configure() calls, which were indeed masking the new migration stuff. yamltest now runs migrations as part of setting up test CAs; still need to decide whether running migrations in production is something that should be handled explicitly via rpki-manage or should be bundled into rpki-sql-setup. Old settings.configure() code still present as a trail of breadcrumbs to follow when backing out the rest of the tortuous startup sequence required by the old way of doing things. svn path=/branches/tk713/; revision=5950 --- ca/tests/yamltest.py | 62 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 10 deletions(-) (limited to 'ca/tests/yamltest.py') diff --git a/ca/tests/yamltest.py b/ca/tests/yamltest.py index 62b1252b..e265fd46 100644 --- a/ca/tests/yamltest.py +++ b/ca/tests/yamltest.py @@ -73,13 +73,14 @@ def cleanpath(*names): this_dir = os.getcwd() test_dir = cleanpath(this_dir, "yamltest.dir") -rpkid_dir = cleanpath(this_dir, "..") +ca_dir = cleanpath(this_dir, "..") -prog_rpkic = cleanpath(rpkid_dir, "rpkic") -prog_rpkid = cleanpath(rpkid_dir, "rpkid") -prog_irdbd = cleanpath(rpkid_dir, "irdbd") -prog_pubd = cleanpath(rpkid_dir, "pubd") -prog_rootd = cleanpath(rpkid_dir, "rootd") +prog_rpkic = cleanpath(ca_dir, "rpkic") +prog_rpkid = cleanpath(ca_dir, "rpkid") +prog_irdbd = cleanpath(ca_dir, "irdbd") +prog_pubd = cleanpath(ca_dir, "pubd") +prog_rootd = cleanpath(ca_dir, "rootd") +prog_rpki_manage = cleanpath(ca_dir, "rpki-manage") class roa_request(object): """ @@ -129,7 +130,7 @@ class router_cert(object): def __init__(self, asn, router_id): self.asn = rpki.resource_set.resource_set_as("".join(str(asn).split())) self.router_id = router_id - self.keypair = rpki.x509.ECDSA.generate(self.ecparams()) + self.keypair = rpki.x509.ECDSA.generate(params = self.ecparams(), quiet = True) self.pkcs10 = rpki.x509.PKCS10.create(keypair = self.keypair) self.gski = self.pkcs10.gSKI() @@ -492,7 +493,7 @@ class allocation(object): print "Writing", f.name section = None - for line in open(cleanpath(rpkid_dir, "examples/rpki.conf")): + for line in open(cleanpath(ca_dir, "examples/rpki.conf")): m = section_regexp.match(line) if m: section = m.group(1) @@ -539,6 +540,7 @@ class allocation(object): """ Run rpkic for this entity. """ + cmd = [prog_rpkic, "-i", self.name] if args.profile: cmd.append("--profile") @@ -550,11 +552,48 @@ class allocation(object): RPKI_CONF = self.path("rpki.conf")) subprocess.check_call(cmd, cwd = self.host.path(), env = env) + def syncdb(self): + """ + Run whatever Django ORM commands are necessary to set up the + database this week. + + This may end up moving back into rpkic as an explicit command, but + for the moment I'm assuming that production use handle this via + rpki-sql-setup and that we therefore must do it ourselves for + testing. We'll see. + """ + + verbosity = 0 # Set to 3 for copious output + use_management_api = False # Set to use internal management API + + # These are equivalent. We probably want to use rpki-manage so we + # can show the user what's happening, but preserve both forms as a + # reference in case we want the internal form in rpki-sql-setup, + # in which case we might want the internal form here too after + # all. Decisions, decisions. + + if use_management_api: + if not os.fork(): + os.environ["RPKI_CONF"] = self.path("rpki.conf") + import django.core.management + django.core.management.call_command("syncdb", verbosity = verbosity, load_initial_data = False, migrate = True) + sys.exit(0) + if os.wait()[1]: + raise RuntimeError("syncdb failed for %s" % self.name) + + else: + cmd = (prog_rpki_manage, "syncdb", "--noinput", "--no-initial-data", "--migrate", "--verbosity", str(verbosity)) + env = os.environ.copy() + env.update(RPKI_CONF = self.path("rpki.conf")) + print 'Running "%s"' % " ".join(cmd) + subprocess.check_call(cmd, cwd = self.host.path(), env = env) + def run_python_daemon(self, prog): """ Start a Python daemon and return a subprocess.Popen object representing the running daemon. """ + basename = os.path.splitext(os.path.basename(prog))[0] cmd = [prog, "--foreground", "--log-level", "debug", "--log-file", self.path(basename + ".log")] @@ -637,8 +676,8 @@ def create_root_certificate(db_root): f.close() - -os.environ["TZ"] = "UTC" +os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings", + TZ = "UTC") time.tzset() parser = argparse.ArgumentParser(description = __doc__) @@ -715,6 +754,7 @@ try: for d in db: if not d.is_hosted: + print "Initializing", d.name os.makedirs(d.path()) d.dump_conf() if d.runs_pubd: @@ -722,7 +762,9 @@ try: d.dump_rsyncd() if d.is_root: os.makedirs(d.path("publication.root")) + d.syncdb() d.run_rpkic("initialize_server_bpki") + print # Initialize resource holding BPKI and generate self-descriptor # for each entity. -- cgit v1.2.3 From bbe4e85e0aa6c3b922788d8d997b92dd8ec8c9f0 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Sun, 14 Sep 2014 03:36:04 +0000 Subject: Add IRDB migration to yamlconf. svn path=/branches/tk713/; revision=5951 --- ca/tests/yamltest.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'ca/tests/yamltest.py') diff --git a/ca/tests/yamltest.py b/ca/tests/yamltest.py index e265fd46..835d25f2 100644 --- a/ca/tests/yamltest.py +++ b/ca/tests/yamltest.py @@ -547,8 +547,8 @@ class allocation(object): cmd.append(self.path("rpkic.%s.prof" % rpki.sundial.now())) cmd.extend(str(a) for a in argv if a is not None) print 'Running "%s"' % " ".join(cmd) - env = os.environ.copy() - env.update(YAMLTEST_RPKIC_COUNTER = self.next_rpkic_counter(), + env = dict(os.environ, + YAMLTEST_RPKIC_COUNTER = self.next_rpkic_counter(), RPKI_CONF = self.path("rpki.conf")) subprocess.check_call(cmd, cwd = self.host.path(), env = env) @@ -576,15 +576,15 @@ class allocation(object): if not os.fork(): os.environ["RPKI_CONF"] = self.path("rpki.conf") import django.core.management - django.core.management.call_command("syncdb", verbosity = verbosity, load_initial_data = False, migrate = True) + django.core.management.call_command("syncdb", migrate = True, verbosity = verbosity, + load_initial_data = False, interactive = False) sys.exit(0) if os.wait()[1]: raise RuntimeError("syncdb failed for %s" % self.name) else: cmd = (prog_rpki_manage, "syncdb", "--noinput", "--no-initial-data", "--migrate", "--verbosity", str(verbosity)) - env = os.environ.copy() - env.update(RPKI_CONF = self.path("rpki.conf")) + env = dict(os.environ, RPKI_CONF = self.path("rpki.conf")) print 'Running "%s"' % " ".join(cmd) subprocess.check_call(cmd, cwd = self.host.path(), env = env) @@ -600,8 +600,7 @@ class allocation(object): if args.profile and basename != "rootd": cmd.extend(( "--profile", self.path(basename + ".prof"))) - env = os.environ.copy() - env.update(RPKI_CONF = self.path("rpki.conf")) + env = dict(os.environ, RPKI_CONF = self.path("rpki.conf")) p = subprocess.Popen(cmd, cwd = self.path(), env = env) print 'Running %s for %s: pid %d process %r' % (" ".join(cmd), self.name, p.pid, p) return p -- cgit v1.2.3 From 238c447318617fc42b38a0396ef9b05022ccfd90 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Tue, 16 Sep 2014 04:53:49 +0000 Subject: Add support for running GUI under yamltest. Result sometimes hangs with browser waiting for response, not sure why, might be ancient abandoned bug #116. Logging control is a bit screwy, everything uses Python's logging library but we have multiple ways of configuring it. svn path=/branches/tk713/; revision=5954 --- ca/tests/yamltest.py | 72 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 26 deletions(-) (limited to 'ca/tests/yamltest.py') diff --git a/ca/tests/yamltest.py b/ca/tests/yamltest.py index 835d25f2..a18d287e 100644 --- a/ca/tests/yamltest.py +++ b/ca/tests/yamltest.py @@ -552,7 +552,7 @@ class allocation(object): RPKI_CONF = self.path("rpki.conf")) subprocess.check_call(cmd, cwd = self.host.path(), env = env) - def syncdb(self): + def syncdb(self, run_gui): """ Run whatever Django ORM commands are necessary to set up the database this week. @@ -563,30 +563,19 @@ class allocation(object): testing. We'll see. """ - verbosity = 0 # Set to 3 for copious output - use_management_api = False # Set to use internal management API + if not os.fork(): + os.environ.update(RPKI_CONF = self.path("rpki.conf"), + RPKI_GUI_ENABLE = "yes") + logging.getLogger().setLevel(logging.WARNING) + import django.core.management + django.core.management.call_command("syncdb", migrate = True, verbosity = 0, + load_initial_data = False, interactive = False) + from django.contrib.auth.models import User + User.objects.create_superuser("root", "root@example.org", "fnord") + sys.exit(0) - # These are equivalent. We probably want to use rpki-manage so we - # can show the user what's happening, but preserve both forms as a - # reference in case we want the internal form in rpki-sql-setup, - # in which case we might want the internal form here too after - # all. Decisions, decisions. - - if use_management_api: - if not os.fork(): - os.environ["RPKI_CONF"] = self.path("rpki.conf") - import django.core.management - django.core.management.call_command("syncdb", migrate = True, verbosity = verbosity, - load_initial_data = False, interactive = False) - sys.exit(0) - if os.wait()[1]: - raise RuntimeError("syncdb failed for %s" % self.name) - - else: - cmd = (prog_rpki_manage, "syncdb", "--noinput", "--no-initial-data", "--migrate", "--verbosity", str(verbosity)) - env = dict(os.environ, RPKI_CONF = self.path("rpki.conf")) - print 'Running "%s"' % " ".join(cmd) - subprocess.check_call(cmd, cwd = self.host.path(), env = env) + if os.wait()[1]: + raise RuntimeError("Django setup failed for %s" % self.name) def run_python_daemon(self, prog): """ @@ -602,7 +591,7 @@ class allocation(object): "--profile", self.path(basename + ".prof"))) env = dict(os.environ, RPKI_CONF = self.path("rpki.conf")) p = subprocess.Popen(cmd, cwd = self.path(), env = env) - print 'Running %s for %s: pid %d process %r' % (" ".join(cmd), self.name, p.pid, p) + print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p) return p def run_rpkid(self): @@ -638,6 +627,24 @@ class allocation(object): print "Running rsyncd for %s: pid %d process %r" % (self.name, p.pid, p) return p + def run_gui(self): + """ + Start an instance of the RPKI GUI under the Django test server and + return a subprocess.Popen object representing the running daemon. + """ + + port = 8000 + self.engine + cmd = (prog_rpki_manage, "runserver", str(port)) + env = dict(os.environ, + RPKI_CONF = self.path("rpki.conf"), + RPKI_DJANGO_DEBUG = "yes", + ALLOW_PLAIN_HTTP_FOR_TESTING = "I solemnly swear that I am not running this in production") + p = subprocess.Popen(cmd, cwd = self.path(), env = env, + stdout = open(self.path("gui.log"), "w"), stderr = subprocess.STDOUT) + print "Running %s for %s: pid %d process %r" % (" ".join(cmd), self.name, p.pid, p) + return p + + def create_root_certificate(db_root): print "Creating rootd RPKI root certificate" @@ -675,6 +682,8 @@ def create_root_certificate(db_root): f.close() +logger = logging.getLogger(__name__) + os.environ.update(DJANGO_SETTINGS_MODULE = "rpki.django_settings", TZ = "UTC") time.tzset() @@ -696,6 +705,8 @@ parser.add_argument("--synchronize", action = "store_true", help = "synchronize IRDB with daemons") parser.add_argument("--profile", action = "store_true", help = "enable profiling") +parser.add_argument("-g", "--run_gui", action = "store_true", + help = "enable GUI using django-admin runserver") parser.add_argument("yaml_file", type = argparse.FileType("r"), help = "YAML description of test network") args = parser.parse_args() @@ -761,7 +772,7 @@ try: d.dump_rsyncd() if d.is_root: os.makedirs(d.path("publication.root")) - d.syncdb() + d.syncdb(args.run_gui) d.run_rpkic("initialize_server_bpki") print @@ -801,6 +812,8 @@ try: if d.runs_pubd: progs.append(d.run_pubd()) progs.append(d.run_rsyncd()) + if args.run_gui: + progs.append(d.run_gui()) if args.synchronize or not args.skip_config: @@ -869,6 +882,13 @@ try: d.dump_ghostbusters() d.dump_router_certificates() + if args.run_gui: + print + print 'GUI user "root", password "fnord"' + for d in db: + if not d.is_hosted: + print "GUI URL http://127.0.0.1:%d/rpki/ for %s" % (8000 + d.engine, d.name) + # Wait until something terminates. if not args.stop_after_config or args.keep_going: -- cgit v1.2.3 From 7c2fb9ca36f120369797072e534041daae6cd8f9 Mon Sep 17 00:00:00 2001 From: Rob Austein Date: Tue, 16 Sep 2014 13:35:29 +0000 Subject: Add --browser switch to yamltest. Less useful than I had hoped it might be due to some kind of CSRF problem, might be worth figuring out how to disable that for debugging but not terribly urgent. svn path=/branches/tk713/; revision=5955 --- ca/tests/yamltest.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'ca/tests/yamltest.py') diff --git a/ca/tests/yamltest.py b/ca/tests/yamltest.py index a18d287e..bc78c090 100644 --- a/ca/tests/yamltest.py +++ b/ca/tests/yamltest.py @@ -43,6 +43,7 @@ import re import os import logging import argparse +import webbrowser import sys import yaml import signal @@ -155,7 +156,7 @@ class allocation_db(list): def __init__(self, yaml): list.__init__(self) self.root = allocation(yaml, self) - assert self.root.is_root + assert self.root.is_root and not any(a.is_root for a in self if a is not self.root) and self[0] is self.root if self.root.crl_interval is None: self.root.crl_interval = 60 * 60 if self.root.regen_margin is None: @@ -707,6 +708,8 @@ parser.add_argument("--profile", action = "store_true", help = "enable profiling") parser.add_argument("-g", "--run_gui", action = "store_true", help = "enable GUI using django-admin runserver") +parser.add_argument("--browser", action = "store_true", + help = "create web browser tabs for GUI") parser.add_argument("yaml_file", type = argparse.FileType("r"), help = "YAML description of test network") args = parser.parse_args() @@ -887,7 +890,14 @@ try: print 'GUI user "root", password "fnord"' for d in db: if not d.is_hosted: - print "GUI URL http://127.0.0.1:%d/rpki/ for %s" % (8000 + d.engine, d.name) + url = "http://127.0.0.1:%d/rpki/" % (8000 + d.engine) + print "GUI URL", url, "for", d.name + if args.browser: + if d is db.root: + webbrowser.open_new(url) + else: + webbrowser.open_new_tab(url) + time.sleep(2) # Wait until something terminates. -- cgit v1.2.3