Teach rcynicng to read command line defaults from rpki.conf. Some of

this takes the form of new rpki.config.parser methods which we may
want to use in other programs, particularly the daemons.

svn path=/branches/tk705/; revision=6345

2 files changed, 148 insertions, 38 deletions

diff --git a/rp/rcynic/rcynicng b/rp/rcynic/rcynicng
index ccd75913..4648e6c6 100755
--- a/rp/rcynic/rcynicng
+++ b/rp/rcynic/rcynicng
@@ -28,14 +28,16 @@ import tornado.process
 import tornado.httpclient
 
 import rpki.POW
-import rpki.sundial
+import rpki.log
 import rpki.config
-import rpki.autoconf
+import rpki.sundial
 import rpki.relaxng
+import rpki.autoconf
 
 from rpki.oids import id_kp_bgpsec_router
 
-from lxml.etree import ElementTree, Element, SubElement, Comment, XML, DocumentInvalid, XMLSyntaxError, iterparse
+from lxml.etree import (ElementTree, Element, SubElement, Comment,
+                        XML, DocumentInvalid, XMLSyntaxError, iterparse)
 
 logger = logging.getLogger("rcynicng")
 
@@ -502,7 +504,7 @@ class  WalkFrame(object):
 
         if not self.fetcher.needed():
             self.state = self.ready
-        elif args.no_spawn_on_fetch:
+        elif not args.spawn_on_fetch:
             self.state = self.fetch
         else:
             self.state = self.fetch
@@ -662,7 +664,7 @@ class WalkTask(object):
 
 
 def read_tals():
-    for head, dirs, files in os.walk(args.tals):
+    for head, dirs, files in os.walk(args.trust_anchor_locators):
         for fn in files:
             if fn.endswith(".tal"):
                 furi = "file://" + os.path.abspath(os.path.join(head, fn))
@@ -755,7 +757,7 @@ class Fetcher(object):
         return None
 
     def needed(self):
-        if args.no_fetch:
+        if not args.fetch:
             return False
         if self.uri.startswith("rsync://"):
             return self._rsync_needed()
@@ -788,7 +790,7 @@ class Fetcher(object):
     def _rsync_fetch(self):
         assert self.uri.startswith("rsync://") and (self.uri.endswith(".cer") if self.ta else self.uri.endswith("/"))
 
-        if args.no_fetch:
+        if not args.fetch:
             return
         path = self._rsync_split_uri()
         dead = path[0] in self._rsync_deadhosts
@@ -940,7 +942,7 @@ class Fetcher(object):
     @tornado.gen.coroutine
     def _https_fetch_ta(self):
 
-        if args.no_fetch:
+        if not args.fetch:
             return
 
         other = self._https_history.get(self.uri)
@@ -1022,7 +1024,7 @@ class Fetcher(object):
     def _rrdp_fetch(self):
         from django.db import transaction
 
-        if args.no_fetch:
+        if not args.fetch:
             return
 
         other = self._https_history.get(self.uri)
@@ -1305,7 +1307,8 @@ def final_report():
     #
     # Should generate <rsync_history/> elements here too, later
     #
-    ElementTree(doc).write(file = args.xml_file, pretty_print = True)
+    ElementTree(doc).write(file = argparse.FileType("w")(args.xml_file),
+                           pretty_print = True)
 
 
 def final_cleanup():
@@ -1386,33 +1389,60 @@ def main():
     time.tzset()
 
     cfg, parser = rpki.config.argparser(section = "rcynic", doc = __doc__, cfg_optional = True)
-    parser.add_argument("-u", "--unauthenticated", 
-                        default = os.path.join(rpki.autoconf.RCYNIC_DIR, "data", "unauthenticated"))
-    parser.add_argument("-x", "--xml-file",     type = argparse.FileType("w"),
-                        default = os.path.join(rpki.autoconf.RCYNIC_DIR, "data", "rcynic.xml"))
-    parser.add_argument("-t", "--tals",         
-                        default = os.path.join(rpki.autoconf.sysconfdir, "rpki", "trust-anchors"))
-    parser.add_argument("-w", "--workers",      default = 10, type = posint)
-    parser.add_argument("--no-fetch",           action = "store_true")
-    parser.add_argument("--no-spawn-on-fetch",  action = "store_true")
-    parser.add_argument("--no-migrate",         action = "store_true")
-    parser.add_argument("--prefer-rsync",       action = "store_true")
-    parser.add_argument("--fetch-ahead-goal",   default = 2, type = posint)
-    parser.add_argument("--https-timeout",      default = 300,  type = posint)
-    parser.add_argument("--validate-https",     action = "store_true")
-    parser.add_argument("--max-https-body-size", type = posint, default = 512 * 1024 * 1024)
-
-    # We already have a whole bunch of logging control code in
-    # rpki.log, just need to figure out / remember how to use it
-    # properly.  See rpki.log.init() & rpki.log.argparse_setup().
+    rpki.log.argparse_setup(parser)
+
+    cfg.add_argument("-u", "--unauthenticated",
+                     help = "where to store unauthenticated data retrieved via rsycnc",
+                     default = os.path.join(rpki.autoconf.RCYNIC_DIR, "data", "unauthenticated"))
+
+    cfg.add_argument("-x", "--xml-file",
+                     help = "where to write XML log of validation results",
+                     default = os.path.join(rpki.autoconf.RCYNIC_DIR, "data", "rcynic.xml"))
+
+    cfg.add_argument("-t", "--trust-anchor-locators", "--tals", 
+                     help = "where to find trust anchor locators",
+                     default = os.path.join(rpki.autoconf.sysconfdir, "rpki", "trust-anchors"))
+
+    cfg.add_argument("-w", "--workers",      type = posint,
+                     help = "number of worker pseudo-threads to allow",
+                     default = 10)
+
+    cfg.add_argument("--fetch-ahead-goal",   type = posint,
+                     help = "how many deltas we want in the fetch-ahead pipe",
+                     default = 2)
+
+    cfg.add_argument("--https-timeout",      type = posint,
+                     help = "HTTPS connection timeout, in seconds",
+                     default = 300)
+
+    cfg.add_argument("--max-https-body-size", type = posint,
+                     help = "upper limit on byte length of HTTPS message body",
+                     default = 512 * 1024 * 1024)
+
+    cfg.add_boolean_argument("--fetch",             default = True,
+                             help = "whether to fetch data at all")
+
+    cfg.add_boolean_argument("--spawn-on-fetch",    default = True,
+                             help = "whether to spawn new pseudo-threads on fetch")
+
+    cfg.add_boolean_argument("--migrate",           default = True,
+                             help = "whether to migrate the ORM database on startup")
+
+    cfg.add_boolean_argument("--prefer-rsync",      default = False,
+                             help = "whether to prefer rsync over RRDP")
+
+    cfg.add_boolean_argument("--validate-https",    default = False,
+                             help = "whether to validate HTTPS server certificates")
 
     global args
     args = parser.parse_args()
 
+    rpki.log.init("rcynic", args)
+
     import django
     django.setup()
 
-    if not args.no_migrate:
+    if args.migrate:
         # Not sure we should be doing this on every run, but sure simplifies things.
         import django.core.management
         django.core.management.call_command("migrate", verbosity = 0, interactive = False)
@@ -1427,7 +1457,6 @@ def main():
     RRDPSnapshot  = rpki.rcynicdb.models.RRDPSnapshot
     RPKIObject    = rpki.rcynicdb.models.RPKIObject
 
-    logging.basicConfig(level = logging.DEBUG, format = "%(asctime)s  %(message)s", datefmt = "%Y-%m-%d %H:%M:%S")
 
     global authenticated
     authenticated = Authenticated.objects.create(started  = rpki.sundial.datetime.now())
diff --git a/rpki/config.py b/rpki/config.py
index 3cc16626..7550c8f4 100644
--- a/rpki/config.py
+++ b/rpki/config.py
@@ -77,8 +77,10 @@ class parser(object):
         allow_missing = kwargs.pop("allow_missing", False)
         set_filename  = kwargs.pop("set_filename",  None)
         filename      = kwargs.pop("filename",      set_filename)
+        argparser     = kwargs.pop("argparser",     None)
 
-        assert not kwargs, "Unexpected keyword arguments: " + ", ".join("%s = %r" % kv for kv in kwargs.iteritems())
+        assert not kwargs, "Unexpected keyword arguments: {}".format(
+            ", ".join("{} = {!r}".format(k, v) for k, v in kwargs.iteritems()))
 
         if set_filename is not None:
             os.environ[rpki_conf_envname] = set_filename
@@ -87,6 +89,7 @@ class parser(object):
         self.default_section = section
 
         self.filename = filename or os.getenv(rpki_conf_envname) or default_filename
+        self.argparser = argparser
 
         try:
             with open(self.filename, "r") as f:
@@ -129,7 +132,8 @@ class parser(object):
         if self.cfg.has_option(section, option):
             yield self.cfg.get(section, option)
         option += "."
-        matches = [o for o in self.cfg.options(section) if o.startswith(option) and o[len(option):].isdigit()]
+        matches = [o for o in self.cfg.options(section) 
+                   if o.startswith(option) and o[len(option):].isdigit()]
         matches.sort()
         for option in matches:
             yield self.cfg.get(section, option)
@@ -176,7 +180,7 @@ class parser(object):
         if isinstance(v, str):
             v = v.lower()
             if v not in self.cfg._boolean_states:
-                raise ValueError("Not a boolean: %s" % v)
+                raise ValueError("Not boolean: {}".format(v))
             v = self.cfg._boolean_states[v]
         return v
 
@@ -197,6 +201,80 @@ class parser(object):
         return long(self.get(option, default, section))
 
 
+    def add_argument(self, *names, **kwargs):
+        """
+        Combined command line and config file argument.  Takes
+        arguments mostly like ArgumentParser.add_argument(), but also
+        looks in config file for option of the same name.
+
+        The "section" and "default" arguments are used for the config file
+        lookup; the resulting value is used as the "default" parameter for 
+        the argument parser.
+
+        If a "type" argument is specified, it applies to both the value
+        parsed from the config file and the argument parser.
+        """
+
+        section = kwargs.pop("section", None)
+        default = kwargs.pop("default", None)
+
+        for name in names:
+            if name.startswith("--"):
+                name = name[2:]
+                break
+        else:
+            raise ValueError
+
+        default = self.get(name, default = default, section = section)
+
+        if "type" in kwargs:
+            default = kwargs["type"](default)
+
+        kwargs["default"] = default
+
+        return self.argparser.add_argument(*names, **kwargs)
+
+    def add_boolean_argument(self, name, **kwargs):
+        """
+        Combined command line and config file boolean argument.  Takes
+        arguments mostly like ArgumentParser.add_argument(), but also
+        looks in config file for option of the same name.
+
+        The "section" and "default" arguments are used for the config file
+        lookup; the resulting value is used as the default value for 
+        the argument parser.
+
+        Usage is a bit different from the normal ArgumentParser boolean
+        handling: because the command line default is controlled by the
+        config file, the "store_true" / "store_false" semantics don't
+        really work for us.  So, instead, we use the --foo / --no-foo
+        convention, and generate a pair of command line arguments with
+        those names controlling a single "foo" value in the result.
+        """
+
+        section = kwargs.pop("section", None)
+        default = kwargs.pop("default", None)
+
+        if not name.startswith("--"):
+            raise ValueError
+        name = name[2:]
+
+        default = self.getboolean(name, default = default, section = section)
+
+        kwargs["action"] = "store_const"
+        kwargs["dest"] = name.replace("-", "_")
+
+        group = self.argparser.add_mutually_exclusive_group()
+
+        kwargs["const"] = True
+        group.add_argument("--" + name, **kwargs)
+
+        kwargs["const"] = False
+        #kwargs["help"] = argparse.SUPPRESS
+        group.add_argument("--no-" + name, **kwargs)
+
+        self.argparser.set_defaults(**{ kwargs["dest"] : default })
+
     def set_global_flags(self):
         """
         Consolidated control for all the little global control flags
@@ -224,14 +302,16 @@ class parser(object):
             pass
 
         try:
-            rpki.x509.XML_CMS_object.dump_outbound_cms = rpki.x509.DeadDrop(self.get("dump_outbound_cms"))
+            rpki.x509.XML_CMS_object.dump_outbound_cms = rpki.x509.DeadDrop(
+                self.get("dump_outbound_cms"))
         except OSError, e:
             logger.warning("Couldn't initialize mailbox %s: %s", self.get("dump_outbound_cms"), e)
         except ConfigParser.NoOptionError:
             pass
 
         try:
-            rpki.x509.XML_CMS_object.dump_inbound_cms = rpki.x509.DeadDrop(self.get("dump_inbound_cms"))
+            rpki.x509.XML_CMS_object.dump_inbound_cms = rpki.x509.DeadDrop(
+                self.get("dump_inbound_cms"))
         except OSError, e:
             logger.warning("Couldn't initialize mailbox %s: %s", self.get("dump_inbound_cms"), e)
         except ConfigParser.NoOptionError:
@@ -323,10 +403,11 @@ def argparser(section = None, doc = None, cfg_optional = False):
 
     args, remaining_argv = cfgparser.parse_known_args()
 
+    argparser = argparse.ArgumentParser(parents = [topparser], description = doc)
+
     cfg = parser(section       = section,
                  set_filename  = args.config,
+                 argparser     = argparser,
                  allow_missing = cfg_optional or args.help)
 
-    argparser = argparse.ArgumentParser(parents = [topparser], description = doc)
-
     return cfg, argparser