Move more programs to integrated config+argparse framework.

svn path=/branches/tk705/; revision=6346

7 files changed, 100 insertions, 80 deletions

diff --git a/rp/config/rpki-generate-root-certificate b/rp/config/rpki-generate-root-certificate
index a1a4d3a6..d4ee08fd 100755
--- a/rp/config/rpki-generate-root-certificate
+++ b/rp/config/rpki-generate-root-certificate
@@ -21,20 +21,20 @@ import rpki.resource_set
 os.environ["TZ"] = "UTC"
 time.tzset()
 
-cfg, parser = rpki.config.argparser(section = "rootd", doc = __doc__)
+cfg = rpki.config.argparser(section = "rootd", doc = __doc__)
 
 default_certfile = cfg.get("rpki-root-cert-file", "root.cer")
 default_keyfile  = cfg.get("rpki-root-key-file",  "root.key")
 default_talfile  = os.path.splitext(default_certfile)[0] + ".tal"
 
-parser.add_argument("-a", "--asns",  help = "ASN resources",    default = "0-4294967295")
-parser.add_argument("-4", "--ipv4",  help = "IPv4 resources",   default = "0.0.0.0/0")
-parser.add_argument("-6", "--ipv6",  help = "IPv6 resources",   default = "::/0")
-parser.add_argument("--certificate", help = "certificate file", default = default_certfile)
-parser.add_argument("--key",         help = "key file",         default = default_keyfile)
-parser.add_argument("--tal",         help = "TAL file",         default = default_talfile)
+cfg.argparser.add_argument("-a", "--asns",  help = "ASN resources",    default = "0-4294967295")
+cfg.argparser.add_argument("-4", "--ipv4",  help = "IPv4 resources",   default = "0.0.0.0/0")
+cfg.argparser.add_argument("-6", "--ipv6",  help = "IPv6 resources",   default = "::/0")
+cfg.argparser.add_argument("--certificate", help = "certificate file", default = default_certfile)
+cfg.argparser.add_argument("--key",         help = "key file",         default = default_keyfile)
+cfg.argparser.add_argument("--tal",         help = "TAL file",         default = default_talfile)
 
-args = parser.parse_args()
+args = cfg.argparser.parse_args()
 
 resources = rpki.resource_set.resource_bag(
   asn = rpki.resource_set.resource_set_as(args.asns),
diff --git a/rp/config/rpki-sql-backup b/rp/config/rpki-sql-backup
index 097ad0d7..09e5856e 100755
--- a/rp/config/rpki-sql-backup
+++ b/rp/config/rpki-sql-backup
@@ -35,12 +35,12 @@ import rpki.config
 os.environ["TZ"] = "UTC"
 time.tzset()
 
-cfg, parser = rpki.config.argparser(doc = __doc__, section = "myrpki")
-parser.add_argument("-o", "--output", type = argparse.FileType("wb"), default = sys.stdout,
-                    help = "destination for SQL dump (default: stdout)")
-parser.add_argument("-v", "--verbose", action = "store_true",
-                    help = "whistle while you work")
-args = parser.parse_args()
+cfg = rpki.config.argparser(doc = __doc__, section = "myrpki")
+cfg.argparser.add_argument("-o", "--output", type = argparse.FileType("wb"), default = sys.stdout,
+                           help = "destination for SQL dump (default: stdout)")
+cfg.argparser.add_argument("-v", "--verbose", action = "store_true",
+                           help = "whistle while you work")
+args = cfg.argparser.parse_args()
 
 templates = dict(mysql      = "mysqldump --add-drop-database -u{username} -p{password} -B{database}",
                  sqlite3    = "sqlite3 {database} .dump",
diff --git a/rp/rcynic/rcynicng b/rp/rcynic/rcynicng
index 4648e6c6..aee000e1 100755
--- a/rp/rcynic/rcynicng
+++ b/rp/rcynic/rcynicng
@@ -1388,8 +1388,8 @@ def main():
                       DJANGO_SETTINGS_MODULE = "rpki.django_settings.rcynic")
     time.tzset()
 
-    cfg, parser = rpki.config.argparser(section = "rcynic", doc = __doc__, cfg_optional = True)
-    rpki.log.argparse_setup(parser)
+    cfg = rpki.config.argparser(section = "rcynic", doc = __doc__, cfg_optional = True)
+    rpki.log.argparse_setup(cfg.argparser)
 
     cfg.add_argument("-u", "--unauthenticated",
                      help = "where to store unauthenticated data retrieved via rsycnc",
@@ -1435,7 +1435,7 @@ def main():
                              help = "whether to validate HTTPS server certificates")
 
     global args
-    args = parser.parse_args()
+    args = cfg.argparser.parse_args()
 
     rpki.log.init("rcynic", args)
 
diff --git a/rpki/config.py b/rpki/config.py
index 7550c8f4..1aea0132 100644
--- a/rpki/config.py
+++ b/rpki/config.py
@@ -254,6 +254,7 @@ class parser(object):
 
         section = kwargs.pop("section", None)
         default = kwargs.pop("default", None)
+        help    = kwargs.pop("help",    None)
 
         if not name.startswith("--"):
             raise ValueError
@@ -270,7 +271,7 @@ class parser(object):
         group.add_argument("--" + name, **kwargs)
 
         kwargs["const"] = False
-        #kwargs["help"] = argparse.SUPPRESS
+        kwargs["help"] = help
         group.add_argument("--no-" + name, **kwargs)
 
         self.argparser.set_defaults(**{ kwargs["dest"] : default })
@@ -370,13 +371,13 @@ def argparser(section = None, doc = None, cfg_optional = False):
     # drop-in, and should reduce the amount of repetitive code.  There
     # are a couple of special cases which will require attention:
     #
-    # - rpki.rtr: These modules have their own handling of all the
-    #   logging setup, and use an argparse subparser.  I -think- that
-    #   the way they're already handling the logging setup should work
-    #   fine, but there may be a few tricky bits reconciling this code
-    #   with the more generalized version in rpki.log.
+    # - rpki.rtr: The rpki-rtr modules have their own handling of all
+    #   the logging setup, and use an argparse subparser.  I -think-
+    #   that the way they're already handling the logging setup should
+    #   work fine, but there may be a few tricky bits reconciling the
+    #   rpki-rtr logging setup with the generalized version in rpki.log.
     #
-    # - rpki.rpkic: Use of argparse here is very complicated due to
+    # - rpki.rpkic: Use of argparse in rpkic is very complicated due to
     #   support for both the external command line and the internal
     #   command loop.  Overall it works quite well, but the setup is
     #   tricky.  rpki.rpkic.main.top_argparse may need to move outside
@@ -410,4 +411,4 @@ def argparser(section = None, doc = None, cfg_optional = False):
                  argparser     = argparser,
                  allow_missing = cfg_optional or args.help)
 
-    return cfg, argparser
+    return cfg
diff --git a/rpki/irdbd.py b/rpki/irdbd.py
index 44628886..270b4f9f 100644
--- a/rpki/irdbd.py
+++ b/rpki/irdbd.py
@@ -44,9 +44,11 @@ class main(object):
     def handle_list_resources(self, q_pdu, r_msg):
         tenant_handle = q_pdu.get("tenant_handle")
         child_handle  = q_pdu.get("child_handle")
-        child  = rpki.irdb.models.Child.objects.get(issuer__handle = tenant_handle, handle = child_handle)
+        child  = rpki.irdb.models.Child.objects.get(issuer__handle = tenant_handle,
+                                                    handle = child_handle)
         resources = child.resource_bag
-        r_pdu = SubElement(r_msg, rpki.left_right.tag_list_resources, tenant_handle = tenant_handle, child_handle = child_handle,
+        r_pdu = SubElement(r_msg, rpki.left_right.tag_list_resources, 
+                           tenant_handle = tenant_handle, child_handle = child_handle,
                            valid_until = child.valid_until.strftime("%Y-%m-%dT%H:%M:%SZ"))
         for k, v in (("asn",  resources.asn),
                      ("ipv4", resources.v4),
@@ -64,7 +66,8 @@ class main(object):
             AND    irdb_resourceholderca.handle = %s
             """, [tenant_handle]):
             prefix_bag = request.roa_prefix_bag
-            r_pdu = SubElement(r_msg, rpki.left_right.tag_list_roa_requests, tenant_handle = tenant_handle, asn = str(request.asn))
+            r_pdu = SubElement(r_msg, rpki.left_right.tag_list_roa_requests, 
+                               tenant_handle = tenant_handle, asn = str(request.asn))
             for k, v in (("ipv4", prefix_bag.v4),
                          ("ipv6", prefix_bag.v6),
                          ("tag",  q_pdu.get("tag"))):
@@ -74,18 +77,22 @@ class main(object):
     def handle_list_ghostbuster_requests(self, q_pdu, r_msg):
         tenant_handle = q_pdu.get("tenant_handle")
         parent_handle = q_pdu.get("parent_handle")
-        ghostbusters = rpki.irdb.models.GhostbusterRequest.objects.filter(issuer__handle = tenant_handle, parent__handle = parent_handle)
+        ghostbusters = rpki.irdb.models.GhostbusterRequest.objects.filter(
+            issuer__handle = tenant_handle, parent__handle = parent_handle)
         if ghostbusters.count() == 0:
-            ghostbusters = rpki.irdb.models.GhostbusterRequest.objects.filter(issuer__handle = tenant_handle, parent = None)
+            ghostbusters = rpki.irdb.models.GhostbusterRequest.objects.filter(
+                issuer__handle = tenant_handle, parent = None)
         for ghostbuster in ghostbusters:
-            r_pdu = SubElement(r_msg, q_pdu.tag, tenant_handle = tenant_handle, parent_handle = parent_handle)
+            r_pdu = SubElement(r_msg, q_pdu.tag, 
+                               tenant_handle = tenant_handle, parent_handle = parent_handle)
             if q_pdu.get("tag"):
                 r_pdu.set("tag", q_pdu.get("tag"))
             r_pdu.text = ghostbuster.vcard
 
     def handle_list_ee_certificate_requests(self, q_pdu, r_msg):
         tenant_handle = q_pdu.get("tenant_handle")
-        for ee_req in rpki.irdb.models.EECertificateRequest.objects.filter(issuer__handle = tenant_handle):
+        for ee_req in rpki.irdb.models.EECertificateRequest.objects.filter(
+                issuer__handle = tenant_handle):
             resources = ee_req.resource_bag
             r_pdu = SubElement(r_msg, q_pdu.tag, tenant_handle = tenant_handle, gski = ee_req.gski,
                                valid_until = ee_req.valid_until.strftime("%Y-%m-%dT%H:%M:%SZ"),
@@ -111,7 +118,8 @@ class main(object):
             q_msg = q_cms.unwrap((serverCA.certificate, rpkid.certificate))
             self.cms_timestamp = q_cms.check_replay(self.cms_timestamp, request.path)
             if q_msg.get("type") != "query":
-                raise rpki.exceptions.BadQuery("Message type is %s, expected query" % q_msg.get("type"))
+                raise rpki.exceptions.BadQuery("Message type is {}, expected query".format(
+                    q_msg.get("type")))
             r_msg = Element(rpki.left_right.tag_msg, nsmap = rpki.left_right.nsmap,
                             type = "reply", version = rpki.left_right.version)
             try:
@@ -120,12 +128,14 @@ class main(object):
 
             except Exception, e:
                 logger.exception("Exception processing PDU %r", q_pdu)
-                r_pdu = SubElement(r_msg, rpki.left_right.tag_report_error, error_code = e.__class__.__name__)
+                r_pdu = SubElement(r_msg, rpki.left_right.tag_report_error, 
+                                   error_code = e.__class__.__name__)
                 r_pdu.text = str(e)
                 if q_pdu.get("tag") is not None:
                     r_pdu.set("tag", q_pdu.get("tag"))
 
-            request.send_cms_response(rpki.left_right.cms_msg().wrap(r_msg, irdbd.private_key, irdbd.certificate))
+            request.send_cms_response(rpki.left_right.cms_msg().wrap(
+                r_msg, irdbd.private_key, irdbd.certificate))
 
         except Exception, e:
             logger.exception("Unhandled exception while processing HTTP request")
@@ -139,21 +149,22 @@ class main(object):
                           DJANGO_SETTINGS_MODULE = "rpki.django_settings.irdb")
         time.tzset()
 
-        parser = argparse.ArgumentParser(description = __doc__)
-        parser.add_argument("-c", "--config",
-                            help = "override default location of configuration file")
-        parser.add_argument("-f", "--foreground", action = "store_true",
-                            help = "do not daemonize")
-        parser.add_argument("--pidfile",
-                            help = "override default location of pid file")
-        parser.add_argument("--profile",
-                            help = "enable profiling, saving data to PROFILE")
-        rpki.log.argparse_setup(parser)
-        args = parser.parse_args()
+        self.cfg = rpki.config.argparser(section = "irdbd", doc = __doc__)
+        self.cfg.add_boolean_argument("--foreground", 
+                                      default = False,
+                                      help = "whether to daemonize")
+        self.cfg.add_argument("--pidfile",   
+                              default = os.path.join(rpki.daemonize.default_pid_directory, 
+                                                     "irdbd.pid"),
+                              help = "override default location of pid file")
+        self.cfg.add_argument("--profile",
+                              default = "",
+                              help = "enable profiling, saving data to PROFILE")
+        rpki.log.argparse_setup(self.cfg.argparser)
+        args = self.cfg.argparser.parse_args()
 
         rpki.log.init("irdbd", args)
 
-        self.cfg = rpki.config.parser(set_filename = args.config, section = "irdbd")
         self.cfg.set_global_flags()
 
         self.cms_timestamp = None
diff --git a/rpki/pubd.py b/rpki/pubd.py
index fc5dffc7..25c2b551 100644
--- a/rpki/pubd.py
+++ b/rpki/pubd.py
@@ -57,23 +57,24 @@ class main(object):
 
         self.irbe_cms_timestamp = None
 
-        parser = argparse.ArgumentParser(description = __doc__)
-        parser.add_argument("-c", "--config",
-                            help = "override default location of configuration file")
-        parser.add_argument("-f", "--foreground", action = "store_true",
-                            help = "do not daemonize")
-        parser.add_argument("--pidfile",
-                            help = "override default location of pid file")
-        parser.add_argument("--profile",
-                            help = "enable profiling, saving data to PROFILE")
-        rpki.log.argparse_setup(parser)
-        args = parser.parse_args()
+        self.cfg = rpki.config.argparser(section = "pubd", doc = __doc__)
+        self.cfg.add_boolean_argument("--foreground", 
+                                      default = False,
+                                      help = "whether to daemonize")
+        self.cfg.add_argument("--pidfile",   
+                              default = os.path.join(rpki.daemonize.default_pid_directory, 
+                                                     "pubd.pid"),
+                              help = "override default location of pid file")
+        self.cfg.add_argument("--profile",
+                              default = "",
+                              help = "enable profiling, saving data to PROFILE")
+        rpki.log.argparse_setup(self.cfg.argparser)
+        args = self.cfg.argparser.parse_args()
 
         self.profile = args.profile
 
         rpki.log.init("pubd", args)
 
-        self.cfg = rpki.config.parser(set_filename = args.config, section = "pubd")
         self.cfg.set_global_flags()
 
         if not args.foreground:
diff --git a/rpki/rpkid.py b/rpki/rpkid.py
index 4c3c5e7e..869f8bb8 100644
--- a/rpki/rpkid.py
+++ b/rpki/rpkid.py
@@ -75,23 +75,24 @@ class main(object):
 
         self.http_client_serialize = weakref.WeakValueDictionary()
 
-        parser = argparse.ArgumentParser(description = __doc__)
-        parser.add_argument("-c", "--config",
-                            help = "override default location of configuration file")
-        parser.add_argument("-f", "--foreground", action = "store_true",
-                            help = "do not daemonize")
-        parser.add_argument("--pidfile",
-                            help = "override default location of pid file")
-        parser.add_argument("--profile",
-                            help = "enable profiling, saving data to PROFILE")
-        rpki.log.argparse_setup(parser)
-        args = parser.parse_args()
+        self.cfg = rpki.config.argparser(section = "rpkid", doc = __doc__)
+        self.cfg.add_boolean_argument("--foreground", 
+                                      default = False,
+                                      help = "whether to daemonize")
+        self.cfg.add_argument("--pidfile",   
+                              default = os.path.join(rpki.daemonize.default_pid_directory, 
+                                                     "rpkid.pid"),
+                              help = "override default location of pid file")
+        self.cfg.add_argument("--profile",
+                              default = "",
+                              help = "enable profiling, saving data to PROFILE")
+        rpki.log.argparse_setup(self.cfg.argparser)
+        args = self.cfg.argparser.parse_args()
 
         self.profile = args.profile
 
         rpki.log.init("rpkid", args)
 
-        self.cfg = rpki.config.parser(set_filename = args.config, section = "rpkid")
         self.cfg.set_global_flags()
 
         if not args.foreground:
@@ -679,12 +680,14 @@ class publication_queue(object):
                                      type = "query", version = rpki.publication.version)
 
         if uri in self.uris:
-            logger.debug("Removing publication duplicate %r %s hash %s", self.uris[uri], uri, self.uris[uri].get("hash"))
+            logger.debug("Removing publication duplicate %r %s hash %s", 
+                         self.uris[uri], uri, self.uris[uri].get("hash"))
             old_pdu = self.uris.pop(uri)
             self.msgs[rid].remove(old_pdu)
             pdu_hash = old_pdu.get("hash")
             if pdu_hash is None and new_obj is None:
-                logger.debug("Withdrawing object %r which was never published simplifies to no-op", old_pdu)
+                logger.debug("Withdrawing object %r which was never published simplifies to no-op", 
+                             old_pdu)
                 return
         elif old_hash is not None:
             logger.debug("Old hash supplied")                   # XXX Debug log
@@ -719,7 +722,9 @@ class publication_queue(object):
             logger.debug("Calling pubd[%r]", self.repositories[rid])
             try:
                 yield self.repositories[rid].call_pubd(self.rpkid, self.msgs[rid], self.handlers)
-            except (rpki.exceptions.ExistingObjectAtURI, rpki.exceptions.DifferentObjectAtURI, rpki.exceptions.NoObjectAtURI) as e:
+            except (rpki.exceptions.ExistingObjectAtURI, 
+                    rpki.exceptions.DifferentObjectAtURI, 
+                    rpki.exceptions.NoObjectAtURI) as e:
                 logger.warn("Lost synchronization with %r: %s", self.repositories[rid], e)
                 yield self.resync(self.repositories[rid])
         for k in self.uris.iterkeys():
@@ -745,13 +750,14 @@ class publication_queue(object):
         pubd_objs = dict((r_pdu.get("uri"), r_pdu.get("hash")) for r_pdu in r_msg)
 
         our_objs = []
-        for ca_detail in rpki.rpkidb.models.CADetail.objects.filter(ca__parent__tenant = repository.tenant, state = "active"):
+        for ca_detail in rpki.rpkidb.models.CADetail.objects.filter(
+                ca__parent__tenant = repository.tenant, state = "active"):
             our_objs = [(ca_detail.crl_uri,      ca_detail.latest_crl),
                         (ca_detail.manifest_uri, ca_detail.latest_manifest)]
-            our_objs.extend((c.uri, c.cert)         for c in ca_detail.child_certs.all())
-            our_objs.extend((r.uri, r.roa)          for r in ca_detail.roas.filter(roa__isnull = False))
-            our_objs.extend((g.uri, g.ghostbuster)  for g in ca_detail.ghostbusters.all())
-            our_objs.extend((c.uri, c.cert)         for c in ca_detail.ee_certificates.all())
+            our_objs.extend((c.uri, c.cert)      for c in ca_detail.child_certs.all())
+            our_objs.extend((r.uri, r.roa)       for r in ca_detail.roas.filter(roa__isnull = False))
+            our_objs.extend((g.uri, g.ghostbuster) for g in ca_detail.ghostbusters.all())
+            our_objs.extend((c.uri, c.cert)      for c in ca_detail.ee_certificates.all())
 
         q_msg = Element(rpki.publication.tag_msg, nsmap = rpki.publication.nsmap,
                         type = "query", version = rpki.publication.version)
@@ -762,7 +768,8 @@ class publication_queue(object):
             else:
                 h = pubd_objs.pop(uri)
                 if h != rpki.x509.sha256(obj.get_DER()).encode("hex"):
-                    SubElement(q_msg, rpki.publication.tag_publish, uri = uri, hash = h).text = obj.get_Base64()
+                    SubElement(q_msg, rpki.publication.tag_publish, 
+                               uri = uri, hash = h).text = obj.get_Base64()
 
         for uri, h in pubd_objs.iteritems():
             SubElement(q_msg, rpki.publication.tag_withdraw, uri = uri, hash = h)