diff options
| -rwxr-xr-x | rp/rcynic/rcynicng | 91 | ||||
| -rw-r--r-- | rpki/config.py | 95 |
2 files changed, 148 insertions, 38 deletions
diff --git a/rp/rcynic/rcynicng b/rp/rcynic/rcynicng index ccd75913..4648e6c6 100755 --- a/rp/rcynic/rcynicng +++ b/rp/rcynic/rcynicng @@ -28,14 +28,16 @@ import tornado.process import tornado.httpclient import rpki.POW -import rpki.sundial +import rpki.log import rpki.config -import rpki.autoconf +import rpki.sundial import rpki.relaxng +import rpki.autoconf from rpki.oids import id_kp_bgpsec_router -from lxml.etree import ElementTree, Element, SubElement, Comment, XML, DocumentInvalid, XMLSyntaxError, iterparse +from lxml.etree import (ElementTree, Element, SubElement, Comment, + XML, DocumentInvalid, XMLSyntaxError, iterparse) logger = logging.getLogger("rcynicng") @@ -502,7 +504,7 @@ class WalkFrame(object): if not self.fetcher.needed(): self.state = self.ready - elif args.no_spawn_on_fetch: + elif not args.spawn_on_fetch: self.state = self.fetch else: self.state = self.fetch @@ -662,7 +664,7 @@ class WalkTask(object): def read_tals(): - for head, dirs, files in os.walk(args.tals): + for head, dirs, files in os.walk(args.trust_anchor_locators): for fn in files: if fn.endswith(".tal"): furi = "file://" + os.path.abspath(os.path.join(head, fn)) @@ -755,7 +757,7 @@ class Fetcher(object): return None def needed(self): - if args.no_fetch: + if not args.fetch: return False if self.uri.startswith("rsync://"): return self._rsync_needed() @@ -788,7 +790,7 @@ class Fetcher(object): def _rsync_fetch(self): assert self.uri.startswith("rsync://") and (self.uri.endswith(".cer") if self.ta else self.uri.endswith("/")) - if args.no_fetch: + if not args.fetch: return path = self._rsync_split_uri() dead = path[0] in self._rsync_deadhosts @@ -940,7 +942,7 @@ class Fetcher(object): @tornado.gen.coroutine def _https_fetch_ta(self): - if args.no_fetch: + if not args.fetch: return other = self._https_history.get(self.uri) @@ -1022,7 +1024,7 @@ class Fetcher(object): def _rrdp_fetch(self): from django.db import transaction - if args.no_fetch: + if not args.fetch: return other = self._https_history.get(self.uri) @@ -1305,7 +1307,8 @@ def final_report(): # # Should generate <rsync_history/> elements here too, later # - ElementTree(doc).write(file = args.xml_file, pretty_print = True) + ElementTree(doc).write(file = argparse.FileType("w")(args.xml_file), + pretty_print = True) def final_cleanup(): @@ -1386,33 +1389,60 @@ def main(): time.tzset() cfg, parser = rpki.config.argparser(section = "rcynic", doc = __doc__, cfg_optional = True) - parser.add_argument("-u", "--unauthenticated", - default = os.path.join(rpki.autoconf.RCYNIC_DIR, "data", "unauthenticated")) - parser.add_argument("-x", "--xml-file", type = argparse.FileType("w"), - default = os.path.join(rpki.autoconf.RCYNIC_DIR, "data", "rcynic.xml")) - parser.add_argument("-t", "--tals", - default = os.path.join(rpki.autoconf.sysconfdir, "rpki", "trust-anchors")) - parser.add_argument("-w", "--workers", default = 10, type = posint) - parser.add_argument("--no-fetch", action = "store_true") - parser.add_argument("--no-spawn-on-fetch", action = "store_true") - parser.add_argument("--no-migrate", action = "store_true") - parser.add_argument("--prefer-rsync", action = "store_true") - parser.add_argument("--fetch-ahead-goal", default = 2, type = posint) - parser.add_argument("--https-timeout", default = 300, type = posint) - parser.add_argument("--validate-https", action = "store_true") - parser.add_argument("--max-https-body-size", type = posint, default = 512 * 1024 * 1024) - - # We already have a whole bunch of logging control code in - # rpki.log, just need to figure out / remember how to use it - # properly. See rpki.log.init() & rpki.log.argparse_setup(). + rpki.log.argparse_setup(parser) + + cfg.add_argument("-u", "--unauthenticated", + help = "where to store unauthenticated data retrieved via rsycnc", + default = os.path.join(rpki.autoconf.RCYNIC_DIR, "data", "unauthenticated")) + + cfg.add_argument("-x", "--xml-file", + help = "where to write XML log of validation results", + default = os.path.join(rpki.autoconf.RCYNIC_DIR, "data", "rcynic.xml")) + + cfg.add_argument("-t", "--trust-anchor-locators", "--tals", + help = "where to find trust anchor locators", + default = os.path.join(rpki.autoconf.sysconfdir, "rpki", "trust-anchors")) + + cfg.add_argument("-w", "--workers", type = posint, + help = "number of worker pseudo-threads to allow", + default = 10) + + cfg.add_argument("--fetch-ahead-goal", type = posint, + help = "how many deltas we want in the fetch-ahead pipe", + default = 2) + + cfg.add_argument("--https-timeout", type = posint, + help = "HTTPS connection timeout, in seconds", + default = 300) + + cfg.add_argument("--max-https-body-size", type = posint, + help = "upper limit on byte length of HTTPS message body", + default = 512 * 1024 * 1024) + + cfg.add_boolean_argument("--fetch", default = True, + help = "whether to fetch data at all") + + cfg.add_boolean_argument("--spawn-on-fetch", default = True, + help = "whether to spawn new pseudo-threads on fetch") + + cfg.add_boolean_argument("--migrate", default = True, + help = "whether to migrate the ORM database on startup") + + cfg.add_boolean_argument("--prefer-rsync", default = False, + help = "whether to prefer rsync over RRDP") + + cfg.add_boolean_argument("--validate-https", default = False, + help = "whether to validate HTTPS server certificates") global args args = parser.parse_args() + rpki.log.init("rcynic", args) + import django django.setup() - if not args.no_migrate: + if args.migrate: # Not sure we should be doing this on every run, but sure simplifies things. import django.core.management django.core.management.call_command("migrate", verbosity = 0, interactive = False) @@ -1427,7 +1457,6 @@ def main(): RRDPSnapshot = rpki.rcynicdb.models.RRDPSnapshot RPKIObject = rpki.rcynicdb.models.RPKIObject - logging.basicConfig(level = logging.DEBUG, format = "%(asctime)s %(message)s", datefmt = "%Y-%m-%d %H:%M:%S") global authenticated authenticated = Authenticated.objects.create(started = rpki.sundial.datetime.now()) diff --git a/rpki/config.py b/rpki/config.py index 3cc16626..7550c8f4 100644 --- a/rpki/config.py +++ b/rpki/config.py @@ -77,8 +77,10 @@ class parser(object): allow_missing = kwargs.pop("allow_missing", False) set_filename = kwargs.pop("set_filename", None) filename = kwargs.pop("filename", set_filename) + argparser = kwargs.pop("argparser", None) - assert not kwargs, "Unexpected keyword arguments: " + ", ".join("%s = %r" % kv for kv in kwargs.iteritems()) + assert not kwargs, "Unexpected keyword arguments: {}".format( + ", ".join("{} = {!r}".format(k, v) for k, v in kwargs.iteritems())) if set_filename is not None: os.environ[rpki_conf_envname] = set_filename @@ -87,6 +89,7 @@ class parser(object): self.default_section = section self.filename = filename or os.getenv(rpki_conf_envname) or default_filename + self.argparser = argparser try: with open(self.filename, "r") as f: @@ -129,7 +132,8 @@ class parser(object): if self.cfg.has_option(section, option): yield self.cfg.get(section, option) option += "." - matches = [o for o in self.cfg.options(section) if o.startswith(option) and o[len(option):].isdigit()] + matches = [o for o in self.cfg.options(section) + if o.startswith(option) and o[len(option):].isdigit()] matches.sort() for option in matches: yield self.cfg.get(section, option) @@ -176,7 +180,7 @@ class parser(object): if isinstance(v, str): v = v.lower() if v not in self.cfg._boolean_states: - raise ValueError("Not a boolean: %s" % v) + raise ValueError("Not boolean: {}".format(v)) v = self.cfg._boolean_states[v] return v @@ -197,6 +201,80 @@ class parser(object): return long(self.get(option, default, section)) + def add_argument(self, *names, **kwargs): + """ + Combined command line and config file argument. Takes + arguments mostly like ArgumentParser.add_argument(), but also + looks in config file for option of the same name. + + The "section" and "default" arguments are used for the config file + lookup; the resulting value is used as the "default" parameter for + the argument parser. + + If a "type" argument is specified, it applies to both the value + parsed from the config file and the argument parser. + """ + + section = kwargs.pop("section", None) + default = kwargs.pop("default", None) + + for name in names: + if name.startswith("--"): + name = name[2:] + break + else: + raise ValueError + + default = self.get(name, default = default, section = section) + + if "type" in kwargs: + default = kwargs["type"](default) + + kwargs["default"] = default + + return self.argparser.add_argument(*names, **kwargs) + + def add_boolean_argument(self, name, **kwargs): + """ + Combined command line and config file boolean argument. Takes + arguments mostly like ArgumentParser.add_argument(), but also + looks in config file for option of the same name. + + The "section" and "default" arguments are used for the config file + lookup; the resulting value is used as the default value for + the argument parser. + + Usage is a bit different from the normal ArgumentParser boolean + handling: because the command line default is controlled by the + config file, the "store_true" / "store_false" semantics don't + really work for us. So, instead, we use the --foo / --no-foo + convention, and generate a pair of command line arguments with + those names controlling a single "foo" value in the result. + """ + + section = kwargs.pop("section", None) + default = kwargs.pop("default", None) + + if not name.startswith("--"): + raise ValueError + name = name[2:] + + default = self.getboolean(name, default = default, section = section) + + kwargs["action"] = "store_const" + kwargs["dest"] = name.replace("-", "_") + + group = self.argparser.add_mutually_exclusive_group() + + kwargs["const"] = True + group.add_argument("--" + name, **kwargs) + + kwargs["const"] = False + #kwargs["help"] = argparse.SUPPRESS + group.add_argument("--no-" + name, **kwargs) + + self.argparser.set_defaults(**{ kwargs["dest"] : default }) + def set_global_flags(self): """ Consolidated control for all the little global control flags @@ -224,14 +302,16 @@ class parser(object): pass try: - rpki.x509.XML_CMS_object.dump_outbound_cms = rpki.x509.DeadDrop(self.get("dump_outbound_cms")) + rpki.x509.XML_CMS_object.dump_outbound_cms = rpki.x509.DeadDrop( + self.get("dump_outbound_cms")) except OSError, e: logger.warning("Couldn't initialize mailbox %s: %s", self.get("dump_outbound_cms"), e) except ConfigParser.NoOptionError: pass try: - rpki.x509.XML_CMS_object.dump_inbound_cms = rpki.x509.DeadDrop(self.get("dump_inbound_cms")) + rpki.x509.XML_CMS_object.dump_inbound_cms = rpki.x509.DeadDrop( + self.get("dump_inbound_cms")) except OSError, e: logger.warning("Couldn't initialize mailbox %s: %s", self.get("dump_inbound_cms"), e) except ConfigParser.NoOptionError: @@ -323,10 +403,11 @@ def argparser(section = None, doc = None, cfg_optional = False): args, remaining_argv = cfgparser.parse_known_args() + argparser = argparse.ArgumentParser(parents = [topparser], description = doc) + cfg = parser(section = section, set_filename = args.config, + argparser = argparser, allow_missing = cfg_optional or args.help) - argparser = argparse.ArgumentParser(parents = [topparser], description = doc) - return cfg, argparser |