Switch CA daemons to run under rpki-nanny.

This is a transitional version of rpki-nanny: in the long run, the
daemons it runs should take care of reading their own log
configuration from rpki.conf, but that's a yak for another day.

svn path=/branches/tk705/; revision=6366

5 files changed, 276 insertions, 104 deletions

diff --git a/buildtools/debian-skeleton/rpki-ca.default b/buildtools/debian-skeleton/rpki-ca.default
index 503d5fc6..387ff261 100644
--- a/buildtools/debian-skeleton/rpki-ca.default
+++ b/buildtools/debian-skeleton/rpki-ca.default
@@ -6,5 +6,5 @@
 # This is a POSIX shell fragment
 #
 
-# Additional options that are passed to rpki-start-servers.
-STARTER_OPTS="--log-level warning --log-directory /var/log/rpki --log-rotating-file-hours 3 --log-backup-count 56"
+# Additional arguments that are passed to rpki-nanny.
+DAEMON_ARGS="--log-level warning --log-directory /var/log/rpki --log-rotating-file-hours 3 --log-backup-count 56"
diff --git a/buildtools/debian-skeleton/rpki-ca.init.d b/buildtools/debian-skeleton/rpki-ca.init.d
index 8ee4a2c2..3cec6aa6 100644
--- a/buildtools/debian-skeleton/rpki-ca.init.d
+++ b/buildtools/debian-skeleton/rpki-ca.init.d
@@ -10,21 +10,21 @@
 
 # Author: Rob Austein <sra@hactrn.net>
 
-# PATH should only include /usr/* if it runs after the mountnfs.sh script
 PATH=/sbin:/usr/sbin:/bin:/usr/bin
 DESC="rpki-ca"
-NAME=rpki-ca
+NAME=rpki-nanny
 PIDDIR=/var/run/rpki
 LOGDIR=/var/log/rpki
-STARTER=/usr/sbin/rpki-start-servers
-STARTER_OPTS="--log-level warning --log-directory $LOGDIR --log-rotating-file-hours 3 --log-backup-count 56"
-SCRIPTNAME=/etc/init.d/$NAME
+DAEMON=/usr/lib/rpki/$NAME
+DAEMON_ARGS="--log-level warning --log-directory $LOGDIR --log-rotating-file-hours 3 --log-backup-count 56"
+SCRIPTNAME=/etc/init.d/rpki-ca
+PIDFILE=$PIDDIR/$NAME.pid
 
 # Exit if the package is not installed
-test -x "$STARTER" || exit 0
+test -x "$DAEMON" || exit 0
 
 # Read configuration variable file if it is present
-test -r /etc/default/$NAME && . /etc/default/$NAME
+test -r /etc/default/rpki-ca && . /etc/default/rpki-ca
 
 # Load the VERBOSE setting and other rcS variables
 . /lib/init/vars.sh
@@ -35,38 +35,6 @@ test -r /etc/default/$NAME && . /etc/default/$NAME
 . /lib/lsb/init-functions
 
 #
-# Extract list of enabled RPKI daemons from config file.
-#
-
-enabled_daemons()
-{
-    python -c 'if True:
-        import rpki.config
-        cfg = rpki.config.parser(section = "myrpki")
-        enabled = [name for name in ("rpkid", "irdbd", "pubd", "rootd")
-                   if cfg.getboolean("run_{}".format("rpkid" if name == "irdbd" else name))]
-        for name in sorted(enabled):
-            print name
-    '
-}
-
-#
-# Figure out which daemons are actually running at the moment.
-#
-
-running_daemons()
-{
-    for pidfile in $PIDDIR/*.pid
-    do
-	test -f "$pidfile" || continue
-	cmdline=/proc/$(cat $pidfile)/cmdline
-	name=${pidfile##*/}
-	test -f $cmdline &&
-	awk -v name=${name%.pid} 'BEGIN {FS="\0"} $2 ~ ("/" name "$") {print name}' $cmdline
-    done
-}
-
-#
 # Function that starts the daemon/service
 #
 do_start()
@@ -78,45 +46,13 @@ do_start()
 
     test -f /etc/rpki.conf || return 2
 
-    enabled="$(enabled_daemons)"
-    running="$(running_daemons)"
-
-    test "X$enabled" = "X" && return 0
-    test "X$enabled" = "X$running" && return 1
-
-    test -d $PIDDIR || install -d -o rpki -g rpki $PIDDIR || return 2
-    test -d $LOGDIR || install -d -o rpki -g rpki $LOGDIR || return 2
-
-    test -f /usr/share/rpki/bpki/ca.cer   || return 2
-    test -f /usr/share/rpki/bpki/irbe.cer || return 2
-
-    case $enabled in
-	*rpkid*)
-	    test -f /usr/share/rpki/bpki/irdbd.cer || return 2
-	    test -f /usr/share/rpki/bpki/rpkid.cer || return 2
-	    test -f /usr/share/rpki/bpki/rpkid.key || return 2
-    esac
-
-    case $enabled in
-	*pubd*)
-	    test -f /usr/share/rpki/bpki/pubd.cer || return 2
-	    test -f /usr/share/rpki/bpki/pubd.key || return 2
-
-	    for dir in /usr/share/rpki/publication /usr/share/rpki/rrdp-publication
-	    do
-		test -d $dir || install -d -o rpki -g rpki $dir || return 2
-	    done
-    esac
-
-    case $enabled in
-	*rootd*)
-	    test -f /usr/share/rpki/bpki/rootd.cer || return 2
-	    test -f /usr/share/rpki/bpki/rootd.key || return 2
-	    test -f /usr/share/rpki/root.cer       || return 2
-	    test -f /usr/share/rpki/root.key       || return 2
-    esac
+    for dir in $PIDDIR $LOGDIR /usr/share/rpki/publication /usr/share/rpki/rrdp-publication
+    do
+	test -d $dir || install -d -o rpki -g rpki $dir || return 2
+    done
 
-    $STARTER $STARTER_OPTS || return 2
+    start-stop-daemon --start --quiet --pidfile $PIDFILE --startas $DAEMON --name $NAME --test > /dev/null || return 1
+    start-stop-daemon --start --quiet --pidfile $PIDFILE --startas $DAEMON --name $NAME -- $DAEMON_ARGS    || return 2
 }
 
 #
@@ -130,15 +66,7 @@ do_stop()
     #   2 if daemon could not be stopped
     #   other if a failure occurred
 
-    running="$(running_daemons)"
-
-    test "X$running" = "X" && return 1
-
-    for name in $running
-    do
-	kill $(cat $PIDDIR/$name.pid)
-    done
-    return 0
+    start-stop-daemon --stop --quiet --oknodo --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME
 }
 
 case "$1" in
@@ -159,20 +87,7 @@ case "$1" in
 	esac
 	;;
     status)
-	enabled="$(enabled_daemons)"
-	running="$(running_daemons)"
-	if test "X$running" = "X"
-	then
-	    log_success_msg "rpki-ca is not running"
-	    exit 3
-	elif test "X$running" = "X$enabled"
-	then
-	    log_success_msg "rpki-ca is running"
-	    exit 0
-	else
-	    log_success_msg "some rpki-ca daemons are running"
-	    exit 4
-	fi
+	status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $?
 	;;
     restart|force-reload)
 	log_daemon_msg "Restarting $DESC" "$NAME"
diff --git a/buildtools/debian-skeleton/rpki-ca.install b/buildtools/debian-skeleton/rpki-ca.install
index 075a3a32..62cf9922 100644
--- a/buildtools/debian-skeleton/rpki-ca.install
+++ b/buildtools/debian-skeleton/rpki-ca.install
@@ -1,6 +1,5 @@
 usr/lib/rpki
 usr/sbin/irbe_cli
-usr/sbin/rpki-start-servers
 usr/sbin/rpkic
 usr/sbin/rpkigui-query-routes
 usr/share/rpki
diff --git a/ca/rpki-nanny b/ca/rpki-nanny
new file mode 100755
index 00000000..a78c10a6
--- /dev/null
+++ b/ca/rpki-nanny
@@ -0,0 +1,258 @@
+#!/usr/bin/env python
+
+# $Id$
+#
+# Copyright (C) 2014  Dragon Research Labs ("DRL")
+# Portions copyright (C) 2009--2013  Internet Systems Consortium ("ISC")
+# Portions copyright (C) 2007--2008  American Registry for Internet Numbers ("ARIN")
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notices and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND DRL, ISC, AND ARIN DISCLAIM ALL
+# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS.  IN NO EVENT SHALL DRL,
+# ISC, OR ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""
+Start servers, using config file to figure out which servers the user
+wants started.
+"""
+
+import os
+import pwd
+import sys
+import time
+import signal
+import logging
+import argparse
+import subprocess
+
+import rpki.log
+import rpki.config
+import rpki.autoconf
+import rpki.daemonize
+
+from logging.handlers import SysLogHandler
+
+logger = logging.getLogger(__name__)
+
+signames = dict((getattr(signal, sig), sig)
+                for sig in dir(signal)
+                if  sig.startswith("SIG")
+                and sig.isalnum()
+                and sig.isupper()
+                and isinstance(getattr(signal, sig), int))
+
+# TODO:
+#
+# * Logging configuration is a mess.  Daemons should be handling this
+#   for themselves, from rpki.conf, and there should be a way to configure
+#   logging for rpki-nanny itself.
+#
+# * Perhaps we should re-read the config file so we can turn individual
+#   daemons on and off?  Or is that unnecessary complexity?
+#
+# * rpki-nanny should probably daemonize itself before forking.
+
+
+class Daemon(object):
+    """
+    Representation and control of one daemon under our care.
+    """
+
+    def __init__(self, name):
+        self.name = name
+        self.proc = None
+        self.next_restart = 0
+        if cfg.getboolean("start_" + name, False):
+            log_file = os.path.join(args.log_directory, name + ".log")
+            self.cmd = (os.path.join(rpki.autoconf.libexecdir, name), 
+                        "--foreground",
+                        "--log-level", args.log_level)
+            if args.log_file:
+                self.cmd += ("--log-file", log_file)
+            elif args.log_rotating_file_kbytes:
+                self.cmd += ("--log-rotating-file", log_file, 
+                             args.log_rotating_file_kbytes, args.log_backup_count)
+            elif args.log_rotating_file_hours:
+                self.cmd += ("--log-timed-rotating-file", log_file, 
+                             args.log_rotating_file_hours, args.log_backup_count)
+            else:
+                self.cmd += ("--log-syslog", args.log_syslog)
+        else:
+            self.cmd = ()
+
+    def start_maybe(self):
+        if self.cmd and self.proc is None and time.time() > self.next_restart:
+            try:
+                self.proc = subprocess.Popen(self.cmd)
+                self.next_restart = int(time.time() + args.restart_delay)
+                logger.debug("Started  %s[%s]", self.name, self.proc.pid)
+            except:
+                logger.exception("Trouble starting %s", self.name)
+
+    def terminate(self):
+        if self.proc is not None:
+            try:
+                logger.debug("Terminating daemon %s[%s]", self.name, self.proc.pid)
+                self.proc.terminate()
+            except:
+                logger.exception("Trouble terminating %s[%s]", self.name, self.proc.pid)
+
+    def delay(self):
+        return max(0, int(self.next_restart - time.time())) if self.cmd and self.proc is None else 0
+
+    def reap(self):
+        if self.proc is not None and self.proc.poll() is not None:
+            code = self.proc.wait()
+            if code < 0:
+                logger.warn("%s[%s] exited on signal %s",
+                            self.name, self.proc.pid, signames.get(-code, "???"))
+            else:
+                logger.warn("%s[%s] exited with status %s",
+                            self.name, self.proc.pid, code)
+            self.proc = None
+
+
+class Signals(object):
+    """
+
+    Convert POSIX signals into something we can use in a loop at main
+    program level.  Assumes that we use signal.pause() to block, so
+    simply receiving the signal is enough to wake us up.
+
+    Calling the constructed Signals object with one or more signal
+    numbers returns True if any of those signals have been received,
+    and clears the internal flag for the first such signal.
+    """
+
+    def __init__(self, *sigs):
+        self._active = set()
+        for sig in sigs:
+            signal.signal(sig, self._handler)
+
+    def _handler(self, sig, frame):
+        self._active.add(sig)
+        #logger.debug("Received %s", signames.get(sig, "???"))
+
+    def __call__(self, *sigs):
+        for sig in sigs:
+            try:
+                self._active.remove(sig)
+                return True
+            except KeyError:
+                pass
+        return False
+
+
+def non_negative_integer(s):
+    if int(s) < 0:
+        raise ValueError
+    return s
+
+def positive_integer(s):
+    if int(s) <= 0:
+        raise ValueError
+    return s
+
+
+if __name__ == "__main__":
+
+    os.environ.update(TZ = "UTC")
+    time.tzset()
+
+    cfg = rpki.config.argparser(section = "myrpki", doc = __doc__)
+
+    cfg.add_argument("--restart-delay", type = positive_integer, default = 60,
+                     help = "how long to wait before restarting a crashed daemon")
+    cfg.add_argument("--pidfile",   
+                     default = os.path.join(rpki.daemonize.default_pid_directory, "rpki-nanny.pid"),
+                     help = "override default location of pid file")
+    cfg.add_boolean_argument("--daemonize", default = True,
+                             help = "whether to daemonize")
+
+    # This stuff is a mess.  Daemons should control their own logging
+    # via rpki.conf settings, but we haven't written that yet, and
+    # this script is meant to be a replacement for rpki-start-servers,
+    # so leave the mess in place for the moment and clean up later.
+
+    cfg.argparser.add_argument("--log-directory", default = ".",
+                               help = "where to write write log files when not using syslog")
+    cfg.argparser.add_argument("--log-backup-count", default = "7", type = non_negative_integer,
+                               help = "keep this many old log files when rotating")
+    cfg.argparser.add_argument("--log-level", default = "warning",
+                               choices = ("debug", "info", "warning", "error", "critical"),
+                               help = "how verbosely to log")
+    group = cfg.argparser.add_mutually_exclusive_group()
+    group.add_argument("--log-file", action = "store_true",
+                       help = "log to files, reopening if rotated away")
+    group.add_argument("--log-rotating-file-kbytes",type = non_negative_integer,
+                       help = "log to files, rotating after this many kbytes")
+    group.add_argument("--log-rotating-file-hours", type = non_negative_integer,
+                       help = "log to files, rotating after this many hours")
+    group.add_argument("--log-syslog", default = "daemon", nargs = "?",
+                       choices = sorted(SysLogHandler.facility_names.keys()),
+                       help = "log syslog")
+
+    args = cfg.argparser.parse_args()
+
+    # Drop privs before daemonizing or opening log file
+
+    pw = pwd.getpwnam(rpki.autoconf.RPKI_USER)
+    os.setgid(pw.pw_gid)
+    os.setuid(pw.pw_uid)
+
+    # Log control mess here is continuation of log control mess above:
+    # all the good names are taken by the pass-through kludge, we'd
+    # have to reimplement all the common logic to use it ourselves
+    # too.  Just wire to stderr or rotating log file for now, using
+    # same log file scheme as set in /etc/defaults/ on Debian/Ubuntu
+    # and the log level wired to DEBUG, fix the whole logging mess later.
+
+    if args.daemonize:
+        log_handler = lambda: logging.handlers.TimedRotatingFileHandler(
+                filename    = os.path.join(args.log_directory, "rpki-nanny.log"),
+                interval    = 3,
+                backupCount = 56,
+                when        = "H",
+                utc         = True)
+    else:
+        log_handler = logging.StreamHandler
+
+    rpki.log.init(ident = "rpki-nanny", 
+                  args  = argparse.Namespace(log_level   = logging.DEBUG,
+                                             log_handler = log_handler))
+    if args.daemonize:
+        rpki.daemonize.daemon(pidfile = args.pidfile)
+
+    signals = Signals(signal.SIGALRM, signal.SIGCHLD, signal.SIGTERM, signal.SIGINT)
+    daemons = [Daemon(name) for name in ("irdbd", "rpkid", "pubd", "rootd")]
+    exiting = False
+
+    try:
+        while not exiting or not all(daemon.proc is None for daemon in daemons):
+            if not exiting and signals(signal.SIGTERM, signal.SIGINT):
+                logger.info("Received exit signal")
+                exiting = True
+                for daemon in daemons:
+                    daemon.terminate()
+            if not exiting:
+                for daemon in daemons:
+                    daemon.start_maybe()
+                alarms = tuple(daemon.delay() for daemon in daemons)
+                signal.alarm(min(a for a in alarms if a > 0) + 1 if any(alarms) else 0)
+            if not signals(signal.SIGCHLD, signal.SIGALRM):
+                signal.pause()
+            for daemon in daemons:
+                daemon.reap()
+    except:
+        logger.exception("Unhandled exception in main loop")
+        for daemon in daemons:
+            daemon.terminate()
+        sys.exit(1)
diff --git a/setup.py b/setup.py
index 10275ba9..c655dbc8 100644
--- a/setup.py
+++ b/setup.py
@@ -118,7 +118,6 @@ if autoconf.CA_TARGET == "ca":
 
     scripts += [(autoconf.sbindir,
                  ["ca/rpkic",
-                  "ca/rpki-start-servers",
                   "ca/rpkigui-query-routes",
                   "ca/irbe_cli"]),
                 (autoconf.libexecdir,
@@ -126,6 +125,7 @@ if autoconf.CA_TARGET == "ca":
                   "ca/pubd",
                   "ca/rootd",
                   "ca/rpkid",
+                  "ca/rpki-nanny",
                   "ca/rpkigui-import-routes",
                   "ca/rpkigui-check-expired",
                   "ca/rpkigui-rcynic",