Source tree reorg, phase 1. Almost everything moved, no file contents changed.

svn path=/branches/tk685/; revision=5757

1 files changed, 658 insertions, 0 deletions

diff --git a/rp/rcynic/rcynic-html b/rp/rcynic/rcynic-html
new file mode 100755
index 00000000..58e65dde
--- /dev/null
+++ b/rp/rcynic/rcynic-html
@@ -0,0 +1,658 @@
+# $Id$
+# 
+# Copyright (C) 2013--2014  Dragon Research Labs ("DRL")
+# Portions copyright (C) 2009--2012  Internet Systems Consortium ("ISC")
+# 
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notices and this permission notice appear in all copies.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL
+# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS.  IN NO EVENT SHALL DRL OR
+# ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
+# DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA
+# OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+
+"""
+Render rcynic's XML output to basic (X)HTML with some rrdtool graphics.
+"""
+
+import sys
+import urlparse
+import os
+import argparse
+import time
+import subprocess
+import copy
+
+try:
+  from lxml.etree            import (ElementTree, Element, SubElement, Comment)
+except ImportError:
+  from xml.etree.ElementTree import (ElementTree, Element, SubElement, Comment)
+
+session = None
+args = None
+
+def parse_options():
+
+  global args
+
+  try:
+    default_rrdtool_binary = ac_rrdtool_binary
+  except NameError:
+    default_rrdtool_binary = "rrdtool"
+
+  parser = argparse.ArgumentParser(description = __doc__)
+  parser.add_argument("--refresh", type = int, default = 1800,
+                      help = "refresh interval for generated HTML")
+  parser.add_argument("--hide-problems", action = "store_true",
+                      help = "don't generate \"problems\" page")
+  parser.add_argument("--hide-graphs", action = "store_true",
+                      help = "don't generate graphs")
+  parser.add_argument("--hide-object-counts", action = "store_true",
+                      help = "don't display object counts")
+  parser.add_argument("--dont-update-rrds", action = "store_true",
+                      help = "don't add new data to RRD databases")
+  parser.add_argument("--png-height", type = int, default = 190,
+                      help = "height of PNG images")
+  parser.add_argument("--png-width", type = int, default = 1350,
+                      help = "width of PNG images")
+  parser.add_argument("--svg-height", type = int, default = 600,
+                      help = "height of SVG images")
+  parser.add_argument("--svg-width", type = int, default = 1200,
+                      help = "width of SVG images")
+  parser.add_argument("--eps-height", type = int, default = 0,
+                      help = "height of EPS images")
+  parser.add_argument("--eps-width", type = int, default = 0,
+                      help = "width of EPS images")
+  parser.add_argument("--rrdtool-binary", default = default_rrdtool_binary,
+                      help = "location of rrdtool binary")
+  parser.add_argument("input_file", type = argparse.FileType("r"),
+                      help = "XML input file")
+  parser.add_argument("output_directory",
+                      help = "output directory")
+  args = parser.parse_args()
+
+
+def parse_utc(s):
+  return int(time.mktime(time.strptime(s, "%Y-%m-%dT%H:%M:%SZ")))
+
+class Label(object):
+
+  moods = ["bad", "warn", "good"]
+
+  def __init__(self, elt):
+    self.code = elt.tag
+    self.mood = elt.get("kind")
+    self.text = elt.text.strip()
+    self.count = 0
+
+  def get_count(self):
+    return self.count
+
+  @property
+  def sort_key(self):
+    try:
+      return self.moods.index(self.mood)
+    except ValueError:
+      return len(self.moods)
+
+class Validation_Status(object):
+
+  def __init__(self, elt, label_map):
+    self.uri = elt.text.strip()
+    self.timestamp = elt.get("timestamp")
+    self.generation = elt.get("generation")
+    self.hostname = urlparse.urlparse(self.uri).hostname or "[None]"
+    self.fn2 = os.path.splitext(self.uri)[1] or None if self.generation else None
+    self.label = label_map[elt.get("status")]
+
+  def sort_key(self):
+    return (self.label.sort_key, self.timestamp, self.hostname, self.fn2, self.generation)
+
+  @property
+  def code(self):
+    return self.label.code
+
+  @property
+  def mood(self):
+    return self.label.mood
+
+  @property
+  def accepted(self):
+    return self.label.code == "object_accepted"
+
+  @property
+  def rejected(self):
+    return self.label.code == "object_rejected"
+
+  @property
+  def is_current(self):
+    return self.generation == "current"
+
+  @property
+  def is_backup(self):
+    return self.generation == "backup"
+  
+  @property
+  def is_problem(self):
+    return self.label.mood != "good"
+
+  @property
+  def is_connection_problem(self):
+    return self.label.mood != "good" and self.label.code.startswith("rsync_transfer_")
+
+  @property
+  def is_object_problem(self):
+    return self.label.mood != "good" and not self.label.code.startswith("rsync_transfer_")
+
+  @property
+  def is_connection_detail(self):
+    return self.label.code.startswith("rsync_transfer_")
+
+  @property
+  def is_object_detail(self):
+    return not self.label.code.startswith("rsync_transfer_")
+
+class Problem_Mixin(object):
+  
+  @property
+  def connection_problems(self):
+    result = [v for v in self.validation_status if v.is_connection_problem]
+    result.sort(key = Validation_Status.sort_key)
+    return result
+
+  @property
+  def object_problems(self):
+    result = [v for v in self.validation_status if v.is_object_problem]
+    result.sort(key = Validation_Status.sort_key)
+    return result
+
+class Host(Problem_Mixin):
+
+  def __init__(self, hostname, timestamp):
+    self.hostname = hostname
+    self.timestamp = timestamp
+    self.elapsed = 0
+    self.connections = 0
+    self.failures = 0
+    self.uris = set()
+    self.graph = None
+    self.counters = {}
+    self.totals = {}
+    self.validation_status = []
+
+  def add_connection(self, elt):
+    self.elapsed += parse_utc(elt.get("finished")) - parse_utc(elt.get("started"))
+    self.connections += 1
+    if elt.get("error") is not None:
+      self.failures += 1
+
+  def add_validation_status(self, v):
+    self.validation_status.append(v)
+    if v.generation == "current":
+      self.uris.add(v.uri)
+    self.counters[(v.fn2, v.generation, v.label)] = self.get_counter(v.fn2, v.generation, v.label) + 1
+    self.totals[v.label] = self.get_total(v.label) + 1
+    v.label.count += 1
+
+  def get_counter(self, fn2, generation, label):
+    return self.counters.get((fn2, generation, label), 0)
+
+  def get_total(self, label):
+    return self.totals.get(label, 0)
+
+  @property
+  def failed(self):
+    return 1 if self.failures > 0 else 0
+
+  @property
+  def objects(self):
+    return len(self.uris)
+
+  field_table = (("connections", "GAUGE"),
+                 ("objects",     "GAUGE"),
+                 ("elapsed",     "GAUGE"),
+                 ("failed",      "ABSOLUTE"))
+
+  rras = tuple("RRA:AVERAGE:0.5:%s:9600" % steps
+               for steps in (1, 4, 24))
+
+  @classmethod
+  def field_ds_specifiers(cls, heartbeat = 24 * 60 * 60, minimum = 0, maximum = "U"):
+    return ["DS:%s:%s:%s:%s:%s" % (field[0], field[1], heartbeat, minimum, maximum)
+            for field in cls.field_table]
+
+  @property
+  def field_values(self):
+    return tuple(str(getattr(self, field[0])) for field in self.field_table)
+
+  @classmethod
+  def field_defs(cls, filebase):
+    return ["DEF:%s=%s.rrd:%s:AVERAGE" % (field[0], filebase, field[0])
+            for field in cls.field_table]
+
+  graph_opts = (
+    "--vertical-label", "Sync time (seconds)",
+    "--right-axis-label", "Objects (count)",
+    "--lower-limit", "0",
+    "--right-axis", "1:0",
+    "--full-size-mode" )
+
+  graph_cmds = (
+
+    # Split elapsed into separate data sets, so we can color
+    # differently to indicate how succesful transfer was.  Intent is
+    # that exactly one of these be defined for every value in elapsed.
+
+    "CDEF:success=failed,UNKN,elapsed,IF",
+    "CDEF:failure=connections,1,EQ,failed,*,elapsed,UNKN,IF",
+    "CDEF:partial=connections,1,NE,failed,*,elapsed,UNKN,IF",
+
+    # Show connection timing first, as color-coded semi-transparent
+    # areas with opaque borders.  Intent is to make the colors stand
+    # out, since they're a major health indicator.  Transparency is
+    # handled via an alpha channel (fourth octet of color code).  We
+    # draw this stuff first so that later lines can overwrite it.
+
+    "AREA:success#00FF0080:Sync time (success)",
+    "AREA:partial#FFA50080:Sync time (partial failure)",
+    "AREA:failure#FF000080:Sync time (total failure)",
+
+    "LINE1:success#00FF00",             # Green
+    "LINE1:partial#FFA500",             # Orange
+    "LINE1:failure#FF0000",             # Red
+
+    # Now show object counts, as a simple black line.
+
+    "LINE1:objects#000000:Objects",     # Black
+
+    # Add averages over period to chart legend.
+
+    "VDEF:avg_elapsed=elapsed,AVERAGE",
+    "VDEF:avg_connections=connections,AVERAGE",
+    "VDEF:avg_objects=objects,AVERAGE",
+    "COMMENT:\j",
+    "GPRINT:avg_elapsed:Average sync time (seconds)\: %5.2lf",
+    "GPRINT:avg_connections:Average connection count\: %5.2lf",
+    "GPRINT:avg_objects:Average object count\: %5.2lf" )
+
+  graph_periods = (("week",  "-1w"),
+                   ("month", "-31d"),
+                   ("year",  "-1y"))
+
+  def rrd_run(self, cmd):
+    try:
+      cmd = [str(i) for i in cmd]
+      cmd.insert(0, args.rrdtool_binary)
+      subprocess.check_call(cmd, stdout = open("/dev/null", "w"))
+    except OSError, e:
+      sys.exit("Problem running %s, perhaps you need to set --rrdtool-binary? (%s)" % (args.rrdtool_binary, e))
+    except subprocess.CalledProcessError, e:
+      sys.exit("Failure running %s: %s" % (args.rrdtool_binary, e))
+
+  def rrd_update(self):
+    filename = os.path.join(args.output_directory, self.hostname) + ".rrd"
+    if not os.path.exists(filename):
+      cmd = ["create", filename, "--start", self.timestamp - 1, "--step", "3600"]
+      cmd.extend(self.field_ds_specifiers())
+      cmd.extend(self.rras)
+      self.rrd_run(cmd)
+    self.rrd_run(["update", filename,
+                  "%s:%s" % (self.timestamp, ":".join(str(v) for v in self.field_values))])
+
+  def rrd_graph(self, html):
+    filebase = os.path.join(args.output_directory, self.hostname)
+    formats = [format for format in ("png", "svg", "eps")
+               if getattr(args, format + "_width") and getattr(args, format + "_height")]
+    for period, start in self.graph_periods:
+      for format in formats:
+        cmds = [ "graph", "%s_%s.%s" % (filebase, period, format),
+                 "--title", "%s last %s" % (self.hostname, period),
+                 "--start", start,
+                 "--width",  getattr(args, format + "_width"),
+                 "--height", getattr(args, format + "_height"),
+                 "--imgformat", format.upper() ]
+        cmds.extend(self.graph_opts)
+        cmds.extend(self.field_defs(filebase))
+        cmds.extend(self.graph_cmds)
+        self.rrd_run(cmds)
+      img = Element("img", src = "%s_%s.png" % (self.hostname, period),
+                    width  = str(args.png_width),
+                    height = str(args.png_height))
+      if self.graph is None:
+        self.graph = copy.copy(img)
+      html.BodyElement("h2").text = "%s over last %s" % (self.hostname, period)
+      html.BodyElement("a", href = "%s_%s_svg.html" % (self.hostname, period)).append(img)
+      html.BodyElement("br")
+      svg_html = HTML("%s over last %s" % (self.hostname, period),
+                      "%s_%s_svg" % (self.hostname, period))
+      svg_html.BodyElement("img", src = "%s_%s.svg" % (self.hostname, period))
+      svg_html.close()
+
+  
+class Session(Problem_Mixin):
+
+  def __init__(self):
+    self.hosts = {}
+
+    self.root = ElementTree(file = args.input_file).getroot()
+
+    self.rcynic_version = self.root.get("rcynic-version")
+    self.rcynic_date = self.root.get("date")
+    self.timestamp = parse_utc(self.rcynic_date)
+
+    self.labels = [Label(elt) for elt in self.root.find("labels")]
+    self.load_validation_status()
+
+    for elt in self.root.findall("rsync_history"):
+      self.get_host(urlparse.urlparse(elt.text.strip()).hostname).add_connection(elt)
+
+    generations = set()
+    fn2s = set()
+
+    for v in self.validation_status:
+      self.get_host(v.hostname).add_validation_status(v)
+      generations.add(v.generation)
+      fn2s.add(v.fn2)
+
+    self.labels = [l for l in self.labels if l.count > 0]
+
+    self.hostnames   = sorted(self.hosts)
+    self.generations = sorted(generations)
+    self.fn2s        = sorted(fn2s)
+
+  def load_validation_status(self):
+    label_map = dict((label.code, label) for label in self.labels)
+    full_validation_status = [Validation_Status(elt, label_map)
+                              for elt in self.root.findall("validation_status")]
+    accepted_current = set(v.uri for v in full_validation_status
+                           if v.is_current and v.accepted)
+    self.validation_status = [v for v in full_validation_status
+                              if not v.is_backup
+                              or v.uri not in accepted_current]
+
+  def get_host(self, hostname):
+    if hostname not in self.hosts:
+      self.hosts[hostname] = Host(hostname, self.timestamp)
+    return self.hosts[hostname]
+
+  def get_sum(self, fn2, generation, label):
+    return sum(h.get_counter(fn2, generation, label)
+               for h in self.hosts.itervalues())
+
+  def rrd_update(self):
+    if not args.dont_update_rrds:
+      for h in self.hosts.itervalues():
+        h.rrd_update()
+
+css = '''
+    th, td {
+      text-align: center; padding: 4px;
+    }
+
+    td.uri {
+      text-align: left;
+    }
+
+    thead tr th, tfoot tr td {
+      font-weight: bold;
+    }
+
+    .good {
+      background-color: #77ff77;
+    }
+
+    .warn {
+      background-color: yellow;
+    }
+
+    .bad {
+      background-color: #ff5500;
+    }
+
+    body {
+      font-family: arial, helvetica, serif;
+    }
+
+    /* Make background-color inherit like color does. */
+    #nav {
+      background-color: inherit;
+    }
+
+    #nav, #nav ul {
+      float: left;
+      width: 100%;
+      list-style: none;
+      line-height: 1;
+      font-weight: normal;
+      padding: 0;
+      border-color: black;
+      border-style: solid;
+      border-width: 1px 0;
+      margin: 0 0 1em 0;
+    }
+
+    #nav a, #nav span {
+      display: block;
+      background-color: white;
+      color: black;
+      text-decoration: none;
+      padding: 0.25em 0.75em;
+    }
+
+    #nav li {
+      float: left;
+      padding: 0;
+    }
+
+    /* Use <ul style="width: ..."> to set submenu width. */
+    #nav li ul {
+      position: absolute;
+      display: none;
+      height: auto;
+      border-width: 1px;
+      margin: 0;
+    }
+
+    #nav li li {
+      width: 100%;
+    }
+
+    /* Display submenu when hovering. */
+    #nav li:hover ul {
+      display: block;
+    }
+
+    /* Reverse video when hovering. */
+    #nav a:hover, #nav span:hover {
+      color: white;
+      background-color: black;
+    }
+'''
+
+class HTML(object):
+
+  def __init__(self, title, filebase):
+
+    self.filename = os.path.join(args.output_directory, filebase + ".html")
+
+    self.html = Element("html")
+    self.html.append(Comment(" Generators:\n" + 
+                             "  " + session.rcynic_version + "\n" +
+                             "  $Id$\n"))
+    self.head = SubElement(self.html, "head")
+    self.body = SubElement(self.html, "body")
+
+    title += " " + session.rcynic_date
+    SubElement(self.head, "title").text = title
+    SubElement(self.body, "h1").text = title
+    SubElement(self.head, "style", type = "text/css").text = css
+
+    if args.refresh:
+      SubElement(self.head, "meta", { "http-equiv" : "Refresh", "content" : str(args.refresh) })
+
+    hostwidth = max(len(hostname) for hostname in session.hostnames)
+
+    toc = SubElement(self.body, "ul", id = "nav")
+    SubElement(SubElement(toc, "li"), "a", href = "index.html").text = "Overview"
+    li = SubElement(toc, "li")
+    SubElement(li, "span").text = "Repositories"
+    ul = SubElement(li, "ul", style = "width: %sem" % hostwidth)
+    for hostname in session.hostnames:
+      SubElement(SubElement(ul, "li"), "a", href = "%s.html" % hostname).text = hostname
+    SubElement(SubElement(toc, "li"), "a", href = "problems.html").text = "Problems"
+    li = SubElement(toc, "li")
+    SubElement(li, "span").text = "All Details"
+    ul = SubElement(li, "ul", style = "width: 15em")
+    SubElement(SubElement(ul, "li"), "a", href = "connections.html").text = "All Connections"
+    SubElement(SubElement(ul, "li"), "a", href = "objects.html").text = "All Objects"
+    SubElement(self.body, "br")
+
+  def close(self):
+    ElementTree(element = self.html).write(self.filename)
+
+  def BodyElement(self, tag, **attrib):
+    return SubElement(self.body, tag, **attrib)
+
+  def counter_table(self, data_func, total_func):
+    table = self.BodyElement("table", rules = "all", border = "1")
+    thead = SubElement(table, "thead")
+    tfoot = SubElement(table, "tfoot")
+    tbody = SubElement(table, "tbody")
+    tr = SubElement(thead, "tr")
+    SubElement(tr, "th")
+    for label in session.labels:
+      SubElement(tr, "th").text = label.text
+    for fn2 in session.fn2s:
+      for generation in session.generations:
+        counters = [data_func(fn2, generation, label) for label in session.labels]
+        if sum(counters) > 0:
+          tr = SubElement(tbody, "tr")
+          SubElement(tr, "td").text = ((generation or "") + " " + (fn2 or "")).strip()
+          for label, count in zip(session.labels, counters):
+            td = SubElement(tr, "td")
+            if count > 0:
+              td.set("class", label.mood)
+              td.text = str(count)
+    tr = SubElement(tfoot, "tr")
+    SubElement(tr, "td").text = "Total"
+    counters = [total_func(label) for label in session.labels]
+    for label, count in zip(session.labels, counters):
+      td = SubElement(tr, "td")
+      if count > 0:
+        td.set("class", label.mood)
+        td.text = str(count)
+    return table
+
+  def object_count_table(self, session):
+    table = self.BodyElement("table", rules = "all", border = "1")
+    thead = SubElement(table, "thead")
+    tbody = SubElement(table, "tbody")
+    tfoot = SubElement(table, "tfoot")
+    fn2s  = [fn2 for fn2 in session.fn2s if fn2 is not None]
+    total = dict((fn2, 0) for fn2 in fn2s)
+    for hostname in session.hostnames:
+      tr = SubElement(tbody, "tr")
+      SubElement(tr, "td").text = hostname
+      for fn2 in fn2s:
+        td = SubElement(tr, "td")
+        count = sum(uri.endswith(fn2) for uri in session.hosts[hostname].uris)
+        total[fn2] += count
+        if count > 0:
+          td.text = str(count)
+    trhead = SubElement(thead, "tr")
+    trfoot = SubElement(tfoot, "tr")
+    SubElement(trhead, "th").text = "Repository"
+    SubElement(trfoot, "td").text = "Total"
+    for fn2 in fn2s:
+      SubElement(trhead, "th").text = fn2
+      SubElement(trfoot, "td").text = str(total[fn2])
+    return table
+
+  def detail_table(self, records):
+    if records:
+      table = self.BodyElement("table", rules = "all", border = "1")
+      thead = SubElement(table, "thead")
+      tbody = SubElement(table, "tbody")
+      tr = SubElement(thead, "tr")
+      SubElement(tr, "th").text = "Timestamp"
+      SubElement(tr, "th").text = "Generation"
+      SubElement(tr, "th").text = "Status"
+      SubElement(tr, "th").text = "URI"
+      for v in records:
+        tr = SubElement(tbody, "tr", { "class" : v.mood })
+        SubElement(tr, "td").text = v.timestamp
+        SubElement(tr, "td").text = v.generation
+        SubElement(tr, "td").text = v.label.text
+        SubElement(tr, "td", { "class" : "uri"}).text = v.uri
+      return table
+    else:
+      self.BodyElement("p").text = "None found"
+      return None
+
+def main():
+
+  global session
+  
+  os.putenv("TZ", "UTC")
+  time.tzset()
+
+  parse_options()
+
+  session = Session()
+  session.rrd_update()
+
+  for hostname in session.hostnames:
+    html = HTML("Repository details for %s" % hostname, hostname)
+    html.counter_table(session.hosts[hostname].get_counter, session.hosts[hostname].get_total)
+    if not args.hide_graphs:
+      session.hosts[hostname].rrd_graph(html)
+    if not args.hide_problems:
+      html.BodyElement("h2").text = "Connection Problems"
+      html.detail_table(session.hosts[hostname].connection_problems)
+      html.BodyElement("h2").text = "Object Problems"
+      html.detail_table(session.hosts[hostname].object_problems)
+    html.close()
+
+  html = HTML("rcynic summary", "index")
+  html.BodyElement("h2").text = "Grand totals for all repositories"
+  html.counter_table(session.get_sum, Label.get_count)
+  if not args.hide_object_counts:
+    html.BodyElement("br")
+    html.BodyElement("hr")
+    html.BodyElement("br")
+    html.BodyElement("h2").text = "Current total object counts (distinct URIs)"
+    html.object_count_table(session)
+  for hostname in session.hostnames:
+    html.BodyElement("br")
+    html.BodyElement("hr")
+    html.BodyElement("br")
+    html.BodyElement("h2").text = "Overview for repository %s" % hostname
+    html.counter_table(session.hosts[hostname].get_counter, session.hosts[hostname].get_total)
+    if not args.hide_graphs:
+      html.BodyElement("br")
+      html.BodyElement("a", href = "%s.html" % hostname).append(session.hosts[hostname].graph)
+  html.close()
+
+  html = HTML("Problems", "problems")
+  html.BodyElement("h2").text = "Connection Problems"
+  html.detail_table(session.connection_problems)
+  html.BodyElement("h2").text = "Object Problems"
+  html.detail_table(session.object_problems)
+  html.close()
+
+  html = HTML("All connections", "connections")
+  html.detail_table([v for v in session.validation_status if v.is_connection_detail])
+  html.close()
+
+  html = HTML("All objects", "objects")
+  html.detail_table([v for v in session.validation_status if v.is_object_detail])
+  html.close()
+
+
+if __name__ == "__main__":
+  main()