# $Id$
# 
# Copyright (C) 2013--2014  Dragon Research Labs ("DRL")
# Portions copyright (C) 2009--2012  Internet Systems Consortium ("ISC")
# 
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notices and this permission notice appear in all copies.
# 
# THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL
# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS.  IN NO EVENT SHALL DRL OR
# ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
# DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA
# OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.

"""
Render rcynic's XML output to basic (X)HTML with some rrdtool graphics.
"""

import sys
import urlparse
import os
import argparse
import time
import subprocess
import copy

try:
  from lxml.etree            import (ElementTree, Element, SubElement, Comment)
except ImportError:
  from xml.etree.ElementTree import (ElementTree, Element, SubElement, Comment)

session = None
args = None

def parse_options():

  global args

  try:
    default_rrdtool_binary = ac_rrdtool_binary
  except NameError:
    default_rrdtool_binary = "rrdtool"

  parser = argparse.ArgumentParser(description = __doc__)
  parser.add_argument("--refresh", type = int, default = 1800,
                      help = "refresh interval for generated HTML")
  parser.add_argument("--hide-problems", action = "store_true",
                      help = "don't generate \"problems\" page")
  parser.add_argument("--hide-graphs", action = "store_true",
                      help = "don't generate graphs")
  parser.add_argument("--hide-object-counts", action = "store_true",
                      help = "don't display object counts")
  parser.add_argument("--dont-update-rrds", action = "store_true",
                      help = "don't add new data to RRD databases")
  parser.add_argument("--png-height", type = int, default = 190,
                      help = "height of PNG images")
  parser.add_argument("--png-width", type = int, default = 1350,
                      help = "width of PNG images")
  parser.add_argument("--svg-height", type = int, default = 600,
                      help = "height of SVG images")
  parser.add_argument("--svg-width", type = int, default = 1200,
                      help = "width of SVG images")
  parser.add_argument("--eps-height", type = int, default = 0,
                      help = "height of EPS images")
  parser.add_argument("--eps-width", type = int, default = 0,
                      help = "width of EPS images")
  parser.add_argument("--rrdtool-binary", default = default_rrdtool_binary,
                      help = "location of rrdtool binary")
  parser.add_argument("input_file", type = argparse.FileType("r"),
                      help = "XML input file")
  parser.add_argument("output_directory",
                      help = "output directory")
  args = parser.parse_args()


def parse_utc(s):
  return int(time.mktime(time.strptime(s, "%Y-%m-%dT%H:%M:%SZ")))

class Label(object):

  moods = ["bad", "warn", "good"]

  def __init__(self, elt):
    self.code = elt.tag
    self.mood = elt.get("kind")
    self.text = elt.text.strip()
    self.count = 0

  def get_count(self):
    return self.count

  @property
  def sort_key(self):
    try:
      return self.moods.index(self.mood)
    except ValueError:
      return len(self.moods)

class Validation_Status(object):

  def __init__(self, elt, label_map):
    self.uri = elt.text.strip()
    self.timestamp = elt.get("timestamp")
    self.generation = elt.get("generation")
    self.hostname = urlparse.urlparse(self.uri).hostname or "[None]"
    self.fn2 = os.path.splitext(self.uri)[1] or None if self.generation else None
    self.label = label_map[elt.get("status")]

  def sort_key(self):
    return (self.label.sort_key, self.timestamp, self.hostname, self.fn2, self.generation)

  @property
  def code(self):
    return self.label.code

  @property
  def mood(self):
    return self.label.mood

  @property
  def accepted(self):
    return self.label.code == "object_accepted"

  @property
  def rejected(self):
    return self.label.code == "object_rejected"

  @property
  def is_current(self):
    return self.generation == "current"

  @property
  def is_backup(self):
    return self.generation == "backup"
  
  @property
  def is_problem(self):
    return self.label.mood != "good"

  @property
  def is_connection_problem(self):
    return self.label.mood != "good" and self.label.code.startswith("rsync_transfer_")

  @property
  def is_object_problem(self):
    return self.label.mood != "good" and not self.label.code.startswith("rsync_transfer_")

  @property
  def is_connection_detail(self):
    return self.label.code.startswith("rsync_transfer_")

  @property
  def is_object_detail(self):
    return not self.label.code.startswith("rsync_transfer_")

class Problem_Mixin(object):
  
  @property
  def connection_problems(self):
    result = [v for v in self.validation_status if v.is_connection_problem]
    result.sort(key = Validation_Status.sort_key)
    return result

  @property
  def object_problems(self):
    result = [v for v in self.validation_status if v.is_object_problem]
    result.sort(key = Validation_Status.sort_key)
    return result

class Host(Problem_Mixin):

  def __init__(self, hostname, timestamp):
    self.hostname = hostname
    self.timestamp = timestamp
    self.elapsed = 0
    self.connections = 0
    self.failures = 0
    self.uris = set()
    self.graph = None
    self.counters = {}
    self.totals = {}
    self.validation_status = []

  def add_connection(self, elt):
    self.elapsed += parse_utc(elt.get("finished")) - parse_utc(elt.get("started"))
    self.connections += 1
    if elt.get("error") is not None:
      self.failures += 1

  def add_validation_status(self, v):
    self.validation_status.append(v)
    if v.generation == "current":
      self.uris.add(v.uri)
    self.counters[(v.fn2, v.generation, v.label)] = self.get_counter(v.fn2, v.generation, v.label) + 1
    self.totals[v.label] = self.get_total(v.label) + 1
    v.label.count += 1

  def get_counter(self, fn2, generation, label):
    return self.counters.get((fn2, generation, label), 0)

  def get_total(self, label):
    return self.totals.get(label, 0)

  @property
  def failed(self):
    return 1 if self.failures > 0 else 0

  @property
  def objects(self):
    return len(self.uris)

  field_table = (("connections", "GAUGE"),
                 ("objects",     "GAUGE"),
                 ("elapsed",     "GAUGE"),
                 ("failed",      "ABSOLUTE"))

  rras = tuple("RRA:AVERAGE:0.5:%s:9600" % steps
               for steps in (1, 4, 24))

  @classmethod
  def field_ds_specifiers(cls, heartbeat = 24 * 60 * 60, minimum = 0, maximum = "U"):
    return ["DS:%s:%s:%s:%s:%s" % (field[0], field[1], heartbeat, minimum, maximum)
            for field in cls.field_table]

  @property
  def field_values(self):
    return tuple(str(getattr(self, field[0])) for field in self.field_table)

  @classmethod
  def field_defs(cls, filebase):
    return ["DEF:%s=%s.rrd:%s:AVERAGE" % (field[0], filebase, field[0])
            for field in cls.field_table]

  graph_opts = (
    "--vertical-label", "Sync time (seconds)",
    "--right-axis-label", "Objects (count)",
    "--lower-limit", "0",
    "--right-axis", "1:0",
    "--full-size-mode" )

  graph_cmds = (

    # Split elapsed into separate data sets, so we can color
    # differently to indicate how succesful transfer was.  Intent is
    # that exactly one of these be defined for every value in elapsed.

    "CDEF:success=failed,UNKN,elapsed,IF",
    "CDEF:failure=connections,1,EQ,failed,*,elapsed,UNKN,IF",
    "CDEF:partial=connections,1,NE,failed,*,elapsed,UNKN,IF",

    # Show connection timing first, as color-coded semi-transparent
    # areas with opaque borders.  Intent is to make the colors stand
    # out, since they're a major health indicator.  Transparency is
    # handled via an alpha channel (fourth octet of color code).  We
    # draw this stuff first so that later lines can overwrite it.

    "AREA:success#00FF0080:Sync time (success)",
    "AREA:partial#FFA50080:Sync time (partial failure)",
    "AREA:failure#FF000080:Sync time (total failure)",

    "LINE1:success#00FF00",             # Green
    "LINE1:partial#FFA500",             # Orange
    "LINE1:failure#FF0000",             # Red

    # Now show object counts, as a simple black line.

    "LINE1:objects#000000:Objects",     # Black

    # Add averages over period to chart legend.

    "VDEF:avg_elapsed=elapsed,AVERAGE",
    "VDEF:avg_connections=connections,AVERAGE",
    "VDEF:avg_objects=objects,AVERAGE",
    "COMMENT:\j",
    "GPRINT:avg_elapsed:Average sync time (seconds)\: %5.2lf",
    "GPRINT:avg_connections:Average connection count\: %5.2lf",
    "GPRINT:avg_objects:Average object count\: %5.2lf" )

  graph_periods = (("week",  "-1w"),
                   ("month", "-31d"),
                   ("year",  "-1y"))

  def rrd_run(self, cmd):
    try:
      cmd = [str(i) for i in cmd]
      cmd.insert(0, args.rrdtool_binary)
      subprocess.check_call(cmd, stdout = open("/dev/null", "w"))
    except OSError, e:
      sys.exit("Problem running %s, perhaps you need to set --rrdtool-binary? (%s)" % (args.rrdtool_binary, e))
    except subprocess.CalledProcessError, e:
      sys.exit("Failure running %s: %s" % (args.rrdtool_binary, e))

  def rrd_update(self):
    filename = os.path.join(args.output_directory, self.hostname) + ".rrd"
    if not os.path.exists(filename):
      cmd = ["create", filename, "--start", self.timestamp - 1, "--step", "3600"]
      cmd.extend(self.field_ds_specifiers())
      cmd.extend(self.rras)
      self.rrd_run(cmd)
    self.rrd_run(["update", filename,
                  "%s:%s" % (self.timestamp, ":".join(str(v) for v in self.field_values))])

  def rrd_graph(self, html):
    filebase = os.path.join(args.output_directory, self.hostname)
    formats = [format for format in ("png", "svg", "eps")
               if getattr(args, format + "_width") and getattr(args, format + "_height")]
    for period, start in self.graph_periods:
      for format in formats:
        cmds = [ "graph", "%s_%s.%s" % (filebase, period, format),
                 "--title", "%s last %s" % (self.hostname, period),
                 "--start", start,
                 "--width",  getattr(args, format + "_width"),
                 "--height", getattr(args, format + "_height"),
                 "--imgformat", format.upper() ]
        cmds.extend(self.graph_opts)
        cmds.extend(self.field_defs(filebase))
        cmds.extend(self.graph_cmds)
        self.rrd_run(cmds)
      img = Element("img", src = "%s_%s.png" % (self.hostname, period),
                    width  = str(args.png_width),
                    height = str(args.png_height))
      if self.graph is None:
        self.graph = copy.copy(img)
      html.BodyElement("h2").text = "%s over last %s" % (self.hostname, period)
      html.BodyElement("a", href = "%s_%s_svg.html" % (self.hostname, period)).append(img)
      html.BodyElement("br")
      svg_html = HTML("%s over last %s" % (self.hostname, period),
                      "%s_%s_svg" % (self.hostname, period))
      svg_html.BodyElement("img", src = "%s_%s.svg" % (self.hostname, period))
      svg_html.close()

  
class Session(Problem_Mixin):

  def __init__(self):
    self.hosts = {}

    self.root = ElementTree(file = args.input_file).getroot()

    self.rcynic_version = self.root.get("rcynic-version")
    self.rcynic_date = self.root.get("date")
    self.timestamp = parse_utc(self.rcynic_date)

    self.labels = [Label(elt) for elt in self.root.find("labels")]
    self.load_validation_status()

    for elt in self.root.findall("rsync_history"):
      self.get_host(urlparse.urlparse(elt.text.strip()).hostname).add_connection(elt)

    generations = set()
    fn2s = set()

    for v in self.validation_status:
      self.get_host(v.hostname).add_validation_status(v)
      generations.add(v.generation)
      fn2s.add(v.fn2)

    self.labels = [l for l in self.labels if l.count > 0]

    self.hostnames   = sorted(self.hosts)
    self.generations = sorted(generations)
    self.fn2s        = sorted(fn2s)

  def load_validation_status(self):
    label_map = dict((label.code, label) for label in self.labels)
    full_validation_status = [Validation_Status(elt, label_map)
                              for elt in self.root.findall("validation_status")]
    accepted_current = set(v.uri for v in full_validation_status
                           if v.is_current and v.accepted)
    self.validation_status = [v for v in full_validation_status
                              if not v.is_backup
                              or v.uri not in accepted_current]

  def get_host(self, hostname):
    if hostname not in self.hosts:
      self.hosts[hostname] = Host(hostname, self.timestamp)
    return self.hosts[hostname]

  def get_sum(self, fn2, generation, label):
    return sum(h.get_counter(fn2, generation, label)
               for h in self.hosts.itervalues())

  def rrd_update(self):
    if not args.dont_update_rrds:
      for h in self.hosts.itervalues():
        h.rrd_update()

css = '''
    th, td {
      text-align: center; padding: 4px;
    }

    td.uri {
      text-align: left;
    }

    thead tr th, tfoot tr td {
      font-weight: bold;
    }

    .good {
      background-color: #77ff77;
    }

    .warn {
      background-color: yellow;
    }

    .bad {
      background-color: #ff5500;
    }

    body {
      font-family: arial, helvetica, serif;
    }

    /* Make background-color inherit like color does. */
    #nav {
      background-color: inherit;
    }

    #nav, #nav ul {
      float: left;
      width: 100%;
      list-style: none;
      line-height: 1;
      font-weight: normal;
      padding: 0;
      border-color: black;
      border-style: solid;
      border-width: 1px 0;
      margin: 0 0 1em 0;
    }

    #nav a, #nav span {
      display: block;
      background-color: white;
      color: black;
      text-decoration: none;
      padding: 0.25em 0.75em;
    }

    #nav li {
      float: left;
      padding: 0;
    }

    /* Use <ul style="width: ..."> to set submenu width. */
    #nav li ul {
      position: absolute;
      display: none;
      height: auto;
      border-width: 1px;
      margin: 0;
    }

    #nav li li {
      width: 100%;
    }

    /* Display submenu when hovering. */
    #nav li:hover ul {
      display: block;
    }

    /* Reverse video when hovering. */
    #nav a:hover, #nav span:hover {
      color: white;
      background-color: black;
    }
'''

class HTML(object):

  def __init__(self, title, filebase):

    self.filename = os.path.join(args.output_directory, filebase + ".html")

    self.html = Element("html")
    self.html.append(Comment(" Generators:\n" + 
                             "  " + session.rcynic_version + "\n" +
                             "  $Id$\n"))
    self.head = SubElement(self.html, "head")
    self.body = SubElement(self.html, "body")

    title += " " + session.rcynic_date
    SubElement(self.head, "title").text = title
    SubElement(self.body, "h1").text = title
    SubElement(self.head, "style", type = "text/css").text = css

    if args.refresh:
      SubElement(self.head, "meta", { "http-equiv" : "Refresh", "content" : str(args.refresh) })

    hostwidth = max(len(hostname) for hostname in session.hostnames)

    toc = SubElement(self.body, "ul", id = "nav")
    SubElement(SubElement(toc, "li"), "a", href = "index.html").text = "Overview"
    li = SubElement(toc, "li")
    SubElement(li, "span").text = "Repositories"
    ul = SubElement(li, "ul", style = "width: %sem" % hostwidth)
    for hostname in session.hostnames:
      SubElement(SubElement(ul, "li"), "a", href = "%s.html" % hostname).text = hostname
    SubElement(SubElement(toc, "li"), "a", href = "problems.html").text = "Problems"
    li = SubElement(toc, "li")
    SubElement(li, "span").text = "All Details"
    ul = SubElement(li, "ul", style = "width: 15em")
    SubElement(SubElement(ul, "li"), "a", href = "connections.html").text = "All Connections"
    SubElement(SubElement(ul, "li"), "a", href = "objects.html").text = "All Objects"
    SubElement(self.body, "br")

  def close(self):
    ElementTree(element = self.html).write(self.filename)

  def BodyElement(self, tag, **attrib):
    return SubElement(self.body, tag, **attrib)

  def counter_table(self, data_func, total_func):
    table = self.BodyElement("table", rules = "all", border = "1")
    thead = SubElement(table, "thead")
    tfoot = SubElement(table, "tfoot")
    tbody = SubElement(table, "tbody")
    tr = SubElement(thead, "tr")
    SubElement(tr, "th")
    for label in session.labels:
      SubElement(tr, "th").text = label.text
    for fn2 in session.fn2s:
      for generation in session.generations:
        counters = [data_func(fn2, generation, label) for label in session.labels]
        if sum(counters) > 0:
          tr = SubElement(tbody, "tr")
          SubElement(tr, "td").text = ((generation or "") + " " + (fn2 or "")).strip()
          for label, count in zip(session.labels, counters):
            td = SubElement(tr, "td")
            if count > 0:
              td.set("class", label.mood)
              td.text = str(count)
    tr = SubElement(tfoot, "tr")
    SubElement(tr, "td").text = "Total"
    counters = [total_func(label) for label in session.labels]
    for label, count in zip(session.labels, counters):
      td = SubElement(tr, "td")
      if count > 0:
        td.set("class", label.mood)
        td.text = str(count)
    return table

  def object_count_table(self, session):
    table = self.BodyElement("table", rules = "all", border = "1")
    thead = SubElement(table, "thead")
    tbody = SubElement(table, "tbody")
    tfoot = SubElement(table, "tfoot")
    fn2s  = [fn2 for fn2 in session.fn2s if fn2 is not None]
    total = dict((fn2, 0) for fn2 in fn2s)
    for hostname in session.hostnames:
      tr = SubElement(tbody, "tr")
      SubElement(tr, "td").text = hostname
      for fn2 in fn2s:
        td = SubElement(tr, "td")
        count = sum(uri.endswith(fn2) for uri in session.hosts[hostname].uris)
        total[fn2] += count
        if count > 0:
          td.text = str(count)
    trhead = SubElement(thead, "tr")
    trfoot = SubElement(tfoot, "tr")
    SubElement(trhead, "th").text = "Repository"
    SubElement(trfoot, "td").text = "Total"
    for fn2 in fn2s:
      SubElement(trhead, "th").text = fn2
      SubElement(trfoot, "td").text = str(total[fn2])
    return table

  def detail_table(self, records):
    if records:
      table = self.BodyElement("table", rules = "all", border = "1")
      thead = SubElement(table, "thead")
      tbody = SubElement(table, "tbody")
      tr = SubElement(thead, "tr")
      SubElement(tr, "th").text = "Timestamp"
      SubElement(tr, "th").text = "Generation"
      SubElement(tr, "th").text = "Status"
      SubElement(tr, "th").text = "URI"
      for v in records:
        tr = SubElement(tbody, "tr", { "class" : v.mood })
        SubElement(tr, "td").text = v.timestamp
        SubElement(tr, "td").text = v.generation
        SubElement(tr, "td").text = v.label.text
        SubElement(tr, "td", { "class" : "uri"}).text = v.uri
      return table
    else:
      self.BodyElement("p").text = "None found"
      return None

def main():

  global session
  
  os.putenv("TZ", "UTC")
  time.tzset()

  parse_options()

  session = Session()
  session.rrd_update()

  for hostname in session.hostnames:
    html = HTML("Repository details for %s" % hostname, hostname)
    html.counter_table(session.hosts[hostname].get_counter, session.hosts[hostname].get_total)
    if not args.hide_graphs:
      session.hosts[hostname].rrd_graph(html)
    if not args.hide_problems:
      html.BodyElement("h2").text = "Connection Problems"
      html.detail_table(session.hosts[hostname].connection_problems)
      html.BodyElement("h2").text = "Object Problems"
      html.detail_table(session.hosts[hostname].object_problems)
    html.close()

  html = HTML("rcynic summary", "index")
  html.BodyElement("h2").text = "Grand totals for all repositories"
  html.counter_table(session.get_sum, Label.get_count)
  if not args.hide_object_counts:
    html.BodyElement("br")
    html.BodyElement("hr")
    html.BodyElement("br")
    html.BodyElement("h2").text = "Current total object counts (distinct URIs)"
    html.object_count_table(session)
  for hostname in session.hostnames:
    html.BodyElement("br")
    html.BodyElement("hr")
    html.BodyElement("br")
    html.BodyElement("h2").text = "Overview for repository %s" % hostname
    html.counter_table(session.hosts[hostname].get_counter, session.hosts[hostname].get_total)
    if not args.hide_graphs:
      html.BodyElement("br")
      html.BodyElement("a", href = "%s.html" % hostname).append(session.hosts[hostname].graph)
  html.close()

  html = HTML("Problems", "problems")
  html.BodyElement("h2").text = "Connection Problems"
  html.detail_table(session.connection_problems)
  html.BodyElement("h2").text = "Object Problems"
  html.detail_table(session.object_problems)
  html.close()

  html = HTML("All connections", "connections")
  html.detail_table([v for v in session.validation_status if v.is_connection_detail])
  html.close()

  html = HTML("All objects", "objects")
  html.detail_table([v for v in session.validation_status if v.is_object_detail])
  html.close()


if __name__ == "__main__":
  main()