diff options
| -rw-r--r-- | rcynic/rcynic.py | 551 |
1 files changed, 272 insertions, 279 deletions
diff --git a/rcynic/rcynic.py b/rcynic/rcynic.py index 8155129d..c3a86ac2 100644 --- a/rcynic/rcynic.py +++ b/rcynic/rcynic.py @@ -32,6 +32,8 @@ try: except ImportError: from xml.etree.ElementTree import (ElementTree, Element, SubElement, Comment) +session = None + opt = { "refresh" : 1800, "suppress-zero-columns" : True, @@ -39,6 +41,7 @@ opt = { "show-detailed-status" : True, "show-problems" : False, "show-summary" : True, + "show-timestamps" : True, "show-graphs" : False, "suppress-backup-whining" : True, "one-file-per-section" : False, @@ -49,92 +52,83 @@ def usage(msg = 0): f.write("Usage: %s %s [options] [input_file [output_file_or_directory]]\n" % (sys.executable, sys.argv[0])) f.write("Options:\n") for i in sorted(opt): - if not isinstance(opt[i], bool): + if "_" not in i and not isinstance(opt[i], bool): f.write(" --%-30s (%s)\n" % (i + " <value>", opt[i])) for i in sorted(opt): - if isinstance(opt[i], bool): + if "_" not in i and isinstance(opt[i], bool): f.write(" --[no-]%-25s (--%s%s)\n" % (i, "" if opt[i] else "no-", i)) if msg: f.write("\n") sys.exit(msg) -opts = ["help"] -for i in opt: - if isinstance(opt[i], bool): - opts.append(i) - opts.append("no-" + i) - else: - opts.append(i + "=") +def parse_options(): -try: - opts, argv = getopt.getopt(sys.argv[1:], "h?", opts) - for o, a in opts: - if o in ("-?", "-h", "--help"): - usage(0) - negated = o.startswith("--no-") - o = o[5:] if negated else o[2:] - if isinstance(opt[o], bool): - opt[o] = not negated - elif isinstance(opt[o], int): - opt[o] = int(a) + opts = ["help"] + for i in opt: + if isinstance(opt[i], bool): + opts.append(i) + opts.append("no-" + i) else: - opt[o] = a -except Exception, e: - usage("%s: %s" % (e.__class__.__name__, str(e))) - -input_file = argv[0] if len(argv) > 0 else None -output_foo = argv[1] if len(argv) > 1 else None - -if len(argv) > 2: - usage("Unexpected arguments") - -output_directory = output_file = None - -if opt["one-file-per-section"] or opt["show-graphs"]: - output_directory = output_foo - if output_directory is None: - usage("--show-graphs and --one-file-per-section require an output directory") - if not os.path.isdir(output_directory): - os.makedirs(output_directory) -else: - output_file = output_foo - -del output_foo + opts.append(i + "=") + + try: + opts, argv = getopt.getopt(sys.argv[1:], "h?", opts) + for o, a in opts: + if o in ("-?", "-h", "--help"): + usage(0) + negated = o.startswith("--no-") + o = o[5:] if negated else o[2:] + if isinstance(opt[o], bool): + opt[o] = not negated + elif isinstance(opt[o], int): + opt[o] = int(a) + else: + opt[o] = a + except Exception, e: + usage("%s: %s" % (e.__class__.__name__, str(e))) + + opt["input_file"] = argv[0] if len(argv) > 0 and argv[0] != "-" else None + output_foo = argv[1] if len(argv) > 1 and argv[1] != "-" else None + + if len(argv) > 2: + usage("Unexpected arguments") + + opt["output_directory"] = opt["output_file"] = None + + if opt["one-file-per-section"] or opt["show-graphs"]: + opt["output_directory"] = output_foo + else: + opt["output_file"] = output_foo -html = None -body = None + if opt["one-file-per-section"] or opt["show-graphs"]: + if opt["output_directory"] is None: + usage("--show-graphs and --one-file-per-section require an output directory") + if not os.path.isdir(opt["output_directory"]): + os.makedirs(opt["output_directory"]) def parse_utc(s): return int(time.mktime(time.strptime(s, "%Y-%m-%dT%H:%M:%SZ"))) class Label(object): - def __init__(self, elt, pos): + def __init__(self, elt): self.code = elt.tag self.mood = elt.get("kind") self.text = elt.text.strip() - self.pos = pos - self.sum = 0 + self.count = 0 - def __cmp__(self, other): - return cmp(self.pos, other.pos) + def get_count(self): + return self.count class Validation_Status(object): - @classmethod - def set_label_map(cls, labels): - cls.label_map = dict((l.code, l) for l in labels) - - def __init__(self, elt): + def __init__(self, elt, label_map): self.uri = elt.text.strip() self.timestamp = elt.get("timestamp") self.generation = elt.get("generation") self.hostname = urlparse.urlparse(self.uri).hostname or None self.fn2 = os.path.splitext(self.uri)[1] or None if self.generation else None - self.label = self.label_map[elt.get("status")] - - def stand_up_and_be_counted(self): - self.label.sum += 1 + self.label = label_map[elt.get("status")] @property def code(self): @@ -170,6 +164,7 @@ class Host(object): self.uris = set() self.graph = None self.counters = {} + self.totals = {} def add_connection(self, elt): self.elapsed += parse_utc(elt.get("finished")) - parse_utc(elt.get("started")) @@ -180,14 +175,15 @@ class Host(object): def add_validation_status(self, v): if v.generation == "current": self.uris.add(v.uri) - self.counters[(v.fn2, v.generation, v.code)] = self.get_counter(v.fn2, v.generation, v.code) + 1 - self.counters[v.code] = self.get_total(v.code) + 1 + self.counters[(v.fn2, v.generation, v.label)] = self.get_counter(v.fn2, v.generation, v.label) + 1 + self.totals[v.label] = self.get_total(v.label) + 1 + v.label.count += 1 - def get_counter(self, fn2, generation, code): - return self.counters.get((fn2, generation, code), 0) + def get_counter(self, fn2, generation, label): + return self.counters.get((fn2, generation, label), 0) - def get_total(self, code): - return self.counters.get(code, 0) + def get_total(self, label): + return self.totals.get(label, 0) @property def failed(self): @@ -220,42 +216,89 @@ class Host(object): if self.graph is None: self.graph = copy.copy(elt) -class Session(dict): +class Session(object): + + def __init__(self): + self.hosts = {} + + if opt["input_file"] is None: + self.root = ElementTree(file = sys.stdin).getroot() + else: + self.root = ElementTree(file = opt["input_file"]).getroot() + + self.rcynic_version = self.root.get("rcynic-version") + self.rcynic_date = self.root.get("date") + self.timestamp = parse_utc(self.rcynic_date) + + self.labels = [Label(elt) for elt in self.root.find("labels")] + label_map = dict((label.code, label) for label in self.labels) + + self.validation_status = [Validation_Status(elt, label_map) + for elt in self.root.findall("validation_status")] + + if opt["suppress-backup-whining"]: + accepted_current = set(v.uri for v in self.validation_status + if v.is_current and v.accepted) + self.validation_status = [v for v in self.validation_status + if not v.is_backup + or v.uri not in accepted_current] + + for elt in self.root.findall("rsync_history"): + self.add_connection(elt) - def __init__(self, timestamp): - dict.__init__(self) - self.timestamp = timestamp + fn2s = set() + generations = set() + + for v in self.validation_status: + self.maybe_add_host(v.hostname).add_validation_status(v) + fn2s.add(v.fn2) + generations.add(v.generation) + + if opt["suppress-zero-columns"]: + self.labels = [l for l in self.labels if l.count > 0] + + self.unique_hostnames = sorted(self.hosts) + self.unique_fn2s = sorted(fn2s) + self.unique_generations = sorted(generations) def maybe_add_host(self, hostname): - if hostname not in self: - self[hostname] = Host() - return self[hostname] + if hostname not in self.hosts: + self.hosts[hostname] = Host() + return self.hosts[hostname] def add_connection(self, elt): hostname = urlparse.urlparse(elt.text.strip()).hostname self.maybe_add_host(hostname).add_connection(elt) - def add_validation_status(self, v): - self.maybe_add_host(v.hostname).add_validation_status(v) + def get_sum(self, fn2, generation, label): + return sum(h.get_counter(fn2, generation, label) + for h in self.hosts.itervalues()) + + def graph(self): + self.rrd_update() + self.rrd_graph() - def run(self, *cmd): + def rrd_run(self, cmd): try: - return subprocess.check_output([str(i) for i in (opt["rrdtool-binary"],) + cmd]).splitlines() + cmd = [str(i) for i in cmd] + cmd.insert(0, opt["rrdtool-binary"]) + return subprocess.check_output(cmd).splitlines() except OSError, e: - usage("Problem running %s, perhaps you need to set --rrdtool-binary? (%s)" % (opt["rrdtool-binary"], e)) + usage("Problem running %s, perhaps you need to set --rrdtool-binary? (%s)" % ( + opt["rrdtool-binary"], e)) rras = tuple("RRA:AVERAGE:0.5:%s:9600" % steps for steps in (1, 4, 24)) - def save(self): - for hostname, h in self.iteritems(): - filename = os.path.join(output_directory, hostname) + ".rrd" + def rrd_update(self): + for hostname, h in self.hosts.iteritems(): + filename = os.path.join(opt["output_directory"], hostname) + ".rrd" if not os.path.exists(filename): cmd = ["create", filename, "--start", self.timestamp - 1, "--step", "3600"] cmd.extend(Host.field_ds_specifiers()) cmd.extend(self.rras) - self.run(*cmd) - self.run("update", filename, - "%s:%s" % (self.timestamp, ":".join(str(v) for v in h.field_values))) + self.rrd_run(cmd) + self.rrd_run(["update", filename, + "%s:%s" % (self.timestamp, ":".join(str(v) for v in h.field_values))]) graph_opts = ( "--width", "1200", @@ -306,10 +349,10 @@ class Session(dict): ("month", "-31d"), ("year", "-1y")) - def graph(self): - for hostname in self: - start_html("Charts for %s" % hostname) - filebase = os.path.join(output_directory, hostname) + def rrd_graph(self): + for hostname in self.hosts: + html = HTML("Charts for %s" % hostname, "%s_graphs" % hostname) + filebase = os.path.join(opt["output_directory"], hostname) for period, start in self.graph_periods: cmds = [ "graph", "%s_%s.png" % (filebase, period), "--title", hostname, @@ -318,230 +361,180 @@ class Session(dict): cmds.extend(self.graph_opts) cmds.extend(Host.field_defs(filebase)) cmds.extend(self.graph_cmds) - imginfo = [i for i in self.run(*cmds) if i.startswith("@imginfo")] + imginfo = [i for i in self.rrd_run(cmds) if i.startswith("@imginfo")] assert len(imginfo) == 1 filename, width, height = imginfo[0].split()[1:] - SubElement(body, "h2").text = "%s over last %s" % (hostname, period) - img = SubElement(body, "img", src = os.path.basename(filename), width = width, height = height) - self[hostname].save_graph_maybe(img) - SubElement(body, "br") - SubElement(body, "a", href = "index.html").text = "Back" - finish_html("%s_graphs" % hostname) + html.BodyElement("h2").text = "%s over last %s" % (hostname, period) + img = html.BodyElement("img", src = os.path.basename(filename), width = width, height = height) + self.hosts[hostname].save_graph_maybe(img) + html.BodyElement("br") + html.BodyElement("a", href = "index.html").text = "Back" + html.close() # -table_css = { "rules" : "all", "border" : "1"} -uri_css = { "class" : "uri" } +class HTML(object): -def start_html(title): + def __init__(self, title, filebase = "index"): - global html - global body + assert filebase == "index" or opt["output_directory"] is not None + assert opt["output_file"] is None or opt["output_directory"] is None - html = Element("html") - html.append(Comment(" Generators:\n" + - " " + input.getroot().get("rcynic-version") + "\n" + - " $Id$\n")) - head = SubElement(html, "head") - body = SubElement(html, "body") + self.filebase = filebase - title += " " + input.getroot().get("date") - SubElement(head, "title").text = title - SubElement(body, "h1").text = title + self.html = Element("html") + self.html.append(Comment(" Generators:\n" + + " " + session.rcynic_version + "\n" + + " $Id$\n")) + self.head = SubElement(self.html, "head") + self.body = SubElement(self.html, "body") - if opt["refresh"]: - SubElement(head, "meta", { "http-equiv" : "Refresh", "content" : str(opt["refresh"]) }) + title += " " + session.rcynic_date + SubElement(self.head, "title").text = title + SubElement(self.body, "h1").text = title - SubElement(head, "style", type = "text/css").text = ''' - th, td { text-align: center; padding: 4px } - td.uri { text-align: left } - thead tr th, - tfoot tr td { font-weight: bold } -''' + if opt["refresh"]: + SubElement(self.head, "meta", { "http-equiv" : "Refresh", "content" : str(opt["refresh"]) }) - if opt["use-colors"]: - SubElement(head, "style", type = "text/css").text = ''' - .good { background-color: #77ff77 } - .warn { background-color: yellow } - .bad { background-color: #ff5500 } + SubElement(self.head, "style", type = "text/css").text = ''' + table { rules : all; border: 1 } + th, td { text-align: center; padding: 4px } + td.uri { text-align: left } + thead tr th, + tfoot tr td { font-weight: bold } ''' -def finish_html(name = "index"): - global html - global body - assert name == "index" or output_directory is not None - assert output_file is None or output_directory is None - if output_file is not None: - output = output_file - elif output_directory is not None: - output = os.path.join(output_directory, name + ".html") - else: - output = sys.stdout - ElementTree(element = html).write(output) - html = None - body = None - -# Main - -os.putenv("TZ", "UTC") -time.tzset() - -input = ElementTree(file = sys.stdin if input_file is None else input_file) - -session = Session(parse_utc(input.getroot().get("date"))) - -labels = [Label(elt, i) for i, elt in enumerate(input.find("labels"))] - -Validation_Status.set_label_map(labels) -validation_status = [Validation_Status(elt) for elt in input.findall("validation_status")] - -if opt["suppress-backup-whining"]: - accepted_current = set(v.uri for v in validation_status if v.is_current and v.accepted) - validation_status = [v for v in validation_status if not v.is_backup or v.uri not in accepted_current] - del accepted_current - -for elt in input.findall("rsync_history"): - session.add_connection(elt) - -for v in validation_status: - v.stand_up_and_be_counted() - session.add_validation_status(v) - -if opt["suppress-zero-columns"]: - labels = [l for l in labels if l.sum > 0] - -if opt["show-graphs"]: - session.save() - session.graph() - -if not opt["one-file-per-section"]: - start_html("rcynic summary") - -if opt["show-summary"]: + if opt["use-colors"]: + SubElement(self.head, "style", type = "text/css").text = ''' + .good { background-color: #77ff77 } + .warn { background-color: yellow } + .bad { background-color: #ff5500 } +''' - unique_hostnames = sorted(set(v.hostname for v in validation_status)) - unique_fn2s = sorted(set(v.fn2 for v in validation_status)) - unique_generations = sorted(set(v.generation for v in validation_status)) + def close(self): + if opt["output_file"] is not None: + output = opt["output_file"] + elif opt["output_directory"] is not None: + output = os.path.join(opt["output_directory"], self.filebase + ".html") + else: + output = sys.stdout + ElementTree(element = self.html).write(output) - if opt["one-file-per-section"]: - start_html("Grand Totals") - else: - SubElement(body, "br") - SubElement(body, "h2").text = "Grand Totals" - - table = SubElement(body, "table", table_css) - thead = SubElement(table, "thead") - tfoot = SubElement(table, "tfoot") - tbody = SubElement(table, "tbody") - tr = SubElement(thead, "tr") - SubElement(tr, "th") - for l in labels: - SubElement(tr, "th").text = l.text - for fn2 in unique_fn2s: - for generation in unique_generations: - counters = [sum(h.get_counter(fn2, generation, l.code) for h in session.itervalues()) for l in labels] - if sum(counters) > 0: - tr = SubElement(tbody, "tr") - SubElement(tr, "td").text = ((generation or "") + " " + (fn2 or "")).strip() - for l, c in zip(labels, counters): - td = SubElement(tr, "td") - if c > 0: - td.set("class", l.mood) - td.text = str(c) - tr = SubElement(tfoot, "tr") - SubElement(tr, "td").text = "Total" - for l in labels: - SubElement(tr, "td", { "class" : l.mood }).text = str(l.sum) - - if opt["one-file-per-section"]: - finish_html("grand_totals") - else: - SubElement(body, "br") - SubElement(body, "h2").text = "Summaries by Repository Host" + def BodyElement(self, tag, **attrib): + return SubElement(self.body, tag, **attrib) - for hostname in unique_hostnames: - if opt["one-file-per-section"]: - start_html("Summary for %s" % hostname) - else: - SubElement(body, "br") - SubElement(body, "h3").text = hostname - table = SubElement(body, "table", table_css) + def counter_table(self, data_func, total_func): + table = self.BodyElement("table", rules = "all", border = "1") thead = SubElement(table, "thead") tfoot = SubElement(table, "tfoot") tbody = SubElement(table, "tbody") tr = SubElement(thead, "tr") SubElement(tr, "th") - for l in labels: - SubElement(tr, "th").text = l.text - for fn2 in unique_fn2s: - for generation in unique_generations: - counters = [session[hostname].get_counter(fn2, generation, l.code) for l in labels] + for label in session.labels: + SubElement(tr, "th").text = label.text + for fn2 in session.unique_fn2s: + for generation in session.unique_generations: + counters = [data_func(fn2, generation, label) for label in session.labels] if sum(counters) > 0: tr = SubElement(tbody, "tr") SubElement(tr, "td").text = ((generation or "") + " " + (fn2 or "")).strip() - for l, c in zip(labels, counters): + for label, count in zip(session.labels, counters): td = SubElement(tr, "td") - if c > 0: - td.set("class", l.mood) - td.text = str(c) + if count > 0: + td.set("class", label.mood) + td.text = str(count) tr = SubElement(tfoot, "tr") SubElement(tr, "td").text = "Total" - counters = [session[hostname].get_total(l.code) for l in labels] - for l, c in zip(labels, counters): + counters = [total_func(label) for label in session.labels] + for lable, count in zip(session.labels, counters): td = SubElement(tr, "td") - if c > 0: - td.set("class", l.mood) - td.text = str(c) - if opt["show-graphs"]: - SubElement(body, "br") - SubElement(body, "a", href = "%s_graphs.html" % hostname).append(session[hostname].graph) - if opt["one-file-per-section"]: - finish_html("%s_summary" % hostname) - -if opt["show-problems"]: + if count > 0: + td.set("class", label.mood) + td.text = str(count) - if opt["one-file-per-section"]: - start_html("Problems") - else: - SubElement(body, "br") - SubElement(body, "h2").text = "Problems" - table = SubElement(body, "table", table_css) - thead = SubElement(table, "thead") - tbody = SubElement(table, "tbody") - tr = SubElement(thead, "tr") - SubElement(tr, "th").text = "Status" - SubElement(tr, "th").text = "URI" - for v in validation_status: - if v.mood != "good": + def detail_table(self, validation_status): + table = self.BodyElement("table", rules = "all", border = "1") + thead = SubElement(table, "thead") + tbody = SubElement(table, "tbody") + tr = SubElement(thead, "tr") + if opt["show-timestamps"]: + SubElement(tr, "th").text = "Timestamp" + SubElement(tr, "th").text = "Generation" + SubElement(tr, "th").text = "Status" + SubElement(tr, "th").text = "URI" + for v in validation_status: tr = SubElement(tbody, "tr", { "class" : v.mood }) + if opt["show-timestamps"]: + SubElement(tr, "td").text = v.timestamp + SubElement(tr, "td").text = v.generation SubElement(tr, "td").text = v.label.text - SubElement(tr, "td", uri_css).text = v.uri - if opt["one-file-per-section"]: - finish_html("problems") + SubElement(tr, "td", { "class" : "uri"}).text = v.uri -if opt["show-detailed-status"]: - if opt["one-file-per-section"]: - start_html("Validation Status") - else: - SubElement(body, "br") - SubElement(body, "h2").text = "Validation Status" - table = SubElement(body, "table", table_css) - thead = SubElement(table, "thead") - tbody = SubElement(table, "tbody") - tr = SubElement(thead, "tr") - SubElement(tr, "th").text = "Timestamp" - SubElement(tr, "th").text = "Generation" - SubElement(tr, "th").text = "Status" - SubElement(tr, "th").text = "URI" - for v in validation_status: - tr = SubElement(tbody, "tr", { "class" : v.mood }) - SubElement(tr, "td").text = v.timestamp - SubElement(tr, "td").text = v.generation - SubElement(tr, "td").text = v.label.text - SubElement(tr, "td", uri_css).text = v.uri - if opt["one-file-per-section"]: - finish_html("validation_status") - -if not opt["one-file-per-section"]: - finish_html() +def main(): + + global session + + os.putenv("TZ", "UTC") + time.tzset() + + parse_options() + + session = Session() + + if opt["show-graphs"]: + session.graph() + + if not opt["one-file-per-section"]: + html = HTML("rcynic summary") + + if opt["show-summary"]: + if opt["one-file-per-section"]: + html = HTML("Grand Totals", "grand_totals") + else: + html.BodyElement("br") + html.BodyElement("h2").text = "Grand Totals" + html.counter_table(session.get_sum, Label.get_count) + if opt["one-file-per-section"]: + html.close() + else: + html.BodyElement("br") + html.BodyElement("h2").text = "Summaries by Repository Host" + for hostname in session.unique_hostnames: + if opt["one-file-per-section"]: + html = HTML("Summary for %s" % hostname, "%s_summary" % hostname) + else: + html.BodyElement("br") + html.BodyElement("h3").text = hostname + html.counter_table(session.hosts[hostname].get_counter, session.hosts[hostname].get_total) + if opt["show-graphs"]: + html.BodyElement("br") + html.BodyElement("a", href = "%s_graphs.html" % hostname).append(session.hosts[hostname].graph) + if opt["one-file-per-section"]: + html.close() + + if opt["show-problems"]: + if opt["one-file-per-section"]: + html = HTML("Problems", "problems") + else: + html.BodyElement("br") + html.BodyElement("h2").text = "Problems" + html.detail_table((v for v in session.validation_status if v.mood != "good")) + if opt["one-file-per-section"]: + html.close() + + if opt["show-detailed-status"]: + if opt["one-file-per-section"]: + html = HTML("Validation Status", "session.validation_status") + else: + html.BodyElement("br") + html.BodyElement("h2").text = "Validation Status" + html.detail_table(session.validation_status) + if opt["one-file-per-section"]: + html.close() + + if not opt["one-file-per-section"]: + html.close() + +if __name__ == "__main__": + main() |