monitor/bot-status.py

#!/usr/bin/env python3

# This script greps the JSON files for the buildbots on the LLVM official
# build master by name and prints an HTML page with the links to the bots
# and the status.
#
# Multiple masters can be used, as well as multiple groups of bots and
# multiple bots per group, all in a json file. See linaro.json in this
# repository to have an idea how the config file is.

import sys
import os
import argparse
import json
import tempfile
import logging
import shutil
from datetime import datetime, timedelta
# The requests allows HTTP keep-alive which re-uses the same TCP connection
# to download multiple files.
import requests
from textwrap import dedent

from buildkite_status import get_buildkite_bots_status

# The GIT revision length used on 'Commits' error display.
GIT_SHORT_LEN=7

def ignored(s):
  return 'ignore' in s and s['ignore']
def not_ignored(s):
  return not ignored(s)


# Returns the parsed json URL or raises an exception
def wget(session, url):
  got = session.get(url)
  got.raise_for_status()
  return got.json()


# Returns a string with the GIT revision usesd on build BUILDID and
# PREV_BUILDID in the form '<id_buildid>-<id_prev_buildid>'.
def get_bot_failure_changes(session, base_url, buildid, prev_buildid):
  def wget_build_rev(bid):
    try:
      contents = wget(session,
                           "{}/api/v2/builds/{}/changes"
                           .format(base_url, bid))
    except requests.exceptions.RequestException:
      logging.debug("    Couldn't get changes for build {}!".format(buildid))
      return None
    changes = contents['changes']
    if changes:
      return changes[0]['revision']
    return None

  revision = wget_build_rev(buildid)
  if revision is None:
    return ""
  revision = revision[:GIT_SHORT_LEN]

  prev_revision = None
  if prev_buildid is not None:
    prev_revision = wget_build_rev(prev_buildid)

  if prev_revision is None:
    return "{}".format(revision)
  else:
    return "{}-{}".format(revision, prev_revision[:GIT_SHORT_LEN])


# Map from buildbot status codes we want to treat as errors to the color they
# should be shown in. The codes are documented at
# https://docs.buildbot.net/latest/developer/results.html#build-result-codes,
# and these colors match the suggested ones there.
RESULT_COLORS = {
  2: 'red',    # Error
  4: 'purple', # Exception
  5: 'purple', # Retry
  6: 'pink',   # Cancelled
}

def get_bot_failing_steps(session, base_url, buildid):
  try:
    contents = wget(session, "{}/api/v2/builds/{}/steps"
                             .format(base_url, buildid))
  except requests.exceptions.RequestException:
    return ""

  for step in contents["steps"]:
    if step["results"] in RESULT_COLORS:
      yield (step["name"], step["results"])


# Get the status of a individual bot BOT.  Returns a dict with the
# information.
def get_bot_status(session, bot, base_url, builder_url, build_url):
  try:
    builds = wget(session,
                    "{}/api/v2/{}/{}/{}"
                    .format(base_url, builder_url, bot, build_url))
  except requests.exceptions.RequestException as e:
      logging.debug("    Couldn't get builds for bot {}!".format(bot))
      return {'valid': False}

  reversed_builds = iter(sorted(builds['builds'], key=lambda b: -b["number"]))
  for build in reversed_builds:
    if build['complete']:
      time_since = (int(datetime.now().timestamp()) - int(build['complete_at']))
      duration = int(build['complete_at']) - int(build['started_at'])
      agent_url = "{}/#/{}/{}".format(base_url, builder_url, build['builderid'])

      status = {
        'builder_url': agent_url,
        'number': build['number'],
        'build_url':  "{}/builds/{}".format(agent_url, build['number']),
        'state': build['state_string'],
        'time_since': timedelta(seconds=time_since),
        'duration': timedelta(seconds=duration),
        'fail': build['state_string'] != 'build successful',
      }

      if status['fail']:
        buildid = build['buildid']
        prev_buildid = next(reversed_builds, None)['buildid']
        status['changes'] = get_bot_failure_changes(session, base_url,
                                                    buildid,
                                                    prev_buildid)
        status['steps'] = list(get_bot_failing_steps(session, base_url,
                                                     buildid))

      return status


# Get status for all bots named in the config
# Return a dictionary of (base_url, bot name) -> status info
def get_buildbot_bots_status(config):
  session = requests.Session()
  bot_cache = {}

  for server in filter(not_ignored, config):
    if server['name'] == "Buildkite":
        continue

    base_url = server['base_url']
    logging.debug('Parsing server {}...'.format(server['name']))
    for builder in server['builders']:
      logging.debug('  Parsing builders {}...'.format(builder['name']))
      for bot in builder['bots']:
        bot_key = (base_url, bot['name'])
        if bot_key in bot_cache:
          continue

        logging.debug('    Parsing bot {}...'.format(bot['name']))
        status = get_bot_status(session, bot['name'], base_url, server['builder_url'],
                                server['build_url'])
        if status is not None:
          if status.get("valid", True):
            logging.debug("    Bot status: " + ("FAIL" if status['fail'] else "PASS"))
          bot_cache[bot_key] = status

  return bot_cache

def write_bot_status(config, output_file, bots_status):
  temp = tempfile.NamedTemporaryFile(mode='w+', delete=False)
  today = "{}\n".format(datetime.today().ctime())
  # Whether we use the fail favicon or not
  found_failure = False

  temp.write(dedent("""\
          <style>
            /* Combine the border between cells to prevent 1px gaps
               in the row background colour. */
            table, td, th {
              border-collapse: collapse;
            }
            /* Colour every other row in a table body grey. */
            tbody tr:nth-child(even) td {
              background-color: #ededed;
            }
          </style>"""))

  # Dump all servers / bots
  for server in filter(not_ignored, config):
    base_url = server['base_url']
    builder_url = server['builder_url']
    build_url = server['build_url']

    column_titles = [
            "Buildbot",
            "Status",
            "T Since",
            "Duration",
            "Build #",
            "Commits",
            "Failing steps"
    ]
    num_columns = len(column_titles)
    column_titles_html = "<tr>{}</tr>\n".format(
            "".join(["<th>{}</th>".format(t) for t in column_titles]))

    temp.write("<table border=0 cellspacing=1 cellpadding=2>\n")
    temp.write("<tr><td colspan={}>&nbsp;</td><tr>\n".format(num_columns))
    temp.write("<tr><th colspan={}>{} @ {}</td><tr>\n"
               .format(num_columns, server['name'], today))

    for builder in server['builders']:
      temp.write("<tr><td colspan={}>&nbsp;</td><tr>\n".format(num_columns))
      temp.write("<tr><th colspan={}>{}</th><tr>\n".format(num_columns, builder['name']))
      temp.write(column_titles_html)
      temp.write("<tbody>\n")
      for bot in builder['bots']:
        temp.write("<tr>\n")
        logging.debug("Writing out status for {}".format(bot['name']))
        try:
          status = bots_status[(base_url, bot['name'])]
        except KeyError:
          temp.write("  <td colspan={}>{} is offline!</td>\n</tr>\n"
                      .format(num_columns, bot['name']))
          continue
        else:
            if not status.get('valid', True):
                temp.write("  <td colspan={}>Could not read status for {}!</td>\n</tr>\n"
                            .format(num_columns, bot['name']))
                continue

        found_failure |= status['fail']

        temp.write("  <td><a href='{}'>{}</a></td>\n".format(
            status['builder_url'], bot['name']))
        temp.write("  <td><font color='{}'>{}</font></td>\n"
                   .format('red' if status['fail'] else 'green',
                           'FAIL' if status['fail'] else 'PASS'))
        empty_cell="  <td>&nbsp;</td>\n"
        if 'time_since' in status:
          time_since = status['time_since']
          # No build should be taking more than a day
          if time_since > timedelta(hours=24):
              time_since = "<p style=\"color:red\">{}</p>".format(
                      time_since)
          else:
              time_since = str(time_since)

          temp.write("  <td>{}</td>\n".format(time_since))
        else:
          temp.write(empty_cell)
        if 'duration' in status:
          temp.write("  <td>{}</td>\n".format(status['duration']))
        else:
          temp.write(empty_cell)
        if 'number' in status:
          temp.write("  <td><a href='{}'>{}</a></td>\n".format(
              status['build_url'], status['number']))
        else:
          temp.write(empty_cell)
        if 'changes' in status:
          temp.write("  <td>{}</td>\n".format(status['changes']))
        else:
          temp.write(empty_cell)
        if 'steps' in status and status['steps']:
          def render_step(name, result):
            return "<font color='{}'>{}</font>".format(RESULT_COLORS[result], name)
          step_list = ', '.join(render_step(name, result) for name, result in status['steps'])
          temp.write("  <td style=\"text-align:center\">{}</td>\n".format(step_list))
        else:
          temp.write(empty_cell)
        temp.write("</tr>\n")
      temp.write("</tbody>\n")
    temp.write("</table>\n")

  temp.write("<link rel=\"shortcut icon\" href=\"{}\" "
             "type=\"image/x-icon\"/>\n".format(
             'fail.ico' if found_failure else 'ok.ico'))

  # Move temp to main (atomic change)
  temp.close()
  shutil.move(temp.name, output_file)


if __name__ == "__main__":
  parser = argparse.ArgumentParser()
  parser.add_argument('-d', dest='debug', action='store_true',
                      help='show debug log messages')
  parser.add_argument('config_file',
                      help='Bots description in JSON format')
  parser.add_argument('output_file',
                      help='output HTML path')
  args = parser.parse_args()

  if args.debug:
    logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)

  try:
    with open(args.config_file, "r") as f:
      config = json.load(f)
  except IOError as e:
    print("error: failed to read {} config file: {}".format(args.config_file, e))
    sys.exit(os.EX_CONFIG)

  status = get_buildbot_bots_status(config)
  status.update(get_buildkite_bots_status(config))
  write_bot_status(config, args.output_file, status)