monitor/bot-status.py

#!/usr/bin/env python3

# This script greps the JSON files for the buildbots on the LLVM official
# build master by name and prints an HTML page with the links to the bots
# and the status.
#
# Multiple masters can be used, as well as multiple groups of bots and
# multiple bots per group, all in a json file. See linaro.json in this
# repository to have an idea how the config file is.

import sys
import os
import argparse
import json
import tempfile
import logging
import shutil
from datetime import datetime, timedelta
# The requests allows HTTP keep-alive which re-uses the same TCP connection
# to download multiple files.
import requests
from textwrap import dedent

from buildkite_status import get_buildkite_bots_status

def ignored(s):
  return 'ignore' in s and s['ignore']
def not_ignored(s):
  return not ignored(s)


# Returns the parsed json URL or raises an exception
def wget(session, url):
  got = session.get(url)
  got.raise_for_status()
  return got.json()


# Map from buildbot status codes we want to treat as errors to the color they
# should be shown in. The codes are documented at
# https://docs.buildbot.net/latest/developer/results.html#build-result-codes,
# and these colors match the suggested ones there.
RESULT_COLORS = {
  2: 'red',    # Error
  4: 'purple', # Exception
  5: 'purple', # Retry
  6: 'pink',   # Cancelled
}

def get_bot_failing_steps(session, base_url, buildid):
  try:
    contents = wget(session, "{}/api/v2/builds/{}/steps"
                             .format(base_url, buildid))
  except requests.exceptions.RequestException:
    return ""

  for step in contents["steps"]:
    if step["results"] in RESULT_COLORS:
      yield (step["name"], step["results"])


# Get the status of a individual bot BOT.  Returns a dict with the
# information.
def get_bot_status(session, bot, base_url, builder_url, build_url):
  try:
    builds = wget(session,
                    "{}/api/v2/{}/{}/{}"
                    .format(base_url, builder_url, bot, build_url))
  except requests.exceptions.RequestException as e:
      logging.debug("    Couldn't get builds for bot {}!".format(bot))
      return {'valid': False}

  reversed_builds = iter(sorted(builds['builds'], key=lambda b: -b["number"]))
  next_build = None
  for build in reversed_builds:
    if not build['complete']:
      next_build = build
      continue

    time_since = (int(datetime.now().timestamp()) - int(build['complete_at']))
    duration = int(build['complete_at']) - int(build['started_at'])
    agent_url = "{}/#/{}/{}".format(base_url, builder_url, build['builderid'])

    status = {
      'builder_url': agent_url,
      'number': build['number'],
      'build_url':  "{}/builds/{}".format(agent_url, build['number']),
      'state': build['state_string'],
      'time_since': timedelta(seconds=time_since),
      'duration': timedelta(seconds=duration),
      'fail': build['state_string'] != 'build successful',
      'next_in_progress': next_build is not None
    }

    if status['fail']:
      buildid = build['buildid']
      status['steps'] = list(get_bot_failing_steps(session, base_url,
                                                   buildid))

    return status


# Get status for all bots named in the config
# Return a dictionary of (base_url, bot name) -> status info
def get_buildbot_bots_status(config):
  session = requests.Session()
  bot_cache = {}

  for server in filter(not_ignored, config):
    if server['name'] == "Buildkite":
        continue

    base_url = server['base_url']
    logging.debug('Parsing server {}...'.format(server['name']))
    for builder in server['builders']:
      logging.debug('  Parsing builders {}...'.format(builder['name']))
      for bot in builder['bots']:
        bot_key = (base_url, bot['name'])
        if bot_key in bot_cache:
          continue

        logging.debug('    Parsing bot {}...'.format(bot['name']))
        status = get_bot_status(session, bot['name'], base_url, server['builder_url'],
                                server['build_url'])
        if status is not None:
          if status.get("valid", True):
            logging.debug("    Bot status: " + ("FAIL" if status['fail'] else "PASS"))
          bot_cache[bot_key] = status

  return bot_cache

def write_bot_status(config, output_file, bots_status):
  temp = tempfile.NamedTemporaryFile(mode='w+', delete=False)

  temp.write(dedent("""\
          <!DOCTYPE html>
          <style>
            /* Combine the border between cells to prevent 1px gaps
               in the row background colour. */
            table, td, th {
              border-collapse: collapse;
            }
            /* Colour every other row in a table body grey. */
            tbody tr:nth-child(even) td {
              background-color: #ededed;
            }
          </style>"""))

  column_titles = [
          "Buildbot",
          "Status",
          "T Since",
          "Duration",
          "Build",
          "Failing steps",
          "Build In Progress",
  ]
  num_columns = len(column_titles)
  column_titles_html = "<tr>{}</tr>\n".format(
          "".join(["<th>{}</th>".format(t) for t in column_titles]))

  # The first table should also say when this was generated.
  # If we were to put this in its own header only table, it would
  # not align with the rest because it has no content.
  first = True

  # Dump all servers / bots
  for server in filter(not_ignored, config):
    base_url = server['base_url']
    builder_url = server['builder_url']
    build_url = server['build_url']

    temp.write("<table border=0 cellspacing=1 cellpadding=2>\n")
    temp.write("<tr><td colspan={}>&nbsp;</td><tr>\n".format(num_columns))
    if first:
      temp.write("<tr><th colspan={}>Generated {}</td><tr>\n"
               .format(num_columns, datetime.today().ctime()))
      temp.write("<tr><td colspan={}>&nbsp;</td><tr>\n".format(num_columns))
      first = False

    temp.write("<tr><th colspan={}>{}</td><tr>\n"
               .format(num_columns, server['name']))

    for builder in server['builders']:
      temp.write("<tr><td colspan={}>&nbsp;</td><tr>\n".format(num_columns))
      temp.write("<tr><th colspan={}>{}</th><tr>\n".format(num_columns, builder['name']))
      temp.write(column_titles_html)
      temp.write("<tbody>\n")
      for bot in builder['bots']:
        temp.write("<tr>\n")
        logging.debug("Writing out status for {}".format(bot['name']))
        try:
          status = bots_status[(base_url, bot['name'])]
        except KeyError:
          temp.write("  <td colspan={}>{} is offline!</td>\n</tr>\n"
                      .format(num_columns, bot['name']))
          continue
        else:
            if not status.get('valid', True):
                temp.write("  <td colspan={}>Could not read status for {}!</td>\n</tr>\n"
                            .format(num_columns, bot['name']))
                continue

        temp.write("  <td><a href='{}'>{}</a></td>\n".format(
            status['builder_url'], bot['name']))
        temp.write("  <td><font color='{}'>{}</font></td>\n"
                   .format('red' if status['fail'] else 'green',
                           'FAIL' if status['fail'] else 'PASS'))
        empty_cell="  <td>&nbsp;</td>\n"
        if 'time_since' in status:
          time_since = status['time_since']
          # No build should be taking more than a day
          if time_since > timedelta(hours=24):
              time_since = "<p style=\"color:red\">{}</p>".format(
                      time_since)
          else:
              time_since = str(time_since)

          temp.write("  <td>{}</td>\n".format(time_since))
        else:
          temp.write(empty_cell)
        if 'duration' in status:
          temp.write("  <td>{}</td>\n".format(status['duration']))
        else:
          temp.write(empty_cell)
        if 'number' in status:
          temp.write("  <td><a href='{}'>{}</a></td>\n".format(
              status['build_url'], status['number']))
        else:
          temp.write(empty_cell)
        if 'steps' in status and status['steps']:
          def render_step(name, result):
            return "<font color='{}'>{}</font>".format(RESULT_COLORS[result], name)
          step_list = ', '.join(render_step(name, result) for name, result in status['steps'])
          temp.write("  <td style=\"text-align:center\">{}</td>\n".format(step_list))
        else:
          temp.write(empty_cell)
        if 'next_in_progress' in status:
          temp.write("  <td>{}</td>\n".format(
                     "Yes" if status['next_in_progress'] else "No"))
        else:
          # No value means we don't know either way.
          temp.write(empty_cell)
        temp.write("</tr>\n")
      temp.write("</tbody>\n")
    temp.write("</table>\n")

  # Move temp to main (atomic change)
  temp.close()
  shutil.move(temp.name, output_file)


if __name__ == "__main__":
  parser = argparse.ArgumentParser()
  parser.add_argument('-d', dest='debug', action='store_true',
                      help='show debug log messages')
  parser.add_argument('config_file',
                      help='Bots description in JSON format')
  parser.add_argument('output_file',
                      help='output HTML path')
  args = parser.parse_args()

  if args.debug:
    logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)

  try:
    with open(args.config_file, "r") as f:
      config = json.load(f)
  except IOError as e:
    print("error: failed to read {} config file: {}".format(args.config_file, e))
    sys.exit(os.EX_CONFIG)

  status = get_buildbot_bots_status(config)
  status.update(get_buildkite_bots_status(config))
  write_bot_status(config, args.output_file, status)