Move bot-status to python and adapt it to buildbot v2
The perl syntax was giving me headaches, so I rewrote it in python3.
It has exactly the same functionalities, even the debug logging (which
is not enabled through command line).
The resulting HMTL should be slight different placement, but with the
same expected informations. The 'Comments' fields might result in
truncated information because some bots returns a long string with all
the resulting information instead of a more simple status.
I also added a small optimization to use HTTP Keep-alive (through the
python requests library) to avoid open multiple connections to fetch
each json file.
Change-Id: I2bc1e400ac8eabdac876bae7cc2f2344d9fdaa1d
diff --git a/monitor/bot-status.py b/monitor/bot-status.py
new file mode 100755
index 0000000..2536fa4
--- /dev/null
+++ b/monitor/bot-status.py
@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+
+# This script greps the JSON files for the buildbots on the LLVM official
+# build master by name and prints an HTML page with the links to the bots
+# and the status.
+#
+# Multiple masters can be used, as well as multiple groups of bots and
+# multiple bots per group, all in a json file. See linaro.json in this
+# repository to have an idea how the config file is.
+
+import sys
+import os
+import argparse
+import json
+import tempfile
+import logging
+from datetime import datetime, timedelta
+# The requests allows HTTP keep-alive which re-uses the same TCP connection
+# to download multiple files.
+import requests
+
+# The GIT revision length used on 'Commits' error display.
+GIT_SHORT_LEN=7
+
+def ignored(s):
+ return 'ignore' in s and s['ignore']
+def not_ignored(s):
+ return not ignored(s)
+
+
+# Returns the parsed json URL or and error string.
+def wget(session, url):
+ try:
+ req = session.get(url)
+ except requests.exceptions.RequestException as e:
+ return str(e), True
+ return req.json(), False
+
+
+# Returns a string with the GIT revision usesd on build BUILDID and
+# PREV_BUILDID in the form '<id_buildid>-<id_prev_buildid>'.
+def get_bot_failure_changes(session, base_url, buildid, prev_buildid):
+ def wget_build_rev(bid):
+ if bid != -1:
+ contents, err = wget(session,
+ "{}/api/v2/builds/{}/changes"
+ .format(base_url, bid))
+ if err or len(contents['changes']) == 0:
+ return ""
+ return contents['changes'][0]['revision']
+ return ""
+
+ revision = wget_build_rev(buildid)
+ prev_revision = wget_build_rev(prev_buildid)
+ if not prev_revision:
+ return "{:.{width}}".format(revision, width=GIT_SHORT_LEN)
+ else:
+ return "{:.{width}}-{:.{width}}".format(revision, prev_revision,
+ width=GIT_SHORT_LEN)
+
+
+# Get the status of a individual bot BOT. Returns a dict with the
+# information.
+def get_bot_status(session, bot, base_url, builder_url, build_url):
+ (contents, err) = wget(session,
+ "{}/api/v2/{}/{}/{}"
+ .format(base_url, builder_url, bot, build_url))
+ if err:
+ return { 'fail' : err }
+
+ builds = contents
+
+ status = {}
+ reversed_builds = reversed(builds['builds'])
+ for build in reversed_builds:
+ if build['complete']:
+ status['builderid'] = build['builderid']
+ status['number'] = build['number']
+ status['state'] = build['state_string']
+ delta = int(build['complete_at']) - int(build['started_at'])
+ status['time'] = str(timedelta(seconds=delta))
+ if build['state_string'] != 'build successful':
+ status['comments'] = build['state_string']
+ status['fail'] = True
+
+ try:
+ prev_buildid = next(reversed_builds)['buildid']
+ except StopIteration:
+ prev_buildid = -1;
+ status['changes'] = get_bot_failure_changes(session, base_url,
+ build['buildid'],
+ prev_buildid)
+ else:
+ status['fail'] = False
+ break
+ return status
+
+
+def bot_status(config_file, output_file):
+ temp = tempfile.NamedTemporaryFile(mode='w+', delete=False)
+
+ today = "{}\n".format(datetime.today().ctime())
+
+ session = requests.Session()
+
+ # Get status for all bots
+ bot_cache = {}
+ for server in filter(not_ignored, config):
+ base_url = server['base_url']
+ builder_url = server['builder_url']
+ build_url = server['build_url']
+ logging.debug('Parsing server {}...'.format(server['name']))
+ for builder in server['builders']:
+ logging.debug(' Parsing builders {}...'.format(builder['name']))
+ for bot in builder['bots']:
+ bot_key = "{}/{}".format(base_url, bot['name'])
+ if bot_key in bot_cache:
+ continue
+ logging.debug(' Parsing bot {}...'.format(bot['name']))
+ status = get_bot_status(session, bot['name'], base_url, builder_url,
+ build_url)
+ if not_ignored(bot):
+ fail = 'fail' in status
+ logging.debug(" FAIL" if status['fail'] else " PASS")
+ bot_cache[bot_key] = status
+
+ # Dump all servers / bots
+ for server in filter(not_ignored, config):
+ base_url = server['base_url']
+ builder_url = server['builder_url']
+ build_url = server['build_url']
+ favicon = 'fail.ico' if fail else 'ok.ico'
+ temp.write("<link rel=\"shortcut icon\" href=\"{}\" "
+ "type=\"image/x-icon\"/>\n".format(favicon))
+ temp.write("<table cellspacing=1 cellpadding=2>\n")
+ temp.write("<tr><td colspan=5> </td><tr>\n")
+ temp.write("<tr><th colspan=5>{} @ {}</td><tr>\n"
+ .format(server['name'], today))
+
+ for builder in server['builders']:
+ temp.write("<tr><td colspan=5> </td><tr>\n")
+ temp.write("<tr><th colspan=5>{}</td><tr>\n".format(builder['name']))
+ temp.write("<tr><th>Buildbot</th><th>Status</th><th>Time</th>"
+ "<th>Build #</th><th>Commits</th><th>Comments</th></tr>\n")
+ for bot in builder['bots']:
+ temp.write("<tr>\n")
+ status = bot_cache["{}/{}".format(base_url, bot['name'])]
+ url = "{}/#/{}/{}".format(base_url, builder_url, status['builderid'])
+ temp.write(" <td><a href='{}'>{}</a></td>\n".format(url, bot['name']))
+ temp.write(" <td><font color='{}'>{}</font></td>\n"
+ .format('red' if status['fail'] else 'green',
+ 'FAIL' if status['fail'] else 'PASS'))
+ empty_cell=" <td> </td>\n"
+ if 'time' in status:
+ temp.write(" <td>{}</td>\n".format(status['time']))
+ else:
+ temp.write(empty_cell)
+ if 'number' in status:
+ build_url = "{}/builds/{}".format(url, status['number'])
+ temp.write(" <td><a href='{}'>{}</a></td>\n".format(build_url, status['number']))
+ else:
+ temp.write(empty_cell)
+ if 'changes' in status:
+ temp.write(" <td>{}</td>\n".format(status['changes']))
+ else:
+ temp.write(empty_cell)
+ if status['fail']:
+ temp.write(" <td>{:.30}</td>\n".format(status['comments']))
+ else:
+ temp.write(empty_cell)
+ temp.write("</tr>\n")
+ temp.write("</table>\n")
+
+ # Move temp to main (atomic change)
+ temp.close()
+ os.rename(temp.name, sys.argv[2])
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-d', dest='debug', action='store_true')
+ parser.add_argument('config_file',
+ help='Bots description in JSON format')
+ parser.add_argument('output_file',
+ help='output HTML path')
+ args = parser.parse_args()
+
+ if args.debug:
+ logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
+
+ try:
+ with open(args.config_file, "r") as f:
+ config = json.load(f)
+ except IOError as e:
+ print("error: failed to read {} config file: {}".format(sys.argv[1], e))
+ sys.exit(os.EX_CONFIG)
+
+ bot_status(config, args.output_file)