llvmbot monitor: Remove Buildkite support and libcxx bots
libcxx is no longer doing post-commit builds of the main branch.
Pre-commit builds continue but there's no need for us to monitor them,
folks will ping us if there are issues.
Change-Id: I063ec2a22b2c200da5d047bf80998ba4c884df68
diff --git a/monitor/README.txt b/monitor/README.txt
index ad7eda6..fd1e187 100644
--- a/monitor/README.txt
+++ b/monitor/README.txt
@@ -5,11 +5,8 @@
It purely reports the status of the builds. If you want hardware monitoring, look
elsewhere.
-It supports Buildbot (as used by LLVM) and Buildkite (https://buildkite.com/, used by LLVM's
-libcxx project). It does not support LLVM Green Dragon (https://green.lab.llvm.org/green/).
-
-Buildkite does require an API key to be available, see the script for how to do that
-and contact your Builkite org admin to get one.
+It supports Buildbot (as used by LLVM) . It does not support LLVM Green Dragon
+(https://green.lab.llvm.org/green/).
Currently we have one monitor running at http://llvm.validation.linaro.org/.
Bookmark this if you have a need to check bot status at a glance.
@@ -20,9 +17,6 @@
The JSON file describes the bots we want to monitor and which master/build service
they connect to.
-Buildbot JSON Format
---------------------
-
The base structure is a list of masters, which has a few properties and a list
of builder groups, which in turn also have some properties and a list of bots
(which in Buildbot terms are actually called "Builders" but we ended up calling
@@ -62,25 +56,6 @@
* On bots it shows the bot but ignores their status. Meaning that an ignored bot failing
does not make the overall page status failed.
-Buildkite JSON Format
----------------------
-
-The Buildkite format follows the Buildbot format closely with some differences.
-
- * Since Buildkite is a centralised service the "name" is always "Buildkite".
- * The "base_url" is always "https://www.buildkite.com".
- * There is a new key "buildkite_org". This is used to find our particular bots
- via the API.
- * The "builder_url" and "build_url" keys are used to form clickable links,
- and are always "builders" and "builds".
- * Bots have an extra key "buildkite_pipeline" (explained below).
-
-When querying Buildkite we make a request like this:
-"show me the results of the pipeline <buildkite_pipeline> in the organisation <buildkite_org>"
-
-From there the script looks for the bot's name in the last finished build. After that
-all the processing is the same as the Buildbot results.
-
HTML Page
---------
diff --git a/monitor/bot-status.py b/monitor/bot-status.py
index f858ef6..a030d3e 100755
--- a/monitor/bot-status.py
+++ b/monitor/bot-status.py
@@ -25,9 +25,6 @@
from textwrap import dedent
from make_table import Table
-from buildkite_status import get_buildkite_bots_status
-
-
def ignored(s):
return "ignore" in s and s["ignore"]
@@ -132,9 +129,6 @@
bot_cache = {}
for server in filter(not_ignored, config):
- if server["name"] == "Buildkite":
- continue
-
base_url = server["base_url"]
logging.debug("Parsing server {}...".format(server["name"]))
for builder in server["builders"]:
@@ -355,7 +349,6 @@
status = pickle.load(f)
else:
status = get_buildbot_bots_status(config)
- status.update(get_buildkite_bots_status(config))
if args.cachefile:
logging.debug("Writing status to cache file {}".format(args.cachefile))
with open(args.cachefile, "wb") as f:
diff --git a/monitor/buildkite_status.py b/monitor/buildkite_status.py
deleted file mode 100755
index 39706c8..0000000
--- a/monitor/buildkite_status.py
+++ /dev/null
@@ -1,158 +0,0 @@
-#! /usr/bin/env python3
-
-import os
-import logging
-from datetime import timedelta
-from functools import lru_cache
-
-# Base dict order not guaranteed until 3.7+
-from collections import OrderedDict
-from datetime import datetime
-
-# To convert time stamps
-# (use datetime.datetime.fromisoformat in Python3.7+)
-import dateutil.parser
-from pybuildkite.buildkite import Buildkite, BuildState
-
-buildkite = Buildkite()
-
-
-def get_buildkite_token():
- """
- We are assuming that this script is run on a machine
- that can only be accessed by users that would already be
- able to generate such tokens.
-
- We could use an env var but the monitor is run via cron
- which would have to inject the token. So it's going to end
- up on disk anyway.
-
- The format of this file is just a single line which is the
- token.
- """
- tokenfile_path = os.path.join("~", ".buildkite_token")
- err_msg = "{}, Buildkite status will not be fetched."
- token = None
-
- try:
- with open(os.path.expanduser(tokenfile_path)) as f:
- # strip to remove ending newline
- token = f.readline().strip()
- if not token:
- logging.debug(err_msg.format("Buildkite API token file was empty"))
- token = None
- except FileNotFoundError:
- logging.debug(err_msg.format("Couldn't read Buildkite API token file"))
-
- return token
-
-
-@lru_cache(maxsize=None)
-def get_pipeline_builds(organisation, pipeline):
- return buildkite.builds().list_all_for_pipeline(
- organisation,
- pipeline,
- # Finished means passed, failed, blocked or cancelled
- states=[BuildState.FINISHED],
- # There are two kinds of builds. Those triggered from phab reviews
- # and those run perdidically to post-commit build the main and release
- # branches.
- # For the status page we ignore pre-commit and release builds.
- branch="main",
- )
-
-
-def get_last_finished_build_status(organisation, job_name, pipeline):
- """
- Get information about the last finished build of a particular
- job. (where a job is one of the pipeline steps)
- """
- builds = get_pipeline_builds(organisation, pipeline)
-
- for build in builds:
- # Where each "job" is a config e.g. AArch64, MacOS, C++20 etc.
- for job in build["jobs"]:
- # Some jobs won't have a name. I think these are the wait steps
- # between jobs or groups of jobs.
- if job.get("name") != job_name:
- continue
-
- # Some jobs were never started due to failures earlier in the pipeline.
- # This isn't our fault so ignore these.
- started_time = job.get("started_at")
- if started_time is None:
- continue
-
- # We use started because that's when it starts to
- # run on the agent (don't want to include waiting time)
- started = dateutil.parser.isoparse(started_time)
- finished = dateutil.parser.isoparse(job["finished_at"])
- # Buildkite gives us mS resoloution but we don't need that
- duration = timedelta(seconds=int((finished - started).total_seconds()))
- # Time from buildkite ends up as UTC but now has no
- # timezone info by default
- now = datetime.now(finished.tzinfo)
- time_since = timedelta(seconds=int((now - finished).total_seconds()))
-
- state = job["state"]
- failed = state != "passed"
- status = {
- # 'url' is the API url, we want a clickable link
- "builder_url": job["agent"]["web_url"],
- # We use the job's url because it takes you to the
- # specific agent, instead of the build's url
- "build_url": job["web_url"],
- "number": build["number"],
- "state": state,
- "duration": duration,
- "time_since": time_since,
- "fail": failed,
- }
- return status
-
- logging.debug(
- 'Found no finished builds for job "{}" '
- 'on pipeline "{}"'.format(job_name, pipeline)
- )
- return dict(valid=False)
-
-
-def bot_key(job_name):
- return ("https://www.buildkite.com", job_name)
-
-
-# Get status for all Buildkite bots in config
-# Return a dictionary of (base_url, bot name) -> status info
-def get_buildkite_bots_status(config):
- status = dict()
-
- token = get_buildkite_token()
- buildkite.set_access_token(token)
-
- for server in config:
- if server["name"] != "Buildkite":
- continue
-
- if token is None:
- # We cannot query buildkite but we don't want to mark these
- # bots as offline because we don't know that.
- for builder in server["builders"]:
- for job in builder["bots"]:
- status[bot_key(job["name"])] = {"valid": False}
- continue
-
- # If we have a valid token then we can get the real status
- organisation = server.get("buildkite_org")
- if organisation is None:
- raise RuntimeError("Buildkite servers must also have a 'buildkite_org'")
-
- for builder in server["builders"]:
- for job in builder["bots"]:
- job_name = job["name"]
- logging.debug('Getting status for buildkite job "{}"'.format(job_name))
-
- status[bot_key(job_name)] = get_last_finished_build_status(
- organisation, job_name, job["buildkite_pipeline"]
- )
-
- return status
diff --git a/monitor/install.sh b/monitor/install.sh
index 1b8b703..db8e10d 100755
--- a/monitor/install.sh
+++ b/monitor/install.sh
@@ -49,9 +49,6 @@
if ! python3 -c 'import requests' > /dev/null; then
echo 'python3-requests module missing'
fi
-if ! python3 -c 'import pybuildkite' > /dev/null; then
- echo 'python3-pybuildkite module missing'
-fi
############################# Install
diff --git a/monitor/linaro.json b/monitor/linaro.json
index 3f4bf12..637062c 100644
--- a/monitor/linaro.json
+++ b/monitor/linaro.json
@@ -77,25 +77,5 @@
]
}
]
- },
- {
- "name": "Buildkite",
- "base_url": "https://www.buildkite.com",
- "buildkite_org": "llvm-project",
- "builder_url": "builders",
- "build_url": "builds",
- "builders": [
- {
- "name": "Libcxx",
- "bots": [
- { "name": "Armv7", "buildkite_pipeline": "libcxx-ci" },
- { "name": "Armv7 -fno-exceptions", "buildkite_pipeline": "libcxx-ci" },
- { "name": "Armv8", "buildkite_pipeline": "libcxx-ci" },
- { "name": "Armv8 -fno-exceptions", "buildkite_pipeline": "libcxx-ci" },
- { "name": "AArch64", "buildkite_pipeline": "libcxx-ci" },
- { "name": "AArch64 -fno-exceptions", "buildkite_pipeline": "libcxx-ci" }
- ]
- }
- ]
}
]