diff options
author | Renato Golin <rengolin@gmail.com> | 2016-04-26 11:02:23 +0100 |
---|---|---|
committer | Renato Golin <rengolin@gmail.com> | 2016-04-26 11:02:23 +0100 |
commit | 94cc104f044261f74fbff3ff587855df1a05f64d (patch) | |
tree | 7c6664fdf7e7fab54fea926b1fc6196a09b6540f |
Initial Commit, moving from dev-private and removing private stuff
48 files changed, 2980 insertions, 0 deletions
diff --git a/README.txt b/README.txt new file mode 100644 index 0000000..b1cb7de --- /dev/null +++ b/README.txt @@ -0,0 +1,34 @@ +LLVM Development Helper Scripts +=============================== + +These scripts are meant to help you setup and develop around LLVM and its +sub-projects. There are many perks when developing with LLVM, but the scripts +can only cover part of them, while others have to be explained via documents. + +The directory is split in multiple parts: + +Bisect: Scripts to help you bisect a problem in LLVM. They use SVN on both +LLVM and Clang and can spot errors at build, check-all and test-suite time. Due +to the symlink problem, neither git nor svn bisect scripts work well with LLVM. + +Buildbot: Scripts for setting up a buildmaster and buildslaves and +how to manage them in the long run. + +Helpers: Scripts that setup the Git/Svn repositories, check-out, validate +branches, build and test LLVM in its multiple forms. Since the repositories have +an intricate relationship (symlinks inside the llvm tree), dealing with them is +not part of the scripts, and they assume the setup is conciously correct. + +Monitor: Scripts to help you monitor buildbots. + +Release: A simple wrapper around the test-release.sh script that records how +we release for ARM and AArch64. + +Stress: A stress script that just builds LLVM non-stop. This is used to test +experimental hardware for constant prolongued abuse of CPU, memory and disk +resources. + +Test-suite: These scripts prepare and run the LNT test-suite in multiple ways. + +These scripts are not meant to be perfect, nor universal. You may find that only +part of them are useful to you. diff --git a/bisect/README.txt b/bisect/README.txt new file mode 100644 index 0000000..83abfb1 --- /dev/null +++ b/bisect/README.txt @@ -0,0 +1,41 @@ +LLVM Bisect Scripts +=================== + +These scripts help you bisect LLVM/Clang for problems on build, check +and test-suite (on a specific test, not all of it). The only script +you should use is 'bisect.pl', as that will call all others in the +correct way for a given command line options. + +You can rely on the power of run.sh to build and test (either check-all +or the test-suite), but you can also write your own test.sh script and +pass it to bisect.pl. + +Usage: + $ ./bisect.sh [options] good_rev bad_rev + +Options: + -c checkout_script (default: ./checkout.sh) + -b build_script (default: ./run.sh) + -t test_script (default: empty) + +Notes: + +* If the test script is empty, the success status will come from the build + script. So, if you're bisecting a build issue, or your build script is + also testing (ex: ./run.sh -c), leaving the test script empty will work + as expected. + +* There is no support for Libc++, LLd etc. But it shouldn't be + too hard to add it to the list of checkout paths and symlinks on run.sh. + +* It assumes your test-suite checkout is correct and on the same directory, + please see the test-suite scripts to help you set it up. If you have your + own setup, please write an according test script. + +* The script creates a log for each checkout ($rev.log), so that you can tail + it while building. It will also move the log to .good and .bad once it + finishes, given run.sh or your test script's return status. + +* I *strongly* advise you to use ccache, since we *must* clean up the build + directory before *every* build to avoid carrying on problems (one of the + reasons I moved back from a proper bisect). diff --git a/bisect/bisect.pl b/bisect/bisect.pl new file mode 100755 index 0000000..acf6c21 --- /dev/null +++ b/bisect/bisect.pl @@ -0,0 +1,145 @@ +#!/usr/bin/env perl + +# This script drives the bisection of LLVM/Clang/RT regressions +# by receiving checkout, build and test commands. +# +# By default, the lines are: +# checkout: './checkout.sh' +# build: './run.sh' +# test: '' +# +# Which means: success == builds. +# +# If you use a different checkout script, make sure it accepts one +# parameter, the SVN revision. +# +# Since testing is the hard part, you should write your own +# custom test script for each job, but you can reuse the checkout +# and run scripts if you like. +# +# The test script should return 0 when the commit is GOOD and non-zero +# when it is bad. +# +# The build script has a way to quickly test (check-all, test-suite) +# and you can use that as well, as a shortcut for the testing. + +use strict; +use warnings; +use Getopt::Std; +use Scalar::Util qw(looks_like_number); + +################################## Command line parsing and validation +my ($prog) = ($0 =~ /\/([^\/]+)$/); +my $syntax = "Syntax: $prog [-c checkout_script] [-b build_script] [-t test_script] good_rev bad_rev\n"; +die $syntax unless &getopts('c:b:t:'); +our($opt_c, $opt_b, $opt_t); +# Job Statuses +my $Good = 0; +my $CompilerError = 1; +my $TestError = 2; +# Defaults +my ($checkout, $build, $test) = ("./checkout.sh", "./run.sh", ""); +$checkout = $opt_c if defined $opt_c; +$build = $opt_b if defined $opt_b; +$test = $opt_t if defined $opt_t; +# Validate +die "Invalid checkout script '$checkout'\n" unless &validate_script(\$checkout); +die "Invalid run script '$build'\n" unless &validate_script(\$build); +die "Invalid test script '$test'\n" unless not $test or &validate_script(\$test); +# Revs +my $last = scalar @ARGV-1; +die $syntax unless $last == 1; +my $good_rev = $ARGV[$last-1]; +die $syntax unless looks_like_number $good_rev; +my $bad_rev = $ARGV[$last]; +die $syntax unless looks_like_number $bad_rev; +die "Bad rev '$bad_rev' is less than or equal good rev '$good_rev'\n" + unless $bad_rev > $good_rev; +&main(); +exit; + +################################## Main bisect logic + +sub main () { + my ($good, $bad) = ($good_rev, $bad_rev); + + # Header + print "\n==============================================\n". + " Checkout script: $checkout\n". + " Build script: $build\n". + " Test script: $test\n". + " Range: [ $good, $bad ]\n". + "\n==============================================\n"; + + # Iterations + my $step = 1; + my $range = $bad-$good; + my $forced_rev = 0; + while ($bad > $good and $bad != $good+1) { + my $rev = int($good+($bad-$good)/2); + # Use forced rev, from compilation error + if ($forced_rev) { + $rev = $forced_rev; + $forced_rev = 0; + } + print "\n---------------------------\n"; + print "Step $step: $good -> $rev -> $bad\n"; + my $result = &check($rev); + # All good + if ($result == $Good) { + print "Revision '$rev' is good\n"; + rename "$rev.log", "$rev.good"; + $good = $rev; + # Compiler error when test should run + } elsif ($result == $CompilerError and $test) { + print "Revision '$rev' has a compiler error\n". + "Since you're looking for a test error,\n". + "we'll skip a rev\n"; + rename "$rev.log", "$rev.compile"; + $forced_rev = $rev+1; + # Proper error + } else { + print "Revision '$rev' is bad\n"; + rename "$rev.log", "$rev.bad"; + $bad = $rev; + } + $step++; + } + + # Final status + print "\n==============================================\n". + " First bad revision is '$bad'\n". + " Found in $step steps in a range of $range commits". + "\n==============================================\n"; +} + +################################## Checkout, run, test + +sub check ($) { + my ($rev) = @_; + unlink "$rev.log"; + my $log = "2>&1 >> $rev.log"; + print "Checking out rev '$rev'\n"; + die "Error while checking out '$rev'\n" if system("$checkout $rev $log"); + print "Building rev '$rev'\n"; + return $CompilerError if system("$build $log"); + if ($test) { + print "Testing rev '$rev'\n"; + return $TestError if system("$test $log"); + } + return $Good; +} + +################################## Helpers + +sub validate_script($) { + my ($command) = @_; + my ($script) = split /\s+/, $$command; + return 0 if (! -x $script); + if ($script !~ /^[\/\.]/) { + $script = "./$script"; + return 0 if (! -x $script); + $$command = "./$$command"; + } + return 1; +} diff --git a/bisect/checkout.sh b/bisect/checkout.sh new file mode 100755 index 0000000..16fe25c --- /dev/null +++ b/bisect/checkout.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +# This script checks out both LLVM and Clang on the same revision number. +# It can also revert a commit in the past to check for hidden failures. + +validate () { + echo $1 | grep -q "^[0-9]\+$" +} + +checkout () { + name=$1 + dir=$2 + to=$3 + echo $name + if [ -d $dir ]; then + cd $dir + svn revert . + svn up -r $to + cd .. + else + svn co -r $to http://llvm.org/svn/llvm-project/$dir/trunk $dir + fi +} + +revert () { + dir=$1 + minus=$2 + cd $dir + svn merge -c -$minus . + cd .. +} + +if [ "$1" = "" ]; then + echo "Syntax: $0 <revision-to> [<revision-minus>]" + exit 1 +fi +if ! validate $1; then + echo "Invalid SVN revision to $1" + exit 1 +fi +if [ "$2" != "" ] && ! validate $2; then + echo "Invalid SVN revision minus $2" + exit 1 +fi + +checkout "LLVM" llvm $1 +if [ "$2" != "" ]; then + revert llvm $2 +fi + +checkout "Clang" cfe $1 +ln -sf `pwd`/cfe `pwd`/llvm/tools/clang +if [ "$2" != "" ]; then + revert cfe $2 +fi + +checkout "Compiler-RT" compiler-rt $1 +ln -sf `pwd`/compiler-rt `pwd`/llvm/projects/compiler-rt +if [ "$2" != "" ]; then + revert compiler-rt $2 +fi diff --git a/bisect/common.sh b/bisect/common.sh new file mode 100755 index 0000000..82ff7e4 --- /dev/null +++ b/bisect/common.sh @@ -0,0 +1,11 @@ +# Common functions for the bisect execution +# Do not call directly. Feel free to use. + +safe_run() { + CMD="$*" + $CMD + if [[ $? != 0 ]]; then + echo "'$CMD' failed, bailing out" + exit 1 + fi +} diff --git a/bisect/run.sh b/bisect/run.sh new file mode 100755 index 0000000..6219248 --- /dev/null +++ b/bisect/run.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +# This script run one cycle of the bisect, by building (or self-hosting) +# LLVM+Clang, running the compiler tests and the test-suite, if required. +# Please, use the sequential.sh script to drive this one. + +. common.sh + +CORE=`grep "CPU part" /proc/cpuinfo | awk '{print $4}'` +if [[ $CORE = '0xc08' ]]; then + CORE="-mcpu=cortex-a8" +else if [[ $CORE = '0xc09' ]]; then + CORE="-mcpu=cortex-a9" +else if [[ $CORE = '0xc0f' ]]; then + CORE="-mcpu=cortex-a15" +else if [[ $CORE = '0xd03' ]]; then + CORE="-mcpu=cortex-a53" +else if [[ $CORE = '0xd07' ]]; then + CORE="-mcpu=cortex-a57" +else + CORE='' +fi fi fi fi fi + +selfhost=false +check=false +cflags=$CORE +full=false +keepbuild=false +while getopts "sct:hfvk" opt; do + case $opt in + s) + selfhost=true + ;; + c) + check=true + ;; + t) + test=$OPTARG + ;; + h) + cflags="$cflags -mthumb" + ;; + v) + cflags="$cflags -mfpu=vfpv3" + ;; + f) + full=true + ;; + k) + keepbuild=true + ;; + *) + echo "Syntax: $0 [-(s)elfhost | -(c)heck-all | -(t)est <test-suite-name>] | -T(h)umb | -(v)fp3 | -(f)ull-with-RT | -(k)eep build dir" + exit 1 + ;; + esac +done + +flush() { + if ! $keepbuild; then + if [[ -d $1 && $1 != '/' ]]; then + rm -rf $1 + fi + fi + mkdir -p $1 + cd $1 +} + +build() { + CC=gcc + CXX=g++ + if [[ $1 != '' ]]; then + CC=$1/bin/clang + CXX=$1/bin/clang++ + fi + CC=$CC CXX=$CXX \ + cmake -G Ninja ../llvm -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS="$cflags" \ + -DCMAKE_CXX_FLAGS="$cflags" \ + -DLLVM_TARGETS_TO_BUILD="ARM;AArch64" \ + -DLLVM_BUILD_TESTS=True \ + -DLLVM_ENABLE_ASSERTIONS=True + safe_run ninja + if $check; then + safe_run ninja check-all + fi +} + +ROOT=`pwd` + +if $full; then + ln -sf `pwd`/compiler-rt `pwd`/llvm/projects/compiler-rt +else + rm -f `pwd`/llvm/projects/compiler-rt +fi + +# Native build +flush $ROOT/build +build + +if $selfhost; then + # Self-hosting build + flush $ROOT/build2 + build $ROOT/build +fi + +if [[ $test != '' ]]; then + cd $ROOT + safe_run ./test.sh $test +fi + +# SVN info +cd $ROOT +svn info llvm diff --git a/bisect/sequential.sh b/bisect/sequential.sh new file mode 100755 index 0000000..bd1d639 --- /dev/null +++ b/bisect/sequential.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# This script is the main driver of the bisection. Please read the +# README.txt in this directory for more info. + +syntax="Syntax: $0 [-(s)tep N | -(o)pts '-hvsc -t TEST'] <first commit> <last commit>" + +step=1 +while getopts "s:o:" opt; do + case $opt in + s) + step=$OPTARG + ;; + o) + run_opts="$run_opts $OPTARG" + ;; + *) + echo -e $syntax + exit 1 + ;; + esac +done +shift $(($OPTIND-1)) + +if [[ $1 = '' || $2 = '' ]]; then + echo -e $syntax + exit 1 +fi + +from=$1 +to=$(($2+1)) +cur=$from + +while [[ $cur < $to ]]; do + echo "Testing $cur..." + ./checkout.sh $cur 2>&1 > $cur.log + ./run.sh $run_opts 2>&1 >> $cur.log + if [[ $? != 0 ]]; then + echo "BAD" + exit 1 + else + echo "GOOD" + fi + cur=$(($cur+$step)) +done diff --git a/bisect/test.sh b/bisect/test.sh new file mode 100755 index 0000000..5d7eb13 --- /dev/null +++ b/bisect/test.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# This script calls the run.sh in the test-suite (see the test-suite dir) +# and bails if the tests fail, giving enough to run.sh to mark that commit +# as BAD and bail. + +if [[ $1 = '' ]]; then + echo "Syntax: $0 <full-test-name>" + exit 1 +fi +testname=$1 +if [ ! -d test ]; then + echo "Missing test directory" + exit 1 +fi +if [ ! -f test/run.sh ]; then + echo "Missing test script" + exit 1 +fi + +# Check which build was built last +build_dir=build +if [ build2/bin/clang -nt build/bin/clang ]; then + build_dir=build2 +fi + +# Update install dir +rm -f install +ln -s $build_dir install + +# Run the test +cd test +./run.sh -t $testname +result=$? +cd .. + +if [[ $result != 0 ]]; then + exit 1 +fi diff --git a/bisect/unit_test.sh b/bisect/unit_test.sh new file mode 100755 index 0000000..f842e7e --- /dev/null +++ b/bisect/unit_test.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +rev_file=.unit.rev +if [ "$1" = "" ]; then + echo "For bisect.pl testing." + echo + echo "Usage:" + echo "./bisect.pl -c "unit_test -c" \\" + echo " -b "unit_test -b BAD_COMPILATION_REV" \\" + echo " -t "unit_test -t BAD_TEST_REV" \\" + echo " start_rev stop_rev" + echo + echo "So that you can make sure that BAD_COMPILATION_REV" + echo "does't break the bisect looking for BAD_TEST_REV." + rm -f $rev_file + exit 0 +fi + +# Replaces checkout.sh REV +# Dummy print revision to temp file +if [ "$1" = "-c" ]; then + echo $2 > $rev_file + exit 0 +fi + +# From here on, we assume the temp file exists +if [ ! -f $rev_file ]; then + echo "Use unit_checkout.sh REV first" + exit 1 +fi +REV=`cat $rev_file` +echo "Testing revision $REV..." + +# Replaces run.sh +# Dummy check for temp rev, compares to fake broken arg +if [ "$1" = "-b" ]; then + if [ "$2" = "$REV" ]; then + echo "Bad compilation" + exit 1 + else + echo "Good compilation" + exit 0 + fi +fi + +# Replaces test.sh +# Dummy check for temp rev, compares to fake broken arg +if [ "$1" = "-t" ]; then + if [ $REV -ge $2 ]; then + echo "Bad test" + exit 1 + else + echo "Good test" + exit 0 + fi +fi diff --git a/buildbot/README.txt b/buildbot/README.txt new file mode 100644 index 0000000..20ad3c1 --- /dev/null +++ b/buildbot/README.txt @@ -0,0 +1,5 @@ +LLVM Buildbot Slave/Master Scripts +================================== + +These scripts are meant to be installed on a buildslave to help with the +setup, from starting Chromebook chroots to restarting services at boot. diff --git a/buildbot/bot b/buildbot/bot new file mode 100755 index 0000000..d20d677 --- /dev/null +++ b/buildbot/bot @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +if [[ $1 != 'start' && $1 != 'stop' ]]; then + echo "Syntax: $0 <start|stop> [bot-dir]" + exit -1 +fi +ACTION=$1 +BOTDIR=/opt/buildbot +if [[ $2 != '' ]]; then + BOTDIR=$2 +fi + +if [[ $ACTION = 'start' ]]; then + sudo -u buildbot buildslave stop $BOTDIR +fi + +sudo -u buildbot buildslave $ACTION $BOTDIR diff --git a/buildbot/llvmbot b/buildbot/llvmbot new file mode 100755 index 0000000..f7c5f0b --- /dev/null +++ b/buildbot/llvmbot @@ -0,0 +1,57 @@ +#! /bin/sh + +### BEGIN INIT INFO +# Provides: LLVM Buildbot Slave +# Required-Start: $local_fs $remote_fs $buildslave +# Required-Stop: +# X-Start-Before: +# Default-Start: 2 3 4 5 +# Default-Stop: +# Short-Description: Runs tests specified by buildmaster +# Description: Runs tests specified by buildmaster +### END INIT INFO + +. /lib/lsb/init-functions + +N=/etc/init.d/llvmbot +USER=linaro +DIR=/home/linaro/devel/buildslave +PID=twistd.pid + +set -e + +start() { + if [ ! -f $DIR/$PID ]; then + su -l -c "buildslave start $DIR" $USER + else + echo "Buildslave on $DIR already started" + fi +} + +stop() { + if [ -f $DIR/$PID ]; then + su -l -c "buildslave stop $DIR" $USER + else + echo "Buildslave on $DIR not started" + fi +} + +case "$1" in + start) start;; + stop) stop;; + restart) + stop; start;; + status) + if [ -f $DIR/twisted.pid ]; then + echo "Buildslave on $DIR active" + else + echo "Buildslave on $DIR stopped" + fi + ;; + *) + echo "Usage: $N {start|stop|restart|status}" >&2 + exit 1 + ;; +esac + +exit 0 diff --git a/buildbot/start b/buildbot/start new file mode 100755 index 0000000..19346ce --- /dev/null +++ b/buildbot/start @@ -0,0 +1,49 @@ +#!/bin/bash + +function psg() { + ps awwux | grep -v grep | grep $1 +} +function botdo() { + sudo -u buildbot $* +} + +if [[ $UID != 0 ]]; then + echo "Not root? Run 'sudo $0'" + exit 1 +fi + +mounted=`mount | grep external` +if [[ $mounted = '' ]]; then + echo "Mounting /external" + in_fstab=`grep external /etc/fstab` + if [[ $in_fstab != '' ]]; then + mount /external + else + mount -t ext4 /dev/sda1 /external + fi +fi + +running=`/etc/init.d/ssh status | grep 'is running'` +if [[ $running = '' ]]; then + echo "Starting SSH daemon" + iptables -P INPUT ACCEPT + /etc/init.d/ssh start +fi + +running=`psg powerd` +if [[ $running != '' ]]; then + echo "Stopping powerd" + stop powerd +fi + +buildbot_root=/external/buildbot +running=`psg buildslave` +if [[ $running = '' ]]; then + echo "Starting buildbot" + if [ -f $buildbot_root/twistd.pid ]; then + botdo buildslave stop $buildbot_root + fi + botdo buildslave start $buildbot_root +fi + +echo "System initialized correctly" diff --git a/helpers/README.txt b/helpers/README.txt new file mode 100644 index 0000000..1875f7a --- /dev/null +++ b/helpers/README.txt @@ -0,0 +1,61 @@ +LLVM Development Helpers +======================== + +These files do almost all needed actions for developing LLVM/Clang. They +create the initial source tree, build and test LLVM, coordinate branches +across repositories, control Git/Svn upstream/origin repositories, etc. + +All scripts assume you use one of two development modes: + +1. Read-Write Git-Svn Mode: + +In this mode, all repositories will be based on Git-Svn and setup that way, +with the LLVM SVN repo as the source, the official LLVM Git repo as 'upstream', +a Linaro git repository as 'origin' (populated at prepare phase). + +The origin repo is meant to be a back-up and share across Linaro developers, +and since this is public, even external developers. We may use branches on +those repos for Jenkins / Buildbot testing in the future. + +All commits will be done via Git-Svn (dcommit) and git-svn-commit will check +if the repo is up-to-date before doing so. It'll also warn if you have more +than one commit, or if you try to commit from master. The former can be +forced, but not the latter. + +2. Read-Only Git Mode: + +In this mode, the prepare script doesn't setup Git-Svn, and relies only on +LLVM's public Git repo for read-only checkout. You can still have your own +local branches, but they will be de-coupled from the Linaro git tree (unless +you cherry-pick or rebase from them manually). + +This type of repo is meant for dev boards, benchmarks and stress tests. + +Environment Variables +--------------------- + +These scripts rely on three environment variables: + +LLVM_SRC : The path to the LLVM source directory. All other repos will be + checked out relative to it in ($LLVM_SRC/../<project>) and linked + to the right place inside $LLVM_SRC. It is recommended that you + set it to a place where all source dirs will be, and only those, + for example ~/devel/llvm/src/llvm. +LLVM_BLD : The path to the (out-of-tree) directory for the release+asserts + build. It is recommended that you set it two levels down from the + source dirs, so you can have multiple build directories (debug, + no-asserts, lld, etc), example ~/devel/llvm/build/llvm. +LLVM_GITRW : Git-Read-Write mode, which also means using Git-Svn or not. + Values can be 'yes' or 'no'. + +Dependencies +------------ + +In order to fully use these scripts, you're going to need to install a few +dependencies. Since people use different Linux distributions, the scripts +don't even try to install that automatically. The dependencies are: + +Scripts/Build: libjson-perl cmake ninja-build perl python2.7 ccache +LLVM: libxml2-dev zlib1g-dev libtinfo-dev python-sphinx binutils-gold +Test-Suite: python-virtualenv bison groff gawk + diff --git a/helpers/git-delete-remote-branch b/helpers/git-delete-remote-branch new file mode 100755 index 0000000..4d03a12 --- /dev/null +++ b/helpers/git-delete-remote-branch @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# This script removes a remote branch. Not that it's hard to do it, but +# it's a very obtuse syntax from git and not straightforward. It's also +# good to have some safety checks. +# +# Syntax: git-delete-remote-branch <remote> <branch> + +remote=$1 +branch=$2 +if [[ $remote = 'upstream' ]]; then + echo "Remote cannot be upstream" + exit 1 +fi +if [[ $branch = 'master' ]]; then + echo "Branch cannot be master" + exit 1 +fi + +git push $remote :$branch diff --git a/helpers/git-hist b/helpers/git-hist new file mode 100755 index 0000000..8feed74 --- /dev/null +++ b/helpers/git-hist @@ -0,0 +1,155 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use Cwd; + +my (@data, @authors, @repos, @changed) = ((), (), (), ()); +# Syntax: git-hist [author (default: whoami)] [number of weeks (default 10 / all 0)]"; +my $author = `whoami`; +my $weeks = 10; +my $rootdir = getcwd; +my $totalcommits = 0; +if (scalar @ARGV) { + foreach my $arg (@ARGV) { + if ($arg =~ /^\d+$/) { + $weeks = $arg; + } else { + $author = $arg; + } + } +} +chomp($author); + +# Make sure there are repositories to use +if (-d ".git") { + push @repos, "."; +} else { + opendir(my $dh, ".") || die "can't open local dir: $!"; + @repos = grep { /^[^\.]/ && -d "$_/.git" } readdir($dh); + closedir $dh; +} +die "Directory (or subs) not git repo\n" unless (scalar @repos); +&main(); +exit; + +##################################################### +sub main() { + # Each week + my $week = 0; + while ($week <= $weeks) { + # Adjust for non-zero week list + $week++ if ($weeks != 0 and $week == 0); + print "\rParsing week: $week" if $week; + my $commits = 0; + # For each repo + for my $repo (@repos) { + #print "REPO: $repo\n"; + my $log = &get_weeklog($repo, $author, $week); + #print "LOG: [$log]\n"; + my $this = &get_commits($log); + #print "COMMITS: $this\n"; + if ($this and ! &find($repo, \@changed)) { + #print "ADDING REPO\n"; + push @changed, $repo; + } + &get_authors($log); + $commits += $this; + } + push @data, $commits; + last if ($weeks == 0); + $week++; + } + print "\n\n"; + + # Log + &print_report(); +} + +###################### +sub print_report() { + return if ($weeks > 0 && scalar @data != $weeks); + + # Weekly histogram + for my $week (0..$weeks-1) { + printf("%02d week [%02d]: ", $week, $data[$week]); + for (1..$data[$week]) { + print "*"; + } + print "\n"; + } + + # Summary + print "\nTotal commits: "; + printf("%3d", $totalcommits); + print "\n Repositories: "; + printf("%3d ", scalar @changed); + &dump(\@changed); + print "\n Authors: "; + printf("%3d ", scalar @authors); + &dump(\@authors); + print "\n"; +} + +###################### +sub get_weeklog() { + my ($repo, $author, $week) = @_; + # Author + my $cmd = "git shortlog -s -n ". + "--author='$author' "; + # And week (if available) + if ($week) { + my $prev = $week-1; + $cmd .= "--since='$week.week' ". + "--until='$prev.week'"; + } + # Run command + chdir($repo); + my $log = `$cmd`; + chdir($rootdir); + return $log; +} + +###################### +sub get_commits() { + my ($log) = @_; + my $commits = 0; + foreach my $line (split(/\n/, $log)) { + my ($this) = ($line =~ /(\d+)/); + $commits += $this; + } + $totalcommits += $commits; + return $commits; +} + +###################### +sub get_authors() { + my ($log) = @_; + foreach my $line (split(/\n/, $log)) { + my ($author) = ($line =~ /\d+\s+(\w.*)/); + if (!&find($author, \@authors)) { + push @authors, $author; + } + } +} + +###################### +sub find() { + my ($elm, $array) = @_; + foreach my $e (@$array) { + return 1 if ($e eq $elm); + } + return 0; +} + +###################### +sub dump() { + my ($array) = @_; + return if (!scalar @{$array}); + my $out = ""; + foreach my $i (@$array) { + $out .= "$i, "; + } + $out =~ s/, $//; + print $out; +} diff --git a/helpers/git-pull b/helpers/git-pull new file mode 100755 index 0000000..3dbd3e1 --- /dev/null +++ b/helpers/git-pull @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# This script pulls a specific branch from the origin repository, making sure +# not to keep the prefix. If the branch already exists, try to merge. If not, +# check out from origin. +# +# Syntax: git-pull linaro-local/[branch] -f + +. llvm-common + +branch=$1 +if [[ $branch = '' ]]; then + branch=`get_branch` +fi +force=$2 +if [[ $force != '-f' ]]; then + force= +fi + +if [[ $branch = 'master' ]]; then + echo "Can't pull the master branch." + echo "Use git-refresh instead." + exit 1 +fi + +prefix="linaro-local/" +if echo $branch | grep -q linaro-local; then + prefix="" +fi + +safe_run git-refresh + +if git branch -a | not grep -q $branch; then + echo "Branch '$branch' doesn't exist in this repository" + exit 2 +fi + +# If the branch exists already, merge +if git branch | grep -q $branch; then + if [[ $force = '-f' ]]; then + echo " + Force check out $branch" + safe_run git checkout master + safe_run git branch -D $branch + safe_run git checkout -b $branch origin/$prefix$branch + exit 0 + fi + + echo " + Merging the origin branch..." + if not git merge --ff-only origin/$prefix$branch; then + echo "Error merging the branch. Use -f to force." + exit 1 + fi +# If not, just check out the branch +else + safe_run git checkout -b $branch origin/$prefix$branch +fi diff --git a/helpers/git-push b/helpers/git-push new file mode 100755 index 0000000..2869dcf --- /dev/null +++ b/helpers/git-push @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# This script pushes the current branch to the origin repository. +# As an added bonus, it refreshes the local master branch from +# the origin, to make sure your branch is in sync with the master of the +# repo you're pushing to. +# +# Syntax: git-push [branch (default=current)] + +. llvm-common + +branch= +if [[ $1 != '' ]]; then + if [[ `git branch | grep $1` = '' ]]; then + echo "Branch '$1' doesn't exist in this repository" + exit 2 + fi + branch=$1 +else + branch=`get_branch` +fi + +if [[ $branch = 'master' ]]; then + echo "Can't push the master branch." + echo "Use git-refresh instead." + exit 1 +fi + +prefix="linaro-local/" +if echo $branch | grep -q linaro-local; then + prefix="" +fi + +echo " ++ Refresh Master from Upstream" +safe_run git-refresh + +echo " ++ Rebase to new master" +safe_run git checkout $branch +safe_run git rebase master + +echo " ++ Push $branch to Origin" +safe_run git push -u origin +$branch:$prefix$branch diff --git a/helpers/git-rebase-all b/helpers/git-rebase-all new file mode 100755 index 0000000..6b09246 --- /dev/null +++ b/helpers/git-rebase-all @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# This script rebases all branches based on the current master. It does not +# pull upstream master beforehand. If you need a fresh master, use git-refresh. +# If you want to rebase only one branch, use the first argument for that. +# To protect broken branches from being merged (and forcing a manual merge), +# rename it to <name>-disabled and it'll be skipped. +# +# Syntax: git-rebase-all [branch (default=all)] + +. llvm-common + +echo "Rebasing all branches" +branch=`get_branch` + +if [[ `git diff | head -1` != '' ]]; then + echo "You have uncommitted changes in your repo, bailing" + echo "Please, stash your changes and run this script again" + exit 2 +fi + +if [[ $1 != '' ]]; then + if [[ `git branch | grep $1` = '' ]]; then + echo "Branch '$1' doesn't exist in this repository" + exit 3 + fi + branch=$1 +fi + +branches=`get_branches` +for br in $branches; do + if [[ $br != 'master' && `echo $br | grep "^disabled"` == '' ]]; then + safe_run git checkout $br + git rebase master + if [[ $? != 0 ]]; then + echo "Rebase failed, aborting rebase..." + safe_run git rebase --abort + safe_run git checkout master + exit 1 + fi + safe_run git checkout master + fi +done + +final_branch=`get_branch` +if [[ $branch != $final_branch ]]; then + echo "Back to original branch" + safe_run git checkout $branch +fi diff --git a/helpers/git-refresh b/helpers/git-refresh new file mode 100755 index 0000000..d27fea5 --- /dev/null +++ b/helpers/git-refresh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# This script refreshes the local repository with the contents of the +# upstream repository. The main usage of this is to pull the +# master branch, and sync git-svn if necessary. +# +# Syntax: git-refresh [all] + +. llvm-common + +# Update from origin to upstream +safe_run git checkout master +echo " + Fetching origin..." +safe_run git fetch origin +if is_git_svn; then + echo " + Rebasing SVN master..." + safe_run git svn rebase -l +else + echo " + Rebasing master..." + safe_run git pull +fi + +# Fetch all other remotes +if [[ $1 = 'all' ]]; then + for remote in `git remote`; do + if [ "$remote" != "origin" ]; then + echo " + Fetching remote $remote..." + safe_run git fetch $remote + fi + done +fi diff --git a/helpers/git-svn-commit b/helpers/git-svn-commit new file mode 100755 index 0000000..49cb0d3 --- /dev/null +++ b/helpers/git-svn-commit @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# This script commits to SVN via Git-Svn dcommit. It also adds some checks to +# make sure you're not trying to commit from master, and warns about multiple +# commits, as they're not always intentional and could mean you're trying to +# commit from the wrong branch. +# +# The script will refresh your master and branches to make sure you have an +# up-to-date branch before committing, as that would mess up your local svn +# setup or would at least block your commit from going through. + +. llvm-common + +force=no +if [[ $1 = '-f' ]]; then + force=yes +fi + +branch=`get_branch` +if [[ $branch = 'master' ]]; then + echo " --- Can't commit master branch" + exit 1 +fi +revs=`git log --pretty=oneline --abbrev-commit --graph --decorate | grep -n master | head -1 | cut -d ":" -f 1` +revs=$((revs-1)) +if [[ $revs = 0 ]]; then + echo " --- No patches in current branch" + exit 1 +fi +if [[ $revs != 1 && $force != 'yes' ]]; then + echo " --- Too many patches in current branch, use -f to force commit" + exit 1 +fi + +# Prepare +safe_run git-refresh +safe_run git checkout $branch +safe_run git rebase master + +# Actuall SVN commit. Try to recover once. +git svn dcommit +if [[ $? != 0 ]]; then + echo " == SVN COMMIT ERROR ==" + # If we're in the base dir, try it again. + if [ -d .git/svn ]; then + echo "We're in base dir, cleaning SVN info and trying again" + rm -rf .git/svn + safe_run git svn rebase + safe_run git svn dcommit + else + echo "Not in base dir, won't try anything. But if you" + echo "saw the message 'resource out of date', remove" + echo "the .git/svn directory and trying again." + exit 1 + fi +fi + +# Refresh / rebase to avoid future conflicts +safe_run git-refresh +safe_run git-rebase-all diff --git a/helpers/git-which-branch b/helpers/git-which-branch new file mode 100755 index 0000000..6ff2cc3 --- /dev/null +++ b/helpers/git-which-branch @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# Simple wrapper for getting the current branch on a git repo. + +. llvm-common + +if is_git > /dev/null 2>&1; then + get_branch +else + echo "You're not in a git repository" + exit 1 +fi diff --git a/helpers/llvm-arm-dis b/helpers/llvm-arm-dis new file mode 100755 index 0000000..a0267bf --- /dev/null +++ b/helpers/llvm-arm-dis @@ -0,0 +1,42 @@ +#!/usr/bin/env perl + +# This script uses llvm-mc to disassemble byte patterns into possible +# x86/x86_64/arm/aarch64 instructions, trying to find a way to represent +# that in any of the instruction sets. + +use strict; +use warnings; +use Data::Dumper; + +#vpush-vpop.s:@ CHECK-ARM: vpop {s8, s9, s10, s11, s12} @ encoding: [0x05,0x4a,0xbd,0xec] +#vpush-vpop.s:@ CHECK-THUMB: vpush {d8, d9, d10, d11, d12} @ encoding: [0x2d,0xed,0x0a,0x8b] +my $syntax = "Syntax: $0 0xFFFF(FFFF)\n"; +my @hexs = &fix_endian($ARGV[0]); +#die Dumper \@hexs; + +foreach (@hexs) { + my $code = $_->{'code'}; + my $arch = $_->{'arch'}; + print "$arch: $code: "; + print `echo "$code" | llvm-mc -disassemble -triple $arch`; +} + +sub fix_endian() { + my ($hex) = @_; + + my ($a, $b, $c, $d) = ($hex =~ /^0?x?(\w{2})(\w{2})(\w{0,2})(\w{0,2})$/); + print "$a $b $c $d\n"; + die $syntax unless $a and $b; # at least two bytes + die $syntax if $c and not $d; # two or four bytes + # ARM/T2/x86 + if ($c) { + return ( + { 'code' => "0x$d 0x$c 0x$b 0x$a", 'arch' => "armv7" }, + { 'code' => "0x$b 0x$a 0x$d 0x$c", 'arch' => "thumbv7" }, + { 'code' => "0x$d 0x$c 0x$b 0x$a", 'arch' => "i686" }, + { 'code' => "0x$d 0x$c 0x$b 0x$a", 'arch' => "x86_64" } + ); + } else { + return ( { 'code' => "0x$b 0x$a", 'arch' => "thumb" } ); + } +} diff --git a/helpers/llvm-branch b/helpers/llvm-branch new file mode 100755 index 0000000..3ed3cef --- /dev/null +++ b/helpers/llvm-branch @@ -0,0 +1,177 @@ +#!/usr/bin/env bash + +# This script helps you check which branches are checked out on git and make +# sure they're coherent with the feature you're developing. For instance, +# if a feature spans across both LLVM and Clang, you could name the branch on +# each repo the same and use this script to seamlessly move between them. + +# Syntax: llvm-branch [-d] [branch] +# -d: delete chosen branch +# branch: find on llvm and clang and check it out +# Without arguments, the script lists all branches and highlight the current one + +. llvm-common + +function print_branches() { + local current=$1 + shift + local space=0 + for each in $*; do + if [[ $space = 1 ]]; then + echo -n " " + fi + if [[ $each = $current ]]; then + echo -n "[$each]" + else + echo -n $each + fi + space=1 + done +} + +function has() { + if [[ $1 = '' || $2 = '' ]]; then + echo no + return + fi + local item=$1 + shift + for each in $*; do + if [[ $item = $each ]]; then + echo yes + return + fi + done + echo no +} + +function switch() { + SRC=$1 + branch=$2 + in=$3 + if [ -d $SRC ]; then + cd $SRC + pwd + if [[ $in = 'yes' ]]; then + if [[ $DEL = '' ]]; then + safe_run git checkout $branch + else + safe_run git checkout master + safe_run git branch $DEL $branch + fi + else + if [[ $DEL = '' ]]; then + safe_run git checkout master + fi + fi + fi +} + +# Gather info +cd $LLVM_SRC +llvm_branch=`get_branch` +llvm_branches=`get_branches` + +CLANG_SRC=$LLVM_SRC/tools/clang +if [ -d $CLANG_SRC ]; then + cd $CLANG_SRC + clang_branch=`get_branch` + clang_branches=`get_branches` +fi + +RT_SRC=$LLVM_SRC/projects/compiler-rt +if [ -d $RT_SRC ]; then + cd $RT_SRC + rt_branch=`get_branch` + rt_branches=`get_branches` +fi + +CXX_SRC=$LLVM_SRC/projects/libcxx +if [ -d $CXX_SRC ]; then + cd $CXX_SRC + cxx_branch=`get_branch` + cxx_branches=`get_branches` +fi + +CXXABI_SRC=$LLVM_SRC/projects/libcxxabi +if [ -d $CXXABI_SRC ]; then + cd $CXXABI_SRC + cxxabi_branch=`get_branch` + cxxabi_branches=`get_branches` +fi + +UNW_SRC=$LLVM_SRC/projects/libunwind +if [ -d $UNW_SRC ]; then + cd $UNW_SRC + unw_branch=`get_branch` + unw_branches=`get_branches` +fi + +# Delete chosen branch +DEL='' +if [[ $1 = '-d' || $1 = '-D' ]]; then + DEL=$1 + shift +fi + +# No branch chosen, list all +if [[ $1 = '' ]]; then + echo -n "LLVM branches: " + print_branches $llvm_branch $llvm_branches + echo + if [ -d $CLANG_SRC ]; then + echo -n "Clang branches: " + print_branches $clang_branch $clang_branches + echo + fi + if [ -d $RT_SRC ]; then + echo -n "Compiler-RT branches: " + print_branches $rt_branch $rt_branches + echo + fi + if [ -d $CXX_SRC ]; then + echo -n "LibC++ branches: " + print_branches $cxx_branch $cxx_branches + echo + fi + if [ -d $CXXABI_SRC ]; then + echo -n "LibC++ABI branches: " + print_branches $cxxabi_branch $cxxabi_branches + echo + fi + if [ -d $UNW_SRC ]; then + echo -n "LibUnwind branches: " + print_branches $unw_branch $unw_branches + echo + fi + exit +fi + +# Search for branch name +branch=$1 + +if [[ $DEL = 1 && $branch = 'master' ]]; then + echo "Cannot delete the master branch" + exit 1 +fi + +# Check which projects the branch is +in_llvm=`has $branch $llvm_branches` +in_clang=`has $branch $clang_branches` +in_rt=`has $branch $rt_branches` +in_cxx=`has $branch $cxx_branches` +in_cxxabi=`has $branch $cxxabi_branches` +in_unw=`has $branch $unw_branches` +if [[ $in_clang = 'no' && $in_llvm = 'no' && $in_rt = 'no' && \ + $in_cxx = 'no' && $in_cxxabi = 'no' && $in_unw = 'no' ]]; then + echo "Branch $branch doesn't exist on any repository" + exit 1 +fi + +# DO IT +switch $LLVM_SRC $branch $in_llvm +switch $CLANG_SRC $branch $in_clang +switch $RT_SRC $branch $in_rt +switch $CXX_SRC $branch $in_cxx +switch $CXXABI_SRC $branch $in_cxxabi +switch $UNW_SRC $branch $in_unw diff --git a/helpers/llvm-build b/helpers/llvm-build new file mode 100755 index 0000000..8d2fab4 --- /dev/null +++ b/helpers/llvm-build @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +# This script helps you build LLVM. It can update the repositories first, +# run the check-all after and even install it in your system. Since the +# LLVM tree changes constantly, it's recommended that you run this script +# with update+check at least once a week, and most importantly, before you +# begin a big change, to make sure the repos are current and stable. + +. llvm-common + +## CMD line options and defaults +CPUs=`grep -c proc /proc/cpuinfo` +build_dir=$LLVM_BLD +install_dir=$LLVM_BLD/../../install +build_type=Release +shared= +targets= +prog=`basename $0` +syntax="Syntax: $prog [-u(pdate)] [-c(check-all)] [-i(nstall)] [-d(debug)]" +update=false +check=false +master=false +debug=false +inst=false +while getopts "ucimd" opt; do + case $opt in + u) + update=true + ;; + c) + check=true + ;; + i) + inst=true + ;; + m) + master=true + ;; + d) + debug=true + build_dir=$LLVM_BLD/../debug + ;; + *) + echo $syntax + exit 1 + ;; + esac +done + +## Run llvm-sync before +if $update; then + echo " + Updating Repositories" + safe_run llvm-sync -a +fi + +## Make sure all branches are on master +if $master; then + echo " + Checking out master" + safe_run llvm-branch master +fi + +## Choose between make and ninja +make=make +ninja= +if which ninja 2>&1 > /dev/null; then + make=ninja + ninja="-G Ninja" +fi + +## Debug mode, make it lighter +if $debug; then + build_type=Debug + shared=-DBUILD_SHARED_LIBS=True + targets=-DLLVM_TARGETS_TO_BUILD="ARM;X86;AArch64" +fi + +## Make sure sure build dir is there +if [ ! -d $build_dir ]; then + mkdir -p $builddir +fi +cd $build_dir + +## Re-run CMake file files are damaged / not there +if [ ! -f build.ninja ] && [ ! -f Makefile ]; then + echo " + Configuring Build" + safe_run cmake $ninja $LLVM_SRC \ + -DCMAKE_BUILD_TYPE=$build_type \ + -DLLVM_BUILD_TESTS=True \ + -DLLVM_ENABLE_ASSERTIONS=True \ + -DPYTHON_EXECUTABLE=/usr/bin/python2 \ + -DCMAKE_INSTALL_PREFIX=$install_dir \ + $targets $shared +fi + +## Build +echo " + Building LLVM" +safe_run $make -j$CPUs + +## Check +if $check; then + echo " + Running Tests" + safe_run $make check-all +fi + +## Install +if $inst; then + echo " + Installing on the System" + safe_run $make install +fi diff --git a/helpers/llvm-common b/helpers/llvm-common new file mode 100755 index 0000000..5cf3f4a --- /dev/null +++ b/helpers/llvm-common @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# The common script is only meant to be included from other LLVM helper scripts + +if [[ $LLVM_SRC = '' ]]; then + echo "Please, define \$LLVM_SRC" + exit -1 +fi +if [[ $LLVM_BLD = '' ]]; then + echo "Please, define \$LLVM_BLD" + exit -1 +fi +if [[ $LLVM_GITRW = '' ]]; then + echo "Please, define \$LLVM_GITRW" + exit -1 +fi +if [[ $LLVM_GITRW = 'yes' ]]; then + if [[ $LLVM_SVNUSER = '' ]] || [[ $LLVM_GITUSER = '' ]]; then + echo "Please, define \$LLVM_GITUSER and \$LLVM_SVNUSER when using GITRW=yes" + echo "GITUSER is your Linaro git user, SVNUSER is your LLVM commit user" + exit 1 + fi +fi + +get_branch() { + branch=`git rev-parse --abbrev-ref HEAD` + if [[ $? != 0 ]]; then + local dir=`pwd` + echo "Source dir '$dir' is not in a git repository" 1>&2 + exit -1 + fi + echo $branch +} + +get_branches() { + branches=`git for-each-ref --sort=-committerdate refs/heads/ --format='%(refname:short)'` + if [[ $? != 0 ]]; then + local dir=`pwd` + echo "Source dir '$dir' is not in a git repository" 1>&2 + exit -1 + fi + echo $branches +} + +safe_run() { + CMD="$*" + $CMD + if [[ $? != 0 ]]; then + echo "'$CMD' failed, bailing out" + exit 1 + fi +} + +is_git() { + test -d `git rev-parse --show-toplevel`/.git +} + +is_git_svn() { + test -f `git rev-parse --show-toplevel`/.git/svn/.metadata +} diff --git a/helpers/llvm-doc b/helpers/llvm-doc new file mode 100755 index 0000000..d8d507f --- /dev/null +++ b/helpers/llvm-doc @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Helper script for re-compiling the documentation after changes. + +if [ ! -f index.rst ]; then + echo "You must be at the LLVM Docs directory to use this script" + exit -1 +fi + +SYNTAX="`basename $0` <doc.rst>" +if [[ $1 = '' ]]; then + echo $SYNTAX + exit -1 +fi +DOC=$1 + +sphinx-build -b html -d _build/doctrees . _build/html $DOC diff --git a/helpers/llvm-irtonew b/helpers/llvm-irtonew new file mode 100755 index 0000000..4e798a4 --- /dev/null +++ b/helpers/llvm-irtonew @@ -0,0 +1,22 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +# This script moves IR from the old syntax to the new one. +# The three biggest changes is to: +# 1. Move "load type object" to "load type, type* object" +# 2. Move "GEP type object" to "GEP type, type* object" +# 3. Clean up metadata + +if (!$ARGV[0] or !-f $ARGV[0]) { + die "Syntax: llvm-irtonew file.ll\n"; +} +open(FH, $ARGV[0]) || die "Can't open file '$ARGV[0]': $!\n"; +foreach my $line (<FH>) { + $line =~ s/ load (.*)\* %/ load $1, $1* %/; + $line =~ s/ getelementptr (.*)\* %/ getelementptr $1, $1* %/; + $line =~ s/,? ?!.*//; + print $line; +} +close FH; diff --git a/helpers/llvm-lit b/helpers/llvm-lit new file mode 100755 index 0000000..32143fb --- /dev/null +++ b/helpers/llvm-lit @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# Helper script for running a single LIT test inside the build tree. + +if [[ $1 = '' ]]; then + echo "Syntax: $0 test-dir" + exit -1 +fi +DIR=$1 + +$LLVM_SRC/utils/lit/lit.py \ + --param build_mode=. \ + -sv \ + --param llvm_site_config=$LLVM_BLD/test/lit.site.cfg \ + --param llvm_unit_site_config=$LLVM_BLD/test/Unit/lit.site.cfg \ + $DIR diff --git a/helpers/llvm-prepare b/helpers/llvm-prepare new file mode 100755 index 0000000..204bfa4 --- /dev/null +++ b/helpers/llvm-prepare @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +# This script creates the source tree as the other LLVM scripts would like to +# find. It's recommended that you use this script before all, and keep the +# format correct, so all the other scripts work accordingly. This script +# honours the LLVM_GITRW requirement and creates the appropriate type. + +. llvm-common + +function prepare() { + src=$1 + link=$2 + origin=$3 + svn=$4 + dir=`basename $src` + # Clone Linaro repo + if [ ! -d $src/.git ]; then + mkdir -p $src + cd $src/.. + safe_run git clone $origin $dir + fi + # Link with upstream SVN + if [[ $LLVM_GITRW = 'yes' && ! -d $src/.git/svn ]]; then + cd $src + safe_run git svn init $svn --username=$LLVM_SVNUSER + safe_run git config svn-remote.svn.fetch :refs/remotes/origin/master + safe_run git svn fetch + fi + # Create the symlink into LLVM's source + if [[ $link != "no" && ! -x $link ]]; then + cd $src + ln -sf $src $link + fi +} + +echo " + Setting up LLVM" +prepare $LLVM_SRC $LLVM_SRC \ + ssh://$LLVM_GITUSER@review.linaro.org:29418/toolchain/llvm/llvm \ + https://llvm.org/svn/llvm-project/llvm/trunk +echo " + Setting up Clang" +prepare $LLVM_SRC/../clang $LLVM_SRC/tools/clang \ + ssh://$LLVM_GITUSER@review.linaro.org:29418/toolchain/llvm/clang \ + https://llvm.org/svn/llvm-project/cfe/trunk +echo " + Setting up Clang Tools Extra" +prepare $LLVM_SRC/../clang-tools-extra $LLVM_SRC/../clang/tools/extra \ + ssh://$LLVM_GITUSER@review.linaro.org:29418/toolchain/llvm/clang-tools-extra \ + https://llvm.org/svn/llvm-project/clang-tools-extra/trunk +echo " + Setting up Compiler-RT" +prepare $LLVM_SRC/../compiler-rt $LLVM_SRC/projects/compiler-rt \ + ssh://$LLVM_GITUSER@review.linaro.org:29418/toolchain/llvm/compiler-rt \ + https://llvm.org/svn/llvm-project/compiler-rt/trunk +echo " + Setting up LLD Linker" +prepare $LLVM_SRC/../lld $LLVM_SRC/tools/lld \ + ssh://$LLVM_GITUSER@review.linaro.org:29418/toolchain/llvm/lld \ + https://llvm.org/svn/llvm-project/lld/trunk +echo " + Setting up LibC++" +prepare $LLVM_SRC/../libcxx no \ + ssh://$LLVM_GITUSER@review.linaro.org:29418/toolchain/llvm/libcxx \ + https://llvm.org/svn/llvm-project/libcxx/trunk +echo " + Setting up LibC++ABI" +prepare $LLVM_SRC/../libcxxabi no \ + ssh://$LLVM_GITUSER@review.linaro.org:29418/toolchain/llvm/libcxxabi \ + https://llvm.org/svn/llvm-project/libcxxabi/trunk +echo " + Setting up LibUnwind" +prepare $LLVM_SRC/../libunwind $LLVM_SRC/projects/libunwind \ + ssh://$LLVM_GITUSER@review.linaro.org:29418/toolchain/llvm/libunwind \ + https://llvm.org/svn/llvm-project/libunwind/trunk +echo " + Setting up LLDB Debugger" +prepare $LLVM_SRC/../lldb $LLVM_SRC/tools/lldb \ + ssh://$LLVM_GITUSER@review.linaro.org:29418/toolchain/llvm/lldb \ + https://llvm.org/svn/llvm-project/lldb/trunk +echo " + Setting up Test Suite" +prepare $LLVM_SRC/../test-suite $LLVM_SRC/projects/test-suite \ + ssh://$LLVM_GITUSER@review.linaro.org:29418/toolchain/llvm/test-suite \ + https://llvm.org/svn/llvm-project/test-suite/trunk +echo " + Setting up LNT Test harness" +prepare $LLVM_SRC/../lnt $LLVM_SRC/projects/lnt \ + ssh://$LLVM_GITUSER@review.linaro.org:29418/toolchain/llvm/lnt \ + https://llvm.org/svn/llvm-project/lnt/trunk +echo " + Setting up Zorg Buildbot Config" +prepare $LLVM_SRC/../zorg $LLVM_SRC/projects/zorg \ + ssh://$LLVM_GITUSER@review.linaro.org:29418/toolchain/llvm/zorg \ + https://llvm.org/svn/llvm-project/zorg/trunk + +mkdir -p $LLVM_BLD + +echo " + Done" diff --git a/helpers/llvm-projs b/helpers/llvm-projs new file mode 100755 index 0000000..e0ff2da --- /dev/null +++ b/helpers/llvm-projs @@ -0,0 +1,150 @@ +#!/usr/bin/env bash + +# This script keeps track of all projects that are linked to the llvm src +# directory. It can detect, enable, disable and map to specific projects, +# so different builds get to see the source dir as they saw originally. + +. llvm-common + +prog=`basename $0` +syntax() { + echo "Syntax: $prog {+/-}[clang|rt|libs|tools|test]" + echo " noarg: list linked projects" + echo " {+/-}: link / unlik projects (default: link)" +} + +# Dirs and links into LLVM +clang_dir=clang +clang_link=tools/clang +rt_dir=compiler-rt +rt_link=projects/compiler-rt +libcxx_dir=libcxx +libcxx_link=projects/libcxx +libcxxabi_dir=libcxxabi +libcxxabi_link=projects/libcxxabi +libunwind_dir=libunwind +libunwind_link=projects/libunwind +lld_dir=lld +lld_link=tools/lld +lldb_dir=lldb +lldb_link=tools/lldb +tests_dir=test-suite +tests_link=projects/test-suite + +# Check if link exists +has() { + link=$1 + [ -s "$LLVM_SRC/$link" ] +} + +# Initialise status +init() { + link=$1 + if has $link; then + echo "true"; + else + echo "false" + fi +} + +# Link/Unlink upon need +update() { + dir=$1 + link=$2 + need=$3 + if $need; then + ln -sf $LLVM_SRC/../$dir $LLVM_SRC/$link + else + rm -f $LLVM_SRC/$link + fi +} + +# Lists linked projects and quit +list_all() { + echo "Projects linked:" + has $clang_link && echo " + Clang" + has $rt_link && echo " + Compiler-RT" + has $libcxx_link && echo " + LibC++" + has $libcxxabi_link && echo " + LibC++abi" + has $libunwind_link && echo " + LibUnwind" + has $lld_link && echo " + LLD" + has $lldb_link && echo " + LLDB" + has $tests_link && echo " + Test-Suite" + echo +} + +# No args, list +if [ "$1" = "" ]; then + echo "Use $prog -h for options" + echo + list_all + exit +fi + +# Need/not need +clang=`init $clang_link` +rt=`init $rt_link` +libcxx=`init $libcxx_link` +libcxxabi=`init $libcxxabi_link` +libunwind=`init $libunwind_link` +lld=`init $lld_link` +lldb=`init $lldb_link` +tests=`init $tests_link` + +# Check all needed projects +while ! test -z $1; do + opt=$1 + sign=${opt:0:1} + flag=true + if [ "$sign" = "-" ]; then + flag=false + opt=${opt:1} + fi + if [ "$sign" = "+" ]; then + opt=${opt:1} + fi + + case $opt in + clang) + clang=$flag + ;; + rt) + rt=$flag + ;; + libs) + libcxx=$flag + libcxxabi=$flag + libunwind=$flag + ;; + tools) + lld=$flag + lldb=$flag + ;; + test) + tests=$flag + ;; + list) + list_all + exit + ;; + -h) + syntax + exit + ;; + *) + syntax + exit 1 + esac + shift +done + +# Update links +update $tests_dir $tests_link $tests +update $lldb_dir $lldb_link $lldb +update $lld_dir $lld_link $lld +update $libunwind_dir $libunwind_link $libunwind +update $libcxxabi_dir $libcxxabi_link $libcxxabi +update $libcxx_dir $libcxx_link $libcxx +update $rt_dir $rt_link $rt +update $clang_dir $clang_link $clang +list_all diff --git a/helpers/llvm-reset b/helpers/llvm-reset new file mode 100755 index 0000000..f219cd8 --- /dev/null +++ b/helpers/llvm-reset @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Helper script for fixing Git-Svn problems. + +. llvm-common + +reset_git_svn() { + name=$1 + dir=$2 + if [[ -d $dir && -d $dir/.git/svn ]]; then + echo " + Resetting $name" + cd $dir + rm -rf .git/svn + safe_run git svn rebase -l + fi +} + +reset_git_svn LLVM $LLVM_SRC +reset_git_svn Clang $LLVM_SRC/../clang +reset_git_svn Compiler-RT $LLVM_SRC/../compiler-rt +reset_git_svn Libc++ $LLVM_SRC/../libcxx +reset_git_svn Libc++abi $LLVM_SRC/../libcxxabi +reset_git_svn LibUnwind $LLVM_SRC/../libunwind +reset_git_svn Linker $LLVM_SRC/../lld +reset_git_svn Debugger $LLVM_SRC/../lldb +reset_git_svn LNT $LLVM_SRC/../lnt +reset_git_svn Zorg $LLVM_SRC/../zorg +reset_git_svn Test-Suite $LLVM_SRC/../test-suite diff --git a/helpers/llvm-sync b/helpers/llvm-sync new file mode 100755 index 0000000..bd46404 --- /dev/null +++ b/helpers/llvm-sync @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +# This script checks out all repositories from upstream and pushes master +# to the origin repo on Linaro's Git server, rebasing all local branches, +# but *not* pushing them, too. + +. llvm-common + +function repo_sync () { + if [ ! -d $2 ]; then return; fi + echo " + Updating $1" + cd $2 + branch=`get_branch` + safe_run git-refresh + safe_run git-rebase-all $branch +} + +prog=`basename $0` +syntax="Syntax: $prog [-compiler(r)T] [-(l)ibs] [-Lin(k)er] [-(d)ebugger] [-(t)ests] [-(w)eb pages] [-(a)ll]" +rt=false +libs=false +linker=false +debug=false +tests=false +web=false + +while getopts "rlkdtwa" opt; do + case $opt in + r) + rt=true + ;; + l) + libs=true + ;; + k) + linker=true + ;; + d) + debug=true + ;; + t) + tests=true + ;; + w) + web=true + ;; + a) + rt=true + libs=true + linker=true + debug=true + tests=true + ;; + *) + echo $syntax + exit 1 + ;; + esac +done + +# Compulsory updates +repo_sync LLVM $LLVM_SRC +repo_sync Clang $LLVM_SRC/../clang + +# Optional updates +if $rt; then + repo_sync RT $LLVM_SRC/../compiler-rt +fi + +if $libs; then + repo_sync LibC++ $LLVM_SRC/../libcxx + repo_sync LibC++ABI $LLVM_SRC/../libcxxabi + repo_sync LibUnwind $LLVM_SRC/../libunwind +fi + +if $linker; then + repo_sync Linker $LLVM_SRC/../lld +fi + +if $debug; then + repo_sync Debugger $LLVM_SRC/../lldb +fi + +if $tests; then + repo_sync Test-Suite $LLVM_SRC/../test-suite + repo_sync LNT $LLVM_SRC/../lnt + repo_sync Zorg $LLVM_SRC/../zorg +fi + +# These are pure SVN repos, not enabled with -a +# You have to manually force with -w +if $web; then + if [ -d $LLVM_SRC/../www ]; then + echo " + Updating WWW" + cd $LLVM_SRC/../www + svn up + fi + if [ -d $LLVM_SRC/../pubs ]; then + echo " + Updating Pubs" + cd $LLVM_SRC/../pubs + svn up + fi +fi diff --git a/monitor/README.txt b/monitor/README.txt new file mode 100644 index 0000000..c5cc6e8 --- /dev/null +++ b/monitor/README.txt @@ -0,0 +1,11 @@ +Monitoring Tools for LLVM Development +===================================== + +These tools are not meant to be used for development or testing, but to be +left running on a server or desktop as monitoring for your buildbots. They +are also meant to be used in conjunction, not as a replacement, to Nagios +and other hardware-level monitoring tools. + +Currently we only have one: bot-monitor, which I keep running on Linaro's +public server (people.linaro.org) and keep it as a bookmark to quickly check +the bot status. It's also a helpful bookmark for all bots we care. diff --git a/monitor/bot-status b/monitor/bot-status new file mode 100755 index 0000000..f2de0ea --- /dev/null +++ b/monitor/bot-status @@ -0,0 +1,244 @@ +#!/usr/bin/env perl + +# This script greps the JSON files for the buildbots on +# the LLVM official build master by name and prints an +# HTML page with the links to the bots and the status. +# +# Multiple masters can be used, as well as multiple groups of bots +# and multiple bots per group, all in a json file. See linaro.json +# in this repository to have an idea how the config file is. +# +# Module JSON needs to be installed, either from cpan or packages. + +push @INC, `dirname $0`; + +use strict; +use warnings; +# Core modules +use File::Temp qw/tempfile/; +use File::Copy; +# This is not part of core, but you really *need* it. +use JSON; +# This can be replaced by `wget/curl` +use LWP; +use LWP::UserAgent; +# We don't have DateTime everywhere... +my $date = `date`; +# DEBUG +my $DEBUG = 0; + +######################################################### Initialisation +# Option checking +my $syntax = "$0 config-file.json output-file.html\n"; +die $syntax unless (scalar @ARGV == 2); +# Read config file +my ($config, $error) = &read_file($ARGV[0]); +die $error if ($error); +($config, $error) = &decode($config); +die $error if ($error); + +# Setup HTML output file +my $output = $ARGV[1]; +my ($temp, $tempname) = tempfile(); + + +######################################################### Main Logic +# Get status for all bots +my %bot_cache; +my $fail = 0; +foreach my $server (@$config) { + next if (defined $server->{'ignore'} and $server->{'ignore'} eq "true"); + my ($BASE_URL, $BUILDER_URL, $BUILD_URL) = + ($server->{'base_url'}, $server->{'builder_url'}, $server->{'build_url'}); + &debug("Parsing server ".$server->{'name'}."...\n"); + foreach my $builder (@{$server->{'builders'}}) { + &debug(" Parsing builder ".$builder->{'name'}."...\n"); + foreach my $bot (@{$builder->{'bots'}}) { + &debug(" Parsing bot ".$bot->{'name'}."...\n"); + next if defined $bot_cache{$bot->{'name'}}; + my $status = &get_status($bot->{'name'}, $BASE_URL, $BUILDER_URL, $BUILD_URL); + if (!defined $bot->{'ignore'} or $bot->{'ignore'} ne "true") { + $fail = 1 if ($status->{'fail'}); + } else { + &debug(" Ignoring...\n"); + } + &debug($status->{'fail'} ? " FAIL\n" : " PASS\n"); + $bot_cache{$bot->{'name'}} = $status; + } + } +} + +# Dump all servers / bots +foreach my $server (@$config) { + next if (defined $server->{'ignore'} and $server->{'ignore'} eq "true"); + my ($BASE_URL, $BUILDER_URL, $BUILD_URL) = + ($server->{'base_url'}, $server->{'builder_url'}, $server->{'build_url'}); + # Favicon + my $favicon = $fail ? "fail.ico" : "ok.ico"; + print $temp "<link rel=\"shortcut icon\" href=\"$favicon\" type=\"image/x-icon\"/>\n"; + # Header + print $temp "<table cellspacing=1 cellpadding=2>\n"; + print $temp "<tr><td colspan=5> </td><tr>\n"; + print $temp "<tr><th colspan=5>$server->{'name'} @ $date</td><tr>\n"; + ## Main loop + foreach my $builder (@{$server->{'builders'}}) { + print $temp "<tr><td colspan=5> </td><tr>\n"; + print $temp "<tr><th colspan=5>$builder->{'name'}</td><tr>\n"; + print $temp "<tr><th>Buildbot</th><th>Status</th><th>Comments</th>". + "<th>Build #</th><th>Commits</th></tr>\n"; + foreach my $bot (@{$builder->{'bots'}}) { + print $temp "<tr>\n"; + my $status = $bot_cache{$bot->{'name'}}; + my $url = "$BASE_URL/$BUILDER_URL/$bot->{'name'}"; + print $temp " <td><a href='$url'>$bot->{'name'}</a></td>\n"; + if ($status->{'fail'}) { + print $temp " <td><font color='red'>FAIL</font></td>\n". + " <td>$status->{'fail'}</td>\n"; + } else { + print $temp " <td><font color='green'>PASS</font></td>\n". + " <td> </td>\n"; + } + if (defined $status->{'build'} and + defined $status->{'from'} and + defined $status->{'to'}) { + print $temp " <td>$status->{'build'}</td>\n". + " <td>$status->{'from'}-$status->{'to'}</td>\n"; + } else { + print $temp " <td colspan=2> </td>\n"; + } + print $temp "</tr>\n"; + } + } + # Footer + print $temp "</table>\n"; +} +close $temp; + +# Move temp to main (atomic change) +move($tempname, $output); +exit; + +######################################################### Subs + +# GET STATUS: get the status of an individual bot +# (botname, base url, builder url, build url) -> (status) +sub get_status() { + my ($bot, $BASE_URL, $BUILDER_URL, $BUILD_URL) = @_; + my ($err, $contents, $json); + my %status; + + # Get buildbot main JSON + ($contents, $err) = wget("$BASE_URL/json/$BUILDER_URL/$bot"); + $status{'fail'} = $err; + return \%status if $err; + ($json, $err) = decode($contents); + $status{'fail'} = $err; + return \%status if $err; + + # Find recent builds + my $cached_builds = scalar @{$json->{'cachedBuilds'}}; + my $running_builds = scalar @{$json->{'currentBuilds'}}; + my $last_build = $json->{'cachedBuilds'}[$cached_builds - $running_builds - 1]; + return \%status if (not defined $last_build); + + # Get most recent build + ($contents, $err) = wget("$BASE_URL/json/$BUILDER_URL/$bot/$BUILD_URL/$last_build"); + $status{'fail'} = $err; + return \%status if $err; + ($json, $err) = decode($contents); + $status{'fail'} = $err; + return \%status if $err; + + # Build number + $status{'build'} = $json->{'number'}; + + # Status of the last build + # "text" : [ "build", "successful" ], + # "text" : [ "failed", "svn-llvm" ], + my $failed = 0; + foreach (@{$json->{'text'}}) { + $status{'fail'} .= $_." " if ($failed); + $failed = 1 if (/failed|exception/); + } + $status{'fail'} =~ s/ $//; + + # Commit range + # [ + # 'revision', + # '238202', + # 'Build' + # ], + my $max = 0; + foreach (@{$json->{'properties'}}) { + if ($_->[0] eq 'revision') { + $max = $_->[1]; + last; + } + } + # Min is in the changes' section + my $min = $max; + foreach (@{$json->{'sourceStamp'}->{'changes'}}) { + my $commit = $_->{'revision'}; + $min = $commit if ($commit < $min); + } + $status{'from'} = $min; + $status{'to'} = $max; + + return \%status; +} + +# WGET: uses LWP to get an URL, returns contents (or error). +# (url) -> (contents, error) +sub wget() { + my ($url) = @_; + my ($contents, $error) = ("", ""); + + my $ua = LWP::UserAgent->new; + $ua->agent("LLVM BotMonitor/0.1"); + my $req = HTTP::Request->new(GET => $url); + my $res = $ua->request($req); + + if ($res->is_success) { + $contents = $res->content; + } else { + $error = $res->status_line; + } + return ($contents, $error); +} + +# READ FILE: Reads a local file, returns contents +# (filename) -> (contents) +sub read_file() { + my ($file) = @_; + my ($contents, $error) = ("", ""); + if (open FH, $file) { + while (<FH>) { $contents .= $_; } + close FH; + } else { + $error = "Can't open config file $file: $!"; + } + return ($contents, $error); +} + +# DECODE: Reads contents, returns JSON output (or error) +# (contents) -> (JSON, error) +sub decode() { + my ($contents) = @_; + my ($json, $error) = ("", ""); + eval { $json = decode_json($contents); }; + if ($@) { + if ($DEBUG) { + $error = $@; + } else { + $error = "JSON error"; + } + } + return ($json, $error); +} + +# DEBUG: Prints debug messages if debug enabled +# (msg) -> () +sub debug () { + my ($msg) = @_; + print STDERR $msg if ($DEBUG); +} diff --git a/monitor/fail.ico b/monitor/fail.ico Binary files differnew file mode 100644 index 0000000..263676d --- /dev/null +++ b/monitor/fail.ico diff --git a/monitor/linaro.json b/monitor/linaro.json new file mode 100644 index 0000000..3cdd7ae --- /dev/null +++ b/monitor/linaro.json @@ -0,0 +1,86 @@ +[ + { + "name": "LLVM Lab", + "base_url": "http://lab.llvm.org:8011", + "builder_url": "builders", + "build_url": "builds", + "builders": [ + { + "name": "Fast Bots", + "bots": [ + { "name": "clang-cmake-armv7-a15" }, + { "name": "clang-cmake-thumbv7-a15" }, + { "name": "clang-cmake-aarch64-quick" }, + { "name": "clang-cmake-aarch64-42vma" } + ] + }, + { + "name": "Full Bots (LLVM, Clang, RT)", + "bots": [ + { "name": "clang-cmake-armv7-a15-full" }, + { "name": "clang-cmake-thumbv7-a15-full-sh" }, + { "name": "clang-cmake-aarch64-42vma" }, + { "name": "clang-cmake-aarch64-full" } + ] + }, + { + "name": "Self Hosting Bots", + "bots": [ + { "name": "clang-cmake-armv7-a15-selfhost" }, + { "name": "clang-cmake-armv7-a15-selfhost-neon" }, + { "name": "clang-cmake-thumbv7-a15-full-sh" }, + { "name": "clang-cmake-aarch64-full" } + ] + }, + { + "name": "Test-Suite Bots", + "bots": [ + { "name": "clang-cmake-aarch64-quick" }, + { "name": "clang-cmake-aarch64-full" }, + { "name": "clang-native-arm-lnt" } + ] + }, + { + "name": "Library Bots", + "bots": [ + { "name": "libcxx-libcxxabi-arm-linux" } + ] + }, + { + "name": "Cross-Compilation Bots", + "bots": [ + { "name": "clang-x86_64-darwin13-cross-arm" } + ] + } + ] + }, + { + "name": "LLVM Silent Master", + "base_url": "http://lab.llvm.org:8014", + "builder_url": "builders", + "build_url": "builds", + "builders": [ + { + "name": "Benchmarking Bots", + "bots": [ + { "name": "clang-native-arm-lnt-perf", "ignore": "true" } + ] + } + ] + }, + { + "name": "Linaro Lab", + "base_url": "http://buildmaster.tcwglab.linaro.org", + "builder_url": "builders", + "build_url": "builds", + "builders": [ + { + "name": "Prototypes", + "bots": [ + { "name": "clang-cmake-armv7-prototype", "ignore": "true" }, + { "name": "clang-cmake-aarch64-prototype", "ignore": "true" } + ] + } + ] + } +] diff --git a/monitor/ok.ico b/monitor/ok.ico Binary files differnew file mode 100644 index 0000000..1acfa53 --- /dev/null +++ b/monitor/ok.ico diff --git a/release/README.txt b/release/README.txt new file mode 100644 index 0000000..1bf265e --- /dev/null +++ b/release/README.txt @@ -0,0 +1,19 @@ +LLVM Release Wrapper Script +=========================== + +The LLVM release is done using the test-release.sh script available on LLVM's +sources, directory utils/release. But the options may change, and to keep track +of how we release LLVM for ARM and AArch64, I wrote a simple script that +remembers what were the options. + +This allows one to quickly setup a machine to produce a release, which can be +used for testing, validation, or replacement of a broken release machine. But +also keeps track of how we released LLVM throughout the years, beginning with +release 3.7.0. + +Usage is simple. With test-release.sh on the same directory, just call: + +$ nohup ./run 3.N.M RC & + +With N as major version, M as minor and RC as release candidate, which is +normally 1, 2, 3, final. diff --git a/release/run.sh b/release/run.sh new file mode 100755 index 0000000..5672fa1 --- /dev/null +++ b/release/run.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +syntax="Syntax: $0 REL RC" + +if [[ $1 = '' ]]; then + echo $syntax + exit 1 +fi +REL=$1 +shift +if [[ $1 = 'final' ]]; then + RC="-final" +else if [[ $1 != '' ]]; then + RC="-rc $1" +else + echo "Syntax: $0 REL RC" + exit 1 +fi fi + +aarch64="`egrep '(AArch64|asimd)' /proc/cpuinfo`" +armv7="`grep ARMv7 /proc/cpuinfo`" +intel="`grep Intel /proc/cpuinfo`" + +if [[ $aarch64 = '' && $armv7 = '' && $intel = '' ]]; then + echo "This is neither of ARMv7, AArch64, Intel" + exit 1 +fi +triple=armv7a-linux-gnueabihf +if [[ $aarch64 != '' ]]; then + triple=aarch64-linux-gnu +fi +if [[ $intel != '' ]]; then + triple=x86_64-linux-gnu +fi + +./test-release.sh -release $REL $RC -triple $triple -j8 -no-libs -no-test-suite diff --git a/stress/README.txt b/stress/README.txt new file mode 100644 index 0000000..40afeb2 --- /dev/null +++ b/stress/README.txt @@ -0,0 +1,13 @@ +Stress Tests for Board Stability +================================ + +This scripts are less helpful to develop LLVM, and more to make sure a +particular platform is good enough to build LLVM on. When choosing machines +to make buildbots or Jenkins slaves, it's important not just to get fast +boxes, but stable and reliable. + +I've ran these scripts on a lot of different machines, local and remote and, +providing you get all the necessary package requirements, they should be very +independent of anything else. The kind of script you'd send to a hardware +manufacturer so they can test their hardware before releasing (or at least +before giving them to the toolchain team to use as builders). diff --git a/stress/build-llvm-forever.sh b/stress/build-llvm-forever.sh new file mode 100755 index 0000000..465a51f --- /dev/null +++ b/stress/build-llvm-forever.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# This script prepares and builds LLVM+Clang in a cycle +# There are two purposes: +# 1. Make sure the board can cope with sequential builds +# for as long as you wish (at least 2 days non-stop) +# 2. Make sure the builds take reasonably the same time +# +# That's why the script has the following features: +# +# * "set +e" to not fail even if the build fails, as a +# subsequent commit can fix that, or even if the build +# fails, we're still stressing the hardware, not the build +# * `date` before each step, so that you can compare +# multiple runs without having to find the `time` output +# * Use -jCPUS, since we're not trying to get it faster, but +# reliable, and that tells us more than -jCPUS+2, etc. +# * Time builds and tests separately, since tests tend to be +# more stable +# * Don't use CCACHE, since that will defeat the purpose + +set +e + +ROOT=`pwd` +CPUS=`grep -c proc /proc/cpuinfo` +LINK_JOBS= +NINJA= +if ninja --version > /dev/null; then + LINK=`free -g | awk '/Mem/ {print $2}'` + LINK_JOBS="-DLLVM_PARALLEL_LINK_JOBS=$LINK" + NINJA="-G Ninja" +fi + +mkdir -p build +if [ ! -d src ]; then + git clone http://llvm.org/git/llvm.git src +fi +if [ ! -d src/tools/clang ]; then + cd src/tools + git clone http://llvm.org/git/clang.git +fi +cd $ROOT/build +if [ ! -f Makefile ]; then + cmake ../src $NINJA \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_BUILD_TESTS=True \ + -DLLVM_ENABLE_ASSERTIONS=True \ + $LINK_JOBS +fi + +while /bin/true; do + echo -n "Updating sources at " + date + cd $ROOT/src + git fetch origin; git pull + cd tools/clang + git fetch origin; git pull + cd $ROOT/build + echo -n "Cleaning at " + date + make -j$CPUS clean + echo -n "Building at " + date + make -j$CPUS + echo -n "Testing at " + date + make -j$CPUS check-all +done diff --git a/test-suite/README.txt b/test-suite/README.txt new file mode 100644 index 0000000..c41d543 --- /dev/null +++ b/test-suite/README.txt @@ -0,0 +1,13 @@ +LLVM Nightly Tests - Test-Suite Scripts +======================================= + +These scripts help running the test-suite, either the whole suite with the +correct options or a single test. The latter is really useful to run as part +of the bisect, or as a debug build to investigate problems, all of which +can be automatically ran using these scripts. + +The entry point is run.sh, and the rest are utulities that are either used +by it or in conjunction with it. The setup script should get your tree +configured correctly in order to run the tests. This is the tree that you +need to setup on the 'test' directory for bisecting problems (see 'bisect' +directory one level down). diff --git a/test-suite/compare_lnt_benchmarks.pl b/test-suite/compare_lnt_benchmarks.pl new file mode 100755 index 0000000..f0d5139 --- /dev/null +++ b/test-suite/compare_lnt_benchmarks.pl @@ -0,0 +1,304 @@ +#!/usr/bin/env perl +# This program handles the results of the LLVM test-suite benchmark. +# There are two stages: +# prepare: collates multiple samples into a single log file containing +# the arithmetic mean of each tests' results. +# compare: compares two collated results and produce a file containing +# the relative values between baseline and target runs, with +# the geomean of compile and execution time at the end. +# +# Suggested usage: +# +# ... point install directory to the unmodified / previous compiler +# $ ./run.sh -b +# $ ./compare_lnt_benchmarks.pl prepare sanbox/build > baseline.txt +# ... change install directory to the modified / current compiler +# $ ./run.sh -b +# $ ./compare_lnt_benchmarks.pl prepare sanbox/build > target.txt +# $ ./compare_lnt_benchmarks.pl compare baseline.txt target.txt > results.txt + +use strict; +use warnings; +use Scalar::Util qw(looks_like_number); +use Statistics::Descriptive; +use Data::Dumper; + +my $syntax = "Syntax: $0 <action> [action options]\n". + "Actions:\n". + "\tprepare: reads all report files in sample-N, collate into one run\n". + "\t options: <sanbox/build> with report.txt or sample-N\n". + "\tcompare: compares two runs, taking geomean of all benchmarks\n". + "\t options: <baseline.report.txt> <target.report.txt>\n"; +my $action = $ARGV[0]; +die $syntax unless defined $action; + +################################################### + +if ($action eq "prepare") { + die $syntax unless defined $ARGV[1]; + &prepare($ARGV[1]); +} elsif ($action eq "compare") { + die $syntax unless defined $ARGV[1] and defined $ARGV[2]; + &compare($ARGV[1], $ARGV[2]); +} else { + die $syntax; +} +exit; + +################################################### + +# prepare a number of logs into a collated results by average +sub prepare($) { + my ($basedir) = @_; + my $logname = "report.simple.txt"; + + # Single report, just copy + if (-f "$basedir/$logname") { + my %results; + my $header = &read_file("$basedir/$logname", \%results); + &dump(\%results, $header); + + # Multiple reports, collate + } elsif (-f "$basedir/sample-0/$logname") { + my @results; + my $header; + + # List all sample files + opendir DIR, $basedir || die "Can't open '$basedir': $!\n"; + my @samples = grep { /sample-\d+/ && -f "$basedir/$_/$logname" } readdir DIR; + closedir DIR; + die "Basedir '$basedir' has no txt logs\n" unless scalar @samples; + + # For each sample, read&push + foreach my $s (@samples) { + my %sample; + $header = &read_file("$basedir/$s/$logname", \%sample); + push @results, \%sample; + } + + # Collate results + my %result; + &collate_results(\@results, \%result); + &dump(\%result, $header); + + } else { + die "Basedir '$basedir' has no txt logs\n" + } +} + +# compare two logs, producing the relative results and geomean +sub compare($$) { + my ($baseline_file, $target_file) = @_; + die "Baseline report file '$baseline_file' doesn't exist or is not a file\n" + unless -f $baseline_file; + die "Target report file '$target_file' doesn't exist or is not a file\n" + unless -f $target_file; + my (%baseline, %target, %result) = ((), (), ()); + + my $header = &read_file($baseline_file, \%baseline); + &read_file($target_file, \%target); + + &compare_results(\%baseline, \%target, \%result); + + &dump(\%result, '', 1); +} + +################################################### + +# reads a file, saving all data indexed by prog name +# returns the header for further use +sub read_file($$) { + my ($filename, $result) = @_; + my $header = ''; + my $max = 1; + open FH, $filename || die "Can't open $filename: $!\n"; + while (<FH>) { + chomp(); + my ($program, $sep, + $cc_pass, $cc_time, $cc_real, + $ex_pass, $ex_time, $ex_real) = split /\s+/; + # Make sure we have the right file + if (!$header) { + die "Invalid header in $filename\n" + unless $program eq "Program" + and $cc_time eq "CC_Time" + and $ex_time eq "Exec_Time"; + $header = $_; + next; + } + # Log each non-header line + $result->{$program} = { + 'cc_pass' => $cc_pass, + 'cc_time' => $cc_time, + 'cc_real' => $cc_real, + 'ex_pass' => $ex_pass, + 'ex_time' => $ex_time, + 'ex_real' => $ex_real, + }; + + # Max length prog name + my $len = length($program); + $max = $len if $len > $max; + } + close FH; + + $result->{'max'} = $max; + + return $header; +} + +# returns "pass" AND "pass" +sub pass_diff($$) { + my ($baseline, $target) = @_; + if (defined $baseline and defined $target and + $baseline eq "pass" and $target eq "pass") { + return "pass"; + } elsif (!$baseline and $target) { + return $target; + } elsif ($baseline and !$target) { + return $baseline; + } else { + return "fail"; + } +} + +# collate multiple reports into one, by getting the +# average of each result into each field +sub collate_results($$) { + my ($results, $result) = @_; + + foreach my $sample (@$results) { + foreach my $prog (keys %$sample) { + next unless ref $sample->{$prog} eq "HASH"; + + # Set up statistics + if (!defined $result->{$prog}) { + $result->{$prog}->{'cc_time'} = new Statistics::Descriptive::Full(); + $result->{$prog}->{'cc_real'} = new Statistics::Descriptive::Full(); + $result->{$prog}->{'ex_time'} = new Statistics::Descriptive::Full(); + $result->{$prog}->{'ex_real'} = new Statistics::Descriptive::Full(); + } + # gather results + $result->{$prog}->{'cc_time'}->add_data($sample->{$prog}->{'cc_time'}); + $result->{$prog}->{'cc_real'}->add_data($sample->{$prog}->{'cc_real'}); + $result->{$prog}->{'ex_time'}->add_data($sample->{$prog}->{'ex_time'}); + $result->{$prog}->{'ex_real'}->add_data($sample->{$prog}->{'ex_real'}); + # collate pass status + $result->{$prog}->{'cc_pass'} = + &pass_diff($result->{$prog}->{'cc_pass'}, + $sample->{$prog}->{'cc_pass'}); + $result->{$prog}->{'ex_pass'} = + &pass_diff($result->{$prog}->{'ex_pass'}, + $sample->{$prog}->{'ex_pass'}); + } + # All max should be the same + $result->{'max'} = $sample->{'max'}; + } + # collate results + foreach my $prog (keys %$result) { + next unless ref $result->{$prog} eq "HASH"; + $result->{$prog}->{'cc_time'} = $result->{$prog}->{'cc_time'}->mean(); + $result->{$prog}->{'cc_real'} = $result->{$prog}->{'cc_real'}->mean(); + $result->{$prog}->{'ex_time'} = $result->{$prog}->{'ex_time'}->mean(); + $result->{$prog}->{'ex_real'} = $result->{$prog}->{'ex_real'}->mean(); + } +} + +# compares two aggregated logs with absolute values +# and creates a final result with relative values +sub compare_results($$$) { + my ($baseline, $target, $result) = @_; + my $cc_geo = new Statistics::Descriptive::Full(); + my $ex_geo = new Statistics::Descriptive::Full(); + + $result->{'cc_pass'} = "pass"; + $result->{'ex_pass'} = "pass"; + foreach my $prog (keys %$baseline) { + next unless ref $baseline->{$prog} eq "HASH"; + die "Program '$prog' in baseline doesn't exist in target\n" + unless defined $target->{$prog}; + + # No zero on divisions + my ($b, $t) = ($baseline->{$prog}, $target->{$prog}); + foreach my $k (keys %$b) { + $b->{$k} = 0.001 if looks_like_number($b->{$k}) + and $b->{$k} == 0.0; + } + foreach my $k (keys %$t) { + $t->{$k} = 0.001 if looks_like_number($t->{$k}) + and $t->{$k} == 0.0; + } + + # Proportional difference + $result->{$prog} = { + 'cc_pass' => &pass_diff($b->{'cc_pass'}, $t->{'cc_pass'}), + 'cc_time' => ($t->{'cc_time'} / $b->{'cc_time'}) * 100, + 'cc_real' => ($t->{'cc_real'} / $b->{'cc_real'}) * 100, + 'ex_pass' => &pass_diff($b->{'ex_pass'}, $t->{'ex_pass'}), + 'ex_time' => ($t->{'ex_time'} / $b->{'ex_time'}) * 100, + 'ex_real' => ($t->{'ex_real'} / $b->{'ex_real'}) * 100, + }; + + # Add data to statistical model (ignore real) + $cc_geo->add_data($result->{$prog}->{'cc_time'}); + $ex_geo->add_data($result->{$prog}->{'ex_time'}); + + # Update global "pass" status + $result->{'cc_pass'} = &pass_diff($result->{'cc_pass'}, + $result->{$prog}->{'cc_pass'}); + $result->{'ex_pass'} = &pass_diff($result->{'ex_pass'}, + $result->{$prog}->{'ex_pass'}); + } + + # Get geomean of all differences / max length of prog name + $result->{'cc_geo'} = $cc_geo->geometric_mean(); + $result->{'ex_geo'} = $ex_geo->geometric_mean(); + $result->{'max'} = $baseline->{'max'}; +} + +# dumps the file in the same format as read +# can dump the header or geomean, if requested +sub dump($$) { + my ($result, $header, $geo) = @_; + my $max = $result->{'max'}; + + # If we have the header, print it + if ($header) { + print $header."\n"; + } + + foreach my $prog (sort keys %$result) { + next unless ref $result->{$prog} eq "HASH"; + + # Dump the program name, with spaces at the end + my $p = $result->{$prog}; + my $spaces = $max - length($prog); + print $prog; + print ' ' x $spaces; + print "\t|\t"; + + # Compile time + print "\t".$p->{'cc_pass'}; + printf("\t%0.4f", $p->{'cc_time'}); + printf("\t%0.4f", $p->{'cc_real'}); + + # Execution time + printf "\t".$p->{'ex_pass'}; + printf("\t%0.4f", $p->{'ex_time'}); + printf("\t%0.4f", $p->{'ex_real'}); + + print "\n"; + } + + # Print the final geomean line + if ($geo) { + print "GEOMEAN"; + print ' ' x ($max - 7); + print "\t|\t"; + print "\t".$result->{'cc_pass'}; + printf("\t%0.2f\t", $result->{'cc_geo'}); + print "\t".$result->{'ex_pass'}; + printf("\t%0.2f", $result->{'ex_geo'}); + print "\n"; + } +} diff --git a/test-suite/compare_lnt_failures.sh b/test-suite/compare_lnt_failures.sh new file mode 100755 index 0000000..314fdd0 --- /dev/null +++ b/test-suite/compare_lnt_failures.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# This script helps comparing two outputs from a specific test failure +# It is a very specialised script and should be used with care. + +syntax="$0 test-dir out-file [skip-pattern]" +if [[ $1 = '' ]]; then + echo $syntax + exit 1 +fi +test_dir=$1 +if [[ $2 = '' ]]; then + echo $syntax + exit 1 +fi +out=$2 +echo "Differences for $test_dir" > $out +echo "LEFT is expected, RIGHT is achieved" >> $out +echo "" >> $out +skip='' +if [[ $3 != '' ]]; then + skip=$3 +fi + +failures=`grep "TEST-FAIL:" $test_dir/test.log | cut -d " " -f 3 | sort -u` +if [[ $failures = '' ]]; then + echo "No failures" + exit 0 +fi + +for f in $failures; do + if [[ $skip != '' && `echo $f | grep $skip` != '' ]]; then + echo "Skipping $f" >> $out + continue + fi + echo "Processing $f" >> $out + dir=`dirname $f` + file=`basename $f` + cd $dir/Output + diff $file.out-nat $file.out-simple >> $out +done diff --git a/test-suite/failures.sh b/test-suite/failures.sh new file mode 100755 index 0000000..c89a488 --- /dev/null +++ b/test-suite/failures.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# This script gathers the failures of a test.log after running the test-suite +# and shows in an easier to understand way. It's meant to be used by run.sh + +if [[ $1 = '' ]]; then + echo "syntax: $0 test.log" + exit -1 +fi +LOG=$1 +DIR="`pwd`/sandbox/test-[^/]\+/" + +grep "TEST-FAIL" $1 | awk '{print $3}' | sed 's%'$DIR'%%' | sort -u > .log + +RET=0 +if [ -s .log ]; then + COUNT=`wc -l .log | awk '{print $1}'` + echo "=========== $COUNT Failures ==============" + cat .log + echo + echo "=========== Full Log ==============" + egrep "(: error:)|(TEST-FAIL)" $LOG | grep -v "llvm\.o" | uniq + RET=1 +fi +rm .log +exit $RET diff --git a/test-suite/run.sh b/test-suite/run.sh new file mode 100755 index 0000000..a4313bb --- /dev/null +++ b/test-suite/run.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +# This script helps you run one or all tests in the LLVM test-suite +# It automatically detects the CPU, the number of cores and uses +# command line options to set up debug or special builds. +# +# The script assumes the install directory is at ../install. It's handy to +# have a symlink to whatever directory you're building to, either a build +# directory or a release/rc1/Phase3/.../*.install + +syntax="Syntax: $0 [-j cpus] [-g(cc)] [-l(ibc++/compiler-rt)] [-d(ebug)] [-t(est) <full-test-name>] [-s(mall)] [-b(enchmark-only)] [-p(erf)]" +CORE=`grep "CPU part" /proc/cpuinfo | awk '{print $4}'` +if [[ $CORE = '0xc08' ]]; then + CORE="-mcpu=cortex-a8" +else if [[ $CORE = '0xc09' ]]; then + CORE="-mcpu=cortex-a9" +else if [[ $CORE = '0xc0f' ]]; then + CORE="-mcpu=cortex-a15" +else if [[ $CORE = '0xd03' ]]; then + CORE="-mcpu=cortex-a53" +else if [[ $CORE = '0xd07' ]]; then + CORE="-mcpu=cortex-a57" +else + CORE='' +fi fi fi fi fi +ROOT=`pwd` +CPUS=`grep -c processor /proc/cpuinfo` +CC=$ROOT/../install/bin/clang +CXX=$ROOT/../install/bin/clang++ +CFLAGS="$CFLAGS $CORE" +CXXFLAGS="$CXXFLAGS $CORE" +LDFLAGS="$LDFLAGS" +TEST= +SMALL= +BENCH= +PERF=false + +while getopts "gldt:j:sbp" opt; do + case $opt in + l) + CFLAGS="$CFLAGS --rtlib=compiler-rt -lc++ -funwind-tables" + CXXFLAGS="$CXXFLAGS --rtlib=compiler-rt -stdlib=libc++ -funwind-tables" + LDFLAGS="$LDFLAGS -L/usr/local/lib -stdlib=libc++ -lc++abi" + ;; + d) + DEBUG="--build-mode Debug --optimize-option -O1" + ;; + t) + TEST="--only-test $OPTARG" + ;; + j) + CPUS="$OPTARG" + ;; + g) + CC=gcc + CXX=g++ + ;; + s) + SMALL="--small" + ;; + b) + BENCH="--benchmarking-only --build-threads=$CPUS" + BENCH="$BENCH --threads=1" + ;; + p) + PERF=true + ;; + *) + echo -e $syntax + exit 1 + ;; + esac +done + +if [ ! -z "$BENCH" ]; then + if $PERF && perf --help > /dev/null; then + # Only use perf is we have it installed and it works + BENCH="$BENCH --use-perf --multisample=4" + else + # Non-perf runs are a lot noisier + BENCH="$BENCH --multisample=8" + fi + # Either way, we want reproducible results + CPUS=1 +fi + +rm -rf sandbox/build + +# CC and CXX need to be duplicated because the --cc only affects the tests +# themselves, not the extra tools, and CFLAGS apply to both, which may break +# if you chose Clang-specific ones +LD_LIBRARY_PATH=$LD_LIBRARY_PATH \ +CFLAGS=$CFLAGS CXXFLAGS=$CXXFLAGS LDFLAGS=$LDFLAGS \ +CC=$CC CXX=$CXX \ +./sandbox/bin/python sandbox/bin/lnt runtest \ + nt \ + $TEST \ + $DEBUG \ + $SMALL \ + $BENCH \ + -j$CPUS \ + --no-timestamp \ + --sandbox sandbox \ + --test-suite $ROOT/test-suite \ + --cc $CC \ + --cxx $CXX + +./failures.sh $ROOT/sandbox/build/test.log +if [ $? != 0 ]; then + exit 1 +fi diff --git a/test-suite/setup.sh b/test-suite/setup.sh new file mode 100755 index 0000000..1d5eeea --- /dev/null +++ b/test-suite/setup.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# This script sets up the LNT test-suite environment +# Use *only* on an empty dir, as it will destroy any data + +set -e + +echo "Checking for python 2.7..." +python=`which python2.7` + +rm -rf sandbox + +gitupdate() { + repo=$1 + git -C $repo checkout master + git -C $repo fetch + git -C $repo pull +} + +echo "Checking out LNT/test-suite repositories..." +if [ ! -e lnt ]; then + git clone http://llvm.org/git/lnt.git +fi +gitupdate lnt +if [ ! -e test-suite ]; then + git clone http://llvm.org/git/test-suite.git +fi +gitupdate test-suite + +echo "Installing Python deps..." +sudo pip install virtualenv + +echo "Setting up the sandbox..." +virtualenv --python=$python sandbox +./sandbox/bin/python ./lnt/setup.py develop + +echo "All done. Use run.sh to run your test(s)" |