path: root/toolchain-tools

#!/bin/bash
#
# This script is used to sync the upstream gcc repository, plus a series of
# other related toolchain repositories.
#
# It is highly dependant on the git setup Linaro has on its server. Setup that
# is done with gitolite and gitweb.
#
# How the mirrorin has been done:
# - mirrorin happens in /srv/mirror on the server
# - depending on the mirror type, we can create a git-svn clone in a separate
#   directory, add a new remote on the "original" repo, and then make sure
#   everything is pushed to the real bare repo.
#
# For more info: https://wiki.linaro.org/Platform/Systems/ToolchainsMirror
#

umask 0022  # cron runs us with 0002

GIT_BIN=`which git`
GITOLITE_BIN="/home/git/bin/gitolite"
REPOSITORIES_DIR="/srv/repositories"
TOOLCHAIN_DIR="$REPOSITORIES_DIR/toolchain"
GCC_REPO="$TOOLCHAIN_DIR/gcc.git"
BINUTILS_GDB_REPO="$TOOLCHAIN_DIR/binutils-gdb.git"
MIRROR_DIR="/srv/mirror"
GCC_MIRROR_DIR="$MIRROR_DIR/gcc.git/"
BZR_BRANCH="bzr/linaro-4.7-branch"
UPSTREAM_48_BRANCH="4.8-branch"
UPSTREAM_47_BRANCH="4.7-branch"
SVN_UPSTREAM_48_BRANCH="$MIRROR_DIR/linaro-gcc-4_8-branch"
SVN_UPSTREAM_47_BRANCH="$MIRROR_DIR/gcc-4_7-branch"
SVN_UPSTREAM_49_BRANCH="$MIRROR_DIR/linaro-gcc-4_9-branch"
LINARO_UPSTREAM_48_BRANCH="linaro-4.8-branch"
LINARO_UPSTREAM_49_BRANCH="linaro-4.9-branch"

NEW_GCC_MIRROR_DIR="$MIRROR_DIR/gcc"
NEW_GCC_TOOLCHAIN_DIR="$NEW_GCC_MIRROR_DIR/toolchain"
NEW_GCC_LOCAL_MIRROR="$NEW_GCC_TOOLCHAIN_DIR/gcc.git"
NEW_GCC_MIRROR_FETCH_DIR="$NEW_GCC_MIRROR_DIR/fetch"
NEW_GCC_BRANCHES_DIR="$NEW_GCC_MIRROR_FETCH_DIR/branches"
NEW_GCC_SVN_BRANCHES_DIR="$NEW_GCC_BRANCHES_DIR/svn"
NEW_GCC_UPSTREAM_DIR="$NEW_GCC_MIRROR_FETCH_DIR/gcc.git"

function errcho() {
    # Print on stderr passed messages.
    >&2 echo $1
}

function logiferr() {
    LOG=$($1 2>&1)
    STATUS=$?
    if [ ${STATUS} != 0 ]
    then
        errcho "CMD: '$1' exited with status ${STATUS}"
        if [ "${LOG}" != "" ]
        then
            errcho "WITH LOG OUTPUT: ${LOG}"
        fi
    fi
    return ${STATUS}
}

# Retry command which tends to have transient failures
function stubborn_do() {
    for i in 5 15 60 180 180; do
        LOG=$("$@" 2>&1)
        STATUS=$?
        if [ $STATUS -eq 0 ]; then
            return 0
        fi
        sleep $i
    done
    errcho "CMD: '$@' exited with status $STATUS"
    errcho "WITH LOG OUTPUT: '$LOG'"
    errcho "Last retry delay: ${i}s"
    return $STATUS
}

function new2_gcc() {
    if [ ! -d $MIRROR_DIR/gcc-upstream-pristine.git ]; then
        git clone --mirror git://gcc.gnu.org/git/gcc.git $MIRROR_DIR/gcc-upstream-pristine.git
    fi

    QUIET="${QUIET---quiet}"
    cd $MIRROR_DIR/gcc-upstream-pristine.git
    stubborn_do git fetch $QUIET
    # Set up gitolite environment for local push
    export GL_BYPASS_ACCESS_CHECKS=1
    export GL_LIBDIR=~/gitolite/src/lib
    git push $QUIET --force /srv/repositories/toolchain/gcc.git 'refs/heads/*' 'refs/tags/*' 2>&1 | grep -v "remote: Sending notification emails to"
}

function new_gcc() {
    new_gcc_update_upstream_mirror
    new_gcc_update_local_mirror
    new_gcc_update_svn_branches

    if [ $? -eq 0 ]; then
        cd $NEW_GCC_LOCAL_MIRROR
        # It's a mirror we are pushing, so no --all/--tag.
        logiferr "$GITOLITE_BIN push"

        if [ $? -ne 0 ]; then
            errcho "Error pushing to final destination"
        fi
    fi
}

function new_gcc_update_svn_branches() {
    # SVN branches need to be created manually.
    # Once cloned, the git-svn repo must have as the remote push origin the
    # path defined in $NEW_GCC_UPSTREAM_DIR
    if [ ! -d $NEW_GCC_SVN_BRANCHES_DIR ]; then
        return 0
    fi

    for git_svn_repo in `find $NEW_GCC_SVN_BRANCHES_DIR -type d -name ".git" -prune -exec dirname {} \;`; do
        cd $git_svn_repo
        branch_name=`basename $git_svn_repo`
        logiferr "$GIT_BIN svn rebase"

        if [ $? -ne 0 ]; then
            errcho "Error updating gcc svn branch $branch_name"
            return 1
        fi

        logiferr "$GIT_BIN push origin master:$branch_name"
        if [ $? -ne 0 ]; then
            errcho "Error pushing gcc svn branch $branch_name"
            return 1
        fi
    done

    return 0
}

function new_gcc_update_upstream_mirror() {
    if [ ! -d $NEW_GCC_UPSTREAM_DIR ]; then
        errcho "Missing upstream gcc mirror, cloning it..."

        mkdir -p $NEW_GCC_MIRROR_FETCH_DIR
        cd $NEW_GCC_MIRROR_FETCH_DIR

        logiferr "$GIT_BIN clone -q --mirror git://gcc.gnu.org/git/gcc.git"

        if [ $? -ne 0 ]; then
            errcho "Error creating gcc upstream mirror (fetch)"
            return 1
        fi

    fi

    cd $NEW_GCC_UPSTREAM_DIR
    logiferr "$GIT_BIN remote update"

    if [ $? -ne 0 ]; then
        errcho "Error updating gcc upstream mirror (fetch)"
        return 1
    fi

    return 0
}

function new_gcc_update_local_mirror() {
    if [ ! -d $NEW_GCC_LOCAL_MIRROR ]; then
        errcho "Missing local gcc mirror, cloning it..."

        mkdir -p $NEW_GCC_TOOLCHAIN_DIR
        cd $NEW_GCC_TOOLCHAIN_DIR

        logiferr "$GIT_BIN clone --mirror $NEW_GCC_UPSTREAM_DIR"

        if [ $? -ne 0 ]; then
            errcho "Error cloning gcc upstream mirror (local)"
            return 1
        fi

        cd $NEW_GCC_LOCAL_MIRROR
        logiferr "$GIT_BIN remote set-url --push origin $TOOLCHAIN_DIR/gcc-new.git"
        if [ $? -ne 0 ]; then
            errcho "Error setting correct remote (local)"
            return 1
        fi
    fi

    cd $NEW_GCC_LOCAL_MIRROR
    logiferr "$GIT_BIN remote update"

    if [ $? -ne 0 ]; then
        errcho "Error updating gcc upstream mirror (local)"
        return 1
    fi

    return 0
}

function push_changes() {
    cd $GCC_MIRROR_DIR
    # gitolite, not git, this is important since the push has to happen on the
    # local filesystem and the repository is controlled by gitolite.
    logiferr "$GITOLITE_BIN push --all"
    logiferr "$GITOLITE_BIN push --tags"
}

function update_server_info() {
    logiferr "git --git-dir=$GCC_REPO update-server-info"
}

function update_bzr_branch() {
    cd $GCC_MIRROR_DIR
    logiferr "git bzr sync $BZR_BRANCH"
    push_changes
    update_server_info
}

function update_upstream_48_branch() {
    cd $GCC_MIRROR_DIR
    logiferr "git checkout $UPSTREAM_48_BRANCH"
    logiferr "git pull --rebase"
    logiferr "git checkout master"
}

function update_linaro_upstream_48_branch() {
    cd $GCC_MIRROR_DIR
    logiferr "git checkout $LINARO_UPSTREAM_48_BRANCH"
    logiferr "git pull --rebase"
    logiferr "git checkout master"
}

function update_linaro_upstream_49_branch() {
    cd $GCC_MIRROR_DIR
    logiferr "git checkout $LINARO_UPSTREAM_49_BRANCH"
    logiferr "git pull --rebase"
    logiferr "git checkout master"
}

function update_svn_upstream_48_branch() {
    cd $SVN_UPSTREAM_48_BRANCH
    logiferr "git svn rebase"
}

function update_svn_upstream_47_branch() {
    cd $SVN_UPSTREAM_47_BRANCH
    logiferr "git svn rebase"
}

function update_svn_upstream_49_branch() {
    cd $SVN_UPSTREAM_49_BRANCH
    logiferr "git svn rebase"
}

function update_upstream_47_branch() {
    cd $GCC_MIRROR_DIR
    logiferr "git checkout $UPSTREAM_47_BRANCH"
    logiferr "git pull --rebase"
    logiferr "git checkout master"
}

function update_gcc_repo_remote() {
    cd $GCC_MIRROR_DIR
    logiferr "git remote update"
    logiferr "git pull --all"
}

function update_newlib_mirror() {
    cd $TOOLCHAIN_DIR/newlib.git
    stubborn_do git remote update
}

function update_glibc_mirror() {
    cd $TOOLCHAIN_DIR/glibc.git
    stubborn_do git remote update
}

function pull_git_changes() {
    update_svn_upstream_47_branch
    update_svn_upstream_48_branch
    update_svn_upstream_49_branch
    update_gcc_repo_remote

    update_upstream_47_branch
    update_upstream_48_branch
    update_linaro_upstream_48_branch
    update_linaro_upstream_49_branch

    push_changes
    update_server_info
}

function update_binutils_gdb() {
    cd $BINUTILS_GDB_REPO
    stubborn_do git remote update
    logiferr "git update-server-info"
}

function update_dejagnu_mirror() {
    cd "$MIRROR_DIR/dejagnu.git"
    stubborn_do git remote update
    if [ $? == 0 ]
    then
        logiferr "$GITOLITE_BIN push"
    fi
}

function all() {
    update_bzr_branch
    pull_git_changes
    update_binutils_gdb
}

$@

all_repos="gcc.git gcc-new-deprecated-201505.git glibc.git dejagnu.git newlib.git binutils-gdb.git"
cd /srv/repositories
for repo in $all_repos ; do
    logiferr "/usr/local/bin/grok-manifest -m /var/www/git.linaro.org/manifest.js.gz -t /srv/repositories/ -v -n toolchain/$repo"
done