path: root/ci-merge

#!/bin/bash

#
# The purpose of this script is to pull different remotes branches and
# merge them into a single one, push the resulting merge to a remote
# repository in order to let the kernelci to compile / boot [ /test ]
# it. The base mechanism relies on the reuse-recorded-resolution, so
# the very first merge will force the caller of this script to resolve
# the conflicts manually. The resolutions will be saved in a cache and
# reused later when the same conflicts appear to solve them
# automatically. In case a conflict was uncorrectly solved at the
# first place, it is up to the user to purge the rerere cache and redo
# a clean resolution conflict.
#
# The merge common point is by default the latest tag, or the latest
# commit id specified in the command line option.
#
# 1. How to use it
#
#  - Create a Linux clean tree
#
#  - Create one configuration file located at (topmost priority):
#    - <linux>/.automerge/config
#    - $HOME/.automerge/config
#    - /etc/automerge.conf
#
#  The ci-config script will return the first configuration file
#  found in the list above.
#
#  - Call the script from the topmost directory of the source tree
#
# 2. CONFIG file format
#
#    <user> <url> <branch>
#
#  - Comments are allowed with '#' format
#
#  - The baseline *must* be specified
#
# eg.
#
# baseline git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git master
# ulf.hansson http://git.linaro.org/people/ulf.hansson/linux-pm.git next
# andy.gross https://git.linaro.org/people/andy.gross/linux-pm.git for-pm-int
#
# By default the baseline name is 'baseline' but its name can be changed
# via the options in the command line parameters.
#
# 3. Conflict resolution
#
# As soon as there is a conflict detected, the merge tool is invoked,
# the default is vimdiff. *Note* that will open a terminal, so if X11
# forwarding is not set with the ssh connection (if any) the
# integration process will fail.
#
# Take care of correctly fix the conflicts, otherwise the bad
# resolutions will be saved and reused later by the rerere mechanism,
# thus introducing the error each time. It is safe in this case to
# wipe out the rerere cache or, if you are used to rerere, use 'git
# rerere forget <resolution>' to remove the bad resolution.
#
# 4. Behavior
#
# If a branch is present in the tree but no present in the CONFIG
# file, it will be removed. So the process of removing an old topic
# branch is automatic, the user has just to remove or comment a line
# in the CONFIG file.
#
# If a branch is not present in the tree but present in the CONFIG
# file, it will be added. So the process of adding a new topic branch
# is automatic, the user has just to add or uncomment a line in the
# CONFIG file.
#
# If no branches have been updated since the last merge, the script
# will suggest to abort integration process [default=y].
#
# When the merge is done, the script propose to push the branch to the
# remote repository [default=y]. *Note* this operation will overwrite
# the previous branch in the remote repository.
#
# In order to prevent polluting the remote repository, only one branch
# 'integ' is pushed, copies of all the previous merges are stored in
# the local repo with the name 'integ-<commit-id>
#

# The default repository directory is the current one
LOCAL_REPO=$PWD

# The default remote repository
REMOTE_REPO=""

# The integration branch
INTEG_BRANCH="integ"
INTEG_BRANCH_NEW_NAME=""

# The baseline remote repo
BASELINE="baseline"

# The baseline branch defaulting to 'master'
BASELINE_BRANCH=master

# Track the tag or HEAD changes
TRACK=tag

# Global cache of rerere resolutions
RERERE_CACHE=""

# Temporary directory to clone the rerere cache into
# This directory is not deleted at the end to allow for new merge resolutions to
# be pushed to git
TMP_DIR=$(mktemp -d --suffix -automerge.rerere)

# Global variable to test if a change occured during the last merge
# or not. If one branch is added, deleted or updated, this variable
# will be different from zero and will lead to the merge operation
# followed by an update of the tree.
CHANGED=0

######################################################################
#
# The help usage function displaying options and default values
#
Help() {
    echo

    echo " The Automerge Continuous Integration script takes all
 remote repositories, merge the <baseline> <branch> into the <integ>
 branch in the <local> working repository as well as the topics
 branches defined in the configuration file and upload the merge result
 to <remote>"

    echo

    echo "	 -h|--help		: Display this help"
    echo "	 -l|--local <path>	: Path to the local repository [$LOCAL_REPO]"
    echo "	 -r|--remote <url>	: URL to the remote repository [$REMOTE_REPO]"
    echo "	 -b|--baseline <name>	: Baseline name [$BASELINE]"
    echo "	 -i|--integ <name>	: Integration branch name [$INTEG_BRANCH]"
    echo "	 -t|--track tag|head	: Changes on the latest tag or HEAD on branch [$TRACK]"
    echo "	 -c|--cache <name>	: URL to rerere cache repository [$RERERE_CACHE]"

    echo
}

######################################################################
#
# Parsing options from command line.
#
options_setup() {

    SHORTOPT="hl:r:b:i:t:c:"
    LONGOPT="help,local:,remote:,baseline:,integ:,track:,cache:"

    OPTS=$(getopt -o $SHORTOPT -l $LONGOPT -- "$@")

    if [ $? != 0 ]; then
	echo "Failed to get options"
	exit 1
    fi

    eval set -- "$OPTS"

    while true; do

	OPT=$1
	shift

	if [ "$OPT" = "" ]; then
	    break
	fi

	case "$OPT" in
	    --help|-h)
		Help
		exit 0 # We just display the help and exit
		;;

	    --local|-l)
		LOCAL_REPO=$1
		;;

	    --remote|-r)
		REMOTE_REPO=$1
		;;

	    --baseline|-b)
		BASELINE=$1
		;;

	    --track|-t)
		if [ "$1" = "tag" -o "$1" = "head" ]; then
		    TRACK=$1
		else
		    echo "Bad tracking keyword"
		fi
		;;

	    --integ|-i)
		INTEG_BRANCH=$1
		;;

	    --cache|-c)
		RERERE_CACHE=$1
		;;

	    --)
		break
		;;
	esac
    done
}


######################################################################
#
# Verify that the local and remote git repos exist so we don't
# end up with strange errors
#
verify_repos() {

    git ls-remote -h $LOCAL_REPO > /dev/null 2>&1
    if [ $? -ne 0 ]; then
	      echo "ERROR: Not found LOCAL_REPO -> $LOCAL_REPO"
	      exit 1
    fi

    if [ ! -z $RERERE_CACHE ]; then
        git ls-remote -h $RERERE_CACHE > /dev/null 2>&1
        if [ $? -ne 0 ]; then
	          echo "ERROR: Not found RERERE_CACHE -> $RERERE_CACHE"
	          exit 1
        fi
    fi

    if [ ! -z $REMOTE_REPO ]; then
        git ls-remote -h $REMOTE_REPO > /dev/null 2>&1
        if [ $? -ne 0 ]; then
	          echo "ERROR: Not found REMOTE_REPO -> $REMOTE_REPO"
	          exit 1
        fi
    fi
    echo "Verified existence of local and remote repos: Success"
}

######################################################################
#
# The merge process can be greatly simplified by using the
# reuse-recorded-resolution (rerere). Make sure it is enabled and create
# of a copy of the shared rerere cache locally.
#
git_rerere_setup() {

    git config --local --bool rerere.enabled true
    git config --local --bool rerere.autoupdate true
    git config --local merge.conflictstyle merge

    # we enable the rerere option
    echo "Reuse-Recorded-Resolution: Enabled"

    git clone -q --depth=1 $RERERE_CACHE $TMP_DIR
    if [ $? -ne 0 ]; then
	echo "WARNING: Unable to get shared rerere cache"
	return 1
    fi

    $(cd $LOCAL_REPO/.git && git archive --remote=$TMP_DIR HEAD | tar xf -)
    echo "Downloaded shared rerere cache"

    return 0
}

######################################################################
#
# Use appropriate terminal
#
terminal() {
    if [ -x /usr/bin/x-terminal-emulator ]; then
	x-terminal-emulator $@
    else
	xterm $@
    fi
}

######################################################################
#
# Check we have a configuration file before continuing
#
config_setup() {

    # The configuration file containing the name + url + branch
    CONFIG=$(ci-config)

    if [ "$CONFIG" == "" ]; then
	echo "No configuration file found, aborting."
	exit 1
    fi
}

######################################################################
#
# Setup path after options and config is done
#
path_setup() {

    # Use pushd here so when the script exits, the CWD will
    # automatically be the initial one before calling this script.
    pushd $LOCAL_REPO
}

######################################################################
#
# Compute the remote branch name from the information we have from the
# config file and the BASELINE name
#
baseline_branch_setup() {

    while read LINE; do

	# Ignore commented line
	echo $LINE | egrep -q '(^#|^\s*$|^\s*\t*#)' && continue

	REMOTE_NAME=$(echo $LINE | awk '{ print $1 }')
	if [ "$REMOTE_NAME" = "$BASELINE" ]; then

	    REMOTE_URL=$(git remote -v | grep "^$REMOTE_NAME\b" | grep fetch | awk '{ print $2 }')
	    REMOTE_BRANCH=$(echo $LINE | awk '{print $3}')
	    if [ "$REMOTE_BRANCH" != "" ]; then
		BASELINE_BRANCH=remotes/$REMOTE_NAME/$REMOTE_BRANCH
		break
	    fi

	fi

    done < $CONFIG
}

######################################################################
#
# Ensure that the local repo is clean
#
clean_local_repo() {
    if [[ $(git status --porcelain) ]]; then
	# Changes
	echo "Local changes present"
	git status --porcelain
	echo -n "Clean it [Y/n]? "
	read RES
	if [ "$RES" == "Y" -o "$RES" == "y" -o "$RES" == "" ]; then
	    git clean -df
	    git reset --hard
	else
	    echo "Please clean the repo before proceeding. Aborting."
	    exit 1
	fi
    else
       # No changes
       echo "Local tree is clean"
    fi
}

######################################################################
#
# This is the global setup entry calling the different setup functions.
# It ensures they are called in the right order.
# !! Make sure to not change the order without double checking the !!
# !! initialized variables dependencies                            !!
#
do_setup() {

    options_setup $@

    verify_repos

    path_setup

    if [ ! -z $RERERE_CACHE ]; then
        git_rerere_setup
    else
        # If we don't use the rerere cache, explicity disable rerere
        # since the user wants to see all merge conflicts
        git config --local --bool rerere.enabled false
        echo "Reuse-Recorded-Resolution: Disabled"
    fi

    config_setup

    baseline_branch_setup

    clean_local_repo
}

######################################################################
#
# When running on a Linux tree, we want to pick the latest tag of the
# Vanilla kernel, not the intermediate state of the Linux tree but a
# relatively stable version.
#
get_last_change_id() {

    if [ "$TRACK" = "tag" ]; then
	git describe --abbrev=0 $BASELINE_BRANCH
    else
	git log -n 1 --pretty=oneline $BASELINE_BRANCH | awk '{ print $1 }'
    fi
}

######################################################################
do_remove_old() {

    echo "Removing old remotes ..."

    #
    # Retrieve all remotes except baseline. The mechanism relies on a
    # clean tree without extra branches other than $BASELINE
    # and $INTEG*.OB
    #
    # All others branches will be asked for ***deletion***.
    #
    REMOVED=0

    for REMOTE_NAME in $(git remote); do

	REMOTE_BRANCH=$(git rev-parse --symbolic-full-name --abbrev-ref --remotes=$REMOTE_NAME | cut -d'/' -f1 --complement)
	REMOTE_URL=$(git remote get-url $REMOTE_NAME)
	FOUND=0

	while read LINE; do

	    # ignore commented lines
	    echo $LINE | egrep -q '(^#|^\s*$|^\s*\t*#)' && continue

	    echo $LINE | grep "^$REMOTE_NAME\b" | grep $REMOTE_URL | grep -q $REMOTE_BRANCH
	    if [ $? -eq 0 ]; then
		FOUND=1
	    fi
	done < $CONFIG

	if [ $FOUND -eq 1 ]; then
	    continue
	fi

	if [ "$REMOTE_NAME" = "$BASELINE" ]; then
	    echo "The baseline $BASELINE is removed from the config file, fix this! (abort)"
	    exit 1
	fi

	echo "The remote $REMOTE_NAME $REMOTE_URL $REMOTE_BRANCH is no longer tracked."
	echo -n "Delete it [Y/n]? "
	read RES
	if [ "$RES" == "Y" -o "$RES" == "y" -o "$RES" == "" ]; then
	    git remote remove $REMOTE_NAME
	    REMOVED=$((REMOVED+1))
	fi

    done

    echo "Done, removed $REMOVED old remote(s)."

    CHANGED=$((REMOVED + CHANGED))
}

######################################################################

do_add_new() {

    echo "Adding all remotes new ..."

    ADDED=0

    while read LINE; do

	# Ignore commented line
	echo $LINE | egrep -q '(^#|^\s*$|^\s*\t*#)' && continue

	REMOTE_NAME=$(echo $LINE | awk '{ print $1 }')
	REMOTE_URL=$(echo $LINE | awk '{print $2}')
	REMOTE_BRANCH=$(echo $LINE | awk '{print $3}')

	git remote | grep -q "$REMOTE_NAME"
	if [ $? -ne 0 ]; then
	    echo "Adding remote $REMOTE_NAME $REMOTE_URL $REMOTE_BRANCH"
	    git remote add -f -t $REMOTE_BRANCH $REMOTE_NAME $REMOTE_URL
	    ADDED=$((ADDED + 1))
	fi
    done < $CONFIG

    echo "Done, added $ADDED new remote(s)."

    CHANGED=$((ADDED + CHANGED))
}

######################################################################

do_update_baseline() {

    ID1=$(get_last_change_id)

    echo "Updating $BASELINE ..."

    git remote update $BASELINE 2>&1

    ID2=$(get_last_change_id)

    echo latest tag/id is $ID2

    if [ "$ID1" != "$ID2" ]; then
	echo "$BASELINE has new tagi/id $ID2."
	CHANGED=$((CHANGED + 1))
    fi

    echo "Done, updated baseline."
}

######################################################################

do_update_remote() {

    echo "Updating the remotes ..."

    UPDATED=0

    while read LINE; do

	echo $LINE | egrep -q '(^#|^\s*$|^\s*\t*#)' && continue

	REMOTE_NAME=$(echo $LINE | awk '{ print $1 }')

	if [ "$REMOTE_NAME" = "$BASELINE" ]; then
		continue
	fi

	echo "Updating $REMOTE_NAME"

	RES=$(git remote update $REMOTE_NAME 2>&1 | wc -l)
	if [ $RES -gt 1 ]; then
	    echo "$REMOTE_NAME has changed."
	    UPDATED=$((UPDATED + 1))
	fi

    done < $CONFIG

    echo "Done, updated $UPDATED remote(s)."

    CHANGED=$((UPDATED + CHANGED))
}

######################################################################

# Do the entire setup, the function makes sure the setup functions are
# called in the right order for variable dependencies
do_setup $@

# Remove all branches present in the git tree but no longer tracked in
# the CONFIG file
do_remove_old

# Add all branches present in the CONFIG file but not present in the
# git tree
do_add_new

# Update all the remote branches
do_update_remote

# Update the baseline
do_update_baseline

# If anything changed in the process (removal, addition, update) then
# it makes sense to create a new integ branch, otherwise let the user
# to choose to continue or abort the process
if [ $CHANGED -eq 0 ]; then
    echo "No branches have changed since last time."
    echo -n "Abort building the integ branch? [Y/n]"
    read RES
    if [ "${RES,,}" == "y" -o "$RES" == "" ]; then
	echo "Integration branch building aborted."
	exit 0
    fi
fi

########################################################

TAG=$(get_last_change_id)
echo "Latest tag is $TAG"

echo "Check the 'integration' branch exists"
git branch | grep -q "$INTEG_BRANCH$"

if [ $? -eq 0 ]; then
	echo "The integration branch already exists."

	INTEG_BRANCH_NEW_NAME=$INTEG_BRANCH-$(date +%Y%m%d-%H%M%S)-$(git describe)

	echo "Rename it to $INTEG_BRANCH_NEW_NAME"
	git branch -M $INTEG_BRANCH $INTEG_BRANCH_NEW_NAME
fi

echo "Create a new integration branch based on $TAG"
git checkout -b $INTEG_BRANCH $TAG

echo "Merging topic branches..."

MERGED=0

while read LINE; do

    echo $LINE | egrep -q '(^#|^\s*$|^\s*\t*#)' && continue
    REMOTE_NAME=$(echo $LINE | awk '{ print $1 }')
    REMOTE_URL=$(echo $LINE | awk '{print $2}')
    REMOTE_BRANCH=$(echo $LINE | awk '{print $3}')

    if [ "$REMOTE_NAME" = "$BASELINE" ]; then
	continue
    fi

    echo "Merging $REMOTE_NAME/$REMOTE_BRANCH"

    git merge -q --no-ff --no-edit $REMOTE_NAME/$REMOTE_BRANCH
    if [ $? -ne 0 ]; then
        if [ -z $DISPLAY ]; then
            # try to commit, may be resolved by rerere
            git commit --no-edit
            commit_ec=$?
            if [ $commit_ec -ne 0 ]; then
                echo "Merge failed, $REMOTE_NAME $REMOTE_URL $REMOTE_BRANCH"
                git merge --abort
                exit $commit_ec
            fi
        else
            echo "Merge failed, manual merge"
            terminal -e "git mergetool -y"
            git commit -a --no-edit
        fi
    fi

    MERGED=$((MERGED+1))

done < $CONFIG

echo "Done, merged $MERGED topic(s)."


if [ ! -z $REMOTE_REPO ]; then
    # The remote branch in git repository
    REMOTE_INTEG_BRANCH="$REMOTE_REPO $INTEG_BRANCH"

    echo -n "Push branch to $REMOTE_INTEG_BRANCH [Y/n]? "
    read RES
    if [ "${RES,,}" == "y" -o "$RES" == "" ]; then
        git push -f $REMOTE_INTEG_BRANCH

        if [ ! -z $INTEG_BRANCH_NEW_NAME ]; then
	          REMOTE_INTEG_BRANCH_NEW_NAME="$REMOTE_REPO $INTEG_BRANCH_NEW_NAME"
	          git push $REMOTE_INTEG_BRANCH_NEW_NAME
        fi
    fi
fi

echo -n "Copy any new merge conflict resolutions to the rerere cache ($TMP_DIR) [Y/n]? "
read RES
if [ "${RES,,}" == "y" -o "$RES" == "" ]; then
    $(cd $LOCAL_REPO/.git && cp -a rr-cache $TMP_DIR)
    echo "Please make sure to _push_ any new conflict resolutions in $TMP_DIR manually to $RERERE_CACHE"
fi