path: root/jenkins_packaged_kernel_build_inst

#! /bin/sh
# Example env variables and its values
# export board_types='panda'
# rootfs_type='nano'
set -xe

# set some defaults
# the : does nothing but the argument is evaluated so this works swell
#
: ${kernel_config:='packaged_kernel_config'}
: ${hwpack_type:='panda'}
: ${board_types:='panda'}
: ${rootfs_type:='nano'}
: ${submit_job:=1}
: ${upstream_git_tree:='git://git.linaro.org/kernel/linux-linaro-3.0.git'}
: ${upstream_git_branch:='master'}
: ${kernel_root:='kernel_root'}
: ${ubuntu_release:='natty'}
: ${git_publish_remote_repo_pull="git://github.com/jcrigby/packaged-linux-linaro-3.0-ci.git"} # read only
: ${git_publish_remote_repo_push="git@github.com:jcrigby/packaged-linux-linaro-3.0-ci.git"}   # r/w
: ${git_publish_remote_branch="omap"}
: ${git_publish_keyfile:="default_git_publish_keyfile"}
: ${git_publish:=1}
lci_b_t=$(pwd)/lci-build-tools
: ${GIT_COMMITTER_NAME:="Linaro CI"}
: ${GIT_COMMITTER_EMAIL:= "<john.rigby@linaro.org>"}
: ${GIT_AUTHOR_NAME:="Linaro CI"}
: ${GIT_AUTHOR_EMAIL:= "<john.rigby@linaro.org>"}
: ${DEBFULLNAME:="Linaro CI"}
: ${DEBEMAIL:="john.rigby@linaro.org"}
: ${force_build:=0}

export GIT_COMMITTER_NAME
export GIT_COMMITTER_EMAIL
export GIT_AUTHOR_NAME
export GIT_AUTHOR_EMAIL
export DEBFULLNAME
export DEBEMAIL

bundle_stream_name=$(echo $JOB_NAME | sed -e 's/_.*//' -e 's/\./_/g')

# Below is the list of values that will be used in the json files
log_info=$BUILD_URL"consoleText"
bundle_stream_name_path="/anonymous/ci-$bundle_stream_name/"

# kernel building helpers
#
fdr() {
    fakeroot ./debian/rules "$@"
}

# grab a value from fdr printenv results
#
fdrpenvget() {
    fdr printenv \
        | grep "^$1[[:blank:]]*=" \
        | sed -e 's/.*=//' -e 's/[ \t]*//g'
}

# grab a value from dpkg-parsechangelog results
#
dpchglget() {
    dpkg-parsechangelog -ldebian.linaro/changelog \
        | grep -Po "^$1:.+" \
        | sed -e "s/$1: //"
}

# prepare cross build
#
prepare_cross_build() {
    # turn off tools, workaround a binutils bug
    sed -i -e 's/do_tools.*=.*/do_tools = false/' debian.linaro/rules.d/armel.mk 
    sed -i -e 's/march=all/march=armv7-a/' arch/arm/boot/compressed/Makefile 
}

# check previous results against packaged and upstream kernels
# to see if a build needs to happen if not then exit with success here
#
exit_if_nothing_todo()
{
    test $force_build = "1" && return 0

    # packaged kernel is HEAD

    # if the results repo does not yet exist then there is something very wrong
    #
    git ls-remote -h $git_publish_remote_repo_pull || \
        exit 1

    # if the results branch does not yet exist then there
    # is most definately something todo
    #
    found=$(git ls-remote -h $git_publish_remote_repo_pull $git_publish_remote_branch |
                wc -l)
    test $found = "0" && new_results_branch=1 && return 0

    # first see how upstream compares to previous results

    # make a remote that points to upstream
    #
    upstream_remote_name="remote_random_$RANDOM"
    git remote add $upstream_remote_name $upstream_git_tree
    git remote update $upstream_remote_name

    # make a remote that points to previous results
    #
    results_remote_name="remote_random_$RANDOM"
    git remote add $results_remote_name $git_publish_remote_repo_pull
    git remote update $results_remote_name

    # count commits in upstream that are not in previous results
    #
    new_upstream_commits=$(git rev-list \
        remotes/$upstream_remote_name/$upstream_git_branch \
        ^remotes/$results_remote_name/$git_publish_remote_branch \
            | wc -l)

    test "$new_upstream_commits" != "0" && \
        echo "new upstream commits so there is something to do!" && \
        return 0

    # counting commits with git-revlist does not work for packaged vs results
    # since the packaged commits in results have been rebased so git-revlist
    # does not know they are the same
    #
    # if the packaged basis of the previous commit is different than
    # current then there is something todo
    #
    packaged_basis_commit_id=$(git log -n1 --pretty=format:%H)
    previous_packaged_basis_commit_id=$(git log -n1 \
        remotes/$results_remote_name/$git_publish_remote_branch | \
        grep packaged_basis_commit_id | \
        sed 's/.*=//')

    test "$packaged_basis_commit_id" != "$previous_packaged_basis_commit_id" && \
        echo "new packaged tree basis so there is something to do!" && \
        return 0

    echo "nothing todo, previous results are up to date"
    exit 0
}

# start a new release 
#
start_new_release() {
    fdr clean
    release=$(fdrpenvget release)
    abinum=$(fdrpenvget abinum)
    uploadnum=$(fdrpenvget uploadnum)
    prev_revision=$(fdrpenvget prev_revision)

    # remove old abi files
    #
    git rm -r -f debian.linaro/abi/$release-$prev_revision

    # for now always increment the abi and the upload numbers
    #
    ((abinum += 1))
    ((uploadnum += 1))
    dch -c debian.linaro/changelog \
        -t "LINARO: CI: Autogenerated Continuous Integration Build" \
        -b -v "$release-$abinum.$uploadnum~ci~$(date +%Y%m%d%H%M%S)"

    sed -i -e "0,/$ubuntu_release/s/$ubuntu_release/UNRELEASED/" \
        debian.linaro/changelog

    prev_revision=$(fdrpenvget prev_revision)
    flavours=$(grep '^flavours[[:blank:]]*=' debian.linaro/rules.d/armel.mk \
        | sed -e "s/.*=[ \t]*//")

    mkdir -p debian.linaro/abi/$release-$prev_revision/armel
    echo $((abinum - 1)) > debian.linaro/abi/$release-$prev_revision/abiname
    git add debian.linaro/abi/$release-$prev_revision/abiname
    echo 1 > debian.linaro/abi/$release-$prev_revision/armel/ignore
    echo 1 > debian.linaro/abi/$release-$prev_revision/armel/ignore.modules 

    # even when ignoring abi files we still need previous version abi files
    # so create some fake ones
    #
    for f in $flavours
    do
        echo "(ignore) ignore/ignore 0x0    ignore" > \
            debian.linaro/abi/$release-$prev_revision/armel/$f
        echo ignore > \
            debian.linaro/abi/$release-$prev_revision/armel/$f.modules
    done
    git add debian.linaro/abi/$release-$prev_revision
    git add debian.linaro/changelog

    git commit -m "LINARO: CI: Start new release"
}

rebase_on_latest_upstream()
{
    # add a remote pointing to upstream and rebase onto it 
    # if there is anything there not already in HEAD
    #
    upstream_remote_name="remote_random_$RANDOM"
    git remote add $upstream_remote_name $upstream_git_tree
    git remote update

    newcommits=$(git rev-list \
        remotes/$upstream_remote_name/$upstream_git_branch \
        ^HEAD \
            | wc -l)
    test $newcommits = "0" && echo "no rebase needed" && return 0

    git rebase remotes/$upstream_remote_name/$upstream_git_branch || true
    # deal with rebase conflicts in the most crude way possible
    # by just skipping them
    #
    echo "Here are all the skipped commits...." > ../skipped-commits
    rm -f ../skipped-commits
    while [ -d .git/rebase-apply/ ]
    do
        git show --pretty=short \
            $(cat .git/rebase-apply/original-commit) \
                >> ../skipped-commits
        git rebase --skip || true
    done
    cat ../skipped-commits

    dch -c debian.linaro/changelog -a "Rebase on new upstream"
    git add debian.linaro/changelog
    git commit -m "LINARO: CI: rebase on new upstream"
    git remote rm $upstream_remote_name
}

update_configs()
{
    # use fdr defaultconfigs instead of fdr updateconfigs because the
    # former uses
    #   yes '' | make oldconfig
    # while the later uses
    #   make silentoldconfig
    # which does not allow piped input so it isn't really appropriate
    # for scripting
    #
    fdr defaultconfigs || true

    # If anything has changed then those changes need to be checked in
    #
    if [ $(git status --short -uno) ]
    then
        dch -c debian.linaro/changelog -a "Update configs"
        git add debian.linaro/changelog
        git commit -a debian.linaro/config -m "LINARO: Update configs"
    fi
}

finalize_changelog()
{
    sed -i -e "0,/UNRELEASED/s/UNRELEASED/$ubuntu_release/" \
        debian.linaro/changelog
    source=$(dpchglget Source)
    version=$(dpchglget Version)
    git add debian.linaro/changelog
    echo "$source $version" > commitmsg
    echo "packaged_basis_commit_id=$packaged_basis_commit_id" >> commitmsg
    git commit -F commitmsg -s
    git_publish_tag_name=$(echo "$source-$version" | sed 's/~/-/g')
    git tag -m "source $version" "$git_publish_tag_name"
}

publish_kernel_source()
{
    test $git_publish = "0" && echo "publishing turned off" && return 0
    # first the branch
    $lci_b_t/git-publish run \
        --local_branch HEAD \
        --remote_repo $git_publish_remote_repo_push \
        --remote_branch $git_publish_remote_branch \
        --keyfile $git_publish_keyfile \
        --force

    # then the tag
    $lci_b_t/git-publish run \
        --tag $git_publish_tag_name \
        --remote_repo $git_publish_remote_repo_push \
        --remote_branch $git_publish_remote_branch \
        --keyfile $git_publish_keyfile \
        --force
    return 0
}

cpu_count=$(cat /proc/cpuinfo  | grep processor | wc -l)
if test x$cpu_count = x; then
cpu_count=1
fi
j_count=$(expr $cpu_count '*' 2)

# ok lets actually build the kernel
#
START=$(date +%s)
{
    cd $kernel_root
    exit_if_nothing_todo
    start_new_release
    rebase_on_latest_upstream
    update_configs
    finalize_changelog
    prepare_cross_build
    fdr clean
    debuild --no-tgz-check --no-lintian -j$j_count \
        -eCROSS_COMPILE=arm-linux-gnueabi-  -b -aarmel -us -uc
    publish_kernel_source
    cd ..
}
END=$(date +%s)
EXECUTION_TIME_IN_SEC=$(( $END - $START ))

if [ $hwpack_type != none ]; then
    kernel_version=$(cd $kernel_root; fdrpenvget revision)

    rm *dbgsym*.deb # this messes up the glob later

    latest_hwpack=$(hwpack_type=$hwpack_type; \
        python lci-build-tools/get_latest_slo_hwpack)
    latest_hwpack_name=$(basename $latest_hwpack)
    wget -c $latest_hwpack
    rm -rf linaro-image-tools
    bzr branch lp:linaro-image-tools
    new_hwpack_url=$(python linaro-image-tools/linaro-hwpack-replace \
        -t $latest_hwpack_name \
        -p ./linux-image*${kernel_version}*.deb \
        -r linux-image \
        -n $BUILD_NUMBER)
    new_hwpack_name=$(basename $new_hwpack_url)
fi

# NB: tabs only indenting in this heredoc
#
cd $kernel_root; cat > ../hwpack_info <<-_HWPACK_INFO_END_
	KERNEL_CONFIG=$kernel_config
	OS_INFO=$(cat /etc/lsb-release | grep DISTRIB_DESCRIPTION | cut -d '=' -f 2)
	LOG=$log_info
	BUILD_ID=$BUILD_NUMBER
	GCC_VERSION=$(dpkg -s gcc | grep Version | cut -d ' ' -f2)
	GCC_CROSS_COMPILER=$(dpkg -s gcc-arm-linux-gnueabi | grep Version | cut -d ' ' -f2)
	BOARD_TYPES=$board_types
	HWPACK_TYPE=$hwpack_type
	ROOTFS_TYPE=$rootfs_type
	BUNDLE_STREAM_NAME=$bundle_stream_name_path
	JOB_URL=http://snapshots.linaro.org/kernel-hwpack/
	KERNEL_VERSION=$(git describe --match='v*')
	KERNEL_GIT=$(cat .git/config|grep -i 'url = ' | cut -d "=" -f2) 
	KERNEL_COMMIT=$GIT_COMMIT 
	COMMIT_TIME=$(git show -s --format=%cD HEAD)
	KERNEL_NAME=$bundle_stream_name
	SUBMIT_LAVA_JOB=$submit_job
	KERNEL_BUILD_URL=$BUILD_URL
	KERNEL_BRANCH=$(git branch | grep -e '*'  | cut -d '*' -f 2)
	KERNEL_JOB_NAME=$JOB_NAME
	GIT_LOG=$(git log -n1 --pretty=oneline)
	HWPACK_NAME=$new_hwpack_name
	URL=$JENKINS_HOME/jobs/$JOB_NAME/workspace/$new_hwpack_name
	EXECUTION_TIME_IN_SEC=$(( $END - $START ))
	HWPACK_BUILD_DATE=$(echo $new_hwpack_name | cut -d '_' -f3)
_HWPACK_INFO_END_
exit 0