path: root/scripts/package_kernel

#!/bin/bash

set -x
set -e
shopt -s extglob

# add directory containing this script to PATH
#
export DEFAULT_PATH="/usr/sbin:/usr/bin:/sbin:/bin"
export CIROOTDIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
export TCBINDIR=${TCBINDIR:-"$HOME/tc/bin"}
export PATH="$CIROOTDIR/scripts:$CIROOTDIR/configs:$PATH"
toolchain_url=${toolchain_url:-}

# exit with error message and bad exit status
#
carp()
{
	echo $1
	exit -1
}

# warn about something but do not exit
#
warn()
{
	echo "WARNING: $1"
	true
}

# exit with message and good exit status
#
infoexit()
{
	echo -e $1
	exit 0
}

info()
{
	echo $1
	true
}

# echo message if being verbose
#
verbose_info()
{
	test "$verbose" && echo $1
	true
}

cleanup()
{
	:
}

usage()
{
	echo "package_kernel var=bar0 --cfg pkgingconfig.cfg [-c pkg.cfg] var=bar1"
	echo "set vars with var=value later settings will override earlier onces"
	echo "or use config files these are sourced and do not need full path if"
	echo "they are in PATH and executable"
	exit 1
}

pvar()
{
	echo "$1=${!1}"
}

fdr()
{
	fakeroot ./debian/rules $@
}

changelog_get()
{
	dpkg-parsechangelog -l$1 \
		| grep -Po "^$2:.+" \
		| sed -e "s/$2: //"
}

set_if_notset()
{
	test "${!1}" && return 0
	info "defaulting $1 to $2"
	eval export $1="$2"
}

must_be_set()
{
	test "${!1}" || carp "$1 must be set in env"
}

only_one_set()
{
	[ "${!1}" -a "${!2}" ] && carp "only $1 or $2 can be set"
	true
}

should_be_set()
{
	test "${!1}" && return 0
	warn "$1 should be set \
		using default ${!2} from $2"
	eval export $1="${!2}"
}

check_settings()
{
	set_if_notset "KARCH" "arm"
	set_if_notset "DEBARCH" "armhf"
	set_if_notset "DEBARCHES" "armhf armel"
	set_if_notset "HOST_PPA" "ppa:linaro-maintainers/staging-overlay"

	must_be_set "kernel_repo"
	must_be_set "kernel_branch"
	must_be_set "linaro_ubuntu_packaging_repo"
	must_be_set "linaro_ubuntu_packaging_branch"
	must_be_set "ubuntu_and_base_config_repo"
	must_be_set "ubuntu_and_base_config_branch"
	must_be_set "distribution"
	must_be_set "SOCFLAVOUR"
	must_be_set "SOCVENDOR"
	must_be_set "SOCFAMILY"
	must_be_set "SAMPLEBOARDS"
	must_be_set "nearby_git"

	should_be_set "board_config_repo" "kernel_repo"
	should_be_set "board_config_branch" "kernel_branch"

	# use frag or frags but never both
	only_one_set "linaro_base_config_frag" "linaro_base_config_frags"
	only_one_set "ubuntu_config_frag" "ubuntu_config_frags"
	only_one_set "board_config_frag" "board_config_frags"

	# set *_frag to defaults
	: ${linaro_base_config_frag:="linaro/configs/linaro-base.conf"}
	: ${ubuntu_config_frag:="linaro/configs/ubuntu.conf"}
	: ${board_config_frag:="linaro/configs/$(echo $SOCFLAVOUR | sed 's/l\?lt-//').conf"}


	# default *_frags to *_frag and then only use them from here on out
	: ${linaro_base_config_frags:="$linaro_base_config_frag"}
	: ${ubuntu_config_frags:="$ubuntu_config_frag"}
	: ${board_config_frags:="$board_config_frag"}

	: ${third_version_digit:=0}
}

cleanup_previous_build()
{
	test -d out || mkdir out
	rm -rf out/*
}

#
# NB - Everything from here until noted runs in kernel_build/linux directory
#
setup_kernel_git()
{
	if [ -d kernel_build/linux ]; then
		mv kernel_build/linux linux-tmp
		rm -rf kernel_build/*
		mv linux-tmp kernel_build/linux
	else
		test -d kernel_build || mkdir kernel_build
		git clone $nearby_git kernel_build/linux
	fi
	cd kernel_build/linux
	for r in $(git remote)
	do
		git remote rm $r
	done
	kernel_desc="Kernel"
	git remote add -t $kernel_branch \
		kernel_remote \
		$kernel_repo
	test "$boot_wrapper_repo" && {
		boot_wrapper_desc="Boot Wrapper"
		rm -rf ../boot-wrapper
		cd ..
			git clone $boot_wrapper_repo boot-wrapper
			cd boot-wrapper
			git branch -r
				git checkout origin/$boot_wrapper_branch
		}
	cd ../linux
	linaro_ubuntu_packaging_desc="Packaging template"
	git remote add -t $linaro_ubuntu_packaging_branch \
		linaro_ubuntu_packaging_remote \
		$linaro_ubuntu_packaging_repo
	board_config_desc="Board config fragment"
	git remote add -t $board_config_branch \
		board_config_remote \
		$board_config_repo
	ubuntu_and_base_config_desc="Ubuntu and Linaro Base config fragments"
	git remote add -t $ubuntu_and_base_config_branch \
		ubuntu_and_base_config_remote \
		$ubuntu_and_base_config_repo
	git remote update
	git checkout -f working_branch ||
	git checkout -b working_branch \
		remotes/kernel_remote/$kernel_branch
	git reset --hard remotes/kernel_remote/$kernel_branch
	git clean -d -f -x
	test -d debian && git rm -r debian
	test -d debian.linaro && git rm -r debian.linaro
	test "$boot_wrapper_repo" && {
		test -d linaro/boot-wrapper && rm -rf linaro/boot-wrapper
		test -d linaro || mkdir linaro
		cp -a ../boot-wrapper/ linaro
		rm -rf linaro/boot-wrapper/.git
		git add linaro/boot-wrapper/
	}
	git checkout remotes/linaro_ubuntu_packaging_remote/$linaro_ubuntu_packaging_branch \
		-- \
		debian \
		debian.linaro
	git checkout remotes/board_config_remote/$board_config_branch \
		-- \
		$board_config_frags
	git checkout remotes/ubuntu_and_base_config_remote/$ubuntu_and_base_config_branch \
		-- \
		$linaro_base_config_frags \
		$ubuntu_config_frags
	git commit -s -m "added packaging template and linaro/configs"
}

fixup_socflavour_contents()
{
	for f in $(grep -r -l SOCFLAVOUR debian.linaro/) debian/rules.d/0-common-vars.mk
	do
		sed -i -e "s/SOCFLAVOUR/$SOCFLAVOUR/g" $f
		git add -f $f
	done
}

fixup_kernel_version()
{
	test "$auto_kernel_version" = "true" && {
		kv=`head -n 1 Makefile |tail -n 1 | cut -d' ' -f3`
		kpl=`head -n 2 Makefile |tail -n 1 | cut -d' ' -f3`
		ksl=`head -n 3 Makefile |tail -n 1 | cut -d' ' -f3`
		kev=`head -n 4 Makefile |tail -n 1 | cut -d' ' -f3 | tr -s "-" "~"`
		sed -i \
			-e "1 s/.*/linux-linaro-SOCFLAVOUR-${kv}.${kpl} (${kv}.${kpl}.0-1.1ubuntu1) UNRELEASED; urgency=low/" \
			debian.linaro/changelog
	}
	true
}

fixup_socflavour_filenames()
{
	for f in $(find debian.linaro/ -type f -name '*SOCFLAVOUR*')
	do
		nf=$(echo $f | sed "s/SOCFLAVOUR/$SOCFLAVOUR/")
		git mv $f $nf
	done
}

fixup_vars_file()
{
	f=debian.linaro/control.d/vars.linaro-$SOCFLAVOUR
	sed -i \
		-e "s/SOCFLAVOUR/$SOCFLAVOUR/g" \
		-e "s/SOCVENDOR/$SOCVENDOR/g" \
		-e "s/SOCFAMILY/$SOCFAMILY/g" \
		-e "s/SAMPLEBOARDS/$SAMPLEBOARDS/g" \
		$f
	git add $f
}

add_config()
{
	: ${linaroconfigsboardflavour:=$(echo $SOCFLAVOUR | sed s/lt-//)}
	ARCH=$KARCH ./scripts/kconfig/merge_config.sh \
		$linaro_base_config_frags \
		$ubuntu_config_frags \
		$board_config_frags
	for a in $DEBARCHES
	do
		cp .config debian.linaro/config/$a/config.flavour.linaro-$SOCFLAVOUR
	done
	fdr clean
	fdr defaultconfigs
	git add debian.linaro/config
}

sourceinfo()
{
	for r in \
		kernel \
		boot_wrapper \
		board_config \
		ubuntu_and_base_config \
		linaro_ubuntu_packaging
	do
		desc=${r}_desc
		test "${!desc}" || continue
		repo=${r}_repo
		branch=${r}_branch
		remote=${r}_remote
		echo
		echo "${!desc}:"
		echo "Repo: ${!repo}:"
		echo "Branch: ${!branch}:"
		case $r in
			board_config)
				echo "Config frag:"
				echo "  $board_config_frags"
				;;
			ubuntu_and_base_config)
				echo "Config frags:"
				echo "  $linaro_base_config_frags"
				echo "  $ubuntu_config_frags"
				;;
		esac
		echo "Head:"
		if [ "$r" == "boot_wrapper" ]; then
			git --git-dir=../boot-wrapper/.git log -1 | sed 's/^/    /'
		else
			git log -1 "${remote}/${!branch}" | sed 's/^/    /'
		fi
	done
}

finish_changelog_and_tag()
{
	dch -c debian.linaro/changelog -t \
		"Packaged version of $SOCFLAVOUR kernel created from:==BODY=="
	perl -i -pe "s#==BODY==#$(sourceinfo | sed -e 's/#/\\#/' -e 's/^/    /')#" \
			debian.linaro/changelog
	sed -i \
		-e "1 s/)/~ci+$(date --utc +%y%m%d%H%M%S))/" \
		-e "s/UNRELEASED/$distribution/" \
			debian.linaro/changelog
	git add debian.linaro/changelog

	git_tag_publish_url="ssh://gh/jcrigby/linaro-ci-kernels.git"
	git_ro_url=$(echo $git_tag_publish_url | sed s#ssh://gh/#git://github.com/#)
	release_tag=$(changelog_get debian.linaro/changelog "Source")
	release_tag+="_"
	release_tag+=$(changelog_get debian.linaro/changelog "Version")
	release_tag=$(echo $release_tag | sed 's/~/--/g')
	sed -i \
		-e "s#Vcs-Git:.*\$#Vcs-Git: $git_ro_url $release_tag#" \
			debian.linaro/control.stub.in
	git add debian.linaro/control.stub.in
	fdr clean
	git add -f debian/changelog debian/control

	git commit -s -m "LINARO: instantiate packaging template for $SOCFLAVOUR"
	git tag $release_tag

	git clean -d -f -x
	git reset --hard HEAD
	true
}

create_source_pkg()
{
	fdr clean
	dpkg-buildpackage -sa -S $keyarg -I -i -a$DEBARCH
}

publish_release_tag()
{
	# set up ssh for pushing to github
	grep -q 'Host gh' ~/.ssh/config || {
		test -e ../../default_git_publish_keyfile && {
			mv ../../default_git_publish_keyfile ~/.ssh/ghkey
			cat <<-__END__ >> ~/.ssh/config
				Host gh
				        Hostname github.com
				        User git
				        IdentityFile ~/.ssh/ghkey
				        StrictHostKeyChecking=no
__END__
		}
	}
	chmod 600 ~/.ssh/config ~/.ssh/ghkey
	git remote add tagpush $git_tag_publish_url
	git push tagpush $release_tag
	true
}

install_custom_toolchain()
{
	test -d ${TCBINDIR} && return 0
	test -z "${toolchain_url}" && return 0
	TOOLCHAIN=`basename ${toolchain_url}`
	test -f ${TOOLCHAIN} || curl -sSO ${toolchain_url}
	mkdir -p `dirname ${TCBINDIR}`
	tar --strip-components=1 -C `dirname ${TCBINDIR}` -xf ${TOOLCHAIN}
	true
}

sbuild_source_pkg()
{
	#this is hacky as it relies on source having been built too - best to get from repo
        cd ../kernel_build/linux
	packagename=$(changelog_get debian.linaro/changelog "Source")
	packagetag=$(changelog_get debian.linaro/changelog "Version")
	cd ../../out
	# -uc & -us are not regognised by sbuild. but -k is
	keyarg=`echo $keyarg | sed -e 's/-uc//g;s/-us//g'`
	#arch-specific hackery: arm64 chroot cannot be exactly the same
	sbuild --host $DEBARCH --chroot-system-setup-commands="configchrootforarch $DEBARCH" --nolog -d $distribution $keyarg -c ${distribution}-amd64-sbuild ${packagename}_${packagetag}
}

test_build_source_pkg()
{
	mkdir test_build
	cd test_build
	dpkg-source -x ../*.dsc linux
	cd linux
	test "$use_ccache" = "true" && {
		sed -i -e 's/CC.*=.*CROSS_COMPILE.gcc/CC = ccache $(CROSS_COMPILE)gcc/' Makefile
	}
	for mk in debian.linaro/rules.d/*.mk
	do
		sed -i -e 's/do_tools.*=.*/do_tools = false/' $mk
	done
	# pass --set-envvar and --prepend-path to set the toolchain directory path
	# for some reason, --prepend-path isn't always enough
	time debuild --set-envvar=PATH="$TCBINDIR:$DEFAULT_PATH" --prepend-path=$TCBINDIR --no-lintian -us -uc -b -a$DEBARCH -j`getconf _NPROCESSORS_ONLN`
	cd ..
	cp -a *.deb *.changes ../../out
	cd ..
}

download_license_protected_file()
{
	local url=$1
	test -f ./linaro-license-protection/tests/license_protected_file_downloader.py || {
		for tries in $(seq 10)
		do
			python -m html2text < /dev/null > /dev/null  && break
			sudo apt-get -y install python-html2text python-beautifulsoup || echo "apt-get install failed, will try again"
			sleep 30
		done
		bzr branch lp:linaro-license-protection
	}
	python ./linaro-license-protection/tests/license_protected_file_downloader.py $url
}

# NB this must match the path munging that happens when the hwpack is copied to s.l.o
# and the subsequently reshuffled
#
get_hwpack_url()
{
	url="http://snapshots.linaro.org/kernel-hwpack/"
	url+=${JOB_NAME/./_}/
	url+=${JOB_NAME}/
	url+=$BUILD_NUMBER/$1
	echo $url
}

get_binary_url()
{
	url="http://snapshots.linaro.org/$distribution/images/nano"
	wget -q -k --no-cookies \
		--header "Cookie: redirectlicensephp=200" \
		$url \
		-O tmp.html
	url=$(grep -Po "(?<=href=\").*$distribution.images.nano.*/\d\d\d?/?(?=\">)" tmp.html | sort -V | tail -1)
	wget -q -k --no-cookies \
		--header "Cookie: redirectlicensephp=200" \
		$url \
		-O tmp.html
	rootfs_url=$(grep -Po '(?<=href=").*gz(?=">)' tmp.html)
	pvar "rootfs_url" >> $OUT/jobinfo.sh
}

create_artifact_dir()
{
	# save source pkg files to be ssh uploaded
	OUT=out/$BUILD_NUMBER
	mkdir -p $OUT
	cat /dev/null > $OUT/jobinfo.sh
	pvar "release_tag" >> $OUT/jobinfo.sh
}


create_hwpack()
{
	# map job_flavour to hwpack_flavour
	#
	job_flavour=${job_flavour/-3.?}
	case $job_flavour in
		arndale)
			hwpack_flavour="arndale"
			;;
		highbank)
			hwpack_flavour="highbank"
			;;
		lt-omap|llt-omap)
			hwpack_flavour="lt-panda"
			;;
		lt-origen|llt-origen)
			hwpack_flavour="leb-origen"
			;;
		lt-u8500|llt-u8500)
			hwpack_flavour="lt-snowball"
			;;
		lt-vexpress|llt-vexpress)
			hwpack_flavour="lt-vexpress"
			;;
		omap)
			hwpack_flavour="panda"
			;;
		origen)
			hwpack_flavour="origen"
			;;
		u8500)
			hwpack_flavour="snowball"
			;;
		vexpress)
			hwpack_flavour="vexpress"
			;;
		vexpress64)
			hwpack_flavour="vexpress64"
			device_type="vexpress64"
			;;
		*)
			carp "unsupported job flavour '$job_flavour'"
			;;
	esac

	url="http://snapshots.linaro.org/$distribution/hwpacks/$hwpack_flavour/latest"
	wget -q -k --no-cookies \
		--header "Cookie: redirectlicensephp=200" \
		$url \
		-O tmp.html
	url=$(grep -Po '(?<=href=")http:.*tar.gz(?=">)' tmp.html)
	rm tmp.html
	orig_hwpack=$(basename $url)
	#wget -N -q $url
	download_license_protected_file $url
	kernel_deb=$(ls kernel_build/test_build/linux-image*.deb | grep -v dbg)
	rm -rf linaro-image-tools
	bzr branch lp:linaro-image-tools
	new_hwpack_url=$(python ./linaro-image-tools/linaro-hwpack-replace \
		-t $orig_hwpack \
		-p $kernel_deb \
		-r linux-image \
		-n $BUILD_NUMBER)
	hwpack_name=$(basename $new_hwpack_url)

	# save for upload
	mv $hwpack_name $OUT
	pvar "hwpack_name" >> $OUT/jobinfo.sh
	hwpack_url=$(get_hwpack_url $hwpack_name)
	pvar "hwpack_url" >> $OUT/jobinfo.sh
	get_binary_url
	true
}

export scriptname=$(basename $0)
trap cleanup EXIT

# default actions are different if running under
# jenkins vs standalone

running_standalone="true"
unset running_in_jenkins

test "$JENKINS_URL" && {
	running_in_jenkins="true"
	unset running_standalone
	do_merge="true"
	do_create_source_pkg="true"
	do_publish_release_tag="true"
	do_test_build_source_pkg="true"
	do_publish_source_pkg="true"
	do_create_artifact_dir="true"
	do_create_hwpack="false"
}

test "$running_standalone" == "true" && {
	do_merge="true"
	do_create_source_pkg="true"
	BUILD_NUMBER=$$
}

# process command line arguments
#
keyarg="-us -uc"
while (( $# > 0 )) ; do
	echo arg is $1
	case $1 in
		-k)
			keyarg="-k$2"
			shift
			;;
		-cfg|--cfg)
			echo "sourcing $2"
			source $2
			job_flavour=${2/.cfg/}
			shift
			;;
		+([[:alpha:]_])*([[:word:]])=+([-[:word:]:/.]))
			eval "export $1"
			;;
		-h|--help|-?)
			usage
			;;
		*)
			usage
			;;
	esac
	shift
done

check_settings

test "$config_check_only" == "true" && {
	exit 0
}

test "$do_merge" == "true" && {
	cleanup_previous_build
	setup_kernel_git	# cd's to kernel_build/linux
	fixup_kernel_version
	fixup_socflavour_contents
	fixup_socflavour_filenames
	fixup_vars_file
	add_config
	finish_changelog_and_tag
	cd ../..
}

test "$do_create_source_pkg" == "true" && {
	cd kernel_build/linux
	create_source_pkg
	cd ..
	cp -a *.tar.gz *.dsc *.changes ../out
	cd ..
}

test "$do_publish_release_tag" == "true" && {
	cd kernel_build/linux
	publish_release_tag
	cd ../..
}

test "$do_test_build_source_pkg" == "true" && {
	install_custom_toolchain
	cd kernel_build
	test_build_source_pkg
	cd ..
}

test "$do_publish_source_pkg" == "true" && {
	if [ "$keyarg" == "-us -uc" ]; then
		echo "skip publish_source_pkg, key not provided"
	else
		cd kernel_build
		dput $HOST_PPA *.changes
		cd ..
	fi
}

test "$do_sbuild_source_pkg" == "true" && {
	cd out
	sbuild_source_pkg
	cd ..
}

test "$do_publish_binary_pkg" == "true" && {
	if [ "$keyarg" == "-us -uc" ]; then
		echo "skip publish_source_pkg, key not provided"
	else
		cd out
		dput $HOST_PPA *$DEBARCH.changes
		cd ..
	fi
}

test "$do_create_artifact_dir" == "true" && {
	create_artifact_dir
}

test "$do_create_hwpack" == "true" && {
	create_hwpack || echo "create_hwpack failed but not failing job"
}

exit 0