#!/bin/bash set -x set -e shopt -s extglob # add directory containing this script to PATH # export DEFAULT_PATH="/usr/sbin:/usr/bin:/sbin:/bin" export CIROOTDIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" export TCBINDIR="$HOME/tc/bin" export TOOLCHAIN="gcc-linaro-aarch64-linux-gnu-4.7-2012.12-20121214_linux.tar.bz2" export PATH="$CIROOTDIR/scripts:$CIROOTDIR/configs:$PATH" # exit with error message and bad exit status # carp() { echo $1 exit -1 } # warn about something but do not exit # warn() { echo "WARNING: $1" true } # exit with message and good exit status # infoexit() { echo -e $1 exit 0 } info() { echo $1 true } # echo message if being verbose # verbose_info() { test "$verbose" && echo $1 true } cleanup() { : } usage() { echo "package_kernel var=bar0 --cfg pkgingconfig.cfg [-c pkg.cfg] var=bar1" echo "set vars with var=value later settings will override earlier onces" echo "or use config files these are sourced and do not need full path if" echo "they are in PATH and executable" exit 1 } pvar() { echo "$1=${!1}" } fdr() { fakeroot ./debian/rules $@ } changelog_get() { dpkg-parsechangelog -l$1 \ | grep -Po "^$2:.+" \ | sed -e "s/$2: //" } set_if_notset() { test "${!1}" && return 0 info "defaulting $1 to $2" eval "export $1=$2" } must_be_set() { test "${!1}" || carp "$1 must be set in env" } only_one_set() { [ "${!1}" -a "${!2}" ] && carp "only $1 or $2 can be set" true } should_be_set() { test "${!1}" && return 0 warn "$1 should be set \ using default ${!2} from $2" eval "export $1=${!2}" } check_settings() { set_if_notset "KARCH" "arm" set_if_notset "DEBARCH" "armhf" set_if_notset "DEBARCHES" "armhf armel" set_if_notset "HOST_PPA" "ppa:linaro-maintainers/staging-overlay" must_be_set "kernel_repo" must_be_set "kernel_branch" must_be_set "linaro_ubuntu_packaging_repo" must_be_set "linaro_ubuntu_packaging_branch" must_be_set "ubuntu_and_base_config_repo" must_be_set "ubuntu_and_base_config_branch" must_be_set "distribution" must_be_set "SOCFLAVOUR" must_be_set "SOCVENDOR" must_be_set "SOCFAMILY" must_be_set "SAMPLEBOARDS" must_be_set "nearby_git" should_be_set "board_config_repo" "kernel_repo" should_be_set "board_config_branch" "kernel_branch" # use frag or frags but never both only_one_set "linaro_base_config_frag" "linaro_base_config_frags" only_one_set "ubuntu_config_frag" "ubuntu_config_frags" only_one_set "board_config_frag" "board_config_frags" # set *_frag to defaults : ${linaro_base_config_frag:="linaro/configs/linaro-base.conf"} : ${ubuntu_config_frag:="linaro/configs/ubuntu.conf"} : ${board_config_frag:="linaro/configs/$(echo $SOCFLAVOUR | sed 's/l\?lt-//').conf"} # default *_frags to *_frag and then only use them from here on out : ${linaro_base_config_frags:="$linaro_base_config_frag"} : ${ubuntu_config_frags:="$ubuntu_config_frag"} : ${board_config_frags:="$board_config_frag"} : ${third_version_digit:=0} } cleanup_previous_build() { test -d out && rm -rf out mkdir out } # # NB - Everything from here until noted runs in kernel_build/linux directory # setup_kernel_git() { test -d kernel_build/linux && { mkdir new_kernel_build mv kernel_build/linux new_kernel_build rm -rf kernel_build mv new_kernel_build kernel_build } test -d kernel_build/linux || { mkdir -p kernel_build/linux git clone $nearby_git kernel_build/linux } cd kernel_build/linux for r in $(git remote) do git remote rm $r done kernel_desc="Kernel" git remote add -t $kernel_branch \ kernel_remote \ $kernel_repo test "$boot_wrapper_repo" && { boot_wrapper_desc="Boot Wrapper" rm -rf ../boot-wrapper cd .. git clone $boot_wrapper_repo boot-wrapper cd boot-wrapper git branch -r git checkout origin/$boot_wrapper_branch } cd ../linux linaro_ubuntu_packaging_desc="Packaging template" git remote add -t $linaro_ubuntu_packaging_branch \ linaro_ubuntu_packaging_remote \ $linaro_ubuntu_packaging_repo board_config_desc="Board config fragment" git remote add -t $board_config_branch \ board_config_remote \ $board_config_repo ubuntu_and_base_config_desc="Ubuntu and Linaro Base config fragments" git remote add -t $ubuntu_and_base_config_branch \ ubuntu_and_base_config_remote \ $ubuntu_and_base_config_repo git remote update git checkout -f working_branch || git checkout -b working_branch \ remotes/kernel_remote/$kernel_branch git reset --hard remotes/kernel_remote/$kernel_branch git clean -d -f -x test -d debian && git rm -r debian test -d debian.linaro && git rm -r debian.linaro test "$boot_wrapper_repo" && { test -d linaro/boot-wrapper && rm -rf linaro/boot-wrapper mkdir -p linaro cp -a ../boot-wrapper/ linaro rm -rf linaro/boot-wrapper/.git git add linaro/boot-wrapper/ } git checkout remotes/linaro_ubuntu_packaging_remote/$linaro_ubuntu_packaging_branch \ -- \ debian \ debian.linaro git checkout remotes/board_config_remote/$board_config_branch \ -- \ $board_config_frags git checkout remotes/ubuntu_and_base_config_remote/$ubuntu_and_base_config_branch \ -- \ $linaro_base_config_frags \ $ubuntu_config_frags git commit -s -m "added packaging template and linaro/configs" } fixup_socflavour_contents() { for f in $(grep -r -l SOCFLAVOUR debian.linaro/) debian/rules.d/0-common-vars.mk do sed -i -e "s/SOCFLAVOUR/$SOCFLAVOUR/g" $f git add -f $f done } fixup_kernel_version() { test "$auto_kernel_version" = "true" && { kv=$(make kernelversion) kv=${kv/.?-rc*/} kv=${kv/.0/} sed -i \ -e "1 s/.*/linux-linaro-SOCFLAVOUR-$kv ($kv.0-1.1ubuntu1) UNRELEASED; urgency=low/" \ debian.linaro/changelog } true } fixup_socflavour_filenames() { for f in $(find debian.linaro/ -type f -name '*SOCFLAVOUR*') do nf=$(echo $f | sed "s/SOCFLAVOUR/$SOCFLAVOUR/") git mv $f $nf done } fixup_vars_file() { f=debian.linaro/control.d/vars.linaro-$SOCFLAVOUR sed -i \ -e "s/SOCFLAVOUR/$SOCFLAVOUR/g" \ -e "s/SOCVENDOR/$SOCVENDOR/g" \ -e "s/SOCFAMILY/$SOCFAMILY/g" \ -e "s/SAMPLEBOARDS/$SAMPLEBOARDS/g" \ $f git add $f } add_config() { : ${linaroconfigsboardflavour:=$(echo $SOCFLAVOUR | sed s/lt-//)} ARCH=$KARCH ./scripts/kconfig/merge_config.sh \ $linaro_base_config_frags \ $ubuntu_config_frags \ $board_config_frags for a in $DEBARCHES do cp .config debian.linaro/config/$a/config.flavour.linaro-$SOCFLAVOUR done fdr clean fdr defaultconfigs git add debian.linaro/config } sourceinfo() { for r in \ kernel \ boot_wrapper \ board_config \ ubuntu_and_base_config \ linaro_ubuntu_packaging do desc=${r}_desc test "${!desc}" || continue repo=${r}_repo branch=${r}_branch remote=${r}_remote echo echo "${!desc}:" echo "Repo: ${!repo}:" echo "Branch: ${!branch}:" case $r in board_config) echo "Config frag:" echo " $board_config_frags" ;; ubuntu_and_base_config) echo "Config frags:" echo " $linaro_base_config_frags" echo " $ubuntu_config_frags" ;; esac echo "Head:" if [ "$r" == "boot_wrapper" ]; then git --git-dir=../boot-wrapper/.git log -1 | sed 's/^/ /' else git log -1 "${remote}/${!branch}" | sed 's/^/ /' fi done } finish_changelog_and_tag() { dch -c debian.linaro/changelog -t \ "Packaged version of $SOCFLAVOUR kernel created from:==BODY==" perl -i -pe "s#==BODY==#$(sourceinfo | sed -e 's/#/\\#/' -e 's/^/ /')#" \ debian.linaro/changelog sed -i \ -e "1 s/)/~ci+$(date --utc +%y%m%d%H%M%S))/" \ -e "s/UNRELEASED/$distribution/" \ debian.linaro/changelog git add debian.linaro/changelog git_tag_publish_url="ssh://gh/jcrigby/linaro-ci-kernels.git" git_ro_url=$(echo $git_tag_publish_url | sed s#ssh://gh/#git://github.com/#) release_tag=$(changelog_get debian.linaro/changelog "Source") release_tag+="_" release_tag+=$(changelog_get debian.linaro/changelog "Version") release_tag=$(echo $release_tag | sed 's/~/--/g') sed -i \ -e "s#Vcs-Git:.*\$#Vcs-Git: $git_ro_url $release_tag#" \ debian.linaro/control.stub.in git add debian.linaro/control.stub.in fdr clean git add -f debian/changelog debian/control git commit -s -m "LINARO: instantiate packaging template for $SOCFLAVOUR" git tag $release_tag git clean -d -f -x git reset --hard HEAD true } create_source_pkg() { fdr clean dpkg-buildpackage -sa -S $keyarg -I -i -a$DEBARCH } publish_release_tag() { # set up ssh for pushing to github grep -q 'Host gh' ~/.ssh/config || { test -e ../../default_git_publish_keyfile && { mv ../../default_git_publish_keyfile ~/.ssh/ghkey cat <<-__END__ >> ~/.ssh/config Host gh Hostname github.com User git IdentityFile ~/.ssh/ghkey StrictHostKeyChecking=no __END__ } } chmod 600 ~/.ssh/config ~/.ssh/ghkey git remote add tagpush $git_tag_publish_url git push tagpush $release_tag true } install_custom_toolchain() { test -d ${TCBINDIR} && return 0 mkdir -p `dirname ${TCBINDIR}` tar --strip-components=1 -C `dirname ${TCBINDIR}` -xf ${TOOLCHAIN} true } test_build_source_pkg() { mkdir -p test_build cd test_build dpkg-source -x ../*.dsc linux cd linux test "$use_ccache" = "true" && { sed -i -e 's/CC.*=.*CROSS_COMPILE.gcc/CC = ccache $(CROSS_COMPILE)gcc/' Makefile } for mk in debian.linaro/rules.d/*.mk do sed -i -e 's/do_tools.*=.*/do_tools = false/' $mk done # pass --set-envvar and --prepend-path to set the toolchain directory path # for some reason, --prepend-path isn't always enough time debuild --set-envvar=PATH="$TCBINDIR:$DEFAULT_PATH" --prepend-path=$TCBINDIR --no-lintian -us -uc -b -a$DEBARCH -j`getconf _NPROCESSORS_ONLN` cd .. ln *.deb *.changes ../../out cd .. } publish_source_pkg() { dput $HOST_PPA *.changes } download_license_protected_file() { local url=$1 test -f ./linaro-license-protection/tests/license_protected_file_downloader.py || { for tries in $(seq 10) do python -m html2text < /dev/null > /dev/null && break sudo apt-get -y install python-html2text python-beautifulsoup || echo "apt-get install failed, will try again" sleep 30 done bzr branch lp:linaro-license-protection } python ./linaro-license-protection/tests/license_protected_file_downloader.py $url } # NB this must match the path munging that happens when the hwpack is copied to s.l.o # and the subsequently reshuffled # get_hwpack_url() { url="http://snapshots.linaro.org/kernel-hwpack/" url+=${JOB_NAME/./_}/ url+=${JOB_NAME}/ url+=$BUILD_NUMBER/$1 echo $url } get_binary_url() { url="http://snapshots.linaro.org/$distribution/images/nano" wget -q -k --no-cookies \ --header "Cookie: redirectlicensephp=200" \ $url \ -O tmp.html url=$(grep -Po "(?<=href=\").*$distribution.images.nano.*/\d\d\d?/?(?=\">)" tmp.html | sort -V | tail -1) wget -q -k --no-cookies \ --header "Cookie: redirectlicensephp=200" \ $url \ -O tmp.html rootfs_url=$(grep -Po '(?<=href=").*gz(?=">)' tmp.html) pvar "rootfs_url" >> $OUT/jobinfo.sh } create_eula() { touch $OUT/EULA.txt } create_artifact_dir() { # save source pkg files to be ssh uploaded # #rm -rf out OUT=out/$BUILD_NUMBER mkdir -p $OUT cat /dev/null > $OUT/jobinfo.sh pvar "release_tag" >> $OUT/jobinfo.sh } create_hwpack() { # map job_flavour to hwpack_flavour # job_flavour=${job_flavour/-3.?} case $job_flavour in lt-omap|llt-omap) hwpack_flavour="lt-panda" ;; lt-origen|llt-origen) hwpack_flavour="leb-origen" ;; lt-u8500|llt-u8500) hwpack_flavour="lt-snowball" ;; lt-vexpress|llt-vexpress) hwpack_flavour="lt-vexpress" ;; omap) hwpack_flavour="panda" ;; origen) hwpack_flavour="origen" ;; u8500) hwpack_flavour="snowball" ;; vexpress) hwpack_flavour="vexpress" ;; vexpress64) hwpack_flavour="vexpress64" device_type="vexpress64" ;; *) carp "unsupported job flavour '$job_flavour'" ;; esac url="http://snapshots.linaro.org/$distribution/hwpacks/$hwpack_flavour/latest" wget -q -k --no-cookies \ --header "Cookie: redirectlicensephp=200" \ $url \ -O tmp.html url=$(grep -Po '(?<=href=")http:.*tar.gz(?=">)' tmp.html) rm tmp.html orig_hwpack=$(basename $url) #wget -N -q $url download_license_protected_file $url kernel_deb=$(ls kernel_build/test_build/linux-image*.deb | grep -v dbg) rm -rf linaro-image-tools bzr branch lp:linaro-image-tools new_hwpack_url=$(python ./linaro-image-tools/linaro-hwpack-replace \ -t $orig_hwpack \ -p $kernel_deb \ -r linux-image \ -n $BUILD_NUMBER) hwpack_name=$(basename $new_hwpack_url) # save for upload mv $hwpack_name $OUT pvar "hwpack_name" >> $OUT/jobinfo.sh hwpack_url=$(get_hwpack_url $hwpack_name) pvar "hwpack_url" >> $OUT/jobinfo.sh get_binary_url create_eula true } export scriptname=$(basename $0) trap cleanup EXIT # default actions are different if running under # jenkins vs standalone running_standalone="true" unset running_in_jenkins test "$JENKINS_URL" && { running_in_jenkins="true" unset running_standalone do_merge="true" do_create_source_pkg="true" do_publish_release_tag="true" do_test_build_source_pkg="true" do_publish_source_pkg="true" do_create_artifact_dir="true" do_create_hwpack="true" } test "$running_standalone" == "true" && { do_merge="true" do_create_source_pkg="true" BUILD_NUMBER=$$ } # process command line arguments # keyarg="-us -uc" while (( $# > 0 )) ; do echo arg is $1 case $1 in -k) keyarg="-k$2" shift ;; -cfg|--cfg) echo "sourcing $2" source $2 job_flavour=${2/.cfg/} shift ;; +([[:alpha:]_])*([[:word:]])=+([-[:word:]:/.])) eval "export $1" ;; -h|--help|-?) usage ;; *) usage ;; esac shift done check_settings test "$config_check_only" == "true" && { exit 0 } test "$do_merge" == "true" && { cleanup_previous_build setup_kernel_git # cd's to kernel_build/linux fixup_kernel_version fixup_socflavour_contents fixup_socflavour_filenames fixup_vars_file add_config finish_changelog_and_tag cd ../.. } test "$do_create_source_pkg" == "true" && { cd kernel_build/linux create_source_pkg cd .. ln *.tar.gz *.dsc *.changes ../out cd .. } test "$do_publish_release_tag" == "true" && { cd kernel_build/linux publish_release_tag cd ../.. } test "$do_test_build_source_pkg" == "true" && { test -f ${TOOLCHAIN} && install_custom_toolchain cd kernel_build test_build_source_pkg cd .. } test "$do_publish_source_pkg" == "true" && { cd kernel_build publish_source_pkg cd .. } test "$do_create_artifact_dir" == "true" && { create_artifact_dir } test "$do_create_hwpack" == "true" && { create_hwpack || echo "create_hwpack failed but not failing job" } exit 0