path: root/tcwg-make-release.yaml

# http://docs.openstack.org/infra/jenkins-job-builder/definition.html
- job:
    name: tcwg-make-release
    project-type: matrix
    defaults: global
    properties:
        - authorization:
            anonymous:
                - job-read
                - job-extended-read
            linaro:
                - job-build
                - job-cancel
        - build-discarder:
            days-to-keep: 30
            num-to-keep: 10
    parameters:
        - string:
            name: gcc_src
            default: 'gcc-linaro-5.3-2016.02.tar.xz'
            description: 'The source of GCC. Specify as it would be passed to abe.sh, either a source tarball, git branch and revision, git tag, or a URL of a tarball e.g., "gcc-linaro-5.3-2016.02.tar.xz"'
        - string:
            name: binutils_src
            default: ''
            description: 'Optional revision of binutils to use for this release, e.g., "binutils-gdb.git~linaro_binutils-2_25-branch"'
        - string:
            name: glibc_src
            default: ''
            description: 'Optional revision of Glibc to use for this release, e.g., "glibc.git~linaro/2.21/master"'
        - string:
            name: release_name
            default: ''
            description: 'Optional release string to use, e.g., "2016.05", "2016.05-rc3", "2016.05-1", "2016.05-1-rc2". The default is to extract the release string from the gcc_src field'
        - string:
            name: toolchain_config
            default: 'default'
            description: 'Toolchain version config, e.g., "default", "gcc5", or "gcc6". Relative to abe config/ dir'
        - string:
            name: target_list
            default: 'aarch64-elf aarch64-linux-gnu aarch64_be-elf aarch64_be-linux-gnu armv8l-linux-gnueabihf arm-eabi arm-linux-gnueabi arm-linux-gnueabihf armeb-eabi armeb-linux-gnueabi armeb-linux-gnueabihf'
            description: 'List of targets to use'
        - string:
            name: builder_arch_list
            default: 'i386 amd64'
            description: 'List of builder architectures to use (i386,amd64)'
        - string:
            name: abe_branch
            default: refs/heads/master
            description: 'ABE revision to test'
        - string:
            name: scripts_branch
            default: master
            description: 'Scripts branch to use'
        - choice:
            name: debug
            choices:
                - none
                - abe
                - yaml
            description: "Level of debug information: none - No debugging information, abe - ABE debugging information, yaml - tcwg-make-release.yaml debug information (and exit)."
        - string:
            name: fileserver
            default: dev-01.tcwglab
            description: 'Remote file server for logs and binaries'
        - bool:
            name: testrelease
            default: 'true'
            description: 'Run test-release if make-release is successful'
        - string:
            name: binariesdir
            default: ''
            description: 'Optional upload dir for binaries on fileserver. If empty, a suitable value is computed by the script'
        - string:
            name: logsdir
            default: ''
            description: 'Optional upload dir for logs on fileserver. If empty, a suitable value is computed by the script'
    disabled: false
    node: tcwg-coordinator
    retry-count: 3
    concurrent: true
    display-name: 'TCWG Make Release'
    scm:
        - git:
            url: https://git-us.linaro.org/toolchain/abe.git
            refspec: +refs/changes/*:refs/remotes/changes/*
            branches:
                - ${abe_branch}
            skip-tag: true
            shallow-clone: true
            basedir: abe
    axes:
        - axis:
            type: slave
            name: label
            values:
                - tcwg-x86_64-build
        - axis:
            type: dynamic
            name: builder_arch
            values:
                - builder_arch_list
        - axis:
            type: dynamic
            name: target
            values:
                - target_list
    execution-strategy:
        sequential: false
    wrappers:
        - timeout:
            timeout: 300
        - timestamps
        - ssh-agent-credentials:
            # tcwg-buildslave user id
            users:
                - 'e0958a95-204f-4c14-a66c-5e2be6c5d50a'
        - build-name:
            name: '#${BUILD_NUMBER}-${ENV,var="release_name"}'
        - workspace-cleanup:
            include:
              - "*"
            dirmatch: true
    builders:
      - shell: |
            #!/bin/bash
            set -e

            if test x"${debug}" = x"abe"; then
              shell="/bin/bash -x"
              set -x
            else
                shell="/bin/bash"
            fi

            if test x"${debug}" = x"yaml"; then
                echo WORKSPACE:  $WORKSPACE
                echo CURRENTDIR: `pwd`
                ls -lR
                exit 0
            fi

            git clone -b $scripts_branch --depth 1 https://git-us.linaro.org/toolchain/jenkins-scripts

            # Start build container
            build_container_type=docker
            build_container_distro=trusty
            builder=$(./jenkins-scripts/nodename2hostname.sh $NODE_NAME)
            bash -x ./jenkins-scripts/start-container-${build_container_type}.sh --arch ${builder_arch} --distro ${build_container_distro} --session-host ${builder} > build-container.sh

            # Define CONTAINER, CONTAINER_CLEANUP, session_host and session_port
            . ./build-container.sh
            BUILD_CONTAINER="${CONTAINER}"
            BUILD_CONTAINER_CLEANUP="${CONTAINER_CLEANUP}"

            # Make sure to cleanup build container if something goes
            # wrong when preparing the rest of the environment
            if [ x"${BUILD_CONTAINER_CLEANUP}" != x ]; then
                CONTAINERS_CLEANUP="${BUILD_CONTAINER_CLEANUP}"
                trap "${CONTAINERS_CLEANUP}" EXIT
            fi

            tcwg_release=${release_name}
            # If there is no release_name set, extract release information from the gcc_src
            if test "${release_name:+set}" != "set"; then
                case "$gcc_src" in
                *.tar.xz)
                    tcwg_release="$(basename $gcc_src | cut -d '-' -f4-)"
                    tcwg_release=${tcwg_release%".tar.xz"}
                    ;;
                gcc.git~*)
                    tcwg_release="$(echo $gcc_src | grep -o -e '-\(20[0-9]\{2\}\.[01][0-9].*\)' )"
                    tcwg_release=${tcwg_release#-}
                    ;;
                *)
                    echo "ERROR: Could not find tcwg release"
                    exit 1
                    ;;
                esac
            fi

            if [ x"${tcwg_release}" = x"" ]; then
                echo "ERROR: Could not find tcwg release"
                exit 1
            fi

            # This job always uses Canadian cross builds and builds 2
            # toolchains at a time when run on a amd64 slave.
            # When run on a i386 host, no need to duplicate the
            # Canadian cross build for mingw.
            canadian=
            case ${builder_arch} in
              amd64)
                host=x86_64
                canadian=--canadian ;;
              i386)
                host=i686
                canadian= ;;
             *)
                echo "ERROR: unsupported label: ${label}"
                exit 1
                ;;
            esac

            # FIXME. MakeRelease copies to a directory, while
            # TestRelease downloads from a URL, so both dirs differ
            # depending on httpd's DocumentRoot
            # Setting url_prefix can be removed once test-release is
            # started after make-release has completed.
            case ${binariesdir} in
              releases/binaries/*)
                url_prefix=$(echo \~${USER}/${binariesdir})
                ;;
              "")
                # Compute default binariesdirs/logsdir in case they
                # are not supplied by the user. Normally they are set
                # by tcwg-make-and-test-release.

                # Extract the gcc_version information from gcc_src to determine
                # which directory to copy the tarballs into.
                case "$gcc_src" in
                    *.tar.*) # src tarballs usually named like this
                        gcc_version=$(basename $gcc_src | awk -F '-' '{ print $3 }')
                        ;;
                    *.git~*) # building from git branch or tag
                        gcc_version=$(basename $gcc_src | awk -F '-' '{ print $2 }')
                        ;;
                    *)
                        echo "ERROR: could not extract version from gcc_src"
                        exit 1
                        ;;
                esac
                if ! echo "${gcc_version}" | grep -xEq '[0-9]+\.[0-9]+(\.[0-9]+)?'; then
                    echo "ERROR: gcc version '${gcc_version}' invalid"
                    exit 1
                fi

                gcc_major="`echo $gcc_version | awk -F '.' '{ print $1 }'`"
                gcc_minor="`echo $gcc_version | awk -F '.' '{ print $2 }'`"
                tcwg_version="${gcc_major}.${gcc_minor}"
                if test ${gcc_major} -lt 5; then
                    gcc_point="`echo $gcc_version | awk -F '.' '{ print $2 }'`"
                    tcwg_version="${gcc_major}.${gcc_minor}.${gcc_point}"
                fi

                buildid=`printf '%04d' ${BUILD_NUMBER}`

                binariesdir=releases/binaries/${tcwg_version}-${tcwg_release}-${buildid}
                logsdir=releases/logs/${tcwg_version}-${tcwg_release}-${buildid}

                url_prefix=$(echo \~${USER}/${binariesdir})
                ;;
              *)
                echo Unsupported binariesdir: ${binariesdir}. Fix the job.
                ;;
            esac

            ${BUILD_CONTAINER} "cd ${WORKSPACE} && ${shell} ${WORKSPACE}/jenkins-scripts/MakeRelease.job --abedir ${WORKSPACE}/abe --workspace ${WORKSPACE} ${canadian} --target ${target}${tcwg_release:+ --release_name ${tcwg_release}} --fileserver ${fileserver} --toolchainconfig ${toolchain_config} --binariesdir ${binariesdir}/${target} --logsdir ${logsdir}/${target} --buildnumber ${BUILD_NUMBER} ${gcc_src:+ --gcc ${gcc_src}} ${binutils_src:+ --binutils ${binutils_src}} ${glibc_src:+ --glibc ${glibc_src}}"
            if test $? -gt 0; then
              exit 1
            fi

            # tarballs are written in ${WORKSPACE}/snapshots, which is
            # not shared between host and container. So, we need to
            # execute the find command inside the container.

            if test x"${canadian}" = x"true"; then
                linux_artifacts=artifacts1.txt
                mingw_artifacts=artifacts2.txt
            else
                linux_artifacts=artifacts2.txt
            fi

            # We always build a Linux tarball, even in canadian cross mode
            tarball_linux=$(grep "^toolchain=" ${WORKSPACE}/${linux_artifacts} | cut -d = -f 2-)
            if test x"${tarball_linux}" = x"" -o ! -f "${tarball_linux}"; then
              echo "ERROR: no binary Linux tarball found!"
              exit 1
            fi

            if test x"${canadian}" = x"true"; then
              tarball_mingw=$(grep "^toolchain=" ${WORKSPACE}/${mingw_artifacts} | cut -d = -f 2-)
              if test x"${tarball_mingw}" = x"" -o ! -f "${tarball_mingw}"; then
                echo "ERROR: no binary MingW tarball found!"
                exit 1
              fi
            fi

            if $testrelease; then
              tarball_linux="`basename ${tarball_linux}`"
              tarball_linux_url="http://${fileserver}/${url_prefix}/${target}/${tarball_linux}"
              cat << EOF > tcwg.params.linux
            tarball_url=${tarball_linux_url}
            release=${tcwg_release}
            target=${host}-${target}
            EOF

              if test x"${canadian}" = x"true"; then
                tarball_mingw="`basename ${tarball_mingw}`"
                tarball_mingw_url="http://${fileserver}/${url_prefix}/${target}/${tarball_mingw}"
                cat << EOF > tcwg.params.mingw
            tarball_url=${tarball_mingw_url}
            release=${tcwg_release}
            target=mingw-${target}
            EOF
              fi
            fi

      - conditional-step:
          condition-kind: file-exists
          condition-filename: tcwg.params.linux
          steps:
              - trigger-builds:
                  - project: 'tcwg-test-release'
                    property-file: tcwg.params.linux
                    current-parameters: true
                    block: false
      - conditional-step:
          condition-kind: file-exists
          condition-filename: tcwg.params.mingw
          steps:
              - trigger-builds:
                  - project: 'tcwg-test-release'
                    property-file: tcwg.params.mingw
                    current-parameters: true
                    block: false
    publishers:
        - archive:
            artifacts: 'MakeRelease*.log.xz'
            latest-only: false
        - email:
            recipients: 'ryan.arnold@linaro.org'
        - email-ext:
            recipients: 'christophe.lyon@linaro.org'
            send-to:
              requester
              recipients