tcwg_gnu_cross.yaml

# Auto generated by ./tcwg/generate-yamlfiles.sh from tcwg_gnu.yaml.in and tcwg_gnu/tcwg_gnu_cross.def. Do not edit.
# Beware: git branch names need the 'refs/heads/' prefix, while tags need 'refs/tags/'
- project:
    name: tcwg_gnu_cross
    type_of_test: ""
    timeout_build: 1000
    # By default, run each CI loop daily.
    cron_master: 'H H * * *'
    docker_security: ''
    ci_project:
      - tcwg_gnu_cross_build
      - tcwg_gnu_cross_check_binutils
      - tcwg_gnu_cross_check_gcc
    binutils_url: git://sourceware.org/git/binutils-gdb.git
    gcc_url: git+ssh://linaroci@gcc.gnu.org/git/gcc.git
    glibc_url: git://sourceware.org/git/glibc.git
    gdb_url: git://sourceware.org/git/binutils-gdb.git
    qemu_url: https://gitlab.com/qemu-project/qemu.git
    target:
      - aarch64
      - arm
    node: tcwg-build && tcwg-x86_64
    jobs:
      - tcwg_gnu_cross-master

- job-group:
    name: tcwg_gnu_cross-master
    toolchain_ver: master
    cron: '{cron_master}'
    binutils_branch: master
    gcc_branch: master
    linux_url: https://gitlab.com/Linaro/lkft/mirrors/torvalds/linux-mainline.git
    linux_branch: master
    glibc_branch: master
    gdb_branch: master
    qemu_branch: master
    distro: default
    jobs:
      - '{ci_project}--{toolchain_ver}-{target}{type_of_test}-build'
      - '{ci_project}--{toolchain_ver}-{target}{type_of_test}-precommit'
      - '{ci_project}--{toolchain_ver}-{target}{type_of_test}-bisect'
    ci_config: '{toolchain_ver}-{target}{type_of_test}'

#BEGIN: tcwg/round-robin.yaml.inc
# -*- mode: Yaml -*-

#BEGIN: tcwg/default.yaml.inc
# -*- mode: Yaml -*-

- property:
    name: default-properties
    properties:
      - authorization:
          anonymous:
            - job-read
            - job-extended-read
          everyone-flat:
            - job-build
            - job-cancel
      - build-discarder:
          days-to-keep: 20

- scm:
    name: jenkins-scripts
    scm:
      - git:
          url: https://gitlab.com/Linaro/tcwg/jenkins-scripts.git
          refspec: +refs/heads/*:refs/remotes/origin/* +refs/changes/*:refs/changes/*
          branches:
            - $scripts_branch
          basedir: jenkins-scripts
          skip-tag: true
          reference-repo: /home/tcwg-buildslave/snapshots-ref/jenkins-scripts.git
          wipe-workspace: false
          clean:
            before: true
          prune: true

#END:   tcwg/default.yaml.inc

- parameter:
    name: default-parameters
    parameters:
      - string:
          name: extra_build_params
          default: ""
          description: "Extra parameters to pass to the build script; can be used to override settings extracted from ci_project/ci_config"
      - string:
          name: distro
          default: '{distro}'
          description: 'Distro image to use'
      - string:
          name: scripts_branch
          default: master
          description: 'Scripts revision to use'

- parameter:
    name: component-parameters
    parameters:
      - string:
          name: '{component}_git'
          default: '{default_git}'
          description: "{component} git_url#branch/sha1 to build, or 'default', 'baseline' or 'jenkins-scm', or pw://series/<id> to build with patches from patchwork ('precommit' mode, use 'pw://series/<id>/retrigger/' to force a rebuild)"

- parameter:
    name: all-component-parameters
    parameters:
      - component-parameters:
          component: binutils
          default_git: '{default_git}'
      - component-parameters:
          component: gcc
          default_git: '{default_git}'
      - component-parameters:
          component: gdb
          default_git: '{default_git}'
      - component-parameters:
          component: glibc
          default_git: '{default_git}'
      - component-parameters:
          component: linux
          default_git: '{default_git}'
      - component-parameters:
          component: qemu
          default_git: '{default_git}'

- parameter:
    name: build-parameters
    parameters:
      - all-component-parameters:
          default_git: '{default_git}'
      - choice:
          name: update_baseline
          choices:
            - onsuccess
            - force
            - init
            - ignore
          description: "What to do with baseline in this build"
      - choice:
          name: notify
          choices:
            - ignore
            - onregression
            - precommit
          description: "Whether to notify developers about the result of this build"
      - string:
          name: rr_stage
          default: build
          description: "Describe the current round-robin stage for this run (informative only)"
      - default-parameters:
          distro: '{distro}'

# Precommit parameters ...
# - allow only "ignore" value for update_baseline parameter;
# - allow string (which should be an email) for notify parameter.
- parameter:
    name: precommit-parameters
    parameters:
      - all-component-parameters:
          default_git: '{default_git}'
      - string:
          name: notify
          default: 'precommit'
          description: "Email to send failure notifications to"
      - default-parameters:
          distro: '{distro}'

- property:
    name: build-properties
    properties:
      - default-properties
      - build-blocker:
          blocking-jobs:
            - '{ci_project}--{ci_config}-bisect'
          queue-scanning: 'ALL'

- property:
    name: precommit-properties
    properties:
      - default-properties
      - build-blocker:
          blocking-jobs:
            - '{ci_project}--{ci_config}-build'
          queue-scanning: 'BUILDABLE'

- property:
    name: bisect-properties
    properties:
      - default-properties
      - build-blocker:
          # Run at most 1 bisect job for all configurations.
          # Bisect jobs can easily run for 5+ hours, and a few of them
          # can hog the entire TCWG build farm for a long time.  Since
          # same regression tends to appear in multiple configurations,
          # run bisections for all configurations in-order.
          blocking-jobs:
            - '{ci_project}--.*-bisect'
          queue-scanning: 'BUILDABLE'

# Beware: git branch names need the 'refs/heads/' prefix while tags
# need 'refs/tags/'.
- scm:
    name: component-scm
    scm:
        - git:
            url: '{url}'
            branches:
              - 'refs/heads/{branch}'
            basedir: '{component}'
            do-not-fetch-tags: true
            reference-repo: '/home/tcwg-buildslave/snapshots-ref/{reference}'
            refspec: '+refs/heads/{branch}:refs/remotes/origin/{branch}'
            shallow-clone: true
            wipe-workspace: false
            clean:
              before: true
            prune: true

# SCMs must start with jenkins-scripts and then be ordered alphabetically.
# Same alphabetical order must be followed in .def files.  The reason behind
# this is that jenkins sets $GIT_COMMIT to sha1 of jenkins-scripts, and then
# sets $GIT_COMMIT_1, $GIT_COMMIT_2, etc. to sha1s of component SCM /in the
# order that they are declared here/.
- scm:
    name: build-scms
    scm:
      - jenkins-scripts
      - component-scm:
          component: binutils
          url: '{binutils_url}'
          branch: '{binutils_branch}'
          reference: binutils-gdb.git
      - component-scm:
          component: gcc
          url: '{gcc_url}'
          branch: '{gcc_branch}'
          reference: gcc.git
      - component-scm:
          component: gdb
          url: '{gdb_url}'
          branch: '{gdb_branch}'
          reference: binutils-gdb.git
      - component-scm:
          component: glibc
          url: '{glibc_url}'
          branch: '{glibc_branch}'
          reference: glibc.git
      - component-scm:
          component: linux
          url: '{linux_url}'
          branch: '{linux_branch}'
          reference: linux.git
      - component-scm:
          component: qemu
          url: '{qemu_url}'
          branch: '{qemu_branch}'
          reference: qemu.git

- wrapper:
    name: build-wrappers
    wrappers:
      - timeout:
          timeout: '{timeout}'
      - timestamps
      - ssh-agent-credentials:
          # tcwg-buildslave user id
          users:
            - 'e0958a95-204f-4c14-a66c-5e2be6c5d50a'
            - 'tcwg-buildslave-20220725'
#            - 'tcwg-build-20220725'
      - credentials-binding:
          - text:
              credential-id: TCWG_JIRA_TOKEN
              variable: TCWG_JIRA_TOKEN

- builder:
    name: trigger-followup-builds-1
    builders:
      - shell: |
          #!/bin/bash
          source artifacts/jenkins/run-build.env
          set -ex

          for i in artifacts/trigger-build-* artifacts/trigger-bisect; do
            if [ -f $i ]; then
              echo "distro=$distro" >> $i
              echo "scripts_branch=$scripts_branch" >> $i
            fi
          done

          trap "" EXIT

- builder:
    name: trigger-followup-builds-2
    builders:
      - trigger-builds:
          - project: '{ci_project}--{ci_config}-build'
            parameter-factories:
              - factory: filebuild
                file-pattern: artifacts/trigger-build-*
          - project: '{ci_project}--{ci_config}-bisect'
            property-file: artifacts/trigger-bisect

- builder:
    name: run-build-notify
    builders:
      - shell: |
          #!/bin/bash
          source artifacts/jenkins/run-build.env
          set -ex

          if [ -f artifacts/failed ]; then
            # Only send notifications for forced successful
            # (aka "onregression") builds; never for failed builds.
            echo "Build failed. Using option notify=ignore."
            notify=ignore
          fi

          # Basic notification -- create entry in interesting-commits.
          # Note that we add --notify to the manifest -- we will need this
          # later in round-robin-baseline.sh when re-writing history
          # to update notifications files in interesting-commits and jira.
          build_container_exec \
            ./jenkins-scripts/rr/round-robin-notify.sh \
            @@rr[top_artifacts] artifacts --notify "$notify" \
            __stage init __verbose true \
            __build_script {build_script} \
            &> ./artifacts/jenkins/notify-init.log &

          if ! wait $!; then
            echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \
              > artifacts/jenkins/error-mail-recipients.txt
            echo -e "$BUILD_URL\nERROR: notify-init failed\n" \
              >> artifacts/jenkins/error-mail-body.txt
            echo "ERROR: notify-init failed"
            exit 1
          fi

          # Full notification
          echo "NOTE: Notify developers about this build"

          build_container_exec \
            ./jenkins-scripts/rr/round-robin-notify.sh \
            @@rr[top_artifacts] artifacts __stage full __verbose true \
            __build_script {build_script} \
            &> ./artifacts/jenkins/notify-full.log &

          if ! wait $!; then
            echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \
              > artifacts/jenkins/error-mail-recipients.txt
            echo -e "$BUILD_URL\nERROR: notify-full failed\n" \
              >> artifacts/jenkins/error-mail-body.txt
            echo "ERROR: notify-full failed"
          fi

          # Failsafe for avoid spamming developers.
          if [ "$notify" = "ignore" ] \
            && [ -f artifacts/jenkins/mail-recipients.txt ]; then
            echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \
              > artifacts/jenkins/error-mail-recipients.txt
            echo -e "$BUILD_URL\nERROR: tried to unexpectedly notify developers\n" \
              >> artifacts/jenkins/error-mail-body.txt
            mv artifacts/jenkins/mail-recipients.txt \
            artifacts/jenkins/mail-recipients.bak
          fi

          trap "" EXIT

- builder:
    name: run-exit-on-failure
    builders:
      - shell: |
          #!/bin/bash
          source artifacts/jenkins/run-build.env
          set -ex

          if [ -f artifacts/failed ]; then
            # Mark failed build as FAILURE; no further steps will run.
            exit $(cat artifacts/failed)
          fi
          # Only successful builds run beyond this point.

          trap "" EXIT

- builder:
    name: run-push-baseline
    builders:
      - shell: |
          #!/bin/bash
          source artifacts/jenkins/run-build.env
          set -ex

          # Update baseline branch of base-artifacts.
          # If we made it this far, then the build is successful
          # (possibly because it was forced to be declared successful).
          if [ "$update_baseline" = "ignore" ]; then
            trap "" EXIT
            exit 0
          fi

          # We may need jira token while rewriting history.
          if [ -f $HOME/.jipdate.yml ]; then
            (
              # BE CAREFUL WITH TCWG_JIRA_TOKEN
              set +x
              build_container_exec \
                sed -i -e "s/#TCWG_JIRA_TOKEN#/$TCWG_JIRA_TOKEN/" \
                "$HOME/.jipdate.yml"
            )
          fi

          build_container_exec \
            ./jenkins-scripts/rr/round-robin-baseline.sh \
            @@rr[top_artifacts] artifacts \
            __build_script {build_script} \
            __push_base_artifacts true \
            __rewrite_base_artifacts true \
            __commit_artifacts {commit_artifacts} \
            __rewrite_num 100 \
            &> ./artifacts/jenkins/rewrite-{commit_artifacts}.log &

          res=0 && wait $! || res=$?

          # Compress big files in artifacts/NN-* directories to save disk space
          # on ci.linaro.org.
          find artifacts/ -size +1M ! -name "*.xz" \
            -path "artifacts/[0-9]*" -print0 \
            | xargs -0 -i@ xz -T0 @

          if [ $res != 0 ]; then
            echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \
              > artifacts/jenkins/error-mail-recipients.txt
            echo -e "$BUILD_URL\nERROR: round-robin-baseline.sh failed" \
              >> artifacts/jenkins/error-mail-body.txt
            echo "ERROR: round-robin-baseline.sh failed"
            # We have failed to update the baseline, so let's fail and
            # not notify developers.
            exit 1
          fi

          trap "" EXIT

- builder:
    name: run-push-lnt-results
    builders:
     - trigger-builds:
        - project: tcwg-update-lnt-results
          predefined-parameters: |
            ci_project={ci_project}
            ci_config={ci_config}

# We place trigger-precommit-builds-2 outside of PRECOMMIT_TRIGGER_yes
# #ifdef to get an upstream/downstream links between build/precommit jobs
# in jenkins UI.
- builder:
    name: trigger-precommit-builds-2
    builders:
      - trigger-builds:
          - project: '{ci_project}--{ci_config}-precommit'
            parameter-factories:
              - factory: filebuild
                file-pattern: artifacts/jenkins/trigger-precommit-*

- builder:
    name: run-build-init
    builders:
      - shell:
          command: |
            #!/bin/bash
            set -ex
            # Delete artifacts now to avoid re-using stale artifacts if
            # below docker-run.sh fails to start up a container.
            rm -rf artifacts/
            mkdir -p artifacts/jenkins

            build_name="#$BUILD_NUMBER"
            echo "$build_name" > artifacts/jenkins/build-name

            git -C jenkins-scripts submodule update --init --remote

            if [ "{precommit}" = "True" ]; then
              cat >> artifacts/jenkins/run-build.env <<EOF
            update_baseline=ignore
            EOF
            fi

            cat >> artifacts/jenkins/run-build.env <<EOF
            build_name="$build_name"
            EOF
          unstable-return: 125

- builder:
    name: run-build-skip-if-queue
    builders:
      - shell:
          command: |
            #!/bin/bash
            source artifacts/jenkins/run-build.env
            set -ex

            # Skip SCM-triggered builds if there are other builds in
            # the queue.  We need to process all pending bisection-triggered
            # builds before fetching a new batch of upstream changes.
            # Re. skipping post-commit build if pre-commit queue is
            # not empty -- it's just a favor to pre-commit builds.
            if [ x"${{BUILD_CAUSE_SCMTRIGGER-false}}" = x"true" ] \
              || [ x"${{BUILD_CAUSE_TIMERTRIGGER-false}}" = x"true" ]; then
              for job_type in build precommit; do
                url="${{JOB_URL%-build/}}-$job_type"
                inQueue=$(curl --retry 5 --max-time 5 -s "$url/api/xml?tree=inQueue" \
                  | sed -e "s#.*<inQueue>\(.*\)</inQueue>.*#\1#")
                if [ "$inQueue" != "false" ]; then
                  echo "SKIP SCM BUILD"
                  touch artifacts/jenkins/skip-scm-build
                  build_name="$build_name-skip-scm-build"
                  echo "$build_name" > artifacts/jenkins/build-name

                  # Mark build unstable aka a skipped build
                  exit 125
                fi
              done
            fi

            trap "" EXIT
          unstable-return: 125

- builder:
    name: run-build-start-manifest
    builders:
      - shell:
          command: |
            #!/bin/bash
            source artifacts/jenkins/run-build.env
            set -ex

            docker_opts=(
              --distro $distro
              --node_labels "$NODE_LABELS"
            )

            ./jenkins-scripts/start-container-docker.sh --prefix build_ \
              "${{docker_opts[@]}}" > build_container.sh
            source build_container.sh
            trap "cleanup_all_containers" EXIT

            # Run the build script to fetch base-artifacts/, which are
            # required to fetch baseline sources to apply patches to in
            # pw-apply.sh below.  This also initializes the build manifest
            # with all below settings; we then import this manifest below.
            #
            # The parameter __finish_after makes the build script stop
            # after fetching the baseline and "__" avoids saving this parameter
            # in the manifest, so that we don't see this option in the main
            # run of the build script below.
            build_container_exec \
              ./jenkins-scripts/{build_script} \
              %%rr[top_artifacts] artifacts \
              ==rr[ci_project] '{ci_project}' \
              ==rr[ci_config] '{ci_config}' \
              $extra_build_params \
              --scripts_branch "$scripts_branch" \
              ==rr[mode] "build" \
              ==rr[update_baseline] "$update_baseline" \
              --BUILD_URL "$BUILD_URL" \
              __finish_at reset_artifacts

            echo "$rr_stage" > artifacts/rr_stage.txt

            cat >> artifacts/jenkins/run-build.env <<EOF
            docker_opts=(${{docker_opts[@]+$(printf "%q$IFS" "${{docker_opts[@]}}")}})
            source build_container.sh
            trap "cleanup_all_containers" EXIT
            EOF

            # Now that we have "trap cleanup_all_containers EXIT" in
            # run-build.env we need to reset the trap at the end of every
            # step to preserve container[s] for the next step.
            trap "" EXIT
          unstable-return: 125

- builder:
    name: run-build-process-params
    builders:
      - shell:
          command: |
            #!/bin/bash
            source artifacts/jenkins/run-build.env
            set -ex

            branch_opt=()
            # 1 to skip jenkins-scripts
            i=1
            for c in binutils gcc gdb glibc linux qemu; do
              eval "g=\$$$${{c}}_git"
              if [ x"$g" = x"default" ]; then
                if [ x"${{BUILD_CAUSE_SCMTRIGGER-false}}" = x"true" ] \
                  || [ x"${{BUILD_CAUSE_TIMERTRIGGER-false}}" = x"true" ]; then
                  g="jenkins-scm"
                else
                  g="baseline"
                fi
              fi
              if [ x"$g" = x"jenkins-scm" ]; then
                eval "gb=\$GIT_BRANCH_$i"
                gb=$(echo $gb | sed 's+origin/++')
                eval "g=\$GIT_URL_$i#$gb"
              fi
              i=$(($i+1))

              if [ x"$g" != x"baseline" ]; then
                build_name="$build_name-$c"
                branch_opt+=("==rr[${{c}}_git]" "$g")
              fi
            done

            echo "$build_name" > artifacts/jenkins/build-name

            cat >> artifacts/jenkins/run-build.env <<EOF
            build_name="$build_name"
            branch_opt=(${{branch_opt[@]+$(printf "%q$IFS" "${{branch_opt[@]}}")}})
            EOF

            trap "" EXIT
          unstable-return: 125

- builder:
    name: run-build-main
    builders:
      - shell:
          command: |
            #!/bin/bash
            source artifacts/jenkins/run-build.env
            set -ex

            # Run the build script to from the manifest created above.
            #
            # The parameter __start_at makes the build script start right
            # where the above run finished.  This allows us to use read-only
            # bind-mount for base-artifacts/, which, otherwise, may require
            # multi-gig rsync.
            build_container_exec \
              ./jenkins-scripts/{build_script} \
              @@rr[top_artifacts] artifacts \
              "${{branch_opt[@]}}" \
              __start_at reset_artifacts+ &
            res=0 && wait $! || res=$?

            # Update build_name
            if [ -f artifacts/results ]; then
              build_name="$build_name-R$(tail -n1 artifacts/results)"
            fi
            if [ "$update_baseline" != "onsuccess" ]; then
              build_name="$build_name-$update_baseline"
            fi

            if [ -f artifacts/trigger-bisect ]; then
              build_name="$build_name-trigger-bisect"
            fi
            echo "$build_name" > artifacts/jenkins/build-name

            # Everything but 0 and $INTERNAL_FAILURE is an unexpected exit code,
            # so stop here and do nothing else.  From jenkins point of view ...
            # - if $res==0: build is successful, so proceed with all
            #   followup steps -- notify, push, trigger precommit.
            # - if $res==123: build has a regression, so proceed with triggering
            #   followup builds and sending precommit notifications; but stop
            #   before run-push-baseline by checking artifacts/failed.
            # - if $res is anything else: build has an unexpected failure, so
            #   skip the rest of the steps by setting UNSTABLE build status.
            #   This will cause all followup conditional-steps to skip due to
            #   "current-status" condition.
            #
            # See comment before round-robin.sh:check_regression() for details.
            case $res in
              0)
                ;;
              123)
                echo 123 > artifacts/failed
                ;;
              *)
                res=125
                ;;
            esac

            if [ $res = 125 ]; then
              # We had unexpected failure in the build scripts, so can't
              # proceed with triggers and notifications.  Mark this build
              # as UNSTABLE, which will skip all following conditional-steps.
              exit 125
            fi

            cat >> artifacts/jenkins/run-build.env <<EOF
            build_name="$build_name"
            EOF

            trap "" EXIT
          unstable-return: 125

- builder:
    name: run-build-fini
    builders:
      - shell:
          command: |
            #!/bin/bash
            source artifacts/jenkins/run-build.env
            set -ex
            # The "trap cleanup_all_containers EXIT" statement in
            # artifacts/jenkins/run-build.env will remove build and precommit
            # containers.

- builder:
    name: run-build
    builders:
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-build-init:
                precommit: false
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-build-skip-if-queue:
                dummy: dummy
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-build-start-manifest:
                ci_project: '{ci_project}'
                ci_config: '{ci_config}'
                build_script: '{build_script}'
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-build-process-params:
                dummy: dummy
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-push-baseline:
                build_script: '{build_script}'
                commit_artifacts: 'false'
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-build-main:
                build_script: '{build_script}'
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
      - conditional-step:
          condition-kind: current-status
          steps:
            - trigger-followup-builds-1:
                dummy: dummy
            - trigger-followup-builds-2:
                ci_project: '{ci_project}'
                ci_config: '{ci_config}'
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-build-notify:
                build_script: '{build_script}'
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-exit-on-failure:
                dummy: dummy
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-push-baseline:
                build_script: '{build_script}'
                commit_artifacts: 'true'
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-push-lnt-results:
                ci_project: '{ci_project}'
                ci_config: '{ci_config}'
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
      - conditional-step:
          condition-kind: current-status
          steps:
            - trigger-precommit-builds-2:
                ci_project: '{ci_project}'
                ci_config: '{ci_config}'
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-build-fini:
                dummy: dummy

- builder:
    name: run-precommit-process-params
    builders:
      - shell:
          command: |
            #!/bin/bash
            source artifacts/jenkins/run-build.env
            set -ex

            ./jenkins-scripts/start-container-docker.sh --prefix precommit_ \
              --task precommit --secondary true "${{docker_opts[@]}}" \
              > precommit_container.sh
            source precommit_container.sh

            precommit_container_rsync -az --del \
              jenkins-scripts/ :jenkins-scripts/

            branch_opt=()
            for c in binutils gcc gdb glibc linux qemu; do
              eval "g=\$$$${{c}}_git"
              case "$g" in
                "pw://"*|"ssh://"*)
                  # Handle pre-commit testing
                  case "$g" in
                    "pw://"*)
                      build_container_exec \
                      ./jenkins-scripts/pw-apply.sh \
                      --ci_bot '{ci_project}--{ci_config}' \
                      --project "$c" --pw_url "$g" __pw_token "$TCWG_PW_TOKEN" \
                      --build_url "$BUILD_URL" \
                      --pw_dir artifacts/jenkins/pw \
                      &> artifacts/jenkins/pw-apply.log &
                      ;;
                    "ssh://"*)
                      build_container_exec \
                      ./jenkins-scripts/precommit-ssh-apply.sh \
                      --project "$c" --pw_url "$g" \
                      --build_url "$BUILD_URL" \
                      --patch_submitter "$notify" \
                      --pw_dir artifacts/jenkins/pw \
                      &> artifacts/jenkins/precommit-ssh-apply.log &
                      ;;
                  esac
                  res=0 && wait $! || res=$?

                  if [ "$res" != "0" ]; then
                    build_name="$build_name-$g-$c-apply-fail"
                    echo "$build_name" > artifacts/jenkins/build-name

                    # Notify developer of failure to apply.
                    for f in mail-body.txt mail-subject.txt \
                      mail-recipients.txt; do
                      # copy the file if exists, and not emtpy.
                      # this is important for mail-recipient.txt, which may
                      # be empty.
                      if [ -s artifacts/jenkins/pw/$f ]; then
                        cp artifacts/jenkins/pw/$f artifacts/jenkins/$f
                      fi
                    done

                    case "$g" in
                      "ssh://"*)
                        # Build was triggered manually, so failure-to-apply
                        # is a real failure.
                        exit 1
                        ;;
                    esac

                    # Skip the build
                    exit 125
                  fi

                  # Check whether we should keep or skip this commit
                  build_container_exec \
                  ./jenkins-scripts/precommit-keep-or-skip.sh \
                  --pw_dir artifacts/jenkins/pw \
                  --ci_project '{ci_project}' \
                  --ci_config '{ci_config}' \
                  --project "$c" &
                  keep=0 && wait $! || keep=$?

                  skip=$(declare -A pw
                             source "artifacts/jenkins/pw/$c"
                             echo "${{pw[${{c}}_skip]}}")

                  if $skip; then
                    build_name="$build_name-$g-$c-test-skip"
                    echo "$build_name" > artifacts/jenkins/build-name

                    # pw-report.sh does not use artifacts/artifacts.precommit,
                    # so it is OK to use unsafe container.
                    build_container_exec \
                    ./jenkins-scripts/pw-report.sh --pw_dir "artifacts/jenkins/pw" \
                    --check test --result skip \
                    &> artifacts/jenkins/pw-report.log

                    # Skip the build
                    exit 125
                  fi

                  patch_id=$(declare -A pw
                             source "artifacts/jenkins/pw/$c"
                             echo "${{pw[${{c}}_patch_id]}}")
                  build_name="$build_name-$g/$patch_id"
                  g=HEAD
                  precommit_container_rsync -az --del \
                    "$c/" ":$c/"
                ;;
              esac

              if [ x"$g" != x"baseline" ]; then
                build_name="$build_name-$c"
                branch_opt+=("==rr[${{c}}_git]" "$g")
              fi
            done

            # Now that artifacts/ include all necessary data (including
            # artifacts/jenkins/pw/ directory -- rsync them into precommit
            # container.
            precommit_container_rsync -az --del \
              artifacts/ :artifacts/

            echo "$build_name" > artifacts/jenkins/build-name

            cat >> artifacts/jenkins/run-build.env <<EOF
            source precommit_container.sh
            build_name="$build_name"
            branch_opt=(${{branch_opt[@]+$(printf "%q$IFS" "${{branch_opt[@]}}")}})
            EOF

            trap "" EXIT
          unstable-return: 125

- builder:
    name: run-precommit-main
    builders:
      - shell:
          command: |
            #!/bin/bash
            source artifacts/jenkins/run-build.env
            set -ex

            # Run the build script to from the manifest created above.
            #
            # The parameter __start_at makes the build script start right
            # where the above run finished.  This allows us to use read-only
            # bind-mount for base-artifacts/, which, otherwise, may require
            # multi-gig rsync.
            precommit_container_exec \
              ./jenkins-scripts/{build_script} \
              @@rr[top_artifacts] artifacts \
              "${{branch_opt[@]}}" \
              __start_at reset_artifacts+ &
            res=0 && wait $! || res=$?

            precommit_container_rsync -az --del \
              :artifacts/ artifacts/artifacts.precommit/
            find artifacts/artifacts.precommit -size +1M ! -name "*.xz" \
              -print0 | xargs -0 -P0 -i@ xz -T0 @

            # Everything but 0 and $INTERNAL_FAILURE is an unexpected exit code,
            # so stop here and do nothing else.  From jenkins point of view ...
            # - if $res==0: build is successful, so proceed with all
            #   followup steps -- notify, push, trigger precommit.
            # - if $res==123: build has a regression, so proceed with triggering
            #   followup builds and sending precommit notifications; but stop
            #   before run-push-baseline by checking artifacts/failed.
            # - if $res is anything else: build has an unexpected failure, so
            #   skip the rest of the steps by setting UNSTABLE build status.
            #   This will cause all followup conditional-steps to skip due to
            #   "current-status" condition.
            #
            # See comment before round-robin.sh:check_regression() for details.
            case $res in
              0)
                pw_result=pass
                ;;
              123)
                echo 123 > artifacts/failed
                pw_result=fail
                ;;
              *)
                res=125
                pw_result=ignore
                ;;
            esac

            # pw-report.sh does not use artifacts/artifacts.precommit,
            # so it is OK to use unsafe container.
            build_container_exec \
              ./jenkins-scripts/pw-report.sh --pw_dir "artifacts/jenkins/pw" \
              --check test --result "$pw_result"
              &> artifacts/jenkins/pw-report.log &
            if ! wait $!; then
              res=125
            fi

            if [ $res = 125 ]; then
              # We had unexpected failure in the build scripts, so can't
              # proceed with triggers and notifications.  Mark this build
              # as UNSTABLE, which will skip all following conditional-steps.
              exit 125
            fi

            cat >> artifacts/jenkins/run-build.env <<EOF
            build_name="$build_name"
            EOF

            trap "" EXIT
          unstable-return: 125

- builder:
    name: run-precommit-notify
    builders:
      - shell: |
          #!/bin/bash
          source artifacts/jenkins/run-build.env
          set -ex

          precommit_container_exec \
            ./jenkins-scripts/rr/round-robin-notify.sh \
            @@rr[top_artifacts] artifacts --notify "$notify" \
            --pw_dir artifacts/jenkins/pw __verbose true \
            __build_script {build_script} \
            &> ./artifacts/jenkins/notify.log &

          if ! wait $!; then
            echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \
              > artifacts/jenkins/error-mail-recipients.txt
            echo -e "$BUILD_URL\nERROR: notify failed\n" \
              >> artifacts/jenkins/error-mail-body.txt
            echo "ERROR: notify failed"
            exit 1
          fi

          precommit_container_rsync -az --del \
            :artifacts/ artifacts/artifacts.precommit/
          # Compress big files in artifacts.precommit/ to save disk space
          # on ci.linaro.org.  We keep all pre-commit builds for a month,
          # which can result in many stored builds.
          find artifacts/artifacts.precommit -size +1M ! -name "*.xz" \
            ! -path "artifacts/artifacts.precommit/notify/*" \
            ! -path "artifacts/artifacts.precommit/jenkins/*" -print0 \
            | xargs -0 -P0 -i@ xz -T0 @

          for f in mail-body.txt mail-subject.txt mail-recipients.txt; do
            # copy the file if exists, and not emtpy.
            # this is important for mail-recipient.txt, which may be empty.
            if [ -s artifacts/artifacts.precommit/jenkins/$f ]; then
              cp artifacts/artifacts.precommit/jenkins/$f artifacts/jenkins/$f
            fi
          done

          trap "" EXIT

- builder:
    name: run-precommit
    builders:
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-build-init:
                precommit: true
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-build-start-manifest:
                ci_project: '{ci_project}'
                ci_config: '{ci_config}'
                build_script: '{build_script}'
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-precommit-process-params:
                ci_project: '{ci_project}'
                ci_config: '{ci_config}'
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-precommit-main:
                build_script: '{build_script}'
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-precommit-notify:
                build_script: '{build_script}'
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-exit-on-failure:
                dummy: dummy
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
      - conditional-step:
          condition-kind: current-status
          steps:
            - run-build-fini:
                dummy: dummy

- publisher:
    name: build-publishers
    publishers:
      - archive:
          artifacts: 'artifacts/**'
          latest-only: false
      - html-publisher:
          name: 'HTML Report'
          dir: 'artifacts/jenkins'
          files: 'status.html'
          keep-all: true
      - email-ext:
          recipients: |
            ${{FILE,path="artifacts/jenkins/mail-recipients.txt"}}
          reply-to: linaro-toolchain@lists.linaro.org
          subject: |
            ${{FILE,path="artifacts/jenkins/mail-subject.txt"}}
          content-type: text
          body: |
            ${{FILE,path="artifacts/jenkins/mail-body.txt"}}
          failure: {email_on_failure}
          success: {email_on_success}
          aborted: false
          send-to:
            - recipients
      - email-ext:
          recipients: |
            ${{FILE,path="artifacts/jenkins/error-mail-recipients.txt"}}
          body: |
            ${{FILE,path="artifacts/jenkins/error-mail-body.txt"}}
          failure: true
          success: true
          aborted: true
          send-to:
            - recipients
      - email-ext:
          recipients: |
            ${{FILE,path="artifacts/jenkins/testresults-mail-recipients.txt"}}
          reply-to: linaro-toolchain@lists.linaro.org
          subject: |
            ${{FILE,path="artifacts/jenkins/testresults-mail-subject.txt"}}
          content-type: text
          body: |
            ${{FILE,path="artifacts/jenkins/testresults-mail-body.txt"}}
          failure: false
          success: true
          aborted: false
          send-to:
            - recipients

- parameter:
    name: bisect-parameters
    parameters:
      - string:
          name: current_project
          default: ""
          description: "Project to bisect"
      - string:
          name: bad_git
          default: ""
          description: "Bad git_url#branch/SHA1"
      - file:
          name: jenkins-scripts/replay_log
          description: "Replay part of bisect using provided bisect log"
      - default-parameters:
          distro: '{distro}'

- wrapper:
    name: bisect-wrappers
    wrappers:
      - build-wrappers:
          timeout: '{timeout}'

- builder:
    name: run-bisect
    builders:
      - shell:
          command: |
            #!/bin/bash
            set -ex
            # Delete artifacts now to avoid re-using stale artifacts if
            # below docker-run.sh fails to start up a container.
            rm -rf artifacts/
            mkdir -p artifacts/jenkins
            build_name="#$BUILD_NUMBER-$current_project"
            echo "$build_name" > artifacts/jenkins/build-name

            cat >> artifacts/jenkins/run-build.env <<EOF
            build_name="$build_name"
            EOF

            ./jenkins-scripts/docker-run.sh \
              --distro $distro \
              --node_labels "$NODE_LABELS" \
              -- \
              ./jenkins-scripts/rr/round-robin-bisect.sh \
              %%rel_artifacts artifacts \
              --BUILD_URL "$BUILD_URL" \
              --current_project "$current_project" \
              --bad_git "$bad_git" \
              --replay_log "$(pwd)/jenkins-scripts/replay_log" \
              --build_script "./jenkins-scripts/{build_script}" \
              -- \
              ==rr[ci_project] '{ci_project}' \
              ==rr[ci_config] '{ci_config}' \
              $extra_build_params \
              --scripts_branch "$scripts_branch" &
            res=0 && wait $! || res=$?

            if [ $res != 0 ]; then
              echo $res > artifacts/failed
            fi
          unstable-return: 125
      - build-name-setter:
          name: 'artifacts/jenkins/build-name'
          file: true
      - conditional-step:
          condition-kind: current-status
          steps:
            - trigger-followup-builds-1:
                dummy: dummy
            - trigger-followup-builds-2:
                ci_project: '{ci_project}'
                ci_config: '{ci_config}'
      - conditional-step:
          condition-kind: current-status
          steps:
            - shell: |
                #!/bin/bash
                set -ex

                if [ -f artifacts/failed ]; then
                  # Mark failed build as FAILURE; no further steps will run.
                  exit $(cat artifacts/failed)
                fi

- publisher:
    name: bisect-publishers
    publishers:
      - archive:
          artifacts: 'artifacts/**'
          latest-only: false
      - email-ext:
          recipients: |
            maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org
          failure: true
          success: false
          aborted: true
          send-to:
            - recipients
#END:   tcwg/round-robin.yaml.inc

- job-template:
    name: '{ci_project}--{toolchain_ver}-{target}{type_of_test}-build'
    project-type: freestyle
    defaults: global
    properties:
      - build-properties:
          ci_project: '{ci_project}'
          ci_config: '{ci_config}'
    parameters:
      - build-parameters:
          default_git: 'default'
          distro: '{distro}'
    disabled: false
    node: '{node}'
    concurrent: false
    display-name: 'TCWG Build {ci_project}/{ci_config}'
    workspace: workspace/tcwg_gnu_$EXECUTOR_NUMBER
    scm:
      - build-scms:
          binutils_url: '{binutils_url}'
          binutils_branch: '{binutils_branch}'
          gcc_url: '{gcc_url}'
          gcc_branch: '{gcc_branch}'
          linux_url: '{linux_url}'
          linux_branch: '{linux_branch}'
          glibc_url: '{glibc_url}'
          glibc_branch: '{glibc_branch}'
          gdb_url: '{gdb_url}'
          gdb_branch: '{gdb_branch}'
          qemu_url: '{qemu_url}'
          qemu_branch: '{qemu_branch}'
    triggers:
      - pollscm:
          cron: '{cron}'
    wrappers:
      - build-wrappers:
          timeout: '{timeout_build}'
    builders:
      - run-build:
          ci_project: '{ci_project}'
          ci_config: '{ci_config}'
          build_script: 'tcwg_gnu-build.sh'
          docker_security: '{docker_security}'
    publishers:
      - build-publishers:
          email_on_failure: false
          email_on_success: true

- job-template:
    name: '{ci_project}--{toolchain_ver}-{target}{type_of_test}-precommit'
    project-type: freestyle
    defaults: global
    properties:
      - precommit-properties:
          ci_project: '{ci_project}'
          ci_config: '{ci_config}'
    parameters:
      - precommit-parameters:
          default_git: 'baseline'
          distro: '{distro}'
    disabled: false
    node: '{node}'
    concurrent: true
    display-name: 'TCWG Precommit {ci_project}/{ci_config}'
    workspace: workspace/tcwg_gnu_$EXECUTOR_NUMBER
    scm:
      - jenkins-scripts
    wrappers:
      - build-wrappers:
          timeout: 300
    builders:
      - run-precommit:
          ci_project: '{ci_project}'
          ci_config: '{ci_config}'
          build_script: 'tcwg_gnu-build.sh'
          docker_security: '{docker_security}'
    publishers:
      - build-publishers:
          email_on_failure: true
          email_on_success: false

- job-template:
    name: '{ci_project}--{toolchain_ver}-{target}{type_of_test}-bisect'
    project-type: freestyle
    defaults: global
    properties:
      - bisect-properties:
          ci_project: '{ci_project}'
          ci_config: '{ci_config}'
    parameters:
      - bisect-parameters:
          distro: '{distro}'
    disabled: false
    node: '{node}'
    concurrent: false
    display-name: 'TCWG Bisect {ci_project}/{ci_config}'
    workspace: workspace/tcwg_gnu_$EXECUTOR_NUMBER
    scm:
      - jenkins-scripts
    wrappers:
      - bisect-wrappers:
          timeout: 6000
    builders:
      - run-bisect:
          ci_project: '{ci_project}'
          ci_config: '{ci_config}'
          build_script: 'tcwg_gnu-build.sh'
          docker_security: '{docker_security}'
    publishers:
      - bisect-publishers
# checksum: a91685819264fa584a8e32f7bb631f04