diff options
Diffstat (limited to 'tcwg_binutils.yaml')
| -rw-r--r-- | tcwg_binutils.yaml | 1274 |
1 files changed, 1078 insertions, 196 deletions
diff --git a/tcwg_binutils.yaml b/tcwg_binutils.yaml index ce1235244f..cbc2ca8c90 100644 --- a/tcwg_binutils.yaml +++ b/tcwg_binutils.yaml @@ -2,34 +2,37 @@ # Beware: git branch names need the 'refs/heads/' prefix, while tags need 'refs/tags/' - project: name: tcwg_binutils - toolchain_name: gnu - ci_project: tcwg_gnu + type_of_test: "" + timeout_build: 1000 + # Trigger configuration that have precommit testing every 6 hours + # to test incoming patches in a timely manner. + cron_master: 'H H/6 * * *' + docker_security: '' + ci_project: + - tcwg_binutils_build + - tcwg_binutils_check binutils_url: git://sourceware.org/git/binutils-gdb.git - toolchain_ver: - - master: - binutils_branch: refs/heads/master - distro: lts - - release: - binutils_branch: refs/heads/binutils-2_36-branch - distro: lts_1 target: - aarch64: - node: tcwg-armv8_64 + node: tcwg-build && tcwg-armv8_64 docker_arch: arm64 - arm: - node: tcwg-armv8_32 + node: tcwg-build && tcwg-armv8_32 docker_arch: armhf - disabled: false - type_of_test: - - check_binutils jobs: - - 'tcwg_binutils-build-{toolchain_name}-{toolchain_ver}-{target}-{type_of_test}' - - 'tcwg_binutils-bisect-{toolchain_name}-{toolchain_ver}-{target}-{type_of_test}' + - tcwg_binutils-master -- view: - name: tcwg_binutils - view-type: list - regex: 'tcwg_binutils-.*' +- job-group: + name: tcwg_binutils-master + toolchain_ver: master + cron: '{cron_master}' + binutils_branch: refs/heads/master + distro: default + jobs: + - '{ci_project}--{toolchain_ver}-{target}{type_of_test}-build' + - '{ci_project}--{toolchain_ver}-{target}{type_of_test}-precommit' + - '{ci_project}--{toolchain_ver}-{target}{type_of_test}-bisect' + ci_config: '{toolchain_ver}-{target}{type_of_test}' #BEGIN: tcwg/round-robin.yaml.inc # -*- mode: Yaml -*- @@ -44,12 +47,11 @@ anonymous: - job-read - job-extended-read - linaro: + everyone-flat: - job-build - job-cancel - build-discarder: days-to-keep: 30 - num-to-keep: 30 - scm: name: jenkins-scripts @@ -73,21 +75,9 @@ name: default-parameters parameters: - string: - name: ci_project - default: '{ci_project}' - description: "CI project ID; used as namespace for git branches" - - string: - name: ci_config - default: '{ci_config}' - description: "CI configuration ID" - - string: name: extra_build_params default: "" - description: "Extra parameters to pass to the build script" - - string: - name: mail_recipients - default: default - description: "Comma-separated list of email recipients; use 'default' unless testing" + description: "Extra parameters to pass to the build script; can be used to override settings extracted from ci_project/ci_config" - string: name: distro default: '{distro}' @@ -101,34 +91,87 @@ name: component-parameters parameters: - string: - name: '{component}_url' - default: '{url}' - description: "{component} git URL" - - string: - name: '{component}_branch' - default: '{branch}' - description: "{component} branch or sha1 to build, or 'baseline'" + name: '{component}_git' + default: '{default_git}' + description: "{component} git_url#branch/sha1 to build, or 'default', 'baseline' or 'jenkins-scm', or pw://series/<id> to build with patches from patchwork ('precommit' mode, use 'pw://series/<id>/retrigger/' to force a rebuild)" - parameter: - name: build-parameters + name: all-component-parameters parameters: - component-parameters: component: binutils - url: '{binutils_url}' - branch: '{default_branch}' + default_git: '{default_git}' + +- parameter: + name: build-parameters + parameters: + - all-component-parameters: + default_git: '{default_git}' - choice: name: update_baseline choices: - - update - - reset + - onsuccess + - force - init - - push + - ignore description: "What to do with baseline in this build" + - choice: + name: notify + choices: + - ignore + - onregression + - precommit + description: "Whether to notify developers about the result of this build" + - default-parameters: + distro: '{distro}' + +# Precommit parameters ... +# - allow only "ignore" value for update_baseline parameter; +# - allow string (which should be an email) for notify parameter. +- parameter: + name: precommit-parameters + parameters: + - all-component-parameters: + default_git: '{default_git}' + - string: + name: notify + default: 'precommit' + description: "Email to send failure notifications to" - default-parameters: - ci_project: '{ci_project}' - ci_config: '{ci_config}' distro: '{distro}' +- property: + name: build-properties + properties: + - default-properties + - build-blocker: + blocking-jobs: + - '{ci_project}--{ci_config}-bisect' + queue-scanning: 'ALL' + +- property: + name: precommit-properties + properties: + - default-properties + - build-blocker: + blocking-jobs: + - '{ci_project}--{ci_config}-build' + queue-scanning: 'BUILDABLE' + +- property: + name: bisect-properties + properties: + - default-properties + - build-blocker: + # Run at most 1 bisect job for all configurations. + # Bisect jobs can easily run for 5+ hours, and a few of them + # can hog the entire TCWG build farm for a long time. Since + # same regression tends to appear in multiple configurations, + # run bisections for all configurations in-order. + blocking-jobs: + - '{ci_project}--.*-bisect' + queue-scanning: 'BUILDABLE' + # Beware: git branch names need the 'refs/heads/' prefix while tags # need 'refs/tags/'. - scm: @@ -171,35 +214,259 @@ # tcwg-buildslave user id users: - 'e0958a95-204f-4c14-a66c-5e2be6c5d50a' +# - 'tcwg-build-20220725' + - credentials-binding: + - text: + credential-id: TCWG_JIRA_TOKEN + variable: TCWG_JIRA_TOKEN + - text: + credential-id: TCWG_PW_TOKEN + variable: TCWG_PW_TOKEN - builder: - name: trigger-followup-builds + name: trigger-followup-builds-1 builders: - - build-name-setter: - name: 'artifacts/jenkins/build-name' - file: true - shell: | #!/bin/bash + source artifacts/jenkins/run-build.env set -ex + for i in artifacts/trigger-build-* artifacts/trigger-bisect; do if [ -f $i ]; then - echo "ci_project=$ci_project" >> $i - echo "ci_config=$ci_config" >> $i - echo "mail_recipients=$mail_recipients" >> $i echo "distro=$distro" >> $i echo "scripts_branch=$scripts_branch" >> $i fi done + + trap "" EXIT + +- builder: + name: trigger-followup-builds-2 + builders: - trigger-builds: - - project: '{rr_project}-build-{ci_project_config}' + - project: '{ci_project}--{ci_config}-build' parameter-factories: - factory: filebuild file-pattern: artifacts/trigger-build-* - - project: '{rr_project}-bisect-{ci_project_config}' + - project: '{ci_project}--{ci_config}-bisect' property-file: artifacts/trigger-bisect - builder: - name: run-build + name: run-build-notify + builders: + - shell: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + if [ -f artifacts/failed ]; then + # Only send notifications for forced successful + # (aka "onregression") builds; never for failed builds. + echo "Build failed. Using option notify=ignore." + notify=ignore + fi + + # Basic notification -- create entry in interesting-commits. + # Note that we add --notify to the manifest -- we will need this + # later in round-robin-baseline.sh when re-writing history + # to update notifications files in interesting-commits and jira. + build_container_exec \ + ./jenkins-scripts/round-robin-notify.sh \ + @@rr[top_artifacts] artifacts --notify "$notify" \ + __stage init __verbose true \ + __build_script {build_script} \ + &> ./artifacts/jenkins/notify-init.log & + + if ! wait $!; then + echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \ + > artifacts/jenkins/error-mail-recipients.txt + echo -e "$BUILD_URL\nERROR: notify-init failed\n" \ + >> artifacts/jenkins/error-mail-body.txt + echo "ERROR: notify-init failed" + exit 1 + fi + + # Full notification + echo "NOTE: Notify developers about this build" + + build_container_exec \ + ./jenkins-scripts/round-robin-notify.sh \ + @@rr[top_artifacts] artifacts __stage full __verbose true \ + __build_script {build_script} \ + &> ./artifacts/jenkins/notify-full.log & + + if ! wait $!; then + echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \ + > artifacts/jenkins/error-mail-recipients.txt + echo -e "$BUILD_URL\nERROR: notify-full failed\n" \ + >> artifacts/jenkins/error-mail-body.txt + echo "ERROR: notify-full failed" + fi + + # Failsafe for avoid spamming developers. + if [ "$notify" = "ignore" ] \ + && [ -f artifacts/jenkins/mail-recipients.txt ]; then + echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \ + > artifacts/jenkins/error-mail-recipients.txt + echo -e "$BUILD_URL\nERROR: tried to unexpectedly notify developers\n" \ + >> artifacts/jenkins/error-mail-body.txt + mv artifacts/jenkins/mail-recipients.txt \ + artifacts/jenkins/mail-recipients.bak + fi + + trap "" EXIT + +- builder: + name: run-exit-on-failure + builders: + - shell: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + if [ -f artifacts/failed ]; then + # Mark failed build as FAILURE; no further steps will run. + exit $(cat artifacts/failed) + fi + # Only successful builds run beyond this point. + + trap "" EXIT + +- builder: + name: run-push-baseline + builders: + - shell: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + # Update baseline branch of base-artifacts. + # If we made it this far, then the build is successful + # (possibly because it was forced to be declared successful). + if [ "$update_baseline" = "ignore" ]; then + trap "" EXIT + exit 0 + fi + + # We may need jira token while rewriting history. + if [ -f $HOME/.jipdate.yml ]; then + ( + # BE CAREFUL WITH TCWG_JIRA_TOKEN + set +x + build_container_exec \ + sed -i -e "s/#TCWG_JIRA_TOKEN#/$TCWG_JIRA_TOKEN/" \ + "$HOME/.jipdate.yml" + ) + fi + + build_container_exec \ + ./jenkins-scripts/round-robin-baseline.sh \ + @@rr[top_artifacts] artifacts \ + __build_script {build_script} \ + __push_base_artifacts true \ + __rewrite_base_artifacts true \ + __commit_artifacts {commit_artifacts} \ + __rewrite_num 100 \ + &> ./artifacts/jenkins/rewrite.log & + + if ! wait $!; then + echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \ + > artifacts/jenkins/error-mail-recipients.txt + echo -e "$BUILD_URL\nERROR: round-robin-baseline.sh failed" \ + >> artifacts/jenkins/error-mail-body.txt + echo "ERROR: round-robin-baseline.sh failed" + # We have failed to update the baseline, so let's fail and + # not notify developers. + exit 1 + fi + + trap "" EXIT + +- builder: + name: run-push-lnt-results + builders: + - trigger-builds: + - project: tcwg-update-lnt-results + predefined-parameters: | + ci_project={ci_project} + ci_config={ci_config} + +- builder: + name: trigger-precommit-builds-1 + builders: + - shell: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + # Trigger pre-commit builds if the jenkins queue for + # this job is empty. + # + # Skip triggering pre-commit builds if there are other post-commit + # or pre-commit builds in the queue. + # Re. post-commit builds -- we need to process all pending + # bisection-triggered builds before fetching a new batch of + # upstream changes. + # Re. pre-commit builds -- we need to wait for the previous batch + # to finish to avoid triggering duplicate builds for the queued + # patches. + for job_type in build precommit; do + url="${{JOB_URL%-build/}}-$job_type" + inQueue=$(curl -s "$url/api/xml?tree=inQueue" \ + | sed -e "s#.*<inQueue>\(.*\)</inQueue>.*#\1#") + if [ "$inQueue" != "false" ]; then + echo "SKIPPING PRECOMMIT TRIGGER: $build_type QUEUE IS NOT EMPTY" + trap "" EXIT + exit 0 + fi + done + + # $update_baseline != ignore tells us that we have just + # updated the baseline and this means that we should be + # in great shape to apply pending patches. + # Most likely current baseline revision is master revision + # as of few hours ago. + if [ "$update_baseline" = "ignore" ]; then + echo "SKIPPING PRECOMMIT TRIGGER: DID NOT UPDATE BASELINE" + trap "" EXIT + exit 0 + fi + + for project in binutils; do + # Note that we get here only when all previous pre-commit + # builds have finished and, presumably, updated ci_bot + # bundles at patchworks instance. If this were not + # the case, there would be some builds queued and + # we would have exited above. + build_container_exec \ + ./jenkins-scripts/pw-trigger.sh \ + --ci_bot '{ci_project}--{ci_config}' --project "$project" \ + --out_dir artifacts/jenkins \ + &> artifacts/jenkins/pw-trigger.log + + for i in artifacts/jenkins/trigger-precommit-*; do + if [ -f $i ]; then + echo "distro=$distro" >> $i + echo "scripts_branch=$scripts_branch" >> $i + fi + done + done + + trap "" EXIT +# We place trigger-precommit-builds-2 outside of PRECOMMIT_TRIGGER_yes +# #ifdef to get an upstream/downstream links between build/precommit jobs +# in jenkins UI. +- builder: + name: trigger-precommit-builds-2 + builders: + - trigger-builds: + - project: '{ci_project}--{ci_config}-precommit' + parameter-factories: + - factory: filebuild + file-pattern: artifacts/jenkins/trigger-precommit-* + +- builder: + name: run-build-init builders: - shell: command: | @@ -210,85 +477,676 @@ rm -rf artifacts/ mkdir -p artifacts/jenkins - dryruncmd="" - build_name="#$BUILD_NUMBER" - branch_opt="" + echo "$build_name" > artifacts/jenkins/build-name + + if [ "{precommit}" = "True" ]; then + cat >> artifacts/jenkins/run-build.env <<EOF + update_baseline=ignore + EOF + fi + + cat >> artifacts/jenkins/run-build.env <<EOF + build_name="$build_name" + EOF + unstable-return: 125 + +- builder: + name: run-build-skip-if-queue + builders: + - shell: + command: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + # Skip SCM-triggered builds if there are other builds in + # the queue. We need to process all pending bisection-triggered + # builds before fetching a new batch of upstream changes. + # Re. skipping post-commit build if pre-commit queue is + # not empty -- it's just a favor to pre-commit builds. + if [ x"${{BUILD_CAUSE_SCMTRIGGER-false}}" = x"true" ] \ + || [ x"${{BUILD_CAUSE_TIMERTRIGGER-false}}" = x"true" ]; then + for job_type in build precommit; do + url="${{JOB_URL%-build/}}-$job_type" + inQueue=$(curl -s "$url/api/xml?tree=inQueue" \ + | sed -e "s#.*<inQueue>\(.*\)</inQueue>.*#\1#") + if [ "$inQueue" != "false" ]; then + echo "SKIP SCM BUILD" + touch artifacts/jenkins/skip-scm-build + build_name="$build_name-skip-scm-build" + echo "$build_name" > artifacts/jenkins/build-name + + # Mark build unstable aka a skipped build + exit 125 + fi + done + fi + + trap "" EXIT + unstable-return: 125 + +- builder: + name: run-build-start-manifest + builders: + - shell: + command: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + docker_opts=( + --distro $distro + --arch {docker_arch} + --node $NODE_NAME + ) + + ./jenkins-scripts/start-container-docker.sh --prefix build_ \ + "${{docker_opts[@]}}" > build_container.sh + source build_container.sh + trap "cleanup_all_containers" EXIT + + # Run the build script to fetch base-artifacts/, which are + # required to fetch baseline sources to apply patches to in + # pw-apply.sh below. This also initializes the build manifest + # with all below settings; we then import this manifest below. + # + # The parameter __finish_after makes the build script stop + # after fetching the baseline and "__" avoids saving this parameter + # in the manifest, so that we don't see this option in the main + # run of the build script below. + build_container_exec \ + ./jenkins-scripts/{build_script} \ + %%rr[top_artifacts] artifacts \ + ==rr[ci_project] '{ci_project}' \ + ==rr[ci_config] '{ci_config}' \ + $extra_build_params \ + --scripts_branch "$scripts_branch" \ + ==rr[mode] "build" \ + ==rr[update_baseline] "$update_baseline" \ + --BUILD_URL "$BUILD_URL" \ + __finish_at reset_artifacts + + cat >> artifacts/jenkins/run-build.env <<EOF + docker_opts=(${{docker_opts[@]+$(printf "%q$IFS" "${{docker_opts[@]}}")}}) + source build_container.sh + trap "cleanup_all_containers" EXIT + EOF + + # Now that we have "trap cleanup_all_containers EXIT" in + # run-build.env we need to reset the trap at the end of every + # step to preserve container[s] for the next step. + trap "" EXIT + unstable-return: 125 + +- builder: + name: run-build-process-params + builders: + - shell: + command: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + branch_opt=() # 1 to skip jenkins-scripts i=1 - for c in {components}; do - eval "u=\$$$${{c}}_url" - eval "b=\$$$${{c}}_branch" - if [ x"$b" = x"default" ]; then - eval "b=\$GIT_COMMIT_$i" + for c in binutils; do + eval "g=\$$$${{c}}_git" + if [ x"$g" = x"default" ]; then + if [ x"${{BUILD_CAUSE_SCMTRIGGER-false}}" = x"true" ] \ + || [ x"${{BUILD_CAUSE_TIMERTRIGGER-false}}" = x"true" ]; then + g="jenkins-scm" + else + g="baseline" + fi + fi + if [ x"$g" = x"jenkins-scm" ]; then + eval "gb=\$GIT_BRANCH_$i" + gb=$(echo $gb | sed 's+origin/++') + eval "g=\$GIT_URL_$i#$gb" fi i=$(($i+1)) - if [ x"$b" != x"baseline" ]; then + + if [ x"$g" != x"baseline" ]; then build_name="$build_name-$c" - branch_opt="$branch_opt ==rr[${{c}}_url] $u ==rr[${{c}}_branch] $b" + branch_opt+=("==rr[${{c}}_git]" "$g") fi done - $dryruncmd \ - ./jenkins-scripts/docker-run.sh \ - --distro $distro \ - --arch {docker_arch} \ - --node $NODE_NAME \ - -- \ - ./jenkins-scripts/{build_script} \ - %% artifacts/jenkins/manifest.sh \ - ==rr[ci_project] $ci_project \ - ==rr[ci_config] $ci_config \ - $extra_build_params \ - --scripts_branch "$scripts_branch" \ - $branch_opt \ - ==rr[mode] "jenkins-full" \ - ==rr[update_baseline] "$update_baseline" \ - --BUILD_URL "$BUILD_URL" & - res=0 && wait $! || res=$? + echo "$build_name" > artifacts/jenkins/build-name - # If we didn't run the docker command - if [ -n "$dryruncmd" ]; then - # Mark build unstable aka a skipped config - exit 125 - fi + cat >> artifacts/jenkins/run-build.env <<EOF + build_name="$build_name" + branch_opt=(${{branch_opt[@]+$(printf "%q$IFS" "${{branch_opt[@]}}")}}) + EOF - if [ $res != 0 ]; then - touch artifacts/failures - fi + trap "" EXIT + unstable-return: 125 + +- builder: + name: run-build-main + builders: + - shell: + command: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex - if [ -e artifacts/results ]; then + # Run the build script to from the manifest created above. + # + # The parameter __start_at makes the build script start right + # where the above run finished. This allows us to use read-only + # bind-mount for base-artifacts/, which, otherwise, may require + # multi-gig rsync. + build_container_exec \ + ./jenkins-scripts/{build_script} \ + @@rr[top_artifacts] artifacts \ + "${{branch_opt[@]}}" \ + __start_at reset_artifacts+ & + res=0 && wait $! || res=$? + + # Update build_name + if [ -f artifacts/results ]; then build_name="$build_name-R$(tail -n1 artifacts/results)" fi - if [ x"$update_baseline" != x"update" ]; then + if [ "$update_baseline" != "onsuccess" ]; then build_name="$build_name-$update_baseline" fi + if [ -f artifacts/trigger-bisect ]; then build_name="$build_name-trigger-bisect" fi echo "$build_name" > artifacts/jenkins/build-name + + # Everything but 0 and $INTERNAL_FAILURE is an unexpected exit code, + # so stop here and do nothing else. From jenkins point of view ... + # - if $res==0: build is successful, so proceed with all + # followup steps -- notify, push, trigger precommit. + # - if $res==123: build has a regression, so proceed with triggering + # followup builds and sending precommit notifications; but stop + # before run-push-baseline by checking artifacts/failed. + # - if $res is anything else: build has an unexpected failure, so + # skip the rest of the steps by setting UNSTABLE build status. + # This will cause all followup conditional-steps to skip due to + # "current-status" condition. + # + # See comment before round-robin.sh:check_regression() for details. + case $res in + 0) + ;; + 123) + echo 123 > artifacts/failed + ;; + *) + res=125 + ;; + esac + + if [ $res = 125 ]; then + # We had unexpected failure in the build scripts, so can't + # proceed with triggers and notifications. Mark this build + # as UNSTABLE, which will skip all following conditional-steps. + exit 125 + fi + + cat >> artifacts/jenkins/run-build.env <<EOF + build_name="$build_name" + EOF + + trap "" EXIT unstable-return: 125 + +- builder: + name: run-build-fini + builders: + - shell: + command: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + # The "trap cleanup_all_containers EXIT" statement in + # artifacts/jenkins/run-build.env will remove build and precommit + # containers. + +- builder: + name: run-build + builders: - conditional-step: - # Only run if above succeeded - # Meaning we did not skip a bad config - # (a failed build is indicated by artifacts/failures) condition-kind: current-status steps: - - trigger-followup-builds: - rr_project: '{rr_project}' - ci_project_config: '{ci_project_config}' - - shell: | - #!/bin/bash - if [ -f artifacts/failures ]; then - exit 1 - fi + - run-build-init: + precommit: false + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-build-skip-if-queue: + dummy: dummy + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-build-start-manifest: + ci_project: '{ci_project}' + ci_config: '{ci_config}' + build_script: '{build_script}' + docker_arch: '{docker_arch}' + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-build-process-params: + dummy: dummy + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-push-baseline: + build_script: '{build_script}' + commit_artifacts: 'false' + - conditional-step: + condition-kind: current-status + steps: + - run-build-main: + build_script: '{build_script}' + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - trigger-followup-builds-1: + dummy: dummy + - trigger-followup-builds-2: + ci_project: '{ci_project}' + ci_config: '{ci_config}' + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-build-notify: + build_script: '{build_script}' + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-exit-on-failure: + dummy: dummy + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-push-baseline: + build_script: '{build_script}' + commit_artifacts: 'true' + - conditional-step: + condition-kind: current-status + steps: + - run-push-lnt-results: + ci_project: '{ci_project}' + ci_config: '{ci_config}' + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - trigger-precommit-builds-1: + ci_project: '{ci_project}' + ci_config: '{ci_config}' + - conditional-step: + condition-kind: current-status + steps: + - trigger-precommit-builds-2: + ci_project: '{ci_project}' + ci_config: '{ci_config}' + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-build-fini: + dummy: dummy + +- builder: + name: run-precommit-process-params + builders: + - shell: + command: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + ./jenkins-scripts/start-container-docker.sh --prefix precommit_ \ + --task precommit --secondary true "${{docker_opts[@]}}" \ + > precommit_container.sh + source precommit_container.sh + + precommit_container_rsync -az --del \ + jenkins-scripts/ :jenkins-scripts/ + + branch_opt=() + for c in binutils; do + eval "g=\$$$${{c}}_git" + case "$g" in + "pw://"*|"ssh://"*) + # Handle pre-commit testing + case "$g" in + "pw://"*) + build_container_exec \ + ./jenkins-scripts/pw-apply.sh \ + --ci_bot '{ci_project}--{ci_config}' \ + --project "$c" --pw_url "$g" __pw_token "$TCWG_PW_TOKEN" \ + --build_url "$BUILD_URL" \ + --pw_dir artifacts/jenkins/pw \ + &> artifacts/jenkins/pw-apply.log & + ;; + "ssh://"*) + build_container_exec \ + ./jenkins-scripts/precommit-ssh-apply.sh \ + --project "$c" --pw_url "$g" \ + --build_url "$BUILD_URL" \ + --patch_submitter "$notify" \ + --pw_dir artifacts/jenkins/pw \ + &> artifacts/jenkins/precommit-ssh-apply.log & + ;; + esac + res=0 && wait $! || res=$? + + if [ "$res" != "0" ]; then + build_name="$build_name-$g-$c-apply-fail" + echo "$build_name" > artifacts/jenkins/build-name + + # Notify developer of failure to apply. + for f in mail-body.txt mail-subject.txt \ + mail-recipients.txt; do + # copy the file if exists, and not emtpy. + # this is important for mail-recipient.txt, which may + # be empty. + if [ -s artifacts/jenkins/pw/$f ]; then + cp artifacts/jenkins/pw/$f artifacts/jenkins/$f + fi + done + + case "$g" in + "ssh://"*) + # Build was triggered manually, so failure-to-apply + # is a real failure. + exit 1 + ;; + esac + + # Skip the build + exit 125 + fi + + patch_id=$(declare -A pw + source "artifacts/jenkins/pw/$c" + echo "${{pw[${{c}}_patch_id]}}") + build_name="$build_name-$g/$patch_id" + g=HEAD + precommit_container_rsync -az --del \ + "$c/" ":$c/" + ;; + esac + + if [ x"$g" != x"baseline" ]; then + build_name="$build_name-$c" + branch_opt+=("==rr[${{c}}_git]" "$g") + fi + done + + # Now that artifacts/ include all necessary data (including + # artifacts/jenkins/pw/ directory -- rsync them into precommit + # container. + precommit_container_rsync -az --del \ + artifacts/ :artifacts/ + + echo "$build_name" > artifacts/jenkins/build-name + + cat >> artifacts/jenkins/run-build.env <<EOF + source precommit_container.sh + build_name="$build_name" + branch_opt=(${{branch_opt[@]+$(printf "%q$IFS" "${{branch_opt[@]}}")}}) + EOF + + trap "" EXIT + unstable-return: 125 + +- builder: + name: run-precommit-main + builders: + - shell: + command: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + # Run the build script to from the manifest created above. + # + # The parameter __start_at makes the build script start right + # where the above run finished. This allows us to use read-only + # bind-mount for base-artifacts/, which, otherwise, may require + # multi-gig rsync. + precommit_container_exec \ + ./jenkins-scripts/{build_script} \ + @@rr[top_artifacts] artifacts \ + "${{branch_opt[@]}}" \ + __start_at reset_artifacts+ & + res=0 && wait $! || res=$? + + precommit_container_rsync -az --del \ + :artifacts/ artifacts/artifacts.precommit/ + find artifacts/artifacts.precommit -size +1M ! -name "*.xz" \ + -print0 | xargs -0 -P0 -i@ xz -T0 @ + + # Everything but 0 and $INTERNAL_FAILURE is an unexpected exit code, + # so stop here and do nothing else. From jenkins point of view ... + # - if $res==0: build is successful, so proceed with all + # followup steps -- notify, push, trigger precommit. + # - if $res==123: build has a regression, so proceed with triggering + # followup builds and sending precommit notifications; but stop + # before run-push-baseline by checking artifacts/failed. + # - if $res is anything else: build has an unexpected failure, so + # skip the rest of the steps by setting UNSTABLE build status. + # This will cause all followup conditional-steps to skip due to + # "current-status" condition. + # + # See comment before round-robin.sh:check_regression() for details. + case $res in + 0) + pw_result=pass + ;; + 123) + echo 123 > artifacts/failed + pw_result=fail + ;; + *) + res=125 + pw_result=ignore + ;; + esac + + # pw-report.sh does not use artifacts/artifacts.precommit, + # so it is OK to use unsafe container. + build_container_exec \ + ./jenkins-scripts/pw-report.sh --pw_dir "artifacts/jenkins/pw" \ + --check test --result "$pw_result" + &> artifacts/jenkins/pw-report.log & + if ! wait $!; then + res=125 + fi + + if [ $res = 125 ]; then + # We had unexpected failure in the build scripts, so can't + # proceed with triggers and notifications. Mark this build + # as UNSTABLE, which will skip all following conditional-steps. + exit 125 + fi + + cat >> artifacts/jenkins/run-build.env <<EOF + build_name="$build_name" + EOF + + trap "" EXIT + unstable-return: 125 + +- builder: + name: run-precommit-notify + builders: + - shell: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + precommit_container_exec \ + ./jenkins-scripts/round-robin-notify.sh \ + @@rr[top_artifacts] artifacts --notify "$notify" \ + --pw_dir artifacts/jenkins/pw __verbose true \ + __build_script {build_script} \ + &> ./artifacts/jenkins/notify.log & + + if ! wait $!; then + echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \ + > artifacts/jenkins/error-mail-recipients.txt + echo -e "$BUILD_URL\nERROR: notify failed\n" \ + >> artifacts/jenkins/error-mail-body.txt + echo "ERROR: notify failed" + exit 1 + fi + + precommit_container_rsync -az --del \ + :artifacts/ artifacts/artifacts.precommit/ + # Compress big files in artifacts.precommit/ to save disk space + # on ci.linaro.org. We keep all pre-commit builds for a month, + # which can result in many stored builds. + find artifacts/artifacts.precommit -size +1M ! -name "*.xz" \ + ! -path "artifacts/artifacts.precommit/notify/*" \ + ! -path "artifacts/artifacts.precommit/jenkins/*" -print0 \ + | xargs -0 -P0 -i@ xz -T0 @ + + for f in mail-body.txt mail-subject.txt mail-recipients.txt; do + # copy the file if exists, and not emtpy. + # this is important for mail-recipient.txt, which may be empty. + if [ -s artifacts/artifacts.precommit/jenkins/$f ]; then + cp artifacts/artifacts.precommit/jenkins/$f artifacts/jenkins/$f + fi + done + + trap "" EXIT + +- builder: + name: run-precommit + builders: + - conditional-step: + condition-kind: current-status + steps: + - run-build-init: + precommit: true + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-build-start-manifest: + ci_project: '{ci_project}' + ci_config: '{ci_config}' + build_script: '{build_script}' + docker_arch: '{docker_arch}' + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-precommit-process-params: + ci_project: '{ci_project}' + ci_config: '{ci_config}' + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-precommit-main: + build_script: '{build_script}' + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-precommit-notify: + build_script: '{build_script}' + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-exit-on-failure: + dummy: dummy + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-build-fini: + dummy: dummy - publisher: name: build-publishers publishers: - - archive: - artifacts: 'artifacts/**' - latest-only: false + - archive: + artifacts: 'artifacts/**' + latest-only: false + - html-publisher: + name: 'HTML Report' + dir: 'artifacts/jenkins' + files: 'status.html' + keep-all: true + - email-ext: + recipients: | + ${{FILE,path="artifacts/jenkins/mail-recipients.txt"}} + reply-to: linaro-toolchain@lists.linaro.org + subject: | + ${{FILE,path="artifacts/jenkins/mail-subject.txt"}} + content-type: text + body: | + ${{FILE,path="artifacts/jenkins/mail-body.txt"}} + failure: {email_on_failure} + success: {email_on_success} + aborted: false + send-to: + - recipients + - email-ext: + recipients: | + ${{FILE,path="artifacts/jenkins/error-mail-recipients.txt"}} + body: | + ${{FILE,path="artifacts/jenkins/error-mail-body.txt"}} + failure: true + success: true + aborted: true + send-to: + - recipients - parameter: name: bisect-parameters @@ -298,19 +1156,13 @@ default: "" description: "Project to bisect" - string: - name: bad_url - default: "" - description: "Bad git URL" - - string: - name: bad_branch + name: bad_git default: "" - description: "Bad branch or sha1" + description: "Bad git_url#branch/SHA1" - file: name: jenkins-scripts/replay_log description: "Replay part of bisect using provided bisect log" - default-parameters: - ci_project: '{ci_project}' - ci_config: '{ci_config}' distro: '{distro}' - wrapper: @@ -318,10 +1170,6 @@ wrappers: - build-wrappers: timeout: '{timeout}' - - credentials-binding: - - text: - credential-id: TCWG_JIRA_PASSWORD - variable: TCWG_JIRA_PASSWORD - builder: name: run-bisect @@ -336,9 +1184,10 @@ mkdir -p artifacts/jenkins build_name="#$BUILD_NUMBER-$current_project" echo "$build_name" > artifacts/jenkins/build-name - # artifacts/jenkins/mail-*.txt will be overwritten in successful builds. - echo "prathamesh.kulkarni@linaro.org,tcwg-gcc@linaro.org" > artifacts/jenkins/mail-recipients.txt - echo "Build $BUILD_URL failed" > artifacts/jenkins/mail-body.txt + + cat >> artifacts/jenkins/run-build.env <<EOF + build_name="$build_name" + EOF ./jenkins-scripts/docker-run.sh \ --distro $distro \ @@ -346,117 +1195,149 @@ --node $NODE_NAME \ -- \ ./jenkins-scripts/round-robin-bisect.sh \ - %% artifacts/manifests/bisect.sh \ + %%rel_artifacts artifacts \ --BUILD_URL "$BUILD_URL" \ --current_project "$current_project" \ - --bad_url "$bad_url" \ - --bad_branch "$bad_branch" \ + --bad_git "$bad_git" \ --replay_log "$(pwd)/jenkins-scripts/replay_log" \ --build_script "./jenkins-scripts/{build_script}" \ - --JIRA_PASSWORD "$TCWG_JIRA_PASSWORD" \ -- \ - ==rr[ci_project] $ci_project \ - ==rr[ci_config] $ci_config \ + ==rr[ci_project] '{ci_project}' \ + ==rr[ci_config] '{ci_config}' \ $extra_build_params \ - --scripts_branch "$scripts_branch" + --scripts_branch "$scripts_branch" & + res=0 && wait $! || res=$? - if [ x"$mail_recipients" != x"default" ]; then - echo "$mail_recipients" > artifacts/jenkins/mail-recipients.txt + if [ $res != 0 ]; then + echo $res > artifacts/failed fi - - trigger-followup-builds: - rr_project: '{rr_project}' - ci_project_config: '{ci_project_config}' + unstable-return: 125 + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - trigger-followup-builds-1: + dummy: dummy + - trigger-followup-builds-2: + ci_project: '{ci_project}' + ci_config: '{ci_config}' + - conditional-step: + condition-kind: current-status + steps: + - shell: | + #!/bin/bash + set -ex + + if [ -f artifacts/failed ]; then + # Mark failed build as FAILURE; no further steps will run. + exit $(cat artifacts/failed) + fi - publisher: name: bisect-publishers publishers: - - build-publishers - - email-ext: - recipients: | - ${{FILE,path="artifacts/jenkins/mail-recipients.txt"}} - content-type: text - body: | - ${{FILE,path="artifacts/jenkins/mail-body.txt"}} - failure: true - success: true - aborted: true - send-to: - - recipients + - archive: + artifacts: 'artifacts/**' + latest-only: false + - email-ext: + recipients: | + maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org + failure: true + success: false + aborted: true + send-to: + - recipients #END: tcwg/round-robin.yaml.inc - job-template: - name: tcwg_binutils-build-{toolchain_name}-{toolchain_ver}-{target}-{type_of_test} + name: '{ci_project}--{toolchain_ver}-{target}{type_of_test}-build' project-type: freestyle defaults: global properties: - - default-properties - - build-blocker: - blocking-jobs: - - "tcwg_binutils-bisect-{toolchain_name}-{toolchain_ver}-{target}-{type_of_test}" - queue-scanning: 'ALL' + - build-properties: + ci_project: '{ci_project}' + ci_config: '{ci_config}' parameters: - build-parameters: - ci_project: '{ci_project}' - ci_config: '{toolchain_name}-{toolchain_ver}-{target}-{type_of_test}' - binutils_url: '{binutils_url}' - default_branch: 'default' + default_git: 'default' distro: '{distro}' - disabled: '{disabled}' - node: '{node} && tcwg-build' + disabled: false + node: '{node}' concurrent: false - display-name: 'TCWG Build {ci_project}/{toolchain_name}-{toolchain_ver}-{target}-{type_of_test}' + display-name: 'TCWG Build {ci_project}/{ci_config}' workspace: workspace/tcwg_gnu_$EXECUTOR_NUMBER scm: - build-scms: binutils_url: '{binutils_url}' binutils_branch: '{binutils_branch}' triggers: - - pollscm: - cron: 'H H H/2 * *' + - timed: '{cron}' wrappers: - build-wrappers: - timeout: 1000 + timeout: '{timeout_build}' builders: - run-build: + ci_project: '{ci_project}' + ci_config: '{ci_config}' build_script: 'tcwg_gnu-build.sh' - components: 'binutils' - rr_project: 'tcwg_binutils' - ci_project_config: '{toolchain_name}-{toolchain_ver}-{target}-{type_of_test}' docker_arch: '{docker_arch}' - target: '{target}' - type_of_test: '{type_of_test}' + docker_security: '{docker_security}' publishers: - - build-publishers - - email-ext: - recipients: 'tcwg-gcc@linaro.org' - aborted: true - failure: false - success: false + - build-publishers: + email_on_failure: false + email_on_success: true - job-template: - name: tcwg_binutils-bisect-{toolchain_name}-{toolchain_ver}-{target}-{type_of_test} + name: '{ci_project}--{toolchain_ver}-{target}{type_of_test}-precommit' project-type: freestyle defaults: global properties: - - default-properties - - build-blocker: - # Run at most 1 bisect job for all configurations. - # Bisect jobs can easily run for 5+ hours, and a few of them - # can hog the entire TCWG build farm for a long time. Since - # same regression tends to appear in multiple configurations, - # run bisections for all configurations in-order. - blocking-jobs: - - "tcwg_binutils-bisect-.*" - queue-scanning: 'BUILDABLE' + - precommit-properties: + ci_project: '{ci_project}' + ci_config: '{ci_config}' parameters: - - bisect-parameters: + - precommit-parameters: + default_git: 'baseline' + distro: '{distro}' + disabled: false + node: '{node}' + concurrent: true + display-name: 'TCWG Precommit {ci_project}/{ci_config}' + workspace: workspace/tcwg_gnu_$EXECUTOR_NUMBER + scm: + - jenkins-scripts + wrappers: + - build-wrappers: + timeout: 300 + builders: + - run-precommit: + ci_project: '{ci_project}' + ci_config: '{ci_config}' + build_script: 'tcwg_gnu-build.sh' + docker_arch: '{docker_arch}' + docker_security: '{docker_security}' + publishers: + - build-publishers: + email_on_failure: true + email_on_success: false + +- job-template: + name: '{ci_project}--{toolchain_ver}-{target}{type_of_test}-bisect' + project-type: freestyle + defaults: global + properties: + - bisect-properties: ci_project: '{ci_project}' - ci_config: '{toolchain_name}-{toolchain_ver}-{target}-{type_of_test}' + ci_config: '{ci_config}' + parameters: + - bisect-parameters: distro: '{distro}' - disabled: '{disabled}' - node: '{node} && tcwg-bisect' + disabled: false + node: '{node}' concurrent: false - display-name: 'TCWG Bisect {ci_project}/{toolchain_name}-{toolchain_ver}-{target}-{type_of_test}' + display-name: 'TCWG Bisect {ci_project}/{ci_config}' workspace: workspace/tcwg_gnu_$EXECUTOR_NUMBER scm: - jenkins-scripts @@ -465,10 +1346,11 @@ timeout: 6000 builders: - run-bisect: + ci_project: '{ci_project}' + ci_config: '{ci_config}' build_script: 'tcwg_gnu-build.sh' - rr_project: 'tcwg_binutils' - ci_project_config: '{toolchain_name}-{toolchain_ver}-{target}-{type_of_test}' docker_arch: '{docker_arch}' + docker_security: '{docker_security}' publishers: - bisect-publishers -# checksum: 557ca8a0570213fa83f592d455fdeeb7 +# checksum: 2489b5878c53dee36245a40ea8e11f2a |