diff options
Diffstat (limited to 'tcwg_chromium--masterfast.yaml')
-rw-r--r-- | tcwg_chromium--masterfast.yaml | 864 |
1 files changed, 864 insertions, 0 deletions
diff --git a/tcwg_chromium--masterfast.yaml b/tcwg_chromium--masterfast.yaml new file mode 100644 index 0000000000..ea40510145 --- /dev/null +++ b/tcwg_chromium--masterfast.yaml @@ -0,0 +1,864 @@ +# Auto generated by ./tcwg/generate-yamlfiles.sh from tcwg_chromium.yaml.in and tcwg_chromium/tcwg_chromium--masterfast.def. Do not edit. +# Beware: git branch names need the 'refs/heads/' prefix, while tags need 'refs/tags/' +- project: + name: tcwg_chromium--masterfast + chromium_url: https://chromium.googlesource.com/chromium/tools/depot_tools.git + jobs: + - 'tcwg_chromium--masterfast' + +- job-group: + name: tcwg_chromium--masterfast + chromium_branch: master + distro: default + cron_schedule: 'H H H/3 * *' + jobs: + - 'tcwg_chromium--masterfast-build' + - 'tcwg_chromium--masterfast-bisect' + ci_project: 'tcwg_chromium' + ci_config: 'masterfast' + +#BEGIN: tcwg/round-robin.yaml.inc +# -*- mode: Yaml -*- + +#BEGIN: tcwg/default.yaml.inc +# -*- mode: Yaml -*- + +- property: + name: default-properties + properties: + - authorization: + anonymous: + - job-read + - job-extended-read + everyone-flat: + - job-build + - job-cancel + - build-discarder: + days-to-keep: 30 + +- scm: + name: jenkins-scripts + scm: + - git: + url: https://git.linaro.org/toolchain/jenkins-scripts.git + refspec: +refs/heads/*:refs/remotes/origin/* +refs/changes/*:refs/changes/* + branches: + - $scripts_branch + basedir: jenkins-scripts + skip-tag: true + reference-repo: /home/tcwg-buildslave/snapshots-ref/jenkins-scripts.git + wipe-workspace: false + clean: + before: true + prune: true + +#END: tcwg/default.yaml.inc + +- parameter: + name: default-parameters + parameters: + - string: + name: extra_build_params + default: "" + description: "Extra parameters to pass to the build script; can be used to override settings extracted from ci_project/ci_config" + - string: + name: distro + default: '{distro}' + description: 'Distro image to use' + - string: + name: scripts_branch + default: master + description: 'Scripts revision to use' + +- parameter: + name: component-parameters + parameters: + - string: + name: '{component}_git' + default: '{default_git}' + description: "{component} git_url#branch/sha1 to build, or 'default', 'baseline' or 'jenkins-scm', or pw://series/<id> to build with patches from patchwork ('precommit' mode, use 'pw://series/<id>/retrigger/' to force a rebuild)" + +- parameter: + name: all-component-parameters + parameters: + - string: + name: 'components_to_update' + default: '*' + description: "Component list to update. '*' means all components" + +- parameter: + name: build-parameters + parameters: + - all-component-parameters: + default_git: '{default_git}' + - choice: + name: update_baseline + choices: + - onsuccess + - force + - init + - ignore + description: "What to do with baseline in this build" + - choice: + name: notify + choices: + - ignore + - onregression + - precommit + description: "Whether to notify developers about the result of this build" + - default-parameters: + distro: '{distro}' + + +- property: + name: build-properties + properties: + - default-properties + - build-blocker: + blocking-jobs: + - '{ci_project}--{ci_config}-bisect' + queue-scanning: 'ALL' + + +- property: + name: bisect-properties + properties: + - default-properties + +# Beware: git branch names need the 'refs/heads/' prefix while tags +# need 'refs/tags/'. +- scm: + name: component-scm + scm: + - git: + url: '{url}' + branches: + - '{branch}' + basedir: '{component}' + skip-tag: true + reference-repo: '/home/tcwg-buildslave/snapshots-ref/{reference}' + wipe-workspace: false + clean: + before: true + prune: true + +# SCMs must start with jenkins-scripts and then be ordered alphabetically. +# Same alphabetical order must be followed in .def files. The reason behind +# this is that jenkins sets $GIT_COMMIT to sha1 of jenkins-scripts, and then +# sets $GIT_COMMIT_1, $GIT_COMMIT_2, etc. to sha1s of component SCM /in the +# order that they are declared here/. +- scm: + name: build-scms + scm: + - jenkins-scripts + +- wrapper: + name: build-wrappers + wrappers: + - timeout: + timeout: '{timeout}' + - timestamps + - ssh-agent-credentials: + # tcwg-buildslave user id + users: + - 'e0958a95-204f-4c14-a66c-5e2be6c5d50a' + - credentials-binding: + - text: + credential-id: TCWG_JIRA_TOKEN + variable: TCWG_JIRA_TOKEN + +- builder: + name: trigger-followup-builds-1 + builders: + - shell: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + for i in artifacts/trigger-build-* artifacts/trigger-bisect; do + if [ -f $i ]; then + echo "distro=$distro" >> $i + echo "scripts_branch=$scripts_branch" >> $i + fi + done + + trap "" EXIT + +- builder: + name: trigger-followup-builds-2 + builders: + - trigger-builds: + - project: '{ci_project}--{ci_config}-build' + parameter-factories: + - factory: filebuild + file-pattern: artifacts/trigger-build-* + - project: '{ci_project}--{ci_config}-bisect' + property-file: artifacts/trigger-bisect + +- builder: + name: run-build-notify + builders: + - shell: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + if [ -f artifacts/failed ]; then + # Only send notifications for forced successful + # (aka "onregression") builds; never for failed builds. + echo "Build failed. Using option notify=ignore." + notify=ignore + fi + + # Basic notification -- create entry in interesting-commits. + # Note that we add --notify to the manifest -- we will need this + # later in round-robin-baseline.sh when re-writing history + # to update notifications files in interesting-commits and jira. + build_container_exec \ + ./jenkins-scripts/round-robin-notify.sh \ + @@rr[top_artifacts] artifacts --notify "$notify" \ + __stage init __verbose true \ + __build_script {build_script} \ + &> ./artifacts/jenkins/notify-init.log & + + if ! wait $!; then + echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \ + > artifacts/jenkins/error-mail-recipients.txt + echo -e "$BUILD_URL\nERROR: notify-init failed\n" \ + >> artifacts/jenkins/error-mail-body.txt + echo "ERROR: notify-init failed" + exit 1 + fi + + # Full notification + echo "NOTE: Notify developers about this build" + + build_container_exec \ + ./jenkins-scripts/round-robin-notify.sh \ + @@rr[top_artifacts] artifacts __stage full __verbose true \ + __build_script {build_script} \ + &> ./artifacts/jenkins/notify-full.log & + + if ! wait $!; then + echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \ + > artifacts/jenkins/error-mail-recipients.txt + echo -e "$BUILD_URL\nERROR: notify-full failed\n" \ + >> artifacts/jenkins/error-mail-body.txt + echo "ERROR: notify-full failed" + fi + + # Failsafe for avoid spamming developers. + if [ "$notify" = "ignore" ] \ + && [ -f artifacts/jenkins/mail-recipients.txt ]; then + echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \ + > artifacts/jenkins/error-mail-recipients.txt + echo -e "$BUILD_URL\nERROR: tried to unexpectedly notify developers\n" \ + >> artifacts/jenkins/error-mail-body.txt + mv artifacts/jenkins/mail-recipients.txt \ + artifacts/jenkins/mail-recipients.bak + fi + + trap "" EXIT + +- builder: + name: run-exit-on-failure + builders: + - shell: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + if [ -f artifacts/failed ]; then + # Mark failed build as FAILURE; no further steps will run. + exit $(cat artifacts/failed) + fi + # Only successful builds run beyond this point. + + trap "" EXIT + +- builder: + name: run-push-baseline + builders: + - shell: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + # Update baseline branch of base-artifacts. + # If we made it this far, then the build is successful + # (possibly because it was forced to be declared successful). + if [ "$update_baseline" = "ignore" ]; then + trap "" EXIT + exit 0 + fi + + # We may need jira token while rewriting history. + if [ -f $HOME/.jipdate.yml ]; then + ( + # BE CAREFUL WITH TCWG_JIRA_TOKEN + set +x + build_container_exec \ + sed -i -e "s/#TCWG_JIRA_TOKEN#/$TCWG_JIRA_TOKEN/" \ + "$HOME/.jipdate.yml" + ) + fi + + build_container_exec \ + ./jenkins-scripts/round-robin-baseline.sh \ + @@rr[top_artifacts] artifacts \ + __build_script {build_script} \ + __push_base_artifacts true \ + __rewrite_base_artifacts true \ + __commit_artifacts {commit_artifacts} \ + __rewrite_num 100 \ + &> ./artifacts/jenkins/rewrite.log & + + if ! wait $!; then + echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \ + > artifacts/jenkins/error-mail-recipients.txt + echo -e "$BUILD_URL\nERROR: round-robin-baseline.sh failed" \ + >> artifacts/jenkins/error-mail-body.txt + echo "ERROR: round-robin-baseline.sh failed" + # We have failed to update the baseline, so let's fail and + # not notify developers. + exit 1 + fi + + trap "" EXIT + +- builder: + name: run-push-lnt-results + builders: + - trigger-builds: + - project: tcwg-update-lnt-results + predefined-parameters: | + ci_project={ci_project} + ci_config={ci_config} + + +- builder: + name: run-build-init + builders: + - shell: + command: | + #!/bin/bash + set -ex + # Delete artifacts now to avoid re-using stale artifacts if + # below docker-run.sh fails to start up a container. + rm -rf artifacts/ + mkdir -p artifacts/jenkins + + build_name="#$BUILD_NUMBER" + echo "$build_name" > artifacts/jenkins/build-name + + if [ "{precommit}" = "True" ]; then + cat >> artifacts/jenkins/run-build.env <<EOF + update_baseline=ignore + EOF + fi + + cat >> artifacts/jenkins/run-build.env <<EOF + build_name="$build_name" + EOF + unstable-return: 125 + +- builder: + name: run-build-skip-if-queue + builders: + - shell: + command: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + # Skip SCM-triggered builds if there are other builds in + # the queue. We need to process all pending bisection-triggered + # builds before fetching a new batch of upstream changes. + # Re. skipping post-commit build if pre-commit queue is + # not empty -- it's just a favor to pre-commit builds. + if [ x"${{BUILD_CAUSE_SCMTRIGGER-false}}" = x"true" ] \ + || [ x"${{BUILD_CAUSE_TIMERTRIGGER-false}}" = x"true" ]; then + for job_type in build; do + url="${{JOB_URL%-build/}}-$job_type" + inQueue=$(curl -s "$url/api/xml?tree=inQueue" \ + | sed -e "s#.*<inQueue>\(.*\)</inQueue>.*#\1#") + if [ "$inQueue" != "false" ]; then + echo "SKIP SCM BUILD" + touch artifacts/jenkins/skip-scm-build + build_name="$build_name-skip-scm-build" + echo "$build_name" > artifacts/jenkins/build-name + + # Mark build unstable aka a skipped build + exit 125 + fi + done + fi + + trap "" EXIT + unstable-return: 125 + +- builder: + name: run-build-start-manifest + builders: + - shell: + command: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + docker_opts=( + --distro $distro + ) + + ./jenkins-scripts/start-container-docker.sh --prefix build_ \ + "${{docker_opts[@]}}" > build_container.sh + source build_container.sh + trap "cleanup_all_containers" EXIT + + # Run the build script to fetch base-artifacts/, which are + # required to fetch baseline sources to apply patches to in + # pw-apply.sh below. This also initializes the build manifest + # with all below settings; we then import this manifest below. + # + # The parameter __finish_after makes the build script stop + # after fetching the baseline and "__" avoids saving this parameter + # in the manifest, so that we don't see this option in the main + # run of the build script below. + build_container_exec \ + ./jenkins-scripts/{build_script} \ + %%rr[top_artifacts] artifacts \ + ==rr[ci_project] '{ci_project}' \ + ==rr[ci_config] '{ci_config}' \ + $extra_build_params \ + --scripts_branch "$scripts_branch" \ + ==rr[mode] "build" \ + ==rr[update_baseline] "$update_baseline" \ + ==rr[components_to_update] "$components_to_update" \ + --fast_demo true \ + --BUILD_URL "$BUILD_URL" \ + __finish_at reset_artifacts + + cat >> artifacts/jenkins/run-build.env <<EOF + docker_opts=(${{docker_opts[@]+$(printf "%q$IFS" "${{docker_opts[@]}}")}}) + source build_container.sh + trap "cleanup_all_containers" EXIT + EOF + + # Now that we have "trap cleanup_all_containers EXIT" in + # run-build.env we need to reset the trap at the end of every + # step to preserve container[s] for the next step. + trap "" EXIT + unstable-return: 125 + +- builder: + name: run-build-process-params + builders: + - shell: + command: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + branch_opt=() + # 1 to skip jenkins-scripts + i=1 + # Update build name with the components to update + build_name="$build_name-$components_to_update" + + echo "$build_name" > artifacts/jenkins/build-name + + cat >> artifacts/jenkins/run-build.env <<EOF + build_name="$build_name" + branch_opt=(${{branch_opt[@]+$(printf "%q$IFS" "${{branch_opt[@]}}")}}) + EOF + + trap "" EXIT + unstable-return: 125 + +- builder: + name: run-build-main + builders: + - shell: + command: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + + # Run the build script to from the manifest created above. + # + # The parameter __start_at makes the build script start right + # where the above run finished. This allows us to use read-only + # bind-mount for base-artifacts/, which, otherwise, may require + # multi-gig rsync. + build_container_exec \ + ./jenkins-scripts/{build_script} \ + @@rr[top_artifacts] artifacts \ + "${{branch_opt[@]}}" \ + __start_at reset_artifacts+ & + res=0 && wait $! || res=$? + + # Update build_name + if [ -f artifacts/results ]; then + build_name="$build_name-R$(tail -n1 artifacts/results)" + fi + if [ "$update_baseline" != "onsuccess" ]; then + build_name="$build_name-$update_baseline" + fi + + if [ -f artifacts/trigger-bisect ]; then + build_name="$build_name-trigger-bisect" + fi + echo "$build_name" > artifacts/jenkins/build-name + + # Everything but 0 and $INTERNAL_FAILURE is an unexpected exit code, + # so stop here and do nothing else. From jenkins point of view ... + # - if $res==0: build is successful, so proceed with all + # followup steps -- notify, push, trigger precommit. + # - if $res==123: build has a regression, so proceed with triggering + # followup builds and sending precommit notifications; but stop + # before run-push-baseline by checking artifacts/failed. + # - if $res is anything else: build has an unexpected failure, so + # skip the rest of the steps by setting UNSTABLE build status. + # This will cause all followup conditional-steps to skip due to + # "current-status" condition. + # + # See comment before round-robin.sh:check_regression() for details. + case $res in + 0) + ;; + 123) + echo 123 > artifacts/failed + ;; + *) + res=125 + ;; + esac + + if [ $res = 125 ]; then + # We had unexpected failure in the build scripts, so can't + # proceed with triggers and notifications. Mark this build + # as UNSTABLE, which will skip all following conditional-steps. + exit 125 + fi + + cat >> artifacts/jenkins/run-build.env <<EOF + build_name="$build_name" + EOF + + trap "" EXIT + unstable-return: 125 + +- builder: + name: run-build-fini + builders: + - shell: + command: | + #!/bin/bash + source artifacts/jenkins/run-build.env + set -ex + # The "trap cleanup_all_containers EXIT" statement in + # artifacts/jenkins/run-build.env will remove build and precommit + # containers. + +- builder: + name: run-build + builders: + - conditional-step: + condition-kind: current-status + steps: + - run-build-init: + precommit: false + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-build-skip-if-queue: + dummy: dummy + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-build-start-manifest: + ci_project: '{ci_project}' + ci_config: '{ci_config}' + build_script: '{build_script}' + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-build-process-params: + dummy: dummy + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-push-baseline: + build_script: '{build_script}' + commit_artifacts: 'false' + - conditional-step: + condition-kind: current-status + steps: + - run-build-main: + build_script: '{build_script}' + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - trigger-followup-builds-1: + dummy: dummy + - trigger-followup-builds-2: + ci_project: '{ci_project}' + ci_config: '{ci_config}' + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-build-notify: + build_script: '{build_script}' + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-exit-on-failure: + dummy: dummy + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-push-baseline: + build_script: '{build_script}' + commit_artifacts: 'true' + - conditional-step: + condition-kind: current-status + steps: + - run-push-lnt-results: + ci_project: '{ci_project}' + ci_config: '{ci_config}' + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - run-build-fini: + dummy: dummy + + +- publisher: + name: build-publishers + publishers: + - archive: + artifacts: 'artifacts/**' + latest-only: false + - html-publisher: + name: 'HTML Report' + dir: 'artifacts/jenkins' + files: 'status.html' + keep-all: true + - email-ext: + recipients: | + ${{FILE,path="artifacts/jenkins/mail-recipients.txt"}} + reply-to: linaro-toolchain@lists.linaro.org + subject: | + ${{FILE,path="artifacts/jenkins/mail-subject.txt"}} + content-type: text + body: | + ${{FILE,path="artifacts/jenkins/mail-body.txt"}} + failure: {email_on_failure} + success: {email_on_success} + aborted: false + send-to: + - recipients + - email-ext: + recipients: | + ${{FILE,path="artifacts/jenkins/error-mail-recipients.txt"}} + body: | + ${{FILE,path="artifacts/jenkins/error-mail-body.txt"}} + failure: true + success: true + aborted: true + send-to: + - recipients + +- parameter: + name: bisect-parameters + parameters: + - string: + name: current_project + default: "" + description: "Project to bisect" + - string: + name: bad_git + default: "" + description: "Bad git_url#branch/SHA1" + - file: + name: jenkins-scripts/replay_log + description: "Replay part of bisect using provided bisect log" + - default-parameters: + distro: '{distro}' + +- wrapper: + name: bisect-wrappers + wrappers: + - build-wrappers: + timeout: '{timeout}' + +- builder: + name: run-bisect + builders: + - shell: + command: | + #!/bin/bash + set -ex + # Delete artifacts now to avoid re-using stale artifacts if + # below docker-run.sh fails to start up a container. + rm -rf artifacts/ + mkdir -p artifacts/jenkins + build_name="#$BUILD_NUMBER-$current_project" + echo "$build_name" > artifacts/jenkins/build-name + + cat >> artifacts/jenkins/run-build.env <<EOF + build_name="$build_name" + EOF + + ./jenkins-scripts/docker-run.sh \ + --distro $distro \ + -- \ + ./jenkins-scripts/round-robin-bisect.sh \ + %%rel_artifacts artifacts \ + --BUILD_URL "$BUILD_URL" \ + --current_project "$current_project" \ + --bad_git "$bad_git" \ + --replay_log "$(pwd)/jenkins-scripts/replay_log" \ + --build_script "./jenkins-scripts/{build_script}" \ + -- \ + ==rr[ci_project] '{ci_project}' \ + ==rr[ci_config] '{ci_config}' \ + $extra_build_params \ + --fast_demo true \ + --scripts_branch "$scripts_branch" & + res=0 && wait $! || res=$? + + if [ $res != 0 ]; then + echo $res > artifacts/failed + fi + unstable-return: 125 + - build-name-setter: + name: 'artifacts/jenkins/build-name' + file: true + - conditional-step: + condition-kind: current-status + steps: + - trigger-followup-builds-1: + dummy: dummy + - trigger-followup-builds-2: + ci_project: '{ci_project}' + ci_config: '{ci_config}' + - conditional-step: + condition-kind: current-status + steps: + - shell: | + #!/bin/bash + set -ex + + if [ -f artifacts/failed ]; then + # Mark failed build as FAILURE; no further steps will run. + exit $(cat artifacts/failed) + fi + +- publisher: + name: bisect-publishers + publishers: + - archive: + artifacts: 'artifacts/**' + latest-only: false + - email-ext: + recipients: | + maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org + failure: true + success: false + aborted: true + send-to: + - recipients +#END: tcwg/round-robin.yaml.inc + +- job-template: + name: tcwg_chromium--masterfast-build + project-type: freestyle + defaults: global + properties: + - build-properties: + ci_project: '{ci_project}' + ci_config: '{ci_config}' + parameters: + - build-parameters: + default_git: 'default' + distro: '{distro}' + disabled: false + node: tcwg-best && tcwg-x86_64 && tcwg-fast-disk + concurrent: false + display-name: 'TCWG Build {ci_project}--{ci_config}' + workspace: workspace/tcwg_chromium_$EXECUTOR_NUMBER + scm: + - jenkins-scripts + triggers: + - pollscm: + cron: '{cron_schedule}' + wrappers: + - build-wrappers: + timeout: 600 + builders: + - run-build: + ci_project: '{ci_project}' + ci_config: '{ci_config}' + build_script: 'tcwg_chromium-build.sh' + publishers: + - build-publishers: + email_on_failure: false + email_on_success: true + +- job-template: + name: tcwg_chromium--masterfast-bisect + project-type: freestyle + defaults: global + properties: + - bisect-properties: + ci_project: '{ci_project}' + ci_config: '{ci_config}' + parameters: + - bisect-parameters: + distro: '{distro}' + disabled: false + node: tcwg-best && tcwg-x86_64 && tcwg-fast-disk + concurrent: false + display-name: 'TCWG Bisect {ci_project}--{ci_config}' + workspace: workspace/tcwg_chromium_$EXECUTOR_NUMBER + scm: + - jenkins-scripts + wrappers: + - bisect-wrappers: + timeout: 2880 + builders: + - run-bisect: + ci_project: '{ci_project}' + ci_config: '{ci_config}' + build_script: 'tcwg_chromium-build.sh' + publishers: + - bisect-publishers +# checksum: 677d5db5295c2cea4dd3308083eee485 |