| # Auto generated by ./tcwg/generate-yamlfiles.sh from tcwg_gnu.yaml.in and tcwg_gnu/tcwg_glibc.def. Do not edit. |
| # Beware: git branch names need the 'refs/heads/' prefix, while tags need 'refs/tags/' |
| - project: |
| name: tcwg_glibc |
| type_of_test: "" |
| timeout_build: 1000 |
| # Trigger configuration that have precommit testing every 6 hours |
| # to test incoming patches in a timely manner. |
| cron_master: 'H H/6 * * *' |
| docker_security: '' |
| ci_project: |
| - tcwg_glibc_build |
| - tcwg_glibc_check: |
| # The SYS_ADMIN is required to allow the glibc container tests use mount, chroot, and ptrace syscalls. |
| # Also, docker default apprmor filter disables the mount sycall, so disable it as well. |
| # See https://linaro.atlassian.net/browse/GNU-836 for more details. |
| docker_security: '--cap-add=SYS_ADMIN --security-opt apparmor=unconfined' |
| glibc_url: git://sourceware.org/git/glibc.git |
| target: |
| - aarch64: |
| node: tcwg-build && tcwg-armv8_64 |
| docker_arch: arm64 |
| - arm: |
| node: tcwg-build && tcwg-armv8_32 |
| docker_arch: armhf |
| jobs: |
| - tcwg_glibc-master |
| |
| - job-group: |
| name: tcwg_glibc-master |
| toolchain_ver: master |
| cron: '{cron_master}' |
| glibc_branch: master |
| distro: default |
| jobs: |
| - '{ci_project}--{toolchain_ver}-{target}{type_of_test}-build' |
| - '{ci_project}--{toolchain_ver}-{target}{type_of_test}-precommit' |
| - '{ci_project}--{toolchain_ver}-{target}{type_of_test}-bisect' |
| ci_config: '{toolchain_ver}-{target}{type_of_test}' |
| |
| #BEGIN: tcwg/round-robin.yaml.inc |
| # -*- mode: Yaml -*- |
| |
| #BEGIN: tcwg/default.yaml.inc |
| # -*- mode: Yaml -*- |
| |
| - property: |
| name: default-properties |
| properties: |
| - authorization: |
| anonymous: |
| - job-read |
| - job-extended-read |
| everyone-flat: |
| - job-build |
| - job-cancel |
| - build-discarder: |
| days-to-keep: 20 |
| |
| - scm: |
| name: jenkins-scripts |
| scm: |
| - git: |
| url: https://gitlab.com/Linaro/tcwg/jenkins-scripts.git |
| refspec: +refs/heads/*:refs/remotes/origin/* +refs/changes/*:refs/changes/* |
| branches: |
| - $scripts_branch |
| basedir: jenkins-scripts |
| skip-tag: true |
| reference-repo: /home/tcwg-buildslave/snapshots-ref/jenkins-scripts.git |
| wipe-workspace: false |
| clean: |
| before: true |
| prune: true |
| |
| #END: tcwg/default.yaml.inc |
| |
| - parameter: |
| name: default-parameters |
| parameters: |
| - string: |
| name: extra_build_params |
| default: "" |
| description: "Extra parameters to pass to the build script; can be used to override settings extracted from ci_project/ci_config" |
| - string: |
| name: distro |
| default: '{distro}' |
| description: 'Distro image to use' |
| - string: |
| name: scripts_branch |
| default: master |
| description: 'Scripts revision to use' |
| |
| - parameter: |
| name: component-parameters |
| parameters: |
| - string: |
| name: '{component}_git' |
| default: '{default_git}' |
| description: "{component} git_url#branch/sha1 to build, or 'default', 'baseline' or 'jenkins-scm', or pw://series/<id> to build with patches from patchwork ('precommit' mode, use 'pw://series/<id>/retrigger/' to force a rebuild)" |
| |
| - parameter: |
| name: all-component-parameters |
| parameters: |
| - component-parameters: |
| component: glibc |
| default_git: '{default_git}' |
| |
| - parameter: |
| name: build-parameters |
| parameters: |
| - all-component-parameters: |
| default_git: '{default_git}' |
| - choice: |
| name: update_baseline |
| choices: |
| - onsuccess |
| - force |
| - init |
| - ignore |
| description: "What to do with baseline in this build" |
| - choice: |
| name: notify |
| choices: |
| - ignore |
| - onregression |
| - precommit |
| description: "Whether to notify developers about the result of this build" |
| - string: |
| name: rr_stage |
| default: build |
| description: "Describe the current round-robin stage for this run (informative only)" |
| - default-parameters: |
| distro: '{distro}' |
| |
| # Precommit parameters ... |
| # - allow only "ignore" value for update_baseline parameter; |
| # - allow string (which should be an email) for notify parameter. |
| - parameter: |
| name: precommit-parameters |
| parameters: |
| - all-component-parameters: |
| default_git: '{default_git}' |
| - string: |
| name: notify |
| default: 'precommit' |
| description: "Email to send failure notifications to" |
| - default-parameters: |
| distro: '{distro}' |
| |
| - property: |
| name: build-properties |
| properties: |
| - default-properties |
| - build-blocker: |
| blocking-jobs: |
| - '{ci_project}--{ci_config}-bisect' |
| queue-scanning: 'ALL' |
| |
| - property: |
| name: precommit-properties |
| properties: |
| - default-properties |
| - build-blocker: |
| blocking-jobs: |
| - '{ci_project}--{ci_config}-build' |
| queue-scanning: 'BUILDABLE' |
| |
| - property: |
| name: bisect-properties |
| properties: |
| - default-properties |
| - build-blocker: |
| # Run at most 1 bisect job for all configurations. |
| # Bisect jobs can easily run for 5+ hours, and a few of them |
| # can hog the entire TCWG build farm for a long time. Since |
| # same regression tends to appear in multiple configurations, |
| # run bisections for all configurations in-order. |
| blocking-jobs: |
| - '{ci_project}--.*-bisect' |
| queue-scanning: 'BUILDABLE' |
| |
| # Beware: git branch names need the 'refs/heads/' prefix while tags |
| # need 'refs/tags/'. |
| - scm: |
| name: component-scm |
| scm: |
| - git: |
| url: '{url}' |
| branches: |
| - 'refs/heads/{branch}' |
| basedir: '{component}' |
| do-not-fetch-tags: true |
| reference-repo: '/home/tcwg-buildslave/snapshots-ref/{reference}' |
| refspec: '+refs/heads/{branch}:refs/remotes/origin/{branch}' |
| shallow-clone: true |
| wipe-workspace: false |
| clean: |
| before: true |
| prune: true |
| |
| # SCMs must start with jenkins-scripts and then be ordered alphabetically. |
| # Same alphabetical order must be followed in .def files. The reason behind |
| # this is that jenkins sets $GIT_COMMIT to sha1 of jenkins-scripts, and then |
| # sets $GIT_COMMIT_1, $GIT_COMMIT_2, etc. to sha1s of component SCM /in the |
| # order that they are declared here/. |
| - scm: |
| name: build-scms |
| scm: |
| - jenkins-scripts |
| - component-scm: |
| component: glibc |
| url: '{glibc_url}' |
| branch: '{glibc_branch}' |
| reference: glibc.git |
| |
| - wrapper: |
| name: build-wrappers |
| wrappers: |
| - timeout: |
| timeout: '{timeout}' |
| - timestamps |
| - ssh-agent-credentials: |
| # tcwg-buildslave user id |
| users: |
| - 'e0958a95-204f-4c14-a66c-5e2be6c5d50a' |
| - 'tcwg-buildslave-20220725' |
| # - 'tcwg-build-20220725' |
| - credentials-binding: |
| - text: |
| credential-id: TCWG_JIRA_TOKEN |
| variable: TCWG_JIRA_TOKEN |
| - text: |
| credential-id: TCWG_PW_TOKEN |
| variable: TCWG_PW_TOKEN |
| |
| - builder: |
| name: trigger-followup-builds-1 |
| builders: |
| - shell: | |
| #!/bin/bash |
| source artifacts/jenkins/run-build.env |
| set -ex |
| |
| for i in artifacts/trigger-build-* artifacts/trigger-bisect; do |
| if [ -f $i ]; then |
| echo "distro=$distro" >> $i |
| echo "scripts_branch=$scripts_branch" >> $i |
| fi |
| done |
| |
| trap "" EXIT |
| |
| - builder: |
| name: trigger-followup-builds-2 |
| builders: |
| - trigger-builds: |
| - project: '{ci_project}--{ci_config}-build' |
| parameter-factories: |
| - factory: filebuild |
| file-pattern: artifacts/trigger-build-* |
| - project: '{ci_project}--{ci_config}-bisect' |
| property-file: artifacts/trigger-bisect |
| |
| - builder: |
| name: run-build-notify |
| builders: |
| - shell: | |
| #!/bin/bash |
| source artifacts/jenkins/run-build.env |
| set -ex |
| |
| if [ -f artifacts/failed ]; then |
| # Only send notifications for forced successful |
| # (aka "onregression") builds; never for failed builds. |
| echo "Build failed. Using option notify=ignore." |
| notify=ignore |
| fi |
| |
| # Basic notification -- create entry in interesting-commits. |
| # Note that we add --notify to the manifest -- we will need this |
| # later in round-robin-baseline.sh when re-writing history |
| # to update notifications files in interesting-commits and jira. |
| build_container_exec \ |
| ./jenkins-scripts/rr/round-robin-notify.sh \ |
| @@rr[top_artifacts] artifacts --notify "$notify" \ |
| __stage init __verbose true \ |
| __build_script {build_script} \ |
| &> ./artifacts/jenkins/notify-init.log & |
| |
| if ! wait $!; then |
| echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \ |
| > artifacts/jenkins/error-mail-recipients.txt |
| echo -e "$BUILD_URL\nERROR: notify-init failed\n" \ |
| >> artifacts/jenkins/error-mail-body.txt |
| echo "ERROR: notify-init failed" |
| exit 1 |
| fi |
| |
| # Full notification |
| echo "NOTE: Notify developers about this build" |
| |
| build_container_exec \ |
| ./jenkins-scripts/rr/round-robin-notify.sh \ |
| @@rr[top_artifacts] artifacts __stage full __verbose true \ |
| __build_script {build_script} \ |
| &> ./artifacts/jenkins/notify-full.log & |
| |
| if ! wait $!; then |
| echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \ |
| > artifacts/jenkins/error-mail-recipients.txt |
| echo -e "$BUILD_URL\nERROR: notify-full failed\n" \ |
| >> artifacts/jenkins/error-mail-body.txt |
| echo "ERROR: notify-full failed" |
| fi |
| |
| # Failsafe for avoid spamming developers. |
| if [ "$notify" = "ignore" ] \ |
| && [ -f artifacts/jenkins/mail-recipients.txt ]; then |
| echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \ |
| > artifacts/jenkins/error-mail-recipients.txt |
| echo -e "$BUILD_URL\nERROR: tried to unexpectedly notify developers\n" \ |
| >> artifacts/jenkins/error-mail-body.txt |
| mv artifacts/jenkins/mail-recipients.txt \ |
| artifacts/jenkins/mail-recipients.bak |
| fi |
| |
| trap "" EXIT |
| |
| - builder: |
| name: run-exit-on-failure |
| builders: |
| - shell: | |
| #!/bin/bash |
| source artifacts/jenkins/run-build.env |
| set -ex |
| |
| if [ -f artifacts/failed ]; then |
| # Mark failed build as FAILURE; no further steps will run. |
| exit $(cat artifacts/failed) |
| fi |
| # Only successful builds run beyond this point. |
| |
| trap "" EXIT |
| |
| - builder: |
| name: run-push-baseline |
| builders: |
| - shell: | |
| #!/bin/bash |
| source artifacts/jenkins/run-build.env |
| set -ex |
| |
| # Update baseline branch of base-artifacts. |
| # If we made it this far, then the build is successful |
| # (possibly because it was forced to be declared successful). |
| if [ "$update_baseline" = "ignore" ]; then |
| trap "" EXIT |
| exit 0 |
| fi |
| |
| # We may need jira token while rewriting history. |
| if [ -f $HOME/.jipdate.yml ]; then |
| ( |
| # BE CAREFUL WITH TCWG_JIRA_TOKEN |
| set +x |
| build_container_exec \ |
| sed -i -e "s/#TCWG_JIRA_TOKEN#/$TCWG_JIRA_TOKEN/" \ |
| "$HOME/.jipdate.yml" |
| ) |
| fi |
| |
| build_container_exec \ |
| ./jenkins-scripts/rr/round-robin-baseline.sh \ |
| @@rr[top_artifacts] artifacts \ |
| __build_script {build_script} \ |
| __push_base_artifacts true \ |
| __rewrite_base_artifacts true \ |
| __commit_artifacts {commit_artifacts} \ |
| __rewrite_num 100 \ |
| &> ./artifacts/jenkins/rewrite-{commit_artifacts}.log & |
| |
| res=0 && wait $! || res=$? |
| |
| # Compress big files in artifacts/NN-* directories to save disk space |
| # on ci.linaro.org. |
| find artifacts/ -size +1M ! -name "*.xz" \ |
| -path "artifacts/[0-9]*" -print0 \ |
| | xargs -0 -i@ xz -T0 @ |
| |
| if [ $res != 0 ]; then |
| echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \ |
| > artifacts/jenkins/error-mail-recipients.txt |
| echo -e "$BUILD_URL\nERROR: round-robin-baseline.sh failed" \ |
| >> artifacts/jenkins/error-mail-body.txt |
| echo "ERROR: round-robin-baseline.sh failed" |
| # We have failed to update the baseline, so let's fail and |
| # not notify developers. |
| exit 1 |
| fi |
| |
| trap "" EXIT |
| |
| - builder: |
| name: run-push-lnt-results |
| builders: |
| - trigger-builds: |
| - project: tcwg-update-lnt-results |
| predefined-parameters: | |
| ci_project={ci_project} |
| ci_config={ci_config} |
| |
| - builder: |
| name: trigger-precommit-builds-1 |
| builders: |
| - shell: | |
| #!/bin/bash |
| source artifacts/jenkins/run-build.env |
| set -ex |
| |
| # Trigger pre-commit builds if the jenkins queue for |
| # this job is empty. |
| # |
| # Skip triggering pre-commit builds if there are other post-commit |
| # or pre-commit builds in the queue. |
| # Re. post-commit builds -- we need to process all pending |
| # bisection-triggered builds before fetching a new batch of |
| # upstream changes. |
| # Re. pre-commit builds -- we need to wait for the previous batch |
| # to finish to avoid triggering duplicate builds for the queued |
| # patches. |
| for job_type in build precommit; do |
| url="${{JOB_URL%-build/}}-$job_type" |
| inQueue=$(curl --retry 5 --max-time 5 -s "$url/api/xml?tree=inQueue" \ |
| | sed -e "s#.*<inQueue>\(.*\)</inQueue>.*#\1#") |
| if [ "$inQueue" != "false" ]; then |
| echo "SKIPPING PRECOMMIT TRIGGER: $build_type QUEUE IS NOT EMPTY" |
| trap "" EXIT |
| exit 0 |
| fi |
| done |
| |
| # $update_baseline != ignore tells us that we have just |
| # updated the baseline and this means that we should be |
| # in great shape to apply pending patches. |
| # Most likely current baseline revision is master revision |
| # as of few hours ago. |
| if [ "$update_baseline" = "ignore" ]; then |
| echo "SKIPPING PRECOMMIT TRIGGER: DID NOT UPDATE BASELINE" |
| trap "" EXIT |
| exit 0 |
| fi |
| |
| for project in glibc; do |
| # Note that we get here only when all previous pre-commit |
| # builds have finished and, presumably, updated ci_bot |
| # bundles at patchworks instance. If this were not |
| # the case, there would be some builds queued and |
| # we would have exited above. |
| build_container_exec \ |
| ./jenkins-scripts/pw-trigger.sh \ |
| --ci_bot '{ci_project}--{ci_config}' --project "$project" \ |
| --out_dir artifacts/jenkins \ |
| &> artifacts/jenkins/pw-trigger.log & |
| |
| # ignore these precommit triggers if failing |
| if ! wait $!; then |
| echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \ |
| > artifacts/jenkins/error-mail-recipients.txt |
| echo -e "$BUILD_URL\nWARNING: Triggering precommit failed\n" \ |
| >> artifacts/jenkins/error-mail-body.txt |
| fi |
| |
| for i in artifacts/jenkins/trigger-precommit-*; do |
| if [ -f $i ]; then |
| echo "distro=$distro" >> $i |
| echo "scripts_branch=$scripts_branch" >> $i |
| fi |
| done |
| done |
| |
| trap "" EXIT |
| # We place trigger-precommit-builds-2 outside of PRECOMMIT_TRIGGER_yes |
| # #ifdef to get an upstream/downstream links between build/precommit jobs |
| # in jenkins UI. |
| - builder: |
| name: trigger-precommit-builds-2 |
| builders: |
| - trigger-builds: |
| - project: '{ci_project}--{ci_config}-precommit' |
| parameter-factories: |
| - factory: filebuild |
| file-pattern: artifacts/jenkins/trigger-precommit-* |
| |
| - builder: |
| name: run-build-init |
| builders: |
| - shell: |
| command: | |
| #!/bin/bash |
| set -ex |
| # Delete artifacts now to avoid re-using stale artifacts if |
| # below docker-run.sh fails to start up a container. |
| rm -rf artifacts/ |
| mkdir -p artifacts/jenkins |
| |
| build_name="#$BUILD_NUMBER" |
| echo "$build_name" > artifacts/jenkins/build-name |
| |
| git -C jenkins-scripts submodule update --init --remote |
| |
| if [ "{precommit}" = "True" ]; then |
| cat >> artifacts/jenkins/run-build.env <<EOF |
| update_baseline=ignore |
| EOF |
| fi |
| |
| cat >> artifacts/jenkins/run-build.env <<EOF |
| build_name="$build_name" |
| EOF |
| unstable-return: 125 |
| |
| - builder: |
| name: run-build-skip-if-queue |
| builders: |
| - shell: |
| command: | |
| #!/bin/bash |
| source artifacts/jenkins/run-build.env |
| set -ex |
| |
| # Skip SCM-triggered builds if there are other builds in |
| # the queue. We need to process all pending bisection-triggered |
| # builds before fetching a new batch of upstream changes. |
| # Re. skipping post-commit build if pre-commit queue is |
| # not empty -- it's just a favor to pre-commit builds. |
| if [ x"${{BUILD_CAUSE_SCMTRIGGER-false}}" = x"true" ] \ |
| || [ x"${{BUILD_CAUSE_TIMERTRIGGER-false}}" = x"true" ]; then |
| for job_type in build precommit; do |
| url="${{JOB_URL%-build/}}-$job_type" |
| inQueue=$(curl --retry 5 --max-time 5 -s "$url/api/xml?tree=inQueue" \ |
| | sed -e "s#.*<inQueue>\(.*\)</inQueue>.*#\1#") |
| if [ "$inQueue" != "false" ]; then |
| echo "SKIP SCM BUILD" |
| touch artifacts/jenkins/skip-scm-build |
| build_name="$build_name-skip-scm-build" |
| echo "$build_name" > artifacts/jenkins/build-name |
| |
| # Mark build unstable aka a skipped build |
| exit 125 |
| fi |
| done |
| fi |
| |
| trap "" EXIT |
| unstable-return: 125 |
| |
| - builder: |
| name: run-build-start-manifest |
| builders: |
| - shell: |
| command: | |
| #!/bin/bash |
| source artifacts/jenkins/run-build.env |
| set -ex |
| |
| docker_opts=( |
| --distro $distro |
| --arch {docker_arch} |
| --node $NODE_NAME |
| --security '{docker_security}' |
| --node_labels "$NODE_LABELS" |
| ) |
| |
| ./jenkins-scripts/start-container-docker.sh --prefix build_ \ |
| "${{docker_opts[@]}}" > build_container.sh |
| source build_container.sh |
| trap "cleanup_all_containers" EXIT |
| |
| # Run the build script to fetch base-artifacts/, which are |
| # required to fetch baseline sources to apply patches to in |
| # pw-apply.sh below. This also initializes the build manifest |
| # with all below settings; we then import this manifest below. |
| # |
| # The parameter __finish_after makes the build script stop |
| # after fetching the baseline and "__" avoids saving this parameter |
| # in the manifest, so that we don't see this option in the main |
| # run of the build script below. |
| build_container_exec \ |
| ./jenkins-scripts/{build_script} \ |
| %%rr[top_artifacts] artifacts \ |
| ==rr[ci_project] '{ci_project}' \ |
| ==rr[ci_config] '{ci_config}' \ |
| $extra_build_params \ |
| --scripts_branch "$scripts_branch" \ |
| ==rr[mode] "build" \ |
| ==rr[update_baseline] "$update_baseline" \ |
| --BUILD_URL "$BUILD_URL" \ |
| __finish_at reset_artifacts |
| |
| echo "$rr_stage" > artifacts/rr_stage.txt |
| |
| cat >> artifacts/jenkins/run-build.env <<EOF |
| docker_opts=(${{docker_opts[@]+$(printf "%q$IFS" "${{docker_opts[@]}}")}}) |
| source build_container.sh |
| trap "cleanup_all_containers" EXIT |
| EOF |
| |
| # Now that we have "trap cleanup_all_containers EXIT" in |
| # run-build.env we need to reset the trap at the end of every |
| # step to preserve container[s] for the next step. |
| trap "" EXIT |
| unstable-return: 125 |
| |
| - builder: |
| name: run-build-process-params |
| builders: |
| - shell: |
| command: | |
| #!/bin/bash |
| source artifacts/jenkins/run-build.env |
| set -ex |
| |
| branch_opt=() |
| # 1 to skip jenkins-scripts |
| i=1 |
| for c in glibc; do |
| eval "g=\$$$${{c}}_git" |
| if [ x"$g" = x"default" ]; then |
| if [ x"${{BUILD_CAUSE_SCMTRIGGER-false}}" = x"true" ] \ |
| || [ x"${{BUILD_CAUSE_TIMERTRIGGER-false}}" = x"true" ]; then |
| g="jenkins-scm" |
| else |
| g="baseline" |
| fi |
| fi |
| if [ x"$g" = x"jenkins-scm" ]; then |
| eval "gb=\$GIT_BRANCH_$i" |
| gb=$(echo $gb | sed 's+origin/++') |
| eval "g=\$GIT_URL_$i#$gb" |
| fi |
| i=$(($i+1)) |
| |
| if [ x"$g" != x"baseline" ]; then |
| build_name="$build_name-$c" |
| branch_opt+=("==rr[${{c}}_git]" "$g") |
| fi |
| done |
| |
| echo "$build_name" > artifacts/jenkins/build-name |
| |
| cat >> artifacts/jenkins/run-build.env <<EOF |
| build_name="$build_name" |
| branch_opt=(${{branch_opt[@]+$(printf "%q$IFS" "${{branch_opt[@]}}")}}) |
| EOF |
| |
| trap "" EXIT |
| unstable-return: 125 |
| |
| - builder: |
| name: run-build-main |
| builders: |
| - shell: |
| command: | |
| #!/bin/bash |
| source artifacts/jenkins/run-build.env |
| set -ex |
| |
| # Run the build script to from the manifest created above. |
| # |
| # The parameter __start_at makes the build script start right |
| # where the above run finished. This allows us to use read-only |
| # bind-mount for base-artifacts/, which, otherwise, may require |
| # multi-gig rsync. |
| build_container_exec \ |
| ./jenkins-scripts/{build_script} \ |
| @@rr[top_artifacts] artifacts \ |
| "${{branch_opt[@]}}" \ |
| __start_at reset_artifacts+ & |
| res=0 && wait $! || res=$? |
| |
| # Update build_name |
| if [ -f artifacts/results ]; then |
| build_name="$build_name-R$(tail -n1 artifacts/results)" |
| fi |
| if [ "$update_baseline" != "onsuccess" ]; then |
| build_name="$build_name-$update_baseline" |
| fi |
| |
| if [ -f artifacts/trigger-bisect ]; then |
| build_name="$build_name-trigger-bisect" |
| fi |
| echo "$build_name" > artifacts/jenkins/build-name |
| |
| # Everything but 0 and $INTERNAL_FAILURE is an unexpected exit code, |
| # so stop here and do nothing else. From jenkins point of view ... |
| # - if $res==0: build is successful, so proceed with all |
| # followup steps -- notify, push, trigger precommit. |
| # - if $res==123: build has a regression, so proceed with triggering |
| # followup builds and sending precommit notifications; but stop |
| # before run-push-baseline by checking artifacts/failed. |
| # - if $res is anything else: build has an unexpected failure, so |
| # skip the rest of the steps by setting UNSTABLE build status. |
| # This will cause all followup conditional-steps to skip due to |
| # "current-status" condition. |
| # |
| # See comment before round-robin.sh:check_regression() for details. |
| case $res in |
| 0) |
| ;; |
| 123) |
| echo 123 > artifacts/failed |
| ;; |
| *) |
| res=125 |
| ;; |
| esac |
| |
| if [ $res = 125 ]; then |
| # We had unexpected failure in the build scripts, so can't |
| # proceed with triggers and notifications. Mark this build |
| # as UNSTABLE, which will skip all following conditional-steps. |
| exit 125 |
| fi |
| |
| cat >> artifacts/jenkins/run-build.env <<EOF |
| build_name="$build_name" |
| EOF |
| |
| trap "" EXIT |
| unstable-return: 125 |
| |
| - builder: |
| name: run-build-fini |
| builders: |
| - shell: |
| command: | |
| #!/bin/bash |
| source artifacts/jenkins/run-build.env |
| set -ex |
| # The "trap cleanup_all_containers EXIT" statement in |
| # artifacts/jenkins/run-build.env will remove build and precommit |
| # containers. |
| |
| - builder: |
| name: run-build |
| builders: |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-build-init: |
| precommit: false |
| - build-name-setter: |
| name: 'artifacts/jenkins/build-name' |
| file: true |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-build-skip-if-queue: |
| dummy: dummy |
| - build-name-setter: |
| name: 'artifacts/jenkins/build-name' |
| file: true |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-build-start-manifest: |
| ci_project: '{ci_project}' |
| ci_config: '{ci_config}' |
| build_script: '{build_script}' |
| docker_arch: '{docker_arch}' |
| docker_security: '{docker_security}' |
| - build-name-setter: |
| name: 'artifacts/jenkins/build-name' |
| file: true |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-build-process-params: |
| dummy: dummy |
| - build-name-setter: |
| name: 'artifacts/jenkins/build-name' |
| file: true |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-push-baseline: |
| build_script: '{build_script}' |
| commit_artifacts: 'false' |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-build-main: |
| build_script: '{build_script}' |
| - build-name-setter: |
| name: 'artifacts/jenkins/build-name' |
| file: true |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - trigger-followup-builds-1: |
| dummy: dummy |
| - trigger-followup-builds-2: |
| ci_project: '{ci_project}' |
| ci_config: '{ci_config}' |
| - build-name-setter: |
| name: 'artifacts/jenkins/build-name' |
| file: true |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-build-notify: |
| build_script: '{build_script}' |
| - build-name-setter: |
| name: 'artifacts/jenkins/build-name' |
| file: true |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-exit-on-failure: |
| dummy: dummy |
| - build-name-setter: |
| name: 'artifacts/jenkins/build-name' |
| file: true |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-push-baseline: |
| build_script: '{build_script}' |
| commit_artifacts: 'true' |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-push-lnt-results: |
| ci_project: '{ci_project}' |
| ci_config: '{ci_config}' |
| - build-name-setter: |
| name: 'artifacts/jenkins/build-name' |
| file: true |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - trigger-precommit-builds-1: |
| ci_project: '{ci_project}' |
| ci_config: '{ci_config}' |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - trigger-precommit-builds-2: |
| ci_project: '{ci_project}' |
| ci_config: '{ci_config}' |
| - build-name-setter: |
| name: 'artifacts/jenkins/build-name' |
| file: true |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-build-fini: |
| dummy: dummy |
| |
| - builder: |
| name: run-precommit-process-params |
| builders: |
| - shell: |
| command: | |
| #!/bin/bash |
| source artifacts/jenkins/run-build.env |
| set -ex |
| |
| ./jenkins-scripts/start-container-docker.sh --prefix precommit_ \ |
| --task precommit --secondary true "${{docker_opts[@]}}" \ |
| > precommit_container.sh |
| source precommit_container.sh |
| |
| precommit_container_rsync -az --del \ |
| jenkins-scripts/ :jenkins-scripts/ |
| |
| branch_opt=() |
| for c in glibc; do |
| eval "g=\$$$${{c}}_git" |
| case "$g" in |
| "pw://"*|"ssh://"*|"https://forge.sourceware.org/gcc/gcc-TEST/pulls/"*) |
| # Handle pre-commit testing |
| case "$g" in |
| "pw://"*) |
| build_container_exec \ |
| ./jenkins-scripts/pw-apply.sh \ |
| --ci_bot '{ci_project}--{ci_config}' \ |
| --project "$c" --pw_url "$g" __pw_token "$TCWG_PW_TOKEN" \ |
| --build_url "$BUILD_URL" \ |
| --pw_dir artifacts/jenkins/pw \ |
| &> artifacts/jenkins/pw-apply.log & |
| ;; |
| "ssh://"*) |
| build_container_exec \ |
| ./jenkins-scripts/precommit-ssh-apply.sh \ |
| --project "$c" --pw_url "$g" \ |
| --build_url "$BUILD_URL" \ |
| --patch_submitter "$notify" \ |
| --pw_dir artifacts/jenkins/pw \ |
| &> artifacts/jenkins/precommit-ssh-apply.log & |
| ;; |
| "https://forge.sourceware.org/gcc/gcc-TEST/pulls/"*) |
| build_container_exec \ |
| ./jenkins-scripts/precommit-forge-apply.sh \ |
| --ci_bot '{ci_project}--{ci_config}' \ |
| --project "$c" --pw_url "$g" \ |
| --branch master \ |
| --build_url "$BUILD_URL" \ |
| --patch_submitter "$notify" \ |
| --pw_dir artifacts/jenkins/pw \ |
| &> artifacts/jenkins/precommit-forge-apply.log & |
| ;; |
| esac |
| res=0 && wait $! || res=$? |
| |
| if [ "$res" != "0" ]; then |
| build_name="$build_name-$g-$c-apply-fail" |
| echo "$build_name" > artifacts/jenkins/build-name |
| |
| # Notify developer of failure to apply. |
| for f in mail-body.txt mail-subject.txt \ |
| mail-recipients.txt; do |
| # copy the file if exists, and not emtpy. |
| # this is important for mail-recipient.txt, which may |
| # be empty. |
| if [ -s artifacts/jenkins/pw/$f ]; then |
| cp artifacts/jenkins/pw/$f artifacts/jenkins/$f |
| fi |
| done |
| |
| case "$g" in |
| "ssh://"*) |
| # Build was triggered manually, so failure-to-apply |
| # is a real failure. |
| exit 1 |
| ;; |
| esac |
| |
| # Skip the build |
| exit 125 |
| fi |
| |
| # Check whether we should keep or skip this commit |
| build_container_exec \ |
| ./jenkins-scripts/precommit-keep-or-skip.sh \ |
| --pw_dir artifacts/jenkins/pw \ |
| --ci_project '{ci_project}' \ |
| --ci_config '{ci_config}' \ |
| --project "$c" & |
| keep=0 && wait $! || keep=$? |
| |
| skip=$(declare -A pw |
| source "artifacts/jenkins/pw/$c" |
| echo "${{pw[${{c}}_skip]}}") |
| |
| if $skip; then |
| build_name="$build_name-$g-$c-test-skip" |
| echo "$build_name" > artifacts/jenkins/build-name |
| |
| # pw-report.sh does not use artifacts/artifacts.precommit, |
| # so it is OK to use unsafe container. |
| build_container_exec \ |
| ./jenkins-scripts/pw-report.sh --pw_dir "artifacts/jenkins/pw" \ |
| --check test --result skip \ |
| &> artifacts/jenkins/pw-report.log |
| |
| # Skip the build |
| exit 125 |
| fi |
| |
| patch_id=$(declare -A pw |
| source "artifacts/jenkins/pw/$c" |
| echo "${{pw[${{c}}_patch_id]}}") |
| build_name="$build_name-$g/$patch_id" |
| g=HEAD |
| precommit_container_rsync -az --del \ |
| "$c/" ":$c/" |
| ;; |
| esac |
| |
| if [ x"$g" != x"baseline" ]; then |
| build_name="$build_name-$c" |
| branch_opt+=("==rr[${{c}}_git]" "$g") |
| fi |
| done |
| |
| # Now that artifacts/ include all necessary data (including |
| # artifacts/jenkins/pw/ directory -- rsync them into precommit |
| # container. |
| precommit_container_rsync -az --del \ |
| artifacts/ :artifacts/ |
| |
| echo "$build_name" > artifacts/jenkins/build-name |
| |
| cat >> artifacts/jenkins/run-build.env <<EOF |
| source precommit_container.sh |
| build_name="$build_name" |
| branch_opt=(${{branch_opt[@]+$(printf "%q$IFS" "${{branch_opt[@]}}")}}) |
| EOF |
| |
| trap "" EXIT |
| unstable-return: 125 |
| |
| - builder: |
| name: run-precommit-main |
| builders: |
| - shell: |
| command: | |
| #!/bin/bash |
| source artifacts/jenkins/run-build.env |
| set -ex |
| |
| # Run the build script to from the manifest created above. |
| # |
| # The parameter __start_at makes the build script start right |
| # where the above run finished. This allows us to use read-only |
| # bind-mount for base-artifacts/, which, otherwise, may require |
| # multi-gig rsync. |
| precommit_container_exec \ |
| ./jenkins-scripts/{build_script} \ |
| @@rr[top_artifacts] artifacts \ |
| "${{branch_opt[@]}}" \ |
| __start_at reset_artifacts+ & |
| res=0 && wait $! || res=$? |
| |
| precommit_container_rsync -az --del \ |
| :artifacts/ artifacts/artifacts.precommit/ |
| find artifacts/artifacts.precommit -size +1M ! -name "*.xz" \ |
| -print0 | xargs -0 -P0 -i@ xz -T0 @ |
| |
| # Everything but 0 and $INTERNAL_FAILURE is an unexpected exit code, |
| # so stop here and do nothing else. From jenkins point of view ... |
| # - if $res==0: build is successful, so proceed with all |
| # followup steps -- notify, push, trigger precommit. |
| # - if $res==123: build has a regression, so proceed with triggering |
| # followup builds and sending precommit notifications; but stop |
| # before run-push-baseline by checking artifacts/failed. |
| # - if $res is anything else: build has an unexpected failure, so |
| # skip the rest of the steps by setting UNSTABLE build status. |
| # This will cause all followup conditional-steps to skip due to |
| # "current-status" condition. |
| # |
| # See comment before round-robin.sh:check_regression() for details. |
| case $res in |
| 0) |
| pw_result=pass |
| ;; |
| 123) |
| echo 123 > artifacts/failed |
| pw_result=fail |
| ;; |
| *) |
| res=125 |
| pw_result=ignore |
| ;; |
| esac |
| |
| # pw-report.sh does not use artifacts/artifacts.precommit, |
| # so it is OK to use unsafe container. |
| build_container_exec \ |
| ./jenkins-scripts/pw-report.sh --pw_dir "artifacts/jenkins/pw" \ |
| --check test --result "$pw_result" |
| &> artifacts/jenkins/pw-report.log & |
| if ! wait $!; then |
| res=125 |
| fi |
| |
| if [ $res = 125 ]; then |
| # We had unexpected failure in the build scripts, so can't |
| # proceed with triggers and notifications. Mark this build |
| # as UNSTABLE, which will skip all following conditional-steps. |
| exit 125 |
| fi |
| |
| cat >> artifacts/jenkins/run-build.env <<EOF |
| build_name="$build_name" |
| EOF |
| |
| trap "" EXIT |
| unstable-return: 125 |
| |
| - builder: |
| name: run-precommit-notify |
| builders: |
| - shell: | |
| #!/bin/bash |
| source artifacts/jenkins/run-build.env |
| set -ex |
| |
| precommit_container_exec \ |
| ./jenkins-scripts/rr/round-robin-notify.sh \ |
| @@rr[top_artifacts] artifacts --notify "$notify" \ |
| --pw_dir artifacts/jenkins/pw __verbose true \ |
| __build_script {build_script} \ |
| &> ./artifacts/jenkins/notify.log & |
| |
| if ! wait $!; then |
| echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \ |
| > artifacts/jenkins/error-mail-recipients.txt |
| echo -e "$BUILD_URL\nERROR: notify failed\n" \ |
| >> artifacts/jenkins/error-mail-body.txt |
| echo "ERROR: notify failed" |
| exit 1 |
| fi |
| |
| precommit_container_rsync -az --del \ |
| :artifacts/ artifacts/artifacts.precommit/ |
| # Compress big files in artifacts.precommit/ to save disk space |
| # on ci.linaro.org. We keep all pre-commit builds for a month, |
| # which can result in many stored builds. |
| find artifacts/artifacts.precommit -size +1M ! -name "*.xz" \ |
| ! -path "artifacts/artifacts.precommit/notify/*" \ |
| ! -path "artifacts/artifacts.precommit/jenkins/*" -print0 \ |
| | xargs -0 -P0 -i@ xz -T0 @ |
| |
| for f in mail-body.txt mail-subject.txt mail-recipients.txt; do |
| # copy the file if exists, and not emtpy. |
| # this is important for mail-recipient.txt, which may be empty. |
| if [ -s artifacts/artifacts.precommit/jenkins/$f ]; then |
| cp artifacts/artifacts.precommit/jenkins/$f artifacts/jenkins/$f |
| fi |
| done |
| |
| trap "" EXIT |
| |
| - builder: |
| name: run-precommit |
| builders: |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-build-init: |
| precommit: true |
| - build-name-setter: |
| name: 'artifacts/jenkins/build-name' |
| file: true |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-build-start-manifest: |
| ci_project: '{ci_project}' |
| ci_config: '{ci_config}' |
| build_script: '{build_script}' |
| docker_arch: '{docker_arch}' |
| docker_security: '{docker_security}' |
| - build-name-setter: |
| name: 'artifacts/jenkins/build-name' |
| file: true |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-precommit-process-params: |
| ci_project: '{ci_project}' |
| ci_config: '{ci_config}' |
| - build-name-setter: |
| name: 'artifacts/jenkins/build-name' |
| file: true |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-precommit-main: |
| build_script: '{build_script}' |
| - build-name-setter: |
| name: 'artifacts/jenkins/build-name' |
| file: true |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-precommit-notify: |
| build_script: '{build_script}' |
| - build-name-setter: |
| name: 'artifacts/jenkins/build-name' |
| file: true |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-exit-on-failure: |
| dummy: dummy |
| - build-name-setter: |
| name: 'artifacts/jenkins/build-name' |
| file: true |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - run-build-fini: |
| dummy: dummy |
| |
| - publisher: |
| name: build-publishers |
| publishers: |
| - archive: |
| artifacts: 'artifacts/**' |
| latest-only: false |
| - html-publisher: |
| name: 'HTML Report' |
| dir: 'artifacts/jenkins' |
| files: 'status.html' |
| keep-all: true |
| - email-ext: |
| recipients: | |
| ${{FILE,path="artifacts/jenkins/mail-recipients.txt"}} |
| reply-to: linaro-toolchain@lists.linaro.org |
| subject: | |
| ${{FILE,path="artifacts/jenkins/mail-subject.txt"}} |
| content-type: text |
| body: | |
| ${{FILE,path="artifacts/jenkins/mail-body.txt"}} |
| failure: {email_on_failure} |
| success: {email_on_success} |
| aborted: false |
| send-to: |
| - recipients |
| - email-ext: |
| recipients: | |
| ${{FILE,path="artifacts/jenkins/error-mail-recipients.txt"}} |
| body: | |
| ${{FILE,path="artifacts/jenkins/error-mail-body.txt"}} |
| failure: true |
| success: true |
| aborted: true |
| send-to: |
| - recipients |
| |
| - parameter: |
| name: bisect-parameters |
| parameters: |
| - string: |
| name: current_project |
| default: "" |
| description: "Project to bisect" |
| - string: |
| name: bad_git |
| default: "" |
| description: "Bad git_url#branch/SHA1" |
| - file: |
| name: jenkins-scripts/replay_log |
| description: "Replay part of bisect using provided bisect log" |
| - default-parameters: |
| distro: '{distro}' |
| |
| - wrapper: |
| name: bisect-wrappers |
| wrappers: |
| - build-wrappers: |
| timeout: '{timeout}' |
| |
| - builder: |
| name: run-bisect |
| builders: |
| - shell: |
| command: | |
| #!/bin/bash |
| set -ex |
| # Delete artifacts now to avoid re-using stale artifacts if |
| # below docker-run.sh fails to start up a container. |
| rm -rf artifacts/ |
| mkdir -p artifacts/jenkins |
| build_name="#$BUILD_NUMBER-$current_project" |
| echo "$build_name" > artifacts/jenkins/build-name |
| |
| cat >> artifacts/jenkins/run-build.env <<EOF |
| build_name="$build_name" |
| EOF |
| |
| ./jenkins-scripts/docker-run.sh \ |
| --distro $distro \ |
| --arch {docker_arch} \ |
| --node $NODE_NAME \ |
| --security '{docker_security}' \ |
| --node_labels "$NODE_LABELS" \ |
| -- \ |
| ./jenkins-scripts/rr/round-robin-bisect.sh \ |
| %%rel_artifacts artifacts \ |
| --BUILD_URL "$BUILD_URL" \ |
| --current_project "$current_project" \ |
| --bad_git "$bad_git" \ |
| --replay_log "$(pwd)/jenkins-scripts/replay_log" \ |
| --build_script "./jenkins-scripts/{build_script}" \ |
| -- \ |
| ==rr[ci_project] '{ci_project}' \ |
| ==rr[ci_config] '{ci_config}' \ |
| $extra_build_params \ |
| --scripts_branch "$scripts_branch" & |
| res=0 && wait $! || res=$? |
| |
| if [ $res != 0 ]; then |
| echo $res > artifacts/failed |
| fi |
| unstable-return: 125 |
| - build-name-setter: |
| name: 'artifacts/jenkins/build-name' |
| file: true |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - trigger-followup-builds-1: |
| dummy: dummy |
| - trigger-followup-builds-2: |
| ci_project: '{ci_project}' |
| ci_config: '{ci_config}' |
| - conditional-step: |
| condition-kind: current-status |
| steps: |
| - shell: | |
| #!/bin/bash |
| set -ex |
| |
| if [ -f artifacts/failed ]; then |
| # Mark failed build as FAILURE; no further steps will run. |
| exit $(cat artifacts/failed) |
| fi |
| |
| - publisher: |
| name: bisect-publishers |
| publishers: |
| - archive: |
| artifacts: 'artifacts/**' |
| latest-only: false |
| - email-ext: |
| recipients: | |
| maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org |
| failure: true |
| success: false |
| aborted: true |
| send-to: |
| - recipients |
| #END: tcwg/round-robin.yaml.inc |
| |
| - job-template: |
| name: '{ci_project}--{toolchain_ver}-{target}{type_of_test}-build' |
| project-type: freestyle |
| defaults: global |
| properties: |
| - build-properties: |
| ci_project: '{ci_project}' |
| ci_config: '{ci_config}' |
| parameters: |
| - build-parameters: |
| default_git: 'default' |
| distro: '{distro}' |
| disabled: false |
| node: '{node}' |
| concurrent: false |
| display-name: 'TCWG Build {ci_project}/{ci_config}' |
| workspace: workspace/tcwg_gnu_$EXECUTOR_NUMBER |
| scm: |
| - build-scms: |
| glibc_url: '{glibc_url}' |
| glibc_branch: '{glibc_branch}' |
| triggers: |
| - timed: '{cron}' |
| wrappers: |
| - build-wrappers: |
| timeout: '{timeout_build}' |
| builders: |
| - run-build: |
| ci_project: '{ci_project}' |
| ci_config: '{ci_config}' |
| build_script: 'tcwg_gnu-build.sh' |
| docker_arch: '{docker_arch}' |
| docker_security: '{docker_security}' |
| publishers: |
| - build-publishers: |
| email_on_failure: false |
| email_on_success: true |
| |
| - job-template: |
| name: '{ci_project}--{toolchain_ver}-{target}{type_of_test}-precommit' |
| project-type: freestyle |
| defaults: global |
| properties: |
| - precommit-properties: |
| ci_project: '{ci_project}' |
| ci_config: '{ci_config}' |
| parameters: |
| - precommit-parameters: |
| default_git: 'baseline' |
| distro: '{distro}' |
| disabled: false |
| node: '{node}' |
| concurrent: true |
| display-name: 'TCWG Precommit {ci_project}/{ci_config}' |
| workspace: workspace/tcwg_gnu_$EXECUTOR_NUMBER |
| scm: |
| - jenkins-scripts |
| wrappers: |
| - build-wrappers: |
| timeout: 300 |
| builders: |
| - run-precommit: |
| ci_project: '{ci_project}' |
| ci_config: '{ci_config}' |
| build_script: 'tcwg_gnu-build.sh' |
| docker_arch: '{docker_arch}' |
| docker_security: '{docker_security}' |
| publishers: |
| - build-publishers: |
| email_on_failure: true |
| email_on_success: false |
| |
| - job-template: |
| name: '{ci_project}--{toolchain_ver}-{target}{type_of_test}-bisect' |
| project-type: freestyle |
| defaults: global |
| properties: |
| - bisect-properties: |
| ci_project: '{ci_project}' |
| ci_config: '{ci_config}' |
| parameters: |
| - bisect-parameters: |
| distro: '{distro}' |
| disabled: false |
| node: '{node}' |
| concurrent: false |
| display-name: 'TCWG Bisect {ci_project}/{ci_config}' |
| workspace: workspace/tcwg_gnu_$EXECUTOR_NUMBER |
| scm: |
| - jenkins-scripts |
| wrappers: |
| - bisect-wrappers: |
| timeout: 6000 |
| builders: |
| - run-bisect: |
| ci_project: '{ci_project}' |
| ci_config: '{ci_config}' |
| build_script: 'tcwg_gnu-build.sh' |
| docker_arch: '{docker_arch}' |
| docker_security: '{docker_security}' |
| publishers: |
| - bisect-publishers |
| # checksum: 24f140391a214b58f7facab5a2962e74 |