tcwg-update-bmk-containers.yaml

# Auto generated by ./tcwg/generate-yamlfiles.sh from tcwg-update-containers.yaml.in and tcwg-update-containers/tcwg-update-bmk-containers.def. Do not edit.
#BEGIN: tcwg/default.yaml.inc
# -*- mode: Yaml -*-

- property:
    name: default-properties
    properties:
      - authorization:
          anonymous:
            - job-read
            - job-extended-read
          everyone-flat:
            - job-build
            - job-cancel
      - build-discarder:
          days-to-keep: 20

- scm:
    name: jenkins-scripts
    scm:
      - git:
          url: https://gitlab.com/Linaro/tcwg/jenkins-scripts.git
          refspec: +refs/heads/*:refs/remotes/origin/* +refs/changes/*:refs/changes/*
          branches:
            - $scripts_branch
          basedir: jenkins-scripts
          skip-tag: true
          reference-repo: /home/tcwg-buildslave/snapshots-ref/jenkins-scripts.git
          wipe-workspace: false
          clean:
            before: true
          prune: true

#END:   tcwg/default.yaml.inc

- job:
    name: tcwg-update-bmk-containers
    project-type: freestyle
    defaults: global
    properties:
        - authorization:
            anonymous:
                - job-read
                - job-extended-read
            everyone-flat:
                - job-build
                - job-cancel
        - build-discarder:
            days-to-keep: 30
            num-to-keep: 100
        - throttle:
            max-per-node: 1
            option: project
    parameters:
        - label:
            name: nodes
            default: tcwg-bmk-pool
            all-nodes: true
            matching-label: 'allCases'
            description: 'Machines to run on'
        - string:
            name: distro
            default: 'default'
            description: 'Distro version to use.'
        - bool:
            name: force
            default: 'false'
            description: 'Whether to force update even with no changes in image'
        - bool:
            name: verbose
            default: 'true'
            description: 'Whether to be verbose'
        - string:
            name: scripts_branch
            default: master
            description: 'Scripts revision to use'
    disabled: false
    node: tcwg-coordinator
    concurrent: true
    display-name: 'TCWG CCC Update bmk containers'
    # We need to unshare workspace with $NODE_NAME in the path to
    # correctly run on tcwg-bmk-* nodes.
    workspace: workspace/tcwg-update-bmk-containers_$EXECUTOR_NUMBER/$NODE_NAME
    scm:
      - jenkins-scripts
    triggers:
        - timed: '@daily'
    wrappers:
        - timeout:
            timeout: 600
        - timestamps
        - ssh-agent-credentials:
            users:
                - 'tcwg-benchmark'
                # tcwg-buildslave user id
                # ??? Do we need tcwg-buildslave's keys for BMK containers?
                - 'e0958a95-204f-4c14-a66c-5e2be6c5d50a'
        - build-name:
            name: '#${BUILD_NUMBER}-${NODE_NAME}'
    builders:
      - shell:
          command: |
            #!/bin/bash
            set -ex

            # See: LABEL_SED below
            set -o pipefail

            rm -rf artifacts
            mkdir artifacts

            echo "$BUILD_URL" > artifacts/mail-body.txt
            echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" > artifacts/mail-recipients.txt

            declare -A pids
            for lock in $(set +f; ls -tr $HOME/boards/$NODE_NAME-*.lock \
                                         $HOME/boards/$NODE_NAME-*.bak*); do
              (
                flock -e 9
                touch $lock
                board=$(cat <&9)

                # Start the jenkins container so that it can process
                # all the queued cleanup and maintenance tasks.
                node=$(basename "$board" .tcwglab)

                ./jenkins-scripts/tcwg-update-bmk-containers.sh \
                  --board "$board" \
                  --distro "$distro" \
                  --force "$force" \
                  --node "$node" \
                  --verbose "$verbose"

                # Wait for jenkins container to become idle
                while sleep 60; do
                  n_busy=$(source jenkins-scripts/jenkins-helpers.sh
                           print_number_of_busy_executors "$node")

                  if [ "$n_busy" = "0" ]; then
                    break
                  fi
                done

                # Now stop the jenkins container so that it's not terminated
                # midway some other build by a starting benchmarking job.
                ssh -Snone $board docker stop "$node"
              ) 9<$lock 2>&1 | sed -e "s/^/$(basename $lock): /" &

              # LABEL_SED: We need pipefail to get correct result of sub-shell
              # "( foo ) | sed" instead of always-succeeding "sed".
              pids[$(basename "$lock")]=$!
            done

            n_good_boards=0
            for lock in "${!pids[@]}"; do
              res=0 && wait "${pids[$lock]}" || res=$?
              case "$res:$lock" in
                "0":*".lock")
                  n_good_boards=$(($n_good_boards + 1))
                  echo "$lock: SUCCESS" >> artifacts/mail-body.txt
                  ;;
                "0":*".bak")
                  n_good_boards=$(($n_good_boards + 1))
                  echo "$lock: UNEXPECTED SUCCESS" >> artifacts/mail-body.txt
                  # Move boards with no STG ticket (e.g., no ".bak.STG-1234")
                  # back into service.
                  mv "$HOME/boards/$lock" "$HOME/boards/${lock%.bak}.lock"
                  ;;
                "0":*)
                  # The board appears fine, but it has STG ticket assigned
                  # to it (e.g., ".bak.STG-1234").
                  echo "$lock: UNEXPECTED SUCCESS" >> artifacts/mail-body.txt
                  ;;
                *:*".lock")
                  echo "$lock: UNEXPECTED FAILURE" >> artifacts/mail-body.txt
                  # Move offline boards out of service.
                  mv "$HOME/boards/$lock" "$HOME/boards/${lock%.lock}.bak"
                  ;;
                *:*)
                  echo "$lock: EXPECTED FAILURE" >> artifacts/mail-body.txt
                  ;;
              esac
            done

            # tcwg-benchmark doesn't have ssh access to ci.linaro.org,
            # so use tcwg-buildslave's credentials.
            n_executors=$(ssh -p2222 -l tcwg-buildslave@linaro.org \
                              ci.linaro.org get-node "$NODE_NAME" \
                              | grep "numExecutors")
            n_executors=$(echo "$n_executors" \
                          | sed -e "s#.*<numExecutors>\([0-9]\+\)</numExecutors>.*#\1#")
            if [ x"$n_executors" != x"$n_good_boards" ]; then
              echo "UNEXPECTED: Updating number of executors on $NODE_NAME from $n_executors to $n_good_boards" \
                >> artifacts/mail-body.txt
              if [ "$n_good_boards" = "0" ]; then
                # Setting executors to "0" will mightily confuse jenkins;
                # the node with 0 executors will be stuck in limbo.
                n_good_boards=1
              fi
              ssh -p2222 -l tcwg-buildslave@linaro.org \
                  ci.linaro.org get-node "$NODE_NAME" \
                | sed -e "s#<numExecutors>\([0-9]\+\)</numExecutors>#<numExecutors>$n_good_boards</numExecutors>#" \
                | ssh -p2222 -l tcwg-buildslave@linaro.org \
                      ci.linaro.org update-node "$NODE_NAME"
            fi

            if ! grep -q UNEXPECTED artifacts/mail-body.txt; then
              exit 0
            fi

            exit 1
    publishers:
      - email-ext:
          recipients: |
            ${FILE,path="artifacts/mail-recipients.txt"}
          content-type: text
          body: |
            ${FILE,path="artifacts/mail-body.txt"}
          failure: true
          success: false
          aborted: true
          send-to:
            - recipients

# checksum: 8bb5656c7a705d39fd1efe42261dfd06