aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--README.md61
-rw-r--r--docker_ssh_buildslave.yml8
-rw-r--r--docker_ssh_rsa_buildslave.yml8
-rw-r--r--docker_templates_aarch64.yml48
-rw-r--r--docker_templates_amd64.yml79
-rw-r--r--docker_templates_amd64_android_lkft.yml10
-rw-r--r--docker_templates_aosp_amd64.yml26
-rw-r--r--docker_templates_art_amd64.yml14
-rw-r--r--docker_templates_art_lab_amd64.yml16
-rw-r--r--docker_vol_tcwg.yml2
-rw-r--r--docker_vol_x86_64.yml12
-rw-r--r--hosts152
-rw-r--r--hosts.yml17
-rw-r--r--hosts_template.yml103
-rw-r--r--requirements.txt6
-rw-r--r--templates/configure-yadocker-cloud.groovy-dryrun.j2407
-rw-r--r--templates/configure-yadocker-cloud.groovy.j287
-rwxr-xr-xyadp_builder.py70
-rwxr-xr-xyadp_builder_v2.py86
19 files changed, 616 insertions, 596 deletions
diff --git a/README.md b/README.md
index 57d3d7b..3cc1bd3 100644
--- a/README.md
+++ b/README.md
@@ -1,15 +1,23 @@
Yet Another Docker Plugin config builder
=======
-Yet Another Docker Plugib (yadp) is extremely hard to manage, when running multiple slaves with multiple images. Due to the way Jenkins displays the configuration page. YADP provides a [groovy script](https://github.com/KostyaSha/yet-another-docker-plugin/blob/master/docs/script-console-scripts/configure-yadocker-cloud.groovy) which builds a json array to populate the config in Jenkins.
+Yet Another Docker Plugin (YADP) is extremely hard to manage, when running multiple slaves with multiple images. Due to the way Jenkins displays the configuration page. YADP provides a [groovy script](https://github.com/KostyaSha/yet-another-docker-plugin/blob/master/docs/script-console-scripts/configure-yadocker-cloud.groovy) which builds a JSON array to populate the configuration in Jenkins.
+
+This script uses YAML and Jinja2 to generate a java JSONARRAY to build the configuration, using a !include constructor in the YAML file, allowing the ability to template up docker_images, since many of our slaves run the same image, it lessens repetition.
+
+Features
+=======
+
+1. Ability to !include inside YAML to lessen duplication
+2. Ability to include multiple YAML files with !include [file1.yml, file2.yml]
+3. Ability to include SSH settings via using launch_method:" along with ssh: or jnlp:
-This script uses yaml and jinja2 to generate a java jsonarray to build the configuration, using a !include constructor in the yaml file, allowing the ability to template up docker_images, since many of our slaves run the same image, it lessens repetition.
Usage
=======
-####hosts.yml
-List your jenkin_slaves here
+####hosts
+
```
- host1:
cloud_name: host1.example.org
@@ -17,11 +25,54 @@ List your jenkin_slaves here
docker_templates: !include external_template_file.yml
- host2:
- cloud_name: host2
+ cloud_name: host2.example.org
docker-url: tcp://0.0.0.1:2375
docker_templates:
- xenial-amd64:
docker_image_name: 'ubuntu:latest'
max_instances: '1'
labels: 'docker-ubuntu'
+ launch_method: ssh
+ ssh:
+ launch_ssh_credentials_id: 'random-id'
+ launch_ssh_port: '22'
+
+- host3
+ cloud_name: host3.example.org
+ docker-url: tcp://0.0.0.0:2375
+ docker_templates:
+ !include [external_template_file.yml, external_template_file_2.yml]
```
+
+Limitations
+=======
+
+Due to the nature of YAML and populating the Java JSONARRAY, its important that YAML is phased correctly.
+
+Most of the limitations surround docker_templates.
+
+A list of limitations and pending improvements.
+
+1. Do not mix !include and templates under docker_templates.
+
+Example of broken approach:
+
+- host1:
+ cloud_name: host1.example.org
+ docker-url: tcp://0.0.0.0:2375
+ docker_templates:
+ !include external_template_file.yml
+ - xenial-amd64:
+ docker_image_name: 'ubuntu:latest'
+ max_instances: '1'
+ labels: 'docker-ubuntu'
+
+2. Do not add spaces under the image cloud_name, the must fall inline with the image name
+
+Example of broken approach:
+- xenial-amd64:
+ docker_image_name: 'ubuntu:latest'
+ max_instances: '1'
+ labels: 'docker-ubuntu'
+
+3. Docker labels could be "host" specific. If you are including a template file, the template has no link to the host, so its not possible to do "host based labels". Instead just create another template file with custom settings and include that file.
diff --git a/docker_ssh_buildslave.yml b/docker_ssh_buildslave.yml
new file mode 100644
index 0000000..baeb3aa
--- /dev/null
+++ b/docker_ssh_buildslave.yml
@@ -0,0 +1,8 @@
+---
+# Name: buildslave (buildslave for generic docker slaves)
+# https://ci.linaro.org/credentials/store/system/domain/_/credential/88434794-118e-40e4-9cd9-be1ae5d70eec/
+launch_ssh_credentials_id: '88434794-118e-40e4-9cd9-be1ae5d70eec'
+launch_ssh_port: '22'
+launch_ssh_connection_timeout: '60'
+launch_ssh_max_num_retries: '1000'
+launch_ssh_time_wait_between_retries: '60'
diff --git a/docker_ssh_rsa_buildslave.yml b/docker_ssh_rsa_buildslave.yml
new file mode 100644
index 0000000..b64300c
--- /dev/null
+++ b/docker_ssh_rsa_buildslave.yml
@@ -0,0 +1,8 @@
+---
+# Name: buildslave (buildslave (rsa))
+# https://ci.linaro.org/credentials/store/system/domain/_/credential/2aba7be7-4678-4620-a8e9-3f59eb4d23a0/
+launch_ssh_credentials_id: '2aba7be7-4678-4620-a8e9-3f59eb4d23a0'
+launch_ssh_port: '22'
+launch_ssh_connection_timeout: '60'
+launch_ssh_max_num_retries: '1000'
+launch_ssh_time_wait_between_retries: '60'
diff --git a/docker_templates_aarch64.yml b/docker_templates_aarch64.yml
new file mode 100644
index 0000000..e44b016
--- /dev/null
+++ b/docker_templates_aarch64.yml
@@ -0,0 +1,48 @@
+---
+- bionic-arm64:
+ docker_image_name: 'linaro/jenkins-arm64-ubuntu:bionic'
+ labels: 'docker-bionic docker-bionic-arm64'
+ max_instances: 15
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ volumes: !include docker_vol_x86_64.yml
+
+- bionic-arm64-armnn:
+ docker_image_name: 'linaro/jenkins-arm64-ubuntu-armnn:bionic'
+ labels: 'docker-bionic-arm64-armnn'
+ max_instances: 10
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ volumes: !include docker_vol_x86_64.yml
+
+- stretch-arm64:
+ docker_image_name: 'linaro/jenkins-arm64-debian:stretch'
+ labels: 'docker-stretch-arm64'
+ max_instances: 15
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ volumes: !include docker_vol_x86_64.yml
+
+- buster-arm64:
+ docker_image_name: 'linaro/jenkins-arm64-debian:buster'
+ labels: 'docker-buster docker-buster-arm64'
+ max_instances: 15
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ volumes: !include docker_vol_x86_64.yml
+
+- docker-bullseye:
+ docker_image_name: 'linaro/jenkins-arm64-debian:bullseye'
+ max_instances: 15
+ labels: 'docker-bullseye docker-bullseye-arm64'
+ volumes: !include docker_vol_x86_64.yml
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+
+- centos7-amd64:
+ docker_image_name: 'linaro/jenkins-arm64-centos:7'
+ labels: 'docker-centos7-arm64 docker-centos7'
+ max_instances: 5
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ volumes: !include docker_vol_x86_64.yml
diff --git a/docker_templates_amd64.yml b/docker_templates_amd64.yml
index 2094805..100eb02 100644
--- a/docker_templates_amd64.yml
+++ b/docker_templates_amd64.yml
@@ -1,23 +1,64 @@
-# Test
+---
- xenial-amd64:
- docker_image_name: 'linaro/xenial'
- max_instances: '1'
- stop_container_timeout: 10
- #valid values: pull_latest, pull_always, pull_once, pull_never
- pull_strategy: 'pull_never'
+ docker_image_name: 'linaro/jenkins-amd64-ubuntu:xenial'
+ max_instances: '10'
remote_fs_root: '/home/buildslave'
- labels: 'docker-xenial docker-xenial-amd64 docker-xenial-amd64-08'
- dns: '8.8.8.8'
- #valid values: launch_ssh or launch_jnlp
- launch_method: 'launch_ssh'
- launch_ssh_credentials_id: '88434794-118e-40e4-9cd9-be1ae5d70eec'
- launch_ssh_port: '22'
- launch_ssh_connection_timeout: '60'
- launch_ssh_max_num_retries: '1000'
- launch_ssh_time_wait_between_retries: '60'
- volumes: !include docker_vol_x86_64.yml
-- utopic-amd64:
- docker_image_name: 'linaro/utopic'
+ labels: 'docker-xenial docker-xenial-amd64'
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ volumes: !include docker_vol_x86_64.yml
+
+- docker-stretch:
+ docker_image_name: 'linaro/jenkins-amd64-debian:stretch'
+ max_instances: '10'
+ labels: 'docker-stretch docker-stretch-amd64'
+ volumes: !include docker_vol_x86_64.yml
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+
+- docker-buster:
+ docker_image_name: 'linaro/jenkins-amd64-debian:buster'
max_instances: '10'
- labels: 'docker-utopic docker-utopic-amd64 docker-utopic-amd64-08'
+ labels: 'docker-buster docker-buster-amd64'
+ volumes: !include docker_vol_x86_64.yml
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+
+- docker-bullseye:
+ docker_image_name: 'linaro/jenkins-amd64-debian:bullseye'
+ max_instances: '10'
+ labels: 'docker-bullseye docker-bullseye-amd64'
+ volumes: !include docker_vol_x86_64.yml
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+
+- docker-bookworm:
+ docker_image_name: 'linaro/jenkins-amd64-debian:bookworm'
+ labels: 'docker-bookworm docker-bookworm-amd64'
+ volumes: !include docker_vol_x86_64.yml
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+
+- bionic-amd64:
+ docker_image_name: 'linaro/jenkins-amd64-ubuntu:bionic'
+ labels: 'docker-bionic docker-bionic-amd64'
+ max_instances: '10'
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ volumes: !include docker_vol_x86_64.yml
+
+- focal-amd64:
+ docker_image_name: 'linaro/jenkins-amd64-ubuntu:focal'
+ labels: 'docker-focal docker-focal-amd64'
+ max_instances: '10'
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ volumes: !include docker_vol_x86_64.yml
+
+- centos7-amd64:
+ docker_image_name: 'linaro/jenkins-amd64-centos:7'
+ labels: 'docker-centos7-amd64 docker-centos7'
+ max_instances: '10'
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
volumes: !include docker_vol_x86_64.yml
diff --git a/docker_templates_amd64_android_lkft.yml b/docker_templates_amd64_android_lkft.yml
new file mode 100644
index 0000000..e1a8010
--- /dev/null
+++ b/docker_templates_amd64_android_lkft.yml
@@ -0,0 +1,10 @@
+---
+- bionic-amd64:
+ docker_image_name: 'linaro/jenkins-amd64-ubuntu:bionic'
+ pull_strategy: 'pull_always'
+ remote_fs_root: '/home/buildslave'
+ labels: 'docker-bionic-android-lkft'
+ launch_ssh_prefix_start_slave_command: '. /home/buildslave/.profile &&'
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ volumes: !include docker_vol_x86_64.yml
diff --git a/docker_templates_aosp_amd64.yml b/docker_templates_aosp_amd64.yml
new file mode 100644
index 0000000..2bfb336
--- /dev/null
+++ b/docker_templates_aosp_amd64.yml
@@ -0,0 +1,26 @@
+---
+- xenial-amd64:
+ docker_image_name: 'linaro/jenkins-amd64-ubuntu:focal'
+ remote_fs_root: '/home/buildslave'
+ labels: 'docker-focal-aosp'
+ launch_ssh_prefix_start_slave_command: '. /home/buildslave/.profile &&'
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ volumes: !include docker_vol_x86_64.yml
+
+- xenial-amd64:
+ docker_image_name: 'linaro/jenkins-amd64-ubuntu:xenial'
+ remote_fs_root: '/home/buildslave'
+ labels: 'docker-xenial-aosp'
+ launch_ssh_prefix_start_slave_command: '. /home/buildslave/.profile &&'
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ volumes: !include docker_vol_x86_64.yml
+
+- bionic-amd64-aosp:
+ docker_image_name: 'linaro/jenkins-amd64-ubuntu:bionic'
+ launch_ssh_prefix_start_slave_command: '. /home/buildslave/.profile &&'
+ labels: 'docker-bionic-aosp'
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ volumes: !include docker_vol_x86_64.yml
diff --git a/docker_templates_art_amd64.yml b/docker_templates_art_amd64.yml
new file mode 100644
index 0000000..47f2630
--- /dev/null
+++ b/docker_templates_art_amd64.yml
@@ -0,0 +1,14 @@
+---
+- utopic-amd64-art:
+ docker_image_name: 'linaro/jenkins-amd64-art-ubuntu:utopic'
+ labels: 'docker-utopic-art'
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ volumes: !include docker_vol_x86_64.yml
+
+- bionic-amd64-art:
+ docker_image_name: 'linaro/jenkins-amd64-art-ubuntu:bionic'
+ labels: 'docker-bionic-art'
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ volumes: !include docker_vol_x86_64.yml
diff --git a/docker_templates_art_lab_amd64.yml b/docker_templates_art_lab_amd64.yml
new file mode 100644
index 0000000..f0b6f78
--- /dev/null
+++ b/docker_templates_art_lab_amd64.yml
@@ -0,0 +1,16 @@
+---
+- utopic-amd64-art:
+ docker_image_name: 'linaro/jenkins-amd64-art-ubuntu:utopic'
+ pull_strategy: 'pull_always'
+ labels: 'docker-utopic-art-lab'
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ volumes: !include docker_vol_x86_64.yml
+
+- bionic-amd64-art:
+ docker_image_name: 'linaro/jenkins-amd64-art-ubuntu:bionic'
+ pull_strategy: 'pull_always'
+ labels: 'docker-bionic-art-lab'
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ volumes: !include docker_vol_x86_64.yml
diff --git a/docker_vol_tcwg.yml b/docker_vol_tcwg.yml
deleted file mode 100644
index acca454..0000000
--- a/docker_vol_tcwg.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-'/home/tcwg-buildslave/snapshots-ref:/home/tcwg-buildslave/snapshots-ref:ro
-\n/home/tcwg-buildslave/.jenkins:/home/tcwg-buildslave/.jenkins:rw'
diff --git a/docker_vol_x86_64.yml b/docker_vol_x86_64.yml
index 8b3cb73..042c205 100644
--- a/docker_vol_x86_64.yml
+++ b/docker_vol_x86_64.yml
@@ -1,4 +1,8 @@
-'/home/buildslave/srv:/home/buildslave/srv
-\n/srv/docker/ssh/buildslave.config:/home/buildslave/.ssh/config
-\n/srv/docker/ssh/buildslave.known_hosts:/home/buildslave/.ssh/known_hosts
-\n/srv/docker/ssh/id_rsa_buildslave.pub:/home/buildslave/.ssh/authorized_keys:ro'
+'/home/buildslave/srv:/home/buildslave/srv:rw
+\n/srv/docker/ssh/buildslave.config:/home/buildslave/.ssh/config:ro
+\n/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro
+\n/srv/docker/ssh/buildslave.known_hosts:/home/buildslave/.ssh/known_hosts:ro
+\n/srv/docker/ssh/id_rsa_buildslave.pub:/home/buildslave/.ssh/authorized_keys:ro
+\n/var/lib/libvirt:/var/lib/libvirt:rw
+\n/dev/bus/usb:/dev/bus/usb:ro
+\n'
diff --git a/hosts b/hosts
new file mode 100644
index 0000000..2d85735
--- /dev/null
+++ b/hosts
@@ -0,0 +1,152 @@
+---
+# List of YADP Hosts
+
+- aarch64-09:
+ cloud_name: aarch64-09
+ docker_url: tcp://147.28.196.111:2375
+ max_containers: 40
+ host_credentials_id: docker-access
+ docker_templates: !include docker_templates_aarch64.yml
+
+- aosp-x86_64-09:
+ cloud_name: aosp-x86_64-09
+ docker_url: tcp://49.12.133.85:2375
+ host_credentials_id: docker-access
+ docker_templates: !include [docker_templates_aosp_amd64.yml, docker_templates_amd64_android_lkft.yml]
+
+- aosp-x86_64-10:
+ cloud_name: aosp-x86_64-10
+ docker_url: tcp://168.119.145.200:2375
+ host_credentials_id: docker-access
+ docker_templates: !include [docker_templates_aosp_amd64.yml, docker_templates_amd64_android_lkft.yml]
+
+- aosp-x86_64-13: # ART
+ cloud_name: aosp-x86_64-13
+ docker_url: tcp://95.217.145.175:2375
+ host_credentials_id: docker-access
+ docker_templates: !include [docker_templates_art_amd64.yml, docker_templates_amd64_android_lkft.yml]
+
+# 192.168.36.x runs in Cambridge Lab
+#- aosp-x86_64-11: # ART RPB
+# cloud_name: aosp-x86_64-11
+# docker_url: tcp://192.168.36.10:2375
+# docker_templates: !include docker_templates_art_lab_amd64.yml
+#
+#- aosp-x86_64-12: # ART RPB
+# cloud_name: aosp-x86_64-12
+# docker_url: tcp://192.168.36.15:2375
+# docker_templates: !include docker_templates_art_lab_amd64.yml
+
+#- leg-aarch64-01:
+# cloud_name: leg-aarch64-01
+# docker_url: tcp://10.101.96.20:2375
+# host_credentials_id: docker-access
+# max_containers: '4'
+# docker_templates:
+# - docker-stretch:
+# docker_image_name: 'linaro/jenkins-arm64-debian:stretch'
+# labels: 'docker-stretch-arm64-ldcg'
+# volumes: !include docker_vol_x86_64.yml
+# launch_method: 'launch_ssh'
+# max_instances: '4'
+# ssh: !include docker_ssh_buildslave.yml
+# - docker-buster:
+# docker_image_name: 'linaro/jenkins-arm64-debian:buster'
+# labels: 'docker-buster-arm64-ldcg'
+# volumes: !include docker_vol_x86_64.yml
+# launch_method: 'launch_ssh'
+# max_instances: '4'
+# ssh: !include docker_ssh_buildslave.yml
+# - docker-bullseye:
+# docker_image_name: 'linaro/jenkins-arm64-debian:bullseye'
+# labels: 'docker-bullseye-arm64-ldcg'
+# volumes: !include docker_vol_x86_64.yml
+# launch_method: 'launch_ssh'
+# max_instances: '4'
+# ssh: !include docker_ssh_buildslave.yml
+# - docker-centos-8:
+# docker_image_name: 'linaro/jenkins-arm64-centos:8'
+# labels: 'docker-centos8-arm64-ldcg'
+# volumes: !include docker_vol_x86_64.yml
+# launch_method: 'launch_ssh'
+# max_instances: '4'
+# ssh: !include docker_ssh_buildslave.yml
+
+- ldcg-aarch64-02:
+ cloud_name: ldcg-aarch64-02
+ docker_url: tcp://10.40.88.130:2375
+ host_credentials_id: docker-access
+ max_containers: '4'
+ docker_templates:
+ - docker-stretch:
+ docker_image_name: 'linaro/jenkins-arm64-debian:stretch'
+ labels: 'docker-stretch-arm64-ldcg'
+ volumes: !include docker_vol_x86_64.yml
+ launch_method: 'launch_ssh'
+ max_instances: '4'
+ ssh: !include docker_ssh_buildslave.yml
+ - docker-buster:
+ docker_image_name: 'linaro/jenkins-arm64-debian:buster'
+ labels: 'docker-buster-arm64-ldcg'
+ volumes: !include docker_vol_x86_64.yml
+ launch_method: 'launch_ssh'
+ max_instances: '4'
+ ssh: !include docker_ssh_buildslave.yml
+ - docker-bookworm:
+ docker_image_name: 'linaro/jenkins-arm64-debian:bookworm'
+ labels: 'docker-bookworm-arm64-ldcg'
+ volumes: !include docker_vol_x86_64.yml
+ launch_method: 'launch_ssh'
+ max_instances: '4'
+ ssh: !include docker_ssh_buildslave.yml
+ - docker-bullseye:
+ docker_image_name: 'linaro/jenkins-arm64-debian:bullseye'
+ labels: 'docker-bullseye-arm64-ldcg'
+ volumes: !include docker_vol_x86_64.yml
+ launch_method: 'launch_ssh'
+ max_instances: '4'
+ ssh: !include docker_ssh_buildslave.yml
+ - docker-centos-8:
+ docker_image_name: 'linaro/jenkins-arm64-centos:8'
+ labels: 'docker-centos8-arm64-ldcg'
+ volumes: !include docker_vol_x86_64.yml
+ launch_method: 'launch_ssh'
+ max_instances: '4'
+ ssh: !include docker_ssh_buildslave.yml
+
+- swarm:
+ cloud_name: x86_64-swarm
+ docker_url: tcp://swarm-manager:2375
+ host_credentials_id: docker-access
+ max_containers: 4 # equals the amount of nodes we have in the swarm
+ docker_templates: !include docker_templates_amd64.yml
+
+- x86_64-15:
+ cloud_name: x86_64-15
+ docker_url: tcp://95.216.16.165:2375
+ host_credentials_id: docker-access
+ max_containers: 1
+ docker_templates: !include [docker_templates_amd64_android_lkft.yml]
+
+# Not using docker_templates_amd64.yml due to using custom number of
+# max_containers/max_instances
+- x86_64-13:
+ cloud_name: x86_64-13
+ docker_url: tcp://88.99.59.232:2375
+ host_credentials_id: docker-access
+ max_containers: 6
+ docker_templates:
+ - xenial-amd64:
+ docker_image_name: 'linaro/jenkins-amd64-ubuntu:xenial'
+ labels: 'docker-xenial-amd64-13'
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ max_instances: '10'
+ volumes: !include docker_vol_x86_64.yml
+ - focal-amd64:
+ docker_image_name: 'linaro/jenkins-amd64-ubuntu:focal'
+ labels: 'docker-focal-amd64-lite'
+ launch_method: 'launch_ssh'
+ ssh: !include docker_ssh_buildslave.yml
+ max_instances: '10'
+ volumes: !include docker_vol_x86_64.yml
diff --git a/hosts.yml b/hosts.yml
deleted file mode 100644
index 6376209..0000000
--- a/hosts.yml
+++ /dev/null
@@ -1,17 +0,0 @@
-# List of YADP Hosts
-
-- x86_64-07:
- cloud_name: x86_64-07
- docker_url: tcp://0.0.0.0:2375
- max_containers: 1
- docker_templates: !include docker_templates_amd64.yml
-
-- lhg01:
- cloud_name: lhg-01
- docker-url: tcp://lhg-01:2375
- docker_templates:
- - xenial-amd64:
- docker_image_name: 'linaro/xenial'
- max_instances: '1'
- labels: 'docker-xenial-lhg'
-
diff --git a/hosts_template.yml b/hosts_template.yml
new file mode 100644
index 0000000..f99866c
--- /dev/null
+++ b/hosts_template.yml
@@ -0,0 +1,103 @@
+---
+# TEMPLATE FILE
+#
+# This is a template file, of all options available
+# If the same information is used on multiple variables, for example docker_templates
+# or volumes, then it is recommended to use the !include constructor. For example
+# !include docker_template_name.yml.
+#
+# Reference:
+# https://github.com/KostyaSha/yet-another-docker-plugin/blob/master/docs/script-console-scripts/configure-yadocker-cloud.groovy
+#
+
+- host1:
+ cloud_name: host1
+ docker_url: tcp://0.0.0.0:2375
+ docker_api_version: '2'
+ host_credentials_id:
+ # valid values: netty, jersey
+ connection_type: netty
+ connect_timeout: 0
+ max_containers: 1
+ docker_templates:
+ - docker-image:
+ max_instances: 1
+ # DOCKER CONTAINER LIFECYCLE
+ docker_image_name: ''
+ # PULL IMAGE SETTINGS
+ # valid values: pull_latest, pull_always, pull_once, pull_never
+ pull_strategy: 'pull_latest'
+ pull_registry_credentials_id: ''
+ # CREATE CONTAINER SETTINGS
+ docker_command: ''
+ hostname: ''
+ dns: '8.8.8.8'
+ #volumes requires /n on next line if more than one volume is used
+ volumes: '/var/example:/var/example:ro
+ \n/var/example2:/var/example2:rw'
+ volumes_from: ''
+ #environment requires /n on nextline if more than one env is used
+ environment: ''
+ port_bindings: ''
+ bind_all_declared_ports: false
+ #0 is unlimited
+ memory_limit_in_mb: 0
+ #0 is unlimited
+ cpu_shares: 0
+ run_container_privileged: false
+ allocate_pseudo_tty: false
+ mac_address: ''
+ #extra_hosts requires /n on nextline if more than one host is used
+ extra_hosts: ''
+ network_mode: ''
+ #devices requires /n on nextline if more than one device is used
+ devices: ''
+ cpuset_constraint_cpus: ''
+ cpuset_constraint_mems: ''
+ #links requires /n on nextline if more than one link is used
+ links: ''
+ #STOP CONTAINER SETTINGS
+ stop_container_timeout: 10
+ remove_volumes: false
+ force_remove_containers: false
+ #JENKINS SLAVE CONFIG
+ remote_fs_root: '/home/jenkins'
+ labels: 'docker'
+ #valid values: exclusive or normal
+ usage: 'exclusive'
+ availability_strategy: 'docker_once_retention_strategy'
+ availability_idle_timeout: 10
+ executors: 1
+ #LAUNCH METHOD
+ #valid values: launch_ssh *OR* launch_jnlp
+ launch_method: 'launch_ssh'
+ #settings specific to launch_ssh (you only need one or the other)
+ ssh:
+ launch_ssh_credentials_id: ''
+ launch_ssh_port: 22
+ launch_ssh_java_path: ''
+ launch_ssh_jvm_options: ''
+ launch_ssh_prefix_start_slave_command: ''
+ launch_ssh_suffix_start_slave_command: ''
+ launch_ssh_connection_timeout: 120
+ launch_ssh_max_num_retries: 10
+ launch_ssh_time_wait_between_retries: 10
+ #settings specific to launch_jnlp
+ launch_method: 'launch_jnlp'
+ jnlp:
+ launch_jnlp_linux_user: 'jenkins'
+ launch_jnlp_lauch_timeout: 120
+ launch_jnlp_slave_jar_options: ''
+ launch_jnlp_slave_jvm_options: ''
+ launch_jnlp_different_jenkins_master_url: ''
+ launch_jnlp_ignore_certificate_check: false
+ #NODE PROPERTIES
+ #environment_variables is a HashMap of key/value pairs
+ environment_variables: '[:]'
+ #tool location key/value pairs from https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/tools/ToolLocationNodeProperty.java
+ #The key is type@name = home where type is typically the class name of the tool and name is the name given in the Global Tools configuration.
+ #For example let's say you have a global tool configuration named OracleJDK8 for JDK installations
+ #tool_locations would be something like ['hudson.model.JDK$DescriptorImpl@OracleJDK8': '/path/to/java_home']
+ #If you're unsure of the tool@name then check config.xml where YADocker configurations are saved.
+ tool_locations: '[:]'
+ remote_fs_root_mapping: ''
diff --git a/requirements.txt b/requirements.txt
index 2dcf4f6..0c9dc92 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
-pyyaml==3.12
-jinja2==2.9.6
-python-jenkins==0.4.15
+pyyaml==5.1
+jinja2==2.10.1
+python-jenkins==1.4.0
diff --git a/templates/configure-yadocker-cloud.groovy-dryrun.j2 b/templates/configure-yadocker-cloud.groovy-dryrun.j2
deleted file mode 100644
index 1f8a0e0..0000000
--- a/templates/configure-yadocker-cloud.groovy-dryrun.j2
+++ /dev/null
@@ -1,407 +0,0 @@
-/*
- Configure Yet Another Docker clouds provided by the Yet Another Docker
- Plugin. This Jenkins Script Console script makes it easy to maintain large
- amounts of configured clouds.
-
- Note: This script deletes all yet another docker cloud configurations from
- Jenkins before configuring the clouds. It does not affect in-process
- builds. However, if there are previously configured yet another docker
- clouds then they will be removed.
-
- Yet Another Docker Plugin 0.1.0-rc31
- */
-
-import com.github.kostyasha.yad.DockerCloud
-import com.github.kostyasha.yad.DockerConnector
-import com.github.kostyasha.yad.DockerContainerLifecycle
-import com.github.kostyasha.yad.DockerSlaveTemplate
-import com.github.kostyasha.yad.commons.DockerCreateContainer
-import com.github.kostyasha.yad.commons.DockerImagePullStrategy
-import com.github.kostyasha.yad.commons.DockerPullImage
-import com.github.kostyasha.yad.commons.DockerRemoveContainer
-import com.github.kostyasha.yad.commons.DockerStopContainer
-import com.github.kostyasha.yad.launcher.DockerComputerJNLPLauncher
-import com.github.kostyasha.yad.launcher.DockerComputerLauncher
-import com.github.kostyasha.yad.launcher.DockerComputerSSHLauncher
-import com.github.kostyasha.yad.other.ConnectorType
-import com.github.kostyasha.yad.strategy.DockerOnceRetentionStrategy
-
-import hudson.model.Node
-import hudson.plugins.sshslaves.SSHConnector
-import hudson.slaves.EnvironmentVariablesNodeProperty
-import hudson.slaves.NodeProperty
-import hudson.slaves.RetentionStrategy
-import hudson.tools.ToolLocationNodeProperty
-import jenkins.model.Jenkins
-import net.sf.json.JSONArray
-import net.sf.json.JSONObject
-
-/*
- Configure the Yet Another Docker Plugin via this clouds_yadocker variable.
-
- This variable can be removed and referenced for other configurations. All
- values are optional with defaults set.
- */
-
-JSONArray clouds_yadocker = [ {% for host in hosts %}
- [
- cloud_name: '{{host.cloud_name}}',
- docker_url: '{{host.docker_url}}',
- docker_api_version: '{{host.docker_api_version}}',
- host_credentials_id: '{{host.host_credentials_id}}',
- //valid values: netty, jersey
- connection_type: '{{host.connection_type}}',
- connect_timeout: '{{host.connect_timeout}}',
- max_containers: '{{host.max_containers}}',
- docker_templates: [
- // List item of templates {% for docker in host.docker_templates %}
- [
- max_instances: '{{docker.max_instances}}',
- //DOCKER CONTAINER LIFECYCLE
- docker_image_name: '{{docker.docker_image_name}}',
- //PULL IMAGE SETTINGS
- //valid values: pull_latest, pull_always, pull_once, pull_never
- pull_strategy: '{{docker.pull_strategy}}',
- pull_registry_credentials_id: '{{docker.pull_registry_credentials_id}}',
- //CREATE CONTAINER SETTINGS
- docker_command: '{{docker.docker_command}}',
- hostname: '{{docker.hostname}}',
- dns: '{{docker.dns}}',
- //volumes can be a string or list of strings
- volumes: '{{docker.volumes}}',
- //volumes_from can be a string or list of strings
- //volumes_from: '{{docker.volumes_from}}',
- //environment can be a string or list of strings
- environment: '{{docker.environment}}',
- port_bindings: '{{docker.port_bindings}}',
- bind_all_declared_ports: '{{docker.bind_all_declared_ports}}',
- //0 is unlimited
- memory_limit_in_mb: '{{docker.memory_limit_in_mb}}',
- //0 is unlimited
- cpu_shares: '{{docker.cpu_shares}}',
- run_container_privileged: '{{docker.run_container_privileged}}',
- allocate_pseudo_tty: '{{docker.allocate_pseudo_tty}}',
- mac_address: '{{docker.mac_address}}',
- //extra_hosts can be a string or list of strings
- extra_hosts: '{{docker.extra_hosts}}',
- network_mode: '{{docker.network_mode}}',
- //devices can be a string or list of strings
- devices: '{{docker.devices}}',
- cpuset_constraint_cpus: '{{docker.cpuset_constraint_cpus}}',
- cpuset_constraint_mems: '{{docker.cpuset_constraint_mems}}',
- //links can be a string or list of strings
- links: '{{docker.links}}',
- //STOP CONTAINER SETTINGS
- stop_container_timeout: '{{docker.stop_container_timeout}}',
- //STOP CONTAINER SETTINGS
- remove_volumes: '{{docker.remove_volumes}}',
- force_remove_containers: '{{docker.force_remove_containers}}',
- //JENKINS SLAVE CONFIG
- remote_fs_root: '{{docker.remote_fs_root}}',
- labels: '{{docker.labels}}',
- //valid values: exclusive or normal
- usage: '{{docker.usage}}',
- availability_strategy: '{{docker.availability_strategy}}',
- availability_idle_timeout: '{{docker.availability_idle_timeout}}',
- executors: '{{docker.executors}}',
- //LAUNCH METHOD
- //valid values: launch_ssh or launch_jnlp
- launch_method: '{{docker.launch_method}}',
- //settings specific to launch_ssh (you only need one or the other)
- launch_ssh_credentials_id: '{{docker.launch_ssh_credentials_id}}',
- launch_ssh_port: '{{docker.launch_ssh_port}}',
- launch_ssh_java_path: '{{docker.launch_ssh_java_path}}',
- launch_ssh_jvm_options: '{{docker.launch_ssh_jvm_options}}',
- launch_ssh_prefix_start_slave_command: '{{docker.launch_ssh_prefix_start_slave_command}}',
- launch_ssh_suffix_start_slave_command: '{{docker.launch_ssh_suffix_start_slave_command}}',
- launch_ssh_connection_timeout: '{{docker.launch_ssh_connection_timeout}}',
- launch_ssh_max_num_retries: '{{docker.launch_ssh_max_num_retries}}',
- launch_ssh_time_wait_between_retries: '{{docker.launch_ssh_time_wait_between_retries}}',
- //settings specific to launch_jnlp
- launch_jnlp_linux_user: '{{docker.launch_jnlp_linux_user}}',
- launch_jnlp_lauch_timeout: '{{docker.launch_jnlp_lauch_timeout}}',
- launch_jnlp_slave_jar_options: '{{docker.launch_jnlp_slave_jar_options}}',
- launch_jnlp_slave_jvm_options: '{{docker.launch_jnlp_slave_jvm_options}}',
- launch_jnlp_different_jenkins_master_url: '{{docker.launch_jnlp_different_jenkins_master_url}}',
- launch_jnlp_ignore_certificate_check: '{{docker.launch_jnlp_ignore_certificate_check}}',
- //NODE PROPERTIES
- //environment_variables is a HashMap of key/value pairs
- environment_variables: [:],
- //tool location key/value pairs from https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/tools/ToolLocationNodeProperty.java
- //The key is type@name = home where type is typically the class name of the tool and name is the name given in the Global Tools configuration.
- //For example let's say you have a global tool configuration named OracleJDK8 for JDK installations
- //tool_locations would be something like ['hudson.model.JDK$DescriptorImpl@OracleJDK8': '/path/to/java_home']
- //If you're unsure of the tool@name then check config.xml where YADocker configurations are saved.
- tool_locations: [:],
- remote_fs_root_mapping: ''
-
- ],
-{% endfor %}
- ],
- ],
-{% endfor %}
- ] as JSONArray
-
-//detect an existing global tool
-def detectGlobalToolExists(String location) {
- def (toolType, toolName) = location.split('@')
- boolean found_installation = Jenkins.instance.getExtensionList(toolType)[0].installations.findAll { it.name == toolName } as boolean
- return found_installation
-}
-
-//return a launcher
-def selectLauncher(String launcherType, JSONObject obj) {
- switch(launcherType) {
- case 'launch_ssh':
- SSHConnector sshConnector = new SSHConnector(obj.optInt('launch_ssh_port', 22),
- obj.optString('launch_ssh_credentials_id'),
- obj.optString('launch_ssh_jvm_options'),
- obj.optString('launch_ssh_java_path'),
- obj.optString('launch_ssh_prefix_start_slave_command'),
- obj.optString('launch_ssh_suffix_start_slave_command'),
- obj.optInt('launch_ssh_connection_timeout'),
- obj.optInt('launch_ssh_max_num_retries'),
- obj.optInt('launch_ssh_time_wait_between_retries')
- )
- return new DockerComputerSSHLauncher(sshConnector)
- case 'launch_jnlp':
- DockerComputerJNLPLauncher dockerComputerJNLPLauncher = new DockerComputerJNLPLauncher()
- dockerComputerJNLPLauncher.setUser(obj.optString('launch_jnlp_linux_user','jenkins'))
- dockerComputerJNLPLauncher.setLaunchTimeout(obj.optLong('launch_jnlp_lauch_timeout', 120L))
- dockerComputerJNLPLauncher.setSlaveOpts(obj.optString('launch_jnlp_slave_jar_options'))
- dockerComputerJNLPLauncher.setJvmOpts(obj.optString('launch_jnlp_slave_jvm_options'))
- dockerComputerJNLPLauncher.setJenkinsUrl(obj.optString('launch_jnlp_different_jenkins_master_url'))
- dockerComputerJNLPLauncher.setNoCertificateCheck(obj.optBoolean('launch_jnlp_ignore_certificate_check', false))
- return dockerComputerJNLPLauncher
- default:
- return null
- }
-}
-
-//create a new instance of the DockerCloud class from a JSONObject
-def newDockerCloud(JSONObject obj) {
- DockerConnector connector = new DockerConnector(obj.optString('docker_url', 'tcp://localhost:2375'))
- if(obj.optInt('connect_timeout')) {
- connector.setConnectTimeout(obj.optInt('connect_timeout'))
- }
- connector.setApiVersion(obj.optString('docker_api_version'))
- connector.setCredentialsId(obj.optString('host_credentials_id'))
- //select connection_type
- List<String> connection_types = ['NETTY', 'JERSEY']
- String connection_type_default = 'NETTY'
- String user_selected_connection_type = obj.optString('connection_type', connection_type_default).toUpperCase()
- String connection_type = (user_selected_connection_type in connection_types)? user_selected_connection_type : connection_type_default
- connector.setConnectorType(ConnectorType."${connection_type}")
-
- DockerCloud cloud = new DockerCloud(obj.optString('cloud_name'),
- bindJSONToList(DockerSlaveTemplate.class, obj.opt('docker_templates')),
- obj.optInt('max_containers', 50),
- connector)
-
- return cloud
-}
-
-def newDockerSlaveTemplate(JSONObject obj) {
- //DockerPullImage
- DockerPullImage pullImage = new DockerPullImage()
- String pullStrategy = obj.optString('pull_strategy', 'PULL_LATEST').toUpperCase()
- if(pullStrategy in ['PULL_LATEST', 'PULL_ALWAYS', 'PULL_ONCE', 'PULL_NEVER']) {
- pullImage.setPullStrategy(DockerImagePullStrategy."${pullStrategy}")
- } else {
- pullImage.setPullStrategy(DockerImagePullStrategy.PULL_LATEST)
- }
- pullImage.setCredentialsId(obj.optString('pull_registry_credentials_id'))
-
- //DockerCreateContainer
- DockerCreateContainer createContainer = new DockerCreateContainer()
- createContainer.setBindPorts(obj.optString('port_bindings'))
- createContainer.setBindAllPorts(obj.optBoolean('bind_all_declared_ports', false))
- createContainer.setDnsString(obj.optString('dns'))
- createContainer.setHostname(obj.optString('hostname'))
- if(obj.optLong('memory_limit_in_mb')) {
- createContainer.setMemoryLimit(obj.optLong('memory_limit_in_mb'))
- }
- createContainer.setPrivileged(obj.optBoolean('run_container_privileged', false))
- createContainer.setTty(obj.optBoolean('allocate_pseudo_tty', false))
- if(obj.optJSONArray('volumes')) {
- createContainer.setVolumes(obj.optJSONArray('volumes') as List<String>)
- } else {
- createContainer.setVolumesString(obj.optString('volumes'))
- }
- if(obj.optJSONArray('volumes_from')) {
- createContainer.setVolumesFrom(obj.optJSONArray('volumes_from') as List<String>)
- } else {
- createContainer.setVolumesFromString(obj.optString('volumes_from'))
- }
- createContainer.setMacAddress(obj.optString('mac_address'))
- if(obj.optInt('cpu_shares')) {
- createContainer.setCpuShares(obj.optInt('cpu_shares'))
- }
- createContainer.setCommand(obj.optString('docker_command'))
- if(obj.optJSONArray('environment')) {
- createContainer.setEnvironment(obj.optJSONArray('environment') as List<String>)
- } else {
- createContainer.setEnvironmentString(obj.optString('environment'))
- }
- if(obj.optJSONArray('extra_hosts')) {
- createContainer.setExtraHosts(obj.optJSONArray('extra_hosts') as List<String>)
- } else {
- createContainer.setExtraHostsString(obj.optString('extra_hosts'))
- }
- createContainer.setNetworkMode(obj.optString('network_mode'))
- if(obj.optJSONArray('devices')) {
- createContainer.setDevices(obj.optJSONArray('devices'))
- } else {
- createContainer.setDevicesString(obj.optString('devices'))
- }
- createContainer.setCpusetCpus(obj.optString('cpuset_constraint_cpus'))
- createContainer.setCpusetMems(obj.optString('cpuset_constraint_mems'))
- if(obj.optJSONArray('links')) {
- createContainer.setLinks(obj.optJSONArray('links'))
- } else {
- createContainer.setLinksString(obj.optString('links'))
- }
-
- //DockerStopContainer
- DockerStopContainer stopContainer = new DockerStopContainer()
- stopContainer.setTimeout(obj.optInt('stop_container_timeout', 10))
-
- //DockerRemoveContainer
- DockerRemoveContainer removeContainer = new DockerRemoveContainer()
- removeContainer.setRemoveVolumes(obj.optBoolean('remove_volumes', false))
- removeContainer.setForce(obj.optBoolean('force_remove_containers', false))
-
- //DockerContainerLifecycle
- DockerContainerLifecycle dockerContainerLifecycle = new DockerContainerLifecycle()
- dockerContainerLifecycle.setImage(obj.optString('docker_image_name'))
- dockerContainerLifecycle.setPullImage(pullImage)
- dockerContainerLifecycle.setCreateContainer(createContainer)
- dockerContainerLifecycle.setStopContainer(stopContainer)
- dockerContainerLifecycle.setRemoveContainer(removeContainer)
-
- //DockerOnceRetentionStrategy
- //this availability_strategy is for "run_once". We can customize it later
- RetentionStrategy retentionStrategy = new DockerOnceRetentionStrategy(obj.optInt('availability_idle_timeout', 10))
-
- //DockerComputerLauncher
- //select a launch method from the list of available launch methods
- List<String> launch_methods = ['launch_ssh', 'launch_jnlp']
- String default_launch_method = 'launch_jnlp'
- String user_selected_launch_method = obj.optString('launch_method', default_launch_method).toLowerCase()
- String launch_method = (user_selected_launch_method in launch_methods)? user_selected_launch_method : default_launch_method
- DockerComputerLauncher launcher = selectLauncher(launch_method, obj)
-
- //DockerSlaveTemplate
- DockerSlaveTemplate dockerSlaveTemplate = new DockerSlaveTemplate()
- dockerSlaveTemplate.setDockerContainerLifecycle(dockerContainerLifecycle)
- dockerSlaveTemplate.setLabelString(obj.optString('labels', 'docker'))
- String node_usage = (obj.optString('usage', 'EXCLUSIVE').toUpperCase().equals('NORMAL'))? 'NORMAL' : 'EXCLUSIVE'
- dockerSlaveTemplate.setMode(Node.Mode."${node_usage}")
- dockerSlaveTemplate.setNumExecutors(obj.optInt('executors', 1))
- dockerSlaveTemplate.setRetentionStrategy(retentionStrategy)
- dockerSlaveTemplate.setLauncher(launcher)
- dockerSlaveTemplate.setRemoteFs(obj.optString('remote_fs_root', '/srv/jenkins'))
- dockerSlaveTemplate.setMaxCapacity(obj.optInt('max_instances', 10))
- //dockerSlaveTemplate.setRemoteFsMapping(obj.optString('remote_fs_root_mapping'))
- //dockerSlaveTemplate.remoteFsMapping = obj.optString('remote_fs_root_mapping')
- //define NODE PROPERTIES
- List<NodeProperty> nodeProperties = [] as List<NodeProperty>
- if(obj.optJSONObject('environment_variables')) {
- HashMap<String,String> env = obj.optJSONObject('environment_variables') as HashMap<String,String>
- List<EnvironmentVariablesNodeProperty.Entry> envEntries = [] as List<EnvironmentVariablesNodeProperty.Entry>
- (env.keySet() as String[]).each { var ->
- envEntries << (new EnvironmentVariablesNodeProperty.Entry(var, env[var]))
- }
- //add environment_variables to nodeProperties
- nodeProperties << (new EnvironmentVariablesNodeProperty(envEntries))
- }
- if(obj.optJSONObject('tool_locations')) {
- HashMap<String,String> tool = obj.optJSONObject('tool_locations') as HashMap<String,String>
- List<ToolLocationNodeProperty.ToolLocation> toolLocations = [] as List<ToolLocationNodeProperty.ToolLocation>
- (tool.keySet() as String[]).each { location ->
- //tool location is only valid if it contains a type i.e. @ symbol
- if(location.contains('@') && detectGlobalToolExists(location)) {
- toolLocations << (new ToolLocationNodeProperty.ToolLocation(location, tool[location]))
- } else {
- //alert the user they configured an invalid tool type or name in tool_locations
- println "WARNING: Invalid tool: '${location}'. Format should be 'type@name' where both type and name already exist in global tool configurations."
- }
- }
- nodeProperties << (new ToolLocationNodeProperty(toolLocations))
- }
- //set NODE PROPERTIES
- if(nodeProperties) {
- dockerSlaveTemplate.setNodeProperties(nodeProperties)
- }
- return dockerSlaveTemplate
-}
-
-def bindJSONToList(Class type, Object src) {
- if(!(type == DockerCloud) && !(type == DockerSlaveTemplate)) {
- throw new Exception("Must use DockerCloud or DockerSlaveTemplate class.")
- }
- //docker_array should be a DockerCloud or DockerSlaveTemplate
- ArrayList<?> docker_array
- if(type == DockerCloud){
- docker_array = new ArrayList<DockerCloud>()
- } else {
- docker_array = new ArrayList<DockerSlaveTemplate>()
- }
- //cast the configuration object to a Docker instance which Jenkins will use in configuration
- if (src instanceof JSONObject) {
- //uses string interpolation to call a method
- //e.g instead of newDockerCloud(src) we use instead...
- docker_array.add("new${type.getSimpleName()}"(src))
- } else if(src instanceof JSONArray) {
- for (Object o : src) {
- if (o instanceof JSONObject) {
- docker_array.add("new${type.getSimpleName()}"(o))
- }
- }
- }
- return docker_array
-}
-
-if(!Jenkins.instance.isQuietingDown()) {
- ArrayList<DockerCloud> clouds = new ArrayList<DockerCloud>()
- clouds = bindJSONToList(DockerCloud.class, clouds_yadocker)
- if(clouds.size() > 0) {
- dockerConfigUpdated = false
-// Jenkins.instance.clouds.removeAll(DockerCloud)
-// Jenkins.instance.clouds.addAll(clouds)
- clouds*.name.each { cloudName ->
- println "DRYRUN: Configured Yet Another Docker cloud ${cloudName}"
- }
- } else {
- println 'Nothing changed. No Yet Another docker clouds to configure.'
- }
-} else {
- println 'Shutdown mode enabled. Configure Yet Another Docker clouds SKIPPED.'
-}
-
-//return null so there's no result value in the script console
-null
-
-/*
- The MIT License (MIT)
-
- Copyright 2017 Sam Gleske - https://github.com/samrocketman/jenkins-bootstrap-jervis
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to
- deal in the Software without restriction, including without limitation the
- rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- sell copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- IN THE SOFTWARE.
- */
diff --git a/templates/configure-yadocker-cloud.groovy.j2 b/templates/configure-yadocker-cloud.groovy.j2
index 5a361a3..2124768 100644
--- a/templates/configure-yadocker-cloud.groovy.j2
+++ b/templates/configure-yadocker-cloud.groovy.j2
@@ -19,15 +19,17 @@ import com.github.kostyasha.yad.commons.DockerCreateContainer
import com.github.kostyasha.yad.commons.DockerImagePullStrategy
import com.github.kostyasha.yad.commons.DockerPullImage
import com.github.kostyasha.yad.commons.DockerRemoveContainer
+import com.github.kostyasha.yad.commons.DockerSSHConnector
import com.github.kostyasha.yad.commons.DockerStopContainer
import com.github.kostyasha.yad.launcher.DockerComputerJNLPLauncher
import com.github.kostyasha.yad.launcher.DockerComputerLauncher
import com.github.kostyasha.yad.launcher.DockerComputerSSHLauncher
import com.github.kostyasha.yad.other.ConnectorType
import com.github.kostyasha.yad.strategy.DockerOnceRetentionStrategy
+import hudson.plugins.sshslaves.verifiers.NonVerifyingKeyVerificationStrategy
+import hudson.plugins.sshslaves.verifiers.SshHostKeyVerificationStrategy
import hudson.model.Node
-import hudson.plugins.sshslaves.SSHConnector
import hudson.slaves.EnvironmentVariablesNodeProperty
import hudson.slaves.NodeProperty
import hudson.slaves.RetentionStrategy
@@ -66,14 +68,14 @@ JSONArray clouds_yadocker = [ {% for host in hosts %}
//CREATE CONTAINER SETTINGS
docker_command: '{{docker.docker_command}}',
hostname: '{{docker.hostname}}',
- dns: '{{docker.dns}}',
+ dns: '{{docker.dns|default('8.8.8.8')}}',
//volumes can be a string or list of strings
volumes: '{{docker.volumes}}',
//volumes_from can be a string or list of strings
//volumes_from: '{{docker.volumes_from}}',
//environment can be a string or list of strings
environment: '{{docker.environment}}',
- port_bindings: '{{docker.port_bindings}}',
+ port_bindings: '{{docker.port_bindings|default('0.0.0.0::22')}}',
bind_all_declared_ports: '{{docker.bind_all_declared_ports}}',
//0 is unlimited
memory_limit_in_mb: '{{docker.memory_limit_in_mb}}',
@@ -97,7 +99,7 @@ JSONArray clouds_yadocker = [ {% for host in hosts %}
remove_volumes: '{{docker.remove_volumes}}',
force_remove_containers: '{{docker.force_remove_containers}}',
//JENKINS SLAVE CONFIG
- remote_fs_root: '{{docker.remote_fs_root}}',
+ remote_fs_root: '{{docker.remote_fs_root|default('/home/buildslave')}}',
labels: '{{docker.labels}}',
//valid values: exclusive or normal
usage: '{{docker.usage}}',
@@ -106,34 +108,40 @@ JSONArray clouds_yadocker = [ {% for host in hosts %}
executors: '{{docker.executors}}',
//LAUNCH METHOD
//valid values: launch_ssh or launch_jnlp
- launch_method: '{{docker.launch_method}}',
+ {%if docker.launch_method == 'launch_ssh' %}
+ launch_method: 'launch_ssh',
//settings specific to launch_ssh (you only need one or the other)
- launch_ssh_credentials_id: '{{docker.launch_ssh_credentials_id}}',
- launch_ssh_port: '{{docker.launch_ssh_port}}',
- launch_ssh_java_path: '{{docker.launch_ssh_java_path}}',
- launch_ssh_jvm_options: '{{docker.launch_ssh_jvm_options}}',
- launch_ssh_prefix_start_slave_command: '{{docker.launch_ssh_prefix_start_slave_command}}',
- launch_ssh_suffix_start_slave_command: '{{docker.launch_ssh_suffix_start_slave_command}}',
- launch_ssh_connection_timeout: '{{docker.launch_ssh_connection_timeout}}',
- launch_ssh_max_num_retries: '{{docker.launch_ssh_max_num_retries}}',
- launch_ssh_time_wait_between_retries: '{{docker.launch_ssh_time_wait_between_retries}}',
+ launch_ssh_credentials_id: '{{docker.ssh.launch_ssh_credentials_id}}',
+ launch_ssh_port: '{{docker.ssh.launch_ssh_port|default('22')}}',
+ launch_ssh_java_path: '{{docker.ssh.launch_ssh_java_path}}',
+ launch_ssh_jvm_options: '{{docker.ssh.launch_ssh_jvm_options}}',
+ launch_ssh_prefix_start_slave_command: '{{docker.ssh.launch_ssh_prefix_start_slave_command}}',
+ launch_ssh_suffix_start_slave_command: '{{docker.ssh.launch_ssh_suffix_start_slave_command}}',
+ launch_ssh_connection_timeout: '{{docker.ssh.launch_ssh_connection_timeout}}',
+ launch_ssh_max_num_retries: '{{docker.ssh.launch_ssh_max_num_retries}}',
+ launch_ssh_time_wait_between_retries: '{{docker.ssh.launch_ssh_time_wait_between_retries}}',
+ {%elif docker.launch_method == 'launch_jnlp' %}
+ launch_method: 'launch_jnlp',
//settings specific to launch_jnlp
- launch_jnlp_linux_user: '{{docker.launch_jnlp_linux_user}}',
- launch_jnlp_lauch_timeout: '{{docker.launch_jnlp_lauch_timeout}}',
- launch_jnlp_slave_jar_options: '{{docker.launch_jnlp_slave_jar_options}}',
- launch_jnlp_slave_jvm_options: '{{docker.launch_jnlp_slave_jvm_options}}',
- launch_jnlp_different_jenkins_master_url: '{{docker.launch_jnlp_different_jenkins_master_url}}',
- launch_jnlp_ignore_certificate_check: '{{docker.launch_jnlp_ignore_certificate_check}}',
+ launch_jnlp_linux_user: '{{docker.jnlp.launch_jnlp_linux_user}}',
+ launch_jnlp_lauch_timeout: '{{docker.jnlp.launch_jnlp_lauch_timeout}}',
+ launch_jnlp_slave_jar_options: '{{docker.jnlp.launch_jnlp_slave_jar_options}}',
+ launch_jnlp_slave_jvm_options: '{{docker.jnlp.launch_jnlp_slave_jvm_options}}',
+ launch_jnlp_different_jenkins_master_url: '{{docker.jnlp.launch_jnlp_different_jenkins_master_url}}',
+ launch_jnlp_ignore_certificate_check: '{{docker.jnlp.launch_jnlp_ignore_certificate_check}}',
+ {%endif%}
//NODE PROPERTIES
//environment_variables is a HashMap of key/value pairs
- environment_variables: [:],
+ environment_variables: [ {% for env in docker.environment_variables %} {% for name, value in env.items() %}
+ '{{name}}': '{{value}}',
+{% endfor %}{% endfor %} ],
//tool location key/value pairs from https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/tools/ToolLocationNodeProperty.java
//The key is type@name = home where type is typically the class name of the tool and name is the name given in the Global Tools configuration.
//For example let's say you have a global tool configuration named OracleJDK8 for JDK installations
//tool_locations would be something like ['hudson.model.JDK$DescriptorImpl@OracleJDK8': '/path/to/java_home']
//If you're unsure of the tool@name then check config.xml where YADocker configurations are saved.
- tool_locations: [:],
- remote_fs_root_mapping: ''
+ tool_locations: '{{docker.tool_locations}}',
+ remote_fs_root_mapping: '{{docker.remote_fs_root_mapping}}'
],
{% endfor %}
@@ -153,15 +161,30 @@ def detectGlobalToolExists(String location) {
def selectLauncher(String launcherType, JSONObject obj) {
switch(launcherType) {
case 'launch_ssh':
- SSHConnector sshConnector = new SSHConnector(obj.optInt('launch_ssh_port', 22),
+ def jenkinsCredentials = com.cloudbees.plugins.credentials.CredentialsProvider.lookupCredentials(
+ com.cloudbees.plugins.credentials.Credentials.class,
+ Jenkins.instance,
+ null,
+ null
+ );
+ def launch_credentials = null
+ for (creds in jenkinsCredentials) {
+ if(creds.id == obj.optString('launch_ssh_credentials_id')){
+ launch_credentials = creds;
+ }
+ }
+ DockerSSHConnector sshConnector = new DockerSSHConnector(obj.optInt('launch_ssh_port', 22),
+ launch_credentials,
obj.optString('launch_ssh_credentials_id'),
obj.optString('launch_ssh_jvm_options'),
obj.optString('launch_ssh_java_path'),
+ null,
obj.optString('launch_ssh_prefix_start_slave_command'),
obj.optString('launch_ssh_suffix_start_slave_command'),
obj.optInt('launch_ssh_connection_timeout'),
obj.optInt('launch_ssh_max_num_retries'),
- obj.optInt('launch_ssh_time_wait_between_retries')
+ obj.optInt('launch_ssh_time_wait_between_retries'),
+ new NonVerifyingKeyVerificationStrategy()
)
return new DockerComputerSSHLauncher(sshConnector)
case 'launch_jnlp':
@@ -188,14 +211,14 @@ def newDockerCloud(JSONObject obj) {
connector.setCredentialsId(obj.optString('host_credentials_id'))
//select connection_type
List<String> connection_types = ['NETTY', 'JERSEY']
- String connection_type_default = 'NETTY'
+ String connection_type_default = 'JERSEY'
String user_selected_connection_type = obj.optString('connection_type', connection_type_default).toUpperCase()
String connection_type = (user_selected_connection_type in connection_types)? user_selected_connection_type : connection_type_default
connector.setConnectorType(ConnectorType."${connection_type}")
DockerCloud cloud = new DockerCloud(obj.optString('cloud_name'),
bindJSONToList(DockerSlaveTemplate.class, obj.opt('docker_templates')),
- obj.optInt('max_containers', 50),
+ obj.optInt('max_containers', 1),
connector)
return cloud
@@ -221,7 +244,7 @@ def newDockerSlaveTemplate(JSONObject obj) {
if(obj.optLong('memory_limit_in_mb')) {
createContainer.setMemoryLimit(obj.optLong('memory_limit_in_mb'))
}
- createContainer.setPrivileged(obj.optBoolean('run_container_privileged', false))
+ createContainer.setPrivileged(obj.optBoolean('run_container_privileged', true))
createContainer.setTty(obj.optBoolean('allocate_pseudo_tty', false))
if(obj.optJSONArray('volumes')) {
createContainer.setVolumes(obj.optJSONArray('volumes') as List<String>)
@@ -301,7 +324,7 @@ def newDockerSlaveTemplate(JSONObject obj) {
dockerSlaveTemplate.setRetentionStrategy(retentionStrategy)
dockerSlaveTemplate.setLauncher(launcher)
dockerSlaveTemplate.setRemoteFs(obj.optString('remote_fs_root', '/srv/jenkins'))
- dockerSlaveTemplate.setMaxCapacity(obj.optInt('max_instances', 10))
+ dockerSlaveTemplate.setMaxCapacity(obj.optInt('max_instances', 1))
//dockerSlaveTemplate.setRemoteFsMapping(obj.optString('remote_fs_root_mapping'))
//dockerSlaveTemplate.remoteFsMapping = obj.optString('remote_fs_root_mapping')
//define NODE PROPERTIES
@@ -366,6 +389,11 @@ if(!Jenkins.instance.isQuietingDown()) {
ArrayList<DockerCloud> clouds = new ArrayList<DockerCloud>()
clouds = bindJSONToList(DockerCloud.class, clouds_yadocker)
if(clouds.size() > 0) {
+{%if dryrun == True %}
+ clouds*.name.each { cloudName ->
+ println "DRYRUN: Configured Yet Another Docker cloud ${cloudName}"
+ }
+{%else%}
dockerConfigUpdated = true
Jenkins.instance.clouds.removeAll(DockerCloud)
Jenkins.instance.clouds.addAll(clouds)
@@ -373,6 +401,7 @@ if(!Jenkins.instance.isQuietingDown()) {
println "Configured Yet Another Docker cloud ${cloudName}"
}
Jenkins.instance.save()
+{%endif%}
} else {
println 'Nothing changed. No Yet Another docker clouds to configure.'
}
diff --git a/yadp_builder.py b/yadp_builder.py
index 786a3b3..1d84154 100755
--- a/yadp_builder.py
+++ b/yadp_builder.py
@@ -1,6 +1,8 @@
+#!/usr/bin/python3
+
import argparse
import yaml
-import os.path
+import os
from jinja2 import FileSystemLoader, Environment
import jenkins
import logging
@@ -32,42 +34,58 @@ class Loader(yaml.Loader, metaclass=LoaderMeta):
super().__init__(stream)
def construct_include(self, node):
- """Include file referenced at node."""
+ if isinstance(node, yaml.ScalarNode):
+ return self.extractFile(self.construct_scalar(node))
+
+ elif isinstance(node, yaml.SequenceNode):
+ result = []
+ for filename in self.construct_sequence(node):
+ result += self.extractFile(filename)
+ return result
+
+ elif isinstance(node, yaml.MappingNode):
+ result = {}
+ for k,v in self.construct_mapping(node).iteritems():
+ result[k] = self.extractFile(v)
+ return result
+
+ else:
+ print("Error:: unrecognised node type in !include statement")
+ raise yaml.constructor.ConstructorError
- filename = os.path.abspath(os.path.join(
- self._root, self.construct_scalar(node)
- ))
- extension = os.path.splitext(filename)[1].lstrip('.')
- with open(filename, 'r') as f:
- if extension in ('yaml', 'yml'):
- return yaml.load(f, Loader)
- else:
- return ''.join(f.readlines())
+ def extractFile(self, filename):
+ filepath = os.path.join(self._root, filename)
+ with open(filepath, 'r') as f:
+ return yaml.load(f, Loader=Loader)
-def jinja2_from_template(directory, template_name, data):
+def jinja2_from_template(directory, template_name, data, dryrun=False):
loader = FileSystemLoader(directory)
env = Environment(loader=loader)
template = env.get_template(template_name)
- return template.render(hosts=data)
+ return template.render(hosts=data, dryrun=dryrun)
def get_parser():
parser = argparse.ArgumentParser()
- parser.add_argument('-u', '--username', type=str, required=True,
+ parser.add_argument('-u', '--username', type=str,
+ default=os.environ.get('JJB_USER'),
help='Username for Jenkins server')
- parser.add_argument('-p', '--password', type=str, required=True,
+ parser.add_argument('-p', '--password', type=str,
+ default=os.environ.get('JJB_PASSWORD'),
help='Password for Jenkins server')
parser.add_argument('-s', '--server', type=str,
default='http://localhost:8080',
help='Jenkins server URL. e.g. http://localhost:8080')
- parser.add_argument('-i', '--inventory', type=str, default='hosts.yml',
+ parser.add_argument('-i', '--inventory', type=str, default='hosts',
help='specify inventory host path')
parser.add_argument('-l', '--loglevel', default='INFO',
help="Setting logging level, default: %(default)s")
parser.add_argument('--dryrun', action='store_true',
help='Do not publish to Jenkins')
+ parser.add_argument('--local', action='store_true',
+ help='Create tmp file only, to be used with dryrun.')
return parser
@@ -76,26 +94,30 @@ if __name__ == '__main__':
args = parser.parse_args()
logging.basicConfig(level=args.loglevel)
with open(args.inventory, 'r') as f:
- data = yaml.load(f, Loader)
+ data = yaml.load(f, Loader=Loader)
logging.debug(data)
template_output = jinja2_from_template(
'./templates',
'configure-yadocker-cloud.groovy.j2', data)
- server = jenkins.Jenkins(args.server, username=args.username,
+
+ if not args.local:
+ server = jenkins.Jenkins(args.server, username=args.username,
password=args.password)
if args.dryrun:
with open('/tmp/configure-yadocker-cloud.groovy', 'w') as fw:
fw.write(template_output)
template_output = jinja2_from_template(
'./templates',
- 'configure-yadocker-cloud.groovy-dryrun.j2', data)
-
- publishdry = server.run_script(template_output)
- if 'error' in publishdry:
+ 'configure-yadocker-cloud.groovy.j2', data, args.dryrun)
+ if not args.local:
+ publishdry = server.run_script(template_output)
+ if 'error' in publishdry:
+ logging.info(publishdry)
+ exit(1)
logging.info(publishdry)
- exit(1)
else:
- logging.info(publishdry)
+ logging.info('Template file created at \
+ /tmp/configure-yadocker-cloud.groovy')
else:
publish = server.run_script(template_output)
logging.info(publish)
diff --git a/yadp_builder_v2.py b/yadp_builder_v2.py
deleted file mode 100755
index 3c815e7..0000000
--- a/yadp_builder_v2.py
+++ /dev/null
@@ -1,86 +0,0 @@
-import argparse
-import yaml
-import os.path
-from jinja2 import FileSystemLoader, Environment
-import jenkins
-import logging
-
-
-class Loader(yaml.Loader):
-
- def __init__(self, *args, **kwargs):
- super(Loader, self).__init__(*args, **kwargs)
- self.add_constructor('!include', type(self).construct_include)
- try:
- self._root = os.path.split(self)[0]
- except AttributeError:
- self._root = os.path.curdir
-
- def construct_include(self, node):
- """Include file referenced at node."""
-
- filename = os.path.abspath(os.path.join(
- self._root, self.construct_scalar(node)
- ))
- extension = os.path.splitext(filename)[1].lstrip('.')
-
- with open(filename, 'r') as f:
- if extension in ('yaml', 'yml'):
- return yaml.load(f, Loader)
- else:
- return ''.join(f.readlines())
-
-
-def jinja2_from_template(directory, template_name, data):
- loader = FileSystemLoader(directory)
- env = Environment(loader=loader)
- template = env.get_template(template_name)
- return template.render(hosts=data)
-
-
-def get_parser():
- parser = argparse.ArgumentParser()
- parser.add_argument('-u', '--username', type=str, required=True,
- help='Username for Jenkins server')
- parser.add_argument('-p', '--password', type=str, required=True,
- help='Password for Jenkins server')
- parser.add_argument('-s', '--server', type=str,
- default='http://localhost:8080',
- help='Jenkins server URL. e.g. http://localhost:8080')
- parser.add_argument('-i', '--inventory', type=str, default='hosts.yml',
- help='specify inventory host path')
- parser.add_argument('-l', '--loglevel', default='INFO',
- help="Setting logging level, default: %(default)s")
- parser.add_argument('--dryrun', action='store_true',
- help='Do not publish to Jenkins')
- return parser
-
-
-if __name__ == '__main__':
- parser = get_parser()
- args = parser.parse_args()
- logging.basicConfig(level=args.loglevel)
- with open(args.inventory, 'r') as f:
- data = yaml.load(f, Loader)
- logging.debug(data)
- template_output = jinja2_from_template(
- './templates',
- 'configure-yadocker-cloud.groovy.j2', data)
- server = jenkins.Jenkins(args.server, username=args.username,
- password=args.password)
- if args.dryrun:
- with open('/tmp/configure-yadocker-cloud.groovy', 'w') as fw:
- fw.write(template_output)
- template_output = jinja2_from_template(
- './templates',
- 'configure-yadocker-cloud.groovy-dryrun.j2', data)
-
- publishdry = server.run_script(template_output)
- if 'error' in publishdry:
- logging.info(publishdry)
- exit(1)
- else:
- logging.info(publishdry)
- else:
- publish = server.run_script(template_output)
- logging.info(publish)
generated by cgit v1.2.3 (git 2.25.1) at 2024-04-20 06:12:01 +0000