aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBenjamin Copeland <ben.copeland@linaro.org>2018-02-23 10:13:26 +0000
committerBenjamin Copeland <ben.copeland@linaro.org>2018-02-23 16:06:49 +0000
commit06c655ce16e49650c3937ac93362dd52781c8880 (patch)
treed86527b4165615a1bee734bfd6c91def7e4bdf05
parent9c0eda17acf2cf3eb2f0380e71a9d976a90a1943 (diff)
downloadci-yadp-builder-master.tar.gz
YADP: Improve script to allow for multiple templatesHEADmaster
It is going to be needed that hosts will need multiple templates. This code change allows for all three YAML nodes (scalar, sequence and mapping). Whilst we will only be using scalar and sequence having mapping might come in handy. Also updated the README file by listing out limitations. Scalar: docker_templates: !include template_foo.yml Sequence: docker_templates: !include [template_foo_.yml, template_foo_2.yml] Mapping: docker_templates: item1: {!include template_foo_.yml, item2: template_foo_2.yml} Change-Id: I68efde207574b14bf9422ce61bcf527861ed420d
-rw-r--r--README.md43
-rwxr-xr-xyadp_builder.py36
-rwxr-xr-xyadp_builder_v2.py36
3 files changed, 89 insertions, 26 deletions
diff --git a/README.md b/README.md
index 57d3d7b..fb5c13f 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,9 @@
Yet Another Docker Plugin config builder
=======
-Yet Another Docker Plugib (yadp) is extremely hard to manage, when running multiple slaves with multiple images. Due to the way Jenkins displays the configuration page. YADP provides a [groovy script](https://github.com/KostyaSha/yet-another-docker-plugin/blob/master/docs/script-console-scripts/configure-yadocker-cloud.groovy) which builds a json array to populate the config in Jenkins.
+Yet Another Docker Plugin (YADP) is extremely hard to manage, when running multiple slaves with multiple images. Due to the way Jenkins displays the configuration page. YADP provides a [groovy script](https://github.com/KostyaSha/yet-another-docker-plugin/blob/master/docs/script-console-scripts/configure-yadocker-cloud.groovy) which builds a JSON array to populate the configuration in Jenkins.
-This script uses yaml and jinja2 to generate a java jsonarray to build the configuration, using a !include constructor in the yaml file, allowing the ability to template up docker_images, since many of our slaves run the same image, it lessens repetition.
+This script uses YAML and jinja2 to generate a java jsonarray to build the configuration, using a !include constructor in the YAML file, allowing the ability to template up docker_images, since many of our slaves run the same image, it lessens repetition.
Usage
=======
@@ -24,4 +24,43 @@ List your jenkin_slaves here
docker_image_name: 'ubuntu:latest'
max_instances: '1'
labels: 'docker-ubuntu'
+
+- host3
+ cloud_name: host3.example.org
+ docker-url: tcp://0.0.0.0:2375
+ docker_templates:
+ !include [external_template_file.yml, external_template_file_2.yml]
```
+
+Limitations
+=======
+
+Due to the nature of YAML and populating the Java JSONARRAY, its important that yaml is phased correctly.
+
+Most of the limitations surround docker_templates.
+
+A list of limitations and pending improvements.
+
+1. Do not mix !include and templates under docker_templates.
+
+Example of broken approach:
+
+- host1:
+ cloud_name: host1.example.org
+ docker-url: tcp://0.0.0.0:2375
+ docker_templates:
+ !include external_template_file.yml
+ - xenial-amd64:
+ docker_image_name: 'ubuntu:latest'
+ max_instances: '1'
+ labels: 'docker-ubuntu'
+
+2. Do not add spaces under the image cloud_name, the must fall inline with the image name
+
+Example of broken approach:
+- xenial-amd64:
+ docker_image_name: 'ubuntu:latest'
+ max_instances: '1'
+ labels: 'docker-ubuntu'
+
+3. Docker labels could be "host" specific. If you are including a template file, the template has no link to the host, so its not possible to do "host based labels". Instead just create another template file with custom settings and include that file.
diff --git a/yadp_builder.py b/yadp_builder.py
index 1b49245..a8e50dc 100755
--- a/yadp_builder.py
+++ b/yadp_builder.py
@@ -32,18 +32,30 @@ class Loader(yaml.Loader, metaclass=LoaderMeta):
super().__init__(stream)
def construct_include(self, node):
- """Include file referenced at node."""
-
- filename = os.path.abspath(os.path.join(
- self._root, self.construct_scalar(node)
- ))
- extension = os.path.splitext(filename)[1].lstrip('.')
-
- with open(filename, 'r') as f:
- if extension in ('yaml', 'yml'):
- return yaml.load(f, Loader)
- else:
- return ''.join(f.readlines())
+ if isinstance(node, yaml.ScalarNode):
+ return self.extractFile(self.construct_scalar(node))
+
+ elif isinstance(node, yaml.SequenceNode):
+ result = []
+ for filename in self.construct_sequence(node):
+ result += self.extractFile(filename)
+ return result
+
+ elif isinstance(node, yaml.MappingNode):
+ result = {}
+ for k,v in self.construct_mapping(node).iteritems():
+ result[k] = self.extractFile(v)
+ return result
+
+ else:
+ print "Error:: unrecognised node type in !include statement"
+ raise yaml.constructor.ConstructorError
+
+
+ def extractFile(self, filename):
+ filepath = os.path.join(self._root, filename)
+ with open(filepath, 'r') as f:
+ return yaml.load(f, Loader)
def jinja2_from_template(directory, template_name, data):
diff --git a/yadp_builder_v2.py b/yadp_builder_v2.py
index 62d60e7..fde8e0e 100755
--- a/yadp_builder_v2.py
+++ b/yadp_builder_v2.py
@@ -17,18 +17,30 @@ class Loader(yaml.Loader):
self._root = os.path.curdir
def construct_include(self, node):
- """Include file referenced at node."""
-
- filename = os.path.abspath(os.path.join(
- self._root, self.construct_scalar(node)
- ))
- extension = os.path.splitext(filename)[1].lstrip('.')
-
- with open(filename, 'r') as f:
- if extension in ('yaml', 'yml'):
- return yaml.load(f, Loader)
- else:
- return ''.join(f.readlines())
+ if isinstance(node, yaml.ScalarNode):
+ return self.extractFile(self.construct_scalar(node))
+
+ elif isinstance(node, yaml.SequenceNode):
+ result = []
+ for filename in self.construct_sequence(node):
+ result += self.extractFile(filename)
+ return result
+
+ elif isinstance(node, yaml.MappingNode):
+ result = {}
+ for k,v in self.construct_mapping(node).iteritems():
+ result[k] = self.extractFile(v)
+ return result
+
+ else:
+ print "Error:: unrecognised node type in !include statement"
+ raise yaml.constructor.ConstructorError
+
+
+ def extractFile(self, filename):
+ filepath = os.path.join(self._root, filename)
+ with open(filepath, 'r') as f:
+ return yaml.load(f, Loader)
def jinja2_from_template(directory, template_name, data):