merge-fragments.py

#!/usr/bin/python

from optparse import OptionParser
import json
import logging
import os
import re
import sys
import urllib2

log = logging.getLogger("jsonmerge")
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)


def download_custom_json(url, auth=None):
    request = urllib2.Request(url)
    if auth:
        request.add_header('Authorization', auth)
    try:
        response = urllib2.urlopen(request)
    except urllib2.URLError, e:
        if hasattr(e, 'reason'):
            log.error('Failed to reach %s.' % url)
            log.error('Reason: ', e.reason)
        elif hasattr(e, 'code'):
            log.error('ci.linaro.org could not fulfill the request: %s' % \
                  url)
            log.error('Error code: ', e.code)
        sys.exit('Failed to get last successful artifact.')
    return response.read()


def substitute_template_variables(template, environ):
    """ Env variable names for substitution
        have to start with SUB_
        Template variables are marked with
        %%VARIABLE%%
        Variable names need to be uppercase
    """
    sub_expr = re.compile("%%(?P<variable>[_A-Z]+)%%")
    for var in sub_expr.finditer(template):
        var_name = var.group('variable')
        log.debug("Substituting: SUB_%s" % (var_name))
        var_value = environ.get("SUB_%s" % var_name)
        if var_value:
            template = template.replace(var.group(0), var_value)
        else:
            log.error("Variable SUB_%s is not set" % (var_name))
            sys.exit("Failed to create valid JSON")
    return template


def main():
    usage = "usage: %prog [OPTIONS]"
    parser = OptionParser(usage=usage)
    parser.add_option("-e", "--environment", dest="environ",
                  help="Environment variable for running script")
    parser.add_option("-m", "--metadata", dest="metadata",
                  help="Metadata in JSON format \
                        to be placed in the output definition")
    parser.add_option("-i", "--input", dest="input_file",
                  help="Input template JSON file")
    parser.add_option("-o", "--output", dest="output",
                  help="Output file to store JSON")
    (options, args) = parser.parse_args()

    if options.environ:
        f = open(options.environ, "r")
        for line in f.readlines():
            split_result = line.split("=", 1)
            if len(split_result) > 1:
                os.environ[split_result[0]] = split_result[1].strip()
        f.close()

    if not options.input_file:
        parser.error("Unable to construct job definition \
                      without the template")

    config_tmpl = open(options.input_file, "r")
    config = substitute_template_variables(config_tmpl.read(), os.environ)
    config_tmpl.close()
    json_config = json.loads(config)

    metadata_json = None
    if options.metadata:
        f = open(options.metadata, "r")
        metadata = f.read()
        try:
            metadata_json = json.loads(metadata)
        except:
            log.error("Unable to parse metadata file")

    metadata_url = os.environ.get("METADATA_URL", None)
    if metadata_url:
        metadata = download_custom_json(metadata_url)
        try:
            metadata_json = json.loads(metadata)
        except:
            log.error("Unable to parse metadata URL")

    if metadata_json:
        if 'metadata' in json_config['actions'][0]:
            json_config['actions'][0]['metadata'].update(metadata_json)
        else:
            json_config['actions'][0]['metadata'] = metadata_json

    if options.output:
        f = open(options.output, "w")
        f.write(json.dumps(json_config, indent=2))
        f.close()
    else:
        print json.dumps(json_config, indent=2)


if __name__ == '__main__':
    main()