diff options
author | Paul Sokolovsky <paul.sokolovsky@linaro.org> | 2013-11-15 18:31:01 +0200 |
---|---|---|
committer | Paul Sokolovsky <paul.sokolovsky@linaro.org> | 2013-11-15 18:31:01 +0200 |
commit | b740657dbdd87b9f74f72ac0dc926d95d5adbc9f (patch) | |
tree | fbb4e9cee2a1262e8b53888a79e80cfb4d9f6301 | |
parent | d28630f9e3391699e45593663e2dd78b3bbad089 (diff) |
Add per-usptream project_list param to get project list via URL.
-rwxr-xr-x | git-gerrit-mirror | 27 | ||||
-rw-r--r-- | mirror.conf | 1 |
2 files changed, 18 insertions, 10 deletions
diff --git a/git-gerrit-mirror b/git-gerrit-mirror index 8e1a1a0..1d24aa6 100755 --- a/git-gerrit-mirror +++ b/git-gerrit-mirror @@ -4,6 +4,7 @@ import sys import time import optparse import logging +import re import urllib import urlparse from xml.dom import minidom @@ -108,7 +109,11 @@ def get_manifest_projects(manifest): dom = minidom.parse(manifest) return [p.getAttribute("name") for p in dom.getElementsByTagName("project")] -def get_cached(get_func, get_params_array, cache_file, cache_ttl=120): +def get_cached(get_func, get_params_array, cache_id=None, cache_ttl=120): + if cache_id is None: + cache_id = get_params_array[0] + cache_file = re.sub(r"[ :/?&]", "_", cache_id) + ".cache" + if os.path.exists(cache_file): age = time.time() - os.stat(cache_file).st_mtime if age < cache_ttl: @@ -136,6 +141,12 @@ def get_gerrit_projects(gerrit_host): f.close() return projects +def get_url_project_list(url): + f = urllib.urlopen(url) + projects = [l.strip() for l in f] + f.close() + return projects + def get_ssh_identity_file_option(): """Return an identity file option to use in SSH command based on config.""" ssh_identity = conf.get_var(options.dest, "ssh_identity") @@ -154,15 +165,13 @@ def get_project_map_for_a_host(host): gerrit_host = conf.get_var(host, "gerrit") if options.manifest: projects = get_manifest_projects(options.manifest) - elif options.gitweb_list: - f = urllib.urlopen(options.gitweb_list) - projects = [l.strip() for l in f.readlines()] - f.close() + elif conf.get_var(host, "project_list"): + url = conf.get_var(host, "project_list") + projects = get_cached(get_url_project_list, (url,)) elif gerrit_host: ssh_supported = conf.get_var(host, "gerrit_ssh", "true") if ssh_supported == "true": - projects = get_cached(get_gerrit_projects, - (gerrit_host,), gerrit_host + ".ls-projects") + projects = get_cached(get_gerrit_projects, (gerrit_host,)) else: log.warn("SSH access disabled for %s, cannot get project list, skipping host", host) projects = [] @@ -240,8 +249,6 @@ optparser.add_option("--config", default="mirror.conf", help="Config file to use (%default)") optparser.add_option("--manifest", help="Use manifest for list of upstream projects (overrides $gerrit in config)") -optparser.add_option("--gitweb-list", - help="Use gitweb txt export for list of upstream projects (overrides $gerrit in config)") optparser.add_option("--upstream", metavar="SUBSTR", default="", help="Process only upstreams matching SUBSTR") optparser.add_option("--dest", default="", @@ -279,7 +286,7 @@ if args[0] == "create": gerrit_port = conf.get_var(options.dest, "gerrit_port") log.debug("Getting list of projects in target Gerrit %s:%s" % (gerrit_host, gerrit_port)) existing_projects = get_cached(get_gerrit_projects, - ("%s:%s" % (gerrit_host, gerrit_port),), gerrit_host + ".ls-projects") + ("%s:%s" % (gerrit_host, gerrit_port),)) for host in conf.get_hosts(options.upstream): log.debug("=== Processing: %s ===", host) projects = get_project_map_for_a_host(host).values() diff --git a/mirror.conf b/mirror.conf index 3b170b0..6bcc967 100644 --- a/mirror.conf +++ b/mirror.conf @@ -30,6 +30,7 @@ $gerrit = review.source.android.com #$active = false $gerrit = android-review.googlesource.com $gerrit_ssh = false +$project_list = https://android.googlesource.com/?format=TEXT [git://android.git.linaro.org] $active = false |