Add per-usptream project_list param to get project list via URL.

2 files changed, 18 insertions, 10 deletions

diff --git a/git-gerrit-mirror b/git-gerrit-mirror
index 8e1a1a0..1d24aa6 100755
--- a/git-gerrit-mirror
+++ b/git-gerrit-mirror
@@ -4,6 +4,7 @@ import sys
 import time
 import optparse
 import logging
+import re
 import urllib
 import urlparse
 from xml.dom import minidom
@@ -108,7 +109,11 @@ def get_manifest_projects(manifest):
     dom = minidom.parse(manifest)
     return [p.getAttribute("name") for p in dom.getElementsByTagName("project")]
 
-def get_cached(get_func, get_params_array, cache_file, cache_ttl=120):
+def get_cached(get_func, get_params_array, cache_id=None, cache_ttl=120):
+    if cache_id is None:
+        cache_id = get_params_array[0]
+    cache_file = re.sub(r"[ :/?&]", "_", cache_id) + ".cache"
+
     if os.path.exists(cache_file):
         age = time.time() - os.stat(cache_file).st_mtime
         if age < cache_ttl:
@@ -136,6 +141,12 @@ def get_gerrit_projects(gerrit_host):
     f.close()
     return projects
 
+def get_url_project_list(url):
+    f = urllib.urlopen(url)
+    projects = [l.strip() for l in f]
+    f.close()
+    return projects
+
 def get_ssh_identity_file_option():
     """Return an identity file option to use in SSH command based on config."""
     ssh_identity = conf.get_var(options.dest, "ssh_identity")
@@ -154,15 +165,13 @@ def get_project_map_for_a_host(host):
     gerrit_host = conf.get_var(host, "gerrit")
     if options.manifest:
         projects = get_manifest_projects(options.manifest)
-    elif options.gitweb_list:
-        f = urllib.urlopen(options.gitweb_list)
-        projects = [l.strip() for l in f.readlines()]
-        f.close()
+    elif conf.get_var(host, "project_list"):
+        url = conf.get_var(host, "project_list")
+        projects = get_cached(get_url_project_list, (url,))
     elif gerrit_host:
         ssh_supported = conf.get_var(host, "gerrit_ssh", "true")
         if ssh_supported == "true":
-            projects = get_cached(get_gerrit_projects,
-                (gerrit_host,), gerrit_host + ".ls-projects")
+            projects = get_cached(get_gerrit_projects, (gerrit_host,))
         else:
             log.warn("SSH access disabled for %s, cannot get project list, skipping host", host)
             projects = []
@@ -240,8 +249,6 @@ optparser.add_option("--config", default="mirror.conf",
                      help="Config file to use (%default)")
 optparser.add_option("--manifest",
                      help="Use manifest for list of upstream projects (overrides $gerrit in config)")
-optparser.add_option("--gitweb-list",
-                     help="Use gitweb txt export for list of upstream projects (overrides $gerrit in config)")
 optparser.add_option("--upstream", metavar="SUBSTR", default="",
                      help="Process only upstreams matching SUBSTR")
 optparser.add_option("--dest", default="",
@@ -279,7 +286,7 @@ if args[0] == "create":
     gerrit_port = conf.get_var(options.dest, "gerrit_port")
     log.debug("Getting list of projects in target Gerrit %s:%s" % (gerrit_host, gerrit_port))
     existing_projects = get_cached(get_gerrit_projects,
-        ("%s:%s" % (gerrit_host, gerrit_port),), gerrit_host + ".ls-projects")
+        ("%s:%s" % (gerrit_host, gerrit_port),))
     for host in conf.get_hosts(options.upstream):
         log.debug("=== Processing: %s ===", host)
         projects = get_project_map_for_a_host(host).values()
diff --git a/mirror.conf b/mirror.conf
index 3b170b0..6bcc967 100644
--- a/mirror.conf
+++ b/mirror.conf
@@ -30,6 +30,7 @@ $gerrit = review.source.android.com
 #$active = false
 $gerrit = android-review.googlesource.com
 $gerrit_ssh = false
+$project_list = https://android.googlesource.com/?format=TEXT
 
 [git://android.git.linaro.org]
 $active = false