#!/usr/bin/python import argparse import logging import os import subprocess import grokmirror EXCLUDES = ( 'gitolite-admin.git', ) logging.basicConfig(level=logging.INFO) log = logging.getLogger('grok-cron') def _can_mirror(full_path): # we have a couple of old symlinks set up, that we don't have # to manage via grokmirror if os.path.isdir(full_path) and not os.path.islink(full_path): if os.path.exists(os.path.join(full_path, 'noweb')): return False if len(os.listdir(os.path.join(full_path, 'refs/heads'))) == 0: # grokmirror won't add these to the manifest (although it should) # these are typicall Gerrit project holders with: # HEAD=refs/meta/config return False return True return False def handle_unmanaged(manifile, manifest_repos, repo_dir, dryrun): '''Handle repos that have been moved and not picked up by grokmirror. This can happen when ITS moves a repo from one user's account to another during an exit procedure. ''' # look at projects.list which is what we serve to the public # see if it exists and isn't known to grokmirror missing = [] repos = subprocess.check_output( ['/home/git/bin/gitolite', 'list-phy-repos']) for repo in repos.splitlines(): repo = repo.strip() + '.git' full_path = os.path.join(repo_dir, repo) if _can_mirror(full_path): if repo not in manifest_repos and repo not in EXCLUDES: missing.append(repo) # grok-manifest is particular about the path format if repo_dir[-1] != '/': repo_dir += '/' logging.debug('fixed repo path: %s', repo_dir) for repo in missing: log.info('Adding %s to manifest', repo) if not dryrun: args = [ '/usr/local/bin/grok-manifest', '-m', manifile, '-t', repo_dir, '-n', repo, ] subprocess.check_call(args, cwd=repo_dir) def main(manifile, repo_dir, no_check_export, dryrun): # uses advisory lock, so its safe even if we die unexpectedly grokmirror.manifest_lock(manifile) changed = False manifest = grokmirror.read_manifest(manifile) for repo in manifest.keys(): # grokmirror doesn't play well with leading / and it also breaks # os.path.join, so just break loudly if this happens (which based # on our ansible deployment is impossible) log.debug('Checking for %s', repo) assert repo[0] != '/' g = os.path.join(repo_dir, repo) gde = os.path.join(g, 'git-daemon-export-ok') if not os.path.exists(g) or \ (not no_check_export and not os.path.exists(gde)): log.info("Removing %s from manifest...", repo) del manifest[repo] changed = True if changed and not dryrun: grokmirror.write_manifest(manifile, manifest) grokmirror.manifest_unlock(manifile) # do this outside the scope of the manifest lock, to avoid a deadlock handle_unmanaged(manifile, manifest.keys(), repo_dir, dryrun) if __name__ == '__main__': parser = argparse.ArgumentParser( description='''Handle tasks that cannot be done in gitolite hooks like purging deleting repos or updating descriptions.''') parser.add_argument('-m', '--manifest', required=True, help='grok manifest file') parser.add_argument('-t', '--toplevel', required=True, help='location of repositories') parser.add_argument('--no-check-export', action='store_true', help='git-daemon-export-ok file not required') parser.add_argument('-n', '--dryrun', action='store_true', help='Make no changes to manifest') parser.add_argument('--log', default='WARN', choices=('WARN', 'INFO', 'DEBUG'), help='Logging level to use. Default=%(default)s') args = parser.parse_args() for l in logging.getLogger().manager.loggerDict.keys(): logging.getLogger(l).setLevel(getattr(logging, args.log)) main(args.manifest, args.toplevel, args.no_check_export, args.dryrun)