#!/usr/bin/env python # Copyright (C) 2013 Linaro Ltd. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see import argparse import logging import os import pwd import subprocess import sys import urlparse from logging.handlers import TimedRotatingFileHandler from tempfile import gettempdir # Default user for operations. DEFAULT_USER = "rhodecode" # Default read-only git URL. BASE_PATH = "http://git.linaro.org/git-ro/" # Path to local bin directory, %s is the user name. LOCAL_BIN_DIR = "/home/%s/.local/bin" # Default API host for RhodeCode. DEFAULT_API_HOST = "http://0.0.0.0:5000" # Name for a lock file. LOCK_FILE_NAME = "mirror-repos.lock" LOCK_FILE = os.path.join(gettempdir(), LOCK_FILE_NAME) FILE_NAME = os.path.basename(__file__) # Default log directory and log file. DEFAULT_LOG_DIR = "/var/log/rhodecode" LOG_FILE_NAME = FILE_NAME + ".log" # When to rotate logs. DEFAULT_ROTATING_TIME = 'midnight' # How many old logs to keep. KEEP_MAX_LOGS = 10 # Default logger. logger = logging.getLogger(FILE_NAME) def args_parser(): """Sets up the argument parser.""" parser = argparse.ArgumentParser() parser.add_argument("--repos-list", required=True, help="File with the repository names to mirror.") parser.add_argument("--checkout-dir", required=True, help="Where git repositories will be cloned.") parser.add_argument("--user", help="User to run the commands as.") parser.add_argument("--rescan-repos", action="store_true", help="If the directory containing repositories " "should be re-scanned when adding new ones.") parser.add_argument("--api-key", help="The RhodeCode API key to use for re-scanning " "the repositories.") parser.add_argument("--api-host", default=DEFAULT_API_HOST, help="The host URL where API interface is located. " "Defaults to '%s'." % DEFAULT_API_HOST) parser.add_argument("--log-dir", default=DEFAULT_LOG_DIR, help="Directory to store logs. Defaults to '%s'." % DEFAULT_LOG_DIR) parser.add_argument("--debug", action="store_true", help="Print debugging statements.") return parser def setup_logging(debug, log_dir): """Sets up logging. :param debug: If the level should be set to DEBUG. :type bool :param log_dir: Where to store file based logs. """ th_formatter = "%(asctime)s %(levelname)-8s %(message)s" log_file = os.path.join(log_dir, LOG_FILE_NAME) timed_handler = TimedRotatingFileHandler(log_file, when=DEFAULT_ROTATING_TIME, backupCount=KEEP_MAX_LOGS) timed_handler.setFormatter(logging.Formatter(th_formatter)) if debug: logger.setLevel(logging.DEBUG) timed_handler.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) timed_handler.setLevel(logging.INFO) logger.addHandler(timed_handler) def check_args(args, parser): """Checks command line arguments passed. :param args: All the command lines as returned by argparse. :param parser: The argparse instance. """ if not os.path.exists(args.repos_list) or \ not os.path.isfile(args.repos_list): print ("Error: file '%s' does not exists or is not a regular file." % args.repos_list) parser.print_usage() sys.exit(1) if not os.path.exists(args.checkout_dir) or \ not os.path.isdir(args.checkout_dir): print ("Error: directory '%s' does not exists or cannot be " "accessed." % args.checkout_dir) parser.print_usage() sys.exit(1) if args.rescan_repos: if not args.api_key: print ("It is necessary to specify the API key of the admin user " "to perform the rescan operation.") parser.print_usage() sys.exit(1) # Just print a warning... if args.api_host == DEFAULT_API_HOST: print ("Warning: default API host will be used: " "%s" % DEFAULT_API_HOST) def mirror_repos(file, dest, user=None): """Clone a mirror copy of a remote repository from git.linaro.org. :param file: The file where to read the repositories to mirror. :param dest: The directory where to clone the repositories into. """ for line in open(file).readlines(): line = line.strip() base_dir = os.path.basename(line) # Git repos need to have a valid name. if base_dir.split(".git")[0]: # Maintain the same directory layout of original git.linaro.org. full_path = os.path.join(dest, line.split(base_dir)[0]) # Skip if repository is already there. if os.path.exists(os.path.join(full_path, base_dir)): logger.debug("Repository '%s' already present." % base_dir) continue # We need to do so, to create the directory as the RhodeCode user # for our installation. cmd_args = ["mkdir", "-p", full_path] execute_command(cmd_args, user=user) # We mirror the original repository, then through a cron job we can # easily update it using the command 'git fetch -q'. full_repo = urlparse.urljoin(BASE_PATH, line) cmd_args = ["git", "clone", "--mirror", full_repo] actual_user = pwd.getpwuid(os.getuid())[0] if actual_user == DEFAULT_USER or not user: as_user = None as_root = False else: as_user = user as_root = True logger.info("Cloning repository %s..." % full_repo) execute_command(cmd_args, work_dir=full_path, as_root=as_root, user=as_user) else: logger.debug("Skipping repository '%s', does not seem a valid " "git one." % base_dir) def rescan_git_directory(api_key, api_host, user=None): """Rescans git directories for new repositories added. :param api_key: The RhodeCode API key. :type str :param api_host: The RhodeCode host where to run the remote command. :type str :param user: The user to run the command as. :type str """ actual_user = pwd.getpwuid(os.getuid())[0] if user: bin_dir = LOCAL_BIN_DIR % user else: # Try to gess a user. bin_dir = LOCAL_BIN_DIR % actual_user api_key_cmd = "--apikey=%s" % api_key api_host_cmd = "--apihost=%s" % api_host api_cmd = os.path.join(bin_dir, "rhodecode-api") cmd_args = [api_cmd, api_key_cmd, api_host_cmd, "rescan_repos"] if actual_user == DEFAULT_USER or not user: execute_command(cmd_args) else: execute_command(cmd_args, as_sudo=True, user=user) def execute_command(cmd_args, as_sudo=False, user=None, work_dir=os.getcwd()): """Executes the command using Popen. :param cmd_args: The list of command and parameters to run. :param as_sudo: If the command has to be run with 'sudo'. :param user: Runs the comand as the specified user. :param work_dir: Where the command should be run from. """ exec_args = [] if not isinstance(cmd_args, list): cmd_args = [cmd_args] if as_sudo: exec_args = ["sudo"] if user and as_sudo: exec_args += ["-u", user, "-H"] exec_args += cmd_args process = subprocess.Popen(exec_args, cwd=work_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p_out, p_err = process.communicate() if process.returncode != 0: logger.error("Error executing the following command: %s" % " ".join(cmd_args)) logger.debug("The full command line is: %s" % " ".join(exec_args)) logger.debug(p_err) if __name__ == '__main__': parser = args_parser() args = parser.parse_args() check_args(args, parser) if os.path.exists(LOCK_FILE): print "Another process is still running: cannot acquire lock." else: setup_logging(args.debug, args.log_dir) try: with open(LOCK_FILE, 'w'): mirror_repos(args.repos_list, args.checkout_dir, user=args.user) if args.rescan_repos: print "Re-scanning git repositories directory..." rescan_git_directory(args.api_key, args.api_host, user=args.user) finally: os.unlink(LOCK_FILE)