diff options
| -rw-r--r-- | INSTALL | 12 | ||||
| -rw-r--r-- | scripts/mirror-repos | 74 | ||||
| -rwxr-xr-x | scripts/rhodecode-setup | 2 | ||||
| -rw-r--r-- | scripts/update-repos | 25 |
4 files changed, 107 insertions, 6 deletions
@@ -55,6 +55,7 @@ Dependencies * python-dev * python-docutils * python-formencode + * python-ldap * python-mailer * python-markdown * python-nose @@ -299,6 +300,17 @@ To clone all git.linaro.org repositories: python scripts/mirror-repos --repos-list config/git_repos.txt \ --checkout-dir /opt/rhodecode/git_repos --user rhodecode +I order for RhodeCode to pick up the newly added respositories, it is necessary +to trigger RhodeCode for a directory re-scan. To do so, the command used before +will look like: + + python scripts/mirror-repos --repos-list config/git_repos.txt \ + --checkout-dir /opt/rhodecode/git_repos --user rhodecode --rescan-repos \ + --api-key $ADMIN_API_KEY + +Where $ADMIN_API_KEY is found in the RhodeCode 'admin' user web profile. + + The script will mirror the repositories maintaining their directory structures. To update the repositories, as a cron job for the 'rhodecode' user, run: diff --git a/scripts/mirror-repos b/scripts/mirror-repos index ffa4d5d..d013b14 100644 --- a/scripts/mirror-repos +++ b/scripts/mirror-repos @@ -6,8 +6,20 @@ import os import subprocess import sys import urlparse +import pwd +from tempfile import gettempdir + + +# Default read-only git URL. BASE_PATH = "http://git.linaro.org/git-ro/" +# Path to local bin directory, %s is the user name. +LOCAL_BIN_DIR = "/home/%s/.local/bin" +# Default API host for RhodeCode. +DEFAULT_API_HOST = "http://0.0.0.0:5000" +# Name for a lock file. +LOCK_FILE_NAME = "mirror-repos.lock" +LOCK_FILE = os.path.join(gettempdir(), LOCK_FILE_NAME) def args_parser(): @@ -21,6 +33,17 @@ def args_parser(): help="Where git repositories will be cloned.") parser.add_argument("--user", help="User to run the commands as.") + parser.add_argument("--rescan-repos", + action="store_true", + help="If the directory containing repositories " + "should be re-scanned when adding new ones.") + parser.add_argument("--api-key", + help="The RhodeCode API key to use for re-scanning " + "the repositories.") + parser.add_argument("--api-host", + default=DEFAULT_API_HOST, + help="The host URL where API interface is located. " + "Defaults to '%s'." % DEFAULT_API_HOST) return parser @@ -44,6 +67,17 @@ def check_args(args, parser): parser.print_usage() sys.exit(1) + if args.rescan_repos: + if not args.api_key: + print ("It is necessary to specify the API key of the admin user " + "to perform the rescan operation.") + parser.print_usage() + sys.exit(1) + # Just print a warning... + if args.api_host == DEFAULT_API_HOST: + print ("Warning: default API host will be used: " + "%s" % DEFAULT_API_HOST) + def mirror_repos(file, dest, user=None): """Clone a mirror copy of a remote repository from git.linaro.org. @@ -76,6 +110,28 @@ def mirror_repos(file, dest, user=None): execute_command(cmd_args, work_dir=full_path, user=user) +def rescan_git_directory(api_key, api_host, user=None): + """Rescans git directories for new repositories added. + + :param api_key: The RhodeCode API key. + :type str + :param api_host: The RhodeCode host where to run the remote command. + :type str + :param user: The user to run the command as. + :type str + """ + if not user: + # Try to gess a user. + user = pwd.getpwuid(os.getuid())[0] + + api_key_cmd = "--apikey=%s" % str(api_key) + api_host_cmd = "--apihost=%s" % api_host + + api_cmd = os.path.join(LOCAL_BIN_DIR % user, "rhodecode-api") + cmd_args = [api_cmd, api_key_cmd, api_host_cmd, "rescan_repos"] + execute_command(cmd_args, user=user) + + def execute_command(cmd_args, as_sudo=True, user=None, work_dir=os.getcwd()): """Executes the command using Popen. @@ -109,4 +165,20 @@ if __name__ == '__main__': parser = args_parser() args = parser.parse_args() check_args(args, parser) - mirror_repos(args.repos_list, args.checkout_dir, user=args.user) + + if os.path.exists(LOCK_FILE): + print "Another process is still running: cannot acquire lock." + else: + try: + with open(LOCK_FILE, 'w'): + mirror_repos(args.repos_list, + args.checkout_dir, + user=args.user) + + if args.rescan_repos: + print "Re-scanning git repositories directory..." + rescan_git_directory(args.api_key, + args.api_host, + user=args.user) + finally: + os.unlink(LOCK_FILE) diff --git a/scripts/rhodecode-setup b/scripts/rhodecode-setup index 23e9e8e..8cd7cd7 100755 --- a/scripts/rhodecode-setup +++ b/scripts/rhodecode-setup @@ -16,7 +16,7 @@ REQUIRED_PACKAGES = ["python-pip", "python-webob", "python-bcrypt", "python-formencode", "python-pylons", "python-dev", "python-pastescript", "python-psycopg2", "python-nose", "build-essential", "python-amqplib", "python-anyjson", - "python-mailer", "apache2", "git"] + "python-mailer", "apache2", "git", "python-ldap"] # Packages required for celery integration. CELERY_REQUIRED_PACKAGES = ["rabbitmq-server"] diff --git a/scripts/update-repos b/scripts/update-repos index 73a6539..154e0fa 100644 --- a/scripts/update-repos +++ b/scripts/update-repos @@ -6,9 +6,14 @@ import os import re import subprocess +from tempfile import gettempdir + # Match a directory that ends with .git, but not only .git GIT_DIRECTORY_ENDS = re.compile(".+\.git$") +# Name for a lock file. +LOCK_FILE_NAME = "update-repos.lock" +LOCK_FILE = os.path.join(gettempdir(), LOCK_FILE_NAME) def args_parser(): @@ -22,10 +27,8 @@ def args_parser(): return parser -if __name__ == '__main__': - parser = args_parser() - args = parser.parse_args() - +def fetch_updates(): + """Traverse the file system and fetc updates.""" for root, dirs, files in os.walk(os.path.abspath(args.repos_dir)): if GIT_DIRECTORY_ENDS.match(root): if files: @@ -49,3 +52,17 @@ if __name__ == '__main__': # git repositories always have a HEAD file, or it means they # are empty. continue + + +if __name__ == '__main__': + parser = args_parser() + args = parser.parse_args() + + if os.path.exists(LOCK_FILE): + print "Another process is still running: cannot acquire lock." + else: + try: + with open(LOCK_FILE, 'w'): + fetch_updates() + finally: + os.unlink(LOCK_FILE) |