aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorStevan Radaković <stevan.radakovic@linaro.org>2013-01-30 17:12:51 +0100
committerStevan Radaković <stevan.radakovic@linaro.org>2013-01-30 17:12:51 +0100
commit993d4884c1854ddf1ff88b4ac87a8e56b655bd59 (patch)
tree6703676a9c48b4e260eb41eddf54b5a771c4cdd1
parenta6cce91d3c0b3a6dc1a494d7bf09e5d9b2afb48d (diff)
parentee97c7680d137458a1a14455ae0f8644170a151b (diff)
downloadrhodecode-config-993d4884c1854ddf1ff88b4ac87a8e56b655bd59.tar.gz
Merge branch 'master' of ssh://git.linaro.org/srv/git.linaro.org/git/infrastructure/rhodecode-config
-rw-r--r--INSTALL12
-rw-r--r--scripts/mirror-repos74
-rwxr-xr-xscripts/rhodecode-setup2
-rw-r--r--scripts/update-repos25
4 files changed, 107 insertions, 6 deletions
diff --git a/INSTALL b/INSTALL
index dbea971..f5be114 100644
--- a/INSTALL
+++ b/INSTALL
@@ -55,6 +55,7 @@ Dependencies
* python-dev
* python-docutils
* python-formencode
+ * python-ldap
* python-mailer
* python-markdown
* python-nose
@@ -299,6 +300,17 @@ To clone all git.linaro.org repositories:
python scripts/mirror-repos --repos-list config/git_repos.txt \
--checkout-dir /opt/rhodecode/git_repos --user rhodecode
+I order for RhodeCode to pick up the newly added respositories, it is necessary
+to trigger RhodeCode for a directory re-scan. To do so, the command used before
+will look like:
+
+ python scripts/mirror-repos --repos-list config/git_repos.txt \
+ --checkout-dir /opt/rhodecode/git_repos --user rhodecode --rescan-repos \
+ --api-key $ADMIN_API_KEY
+
+Where $ADMIN_API_KEY is found in the RhodeCode 'admin' user web profile.
+
+
The script will mirror the repositories maintaining their directory structures.
To update the repositories, as a cron job for the 'rhodecode' user, run:
diff --git a/scripts/mirror-repos b/scripts/mirror-repos
index ffa4d5d..d013b14 100644
--- a/scripts/mirror-repos
+++ b/scripts/mirror-repos
@@ -6,8 +6,20 @@ import os
import subprocess
import sys
import urlparse
+import pwd
+from tempfile import gettempdir
+
+
+# Default read-only git URL.
BASE_PATH = "http://git.linaro.org/git-ro/"
+# Path to local bin directory, %s is the user name.
+LOCAL_BIN_DIR = "/home/%s/.local/bin"
+# Default API host for RhodeCode.
+DEFAULT_API_HOST = "http://0.0.0.0:5000"
+# Name for a lock file.
+LOCK_FILE_NAME = "mirror-repos.lock"
+LOCK_FILE = os.path.join(gettempdir(), LOCK_FILE_NAME)
def args_parser():
@@ -21,6 +33,17 @@ def args_parser():
help="Where git repositories will be cloned.")
parser.add_argument("--user",
help="User to run the commands as.")
+ parser.add_argument("--rescan-repos",
+ action="store_true",
+ help="If the directory containing repositories "
+ "should be re-scanned when adding new ones.")
+ parser.add_argument("--api-key",
+ help="The RhodeCode API key to use for re-scanning "
+ "the repositories.")
+ parser.add_argument("--api-host",
+ default=DEFAULT_API_HOST,
+ help="The host URL where API interface is located. "
+ "Defaults to '%s'." % DEFAULT_API_HOST)
return parser
@@ -44,6 +67,17 @@ def check_args(args, parser):
parser.print_usage()
sys.exit(1)
+ if args.rescan_repos:
+ if not args.api_key:
+ print ("It is necessary to specify the API key of the admin user "
+ "to perform the rescan operation.")
+ parser.print_usage()
+ sys.exit(1)
+ # Just print a warning...
+ if args.api_host == DEFAULT_API_HOST:
+ print ("Warning: default API host will be used: "
+ "%s" % DEFAULT_API_HOST)
+
def mirror_repos(file, dest, user=None):
"""Clone a mirror copy of a remote repository from git.linaro.org.
@@ -76,6 +110,28 @@ def mirror_repos(file, dest, user=None):
execute_command(cmd_args, work_dir=full_path, user=user)
+def rescan_git_directory(api_key, api_host, user=None):
+ """Rescans git directories for new repositories added.
+
+ :param api_key: The RhodeCode API key.
+ :type str
+ :param api_host: The RhodeCode host where to run the remote command.
+ :type str
+ :param user: The user to run the command as.
+ :type str
+ """
+ if not user:
+ # Try to gess a user.
+ user = pwd.getpwuid(os.getuid())[0]
+
+ api_key_cmd = "--apikey=%s" % str(api_key)
+ api_host_cmd = "--apihost=%s" % api_host
+
+ api_cmd = os.path.join(LOCAL_BIN_DIR % user, "rhodecode-api")
+ cmd_args = [api_cmd, api_key_cmd, api_host_cmd, "rescan_repos"]
+ execute_command(cmd_args, user=user)
+
+
def execute_command(cmd_args, as_sudo=True, user=None, work_dir=os.getcwd()):
"""Executes the command using Popen.
@@ -109,4 +165,20 @@ if __name__ == '__main__':
parser = args_parser()
args = parser.parse_args()
check_args(args, parser)
- mirror_repos(args.repos_list, args.checkout_dir, user=args.user)
+
+ if os.path.exists(LOCK_FILE):
+ print "Another process is still running: cannot acquire lock."
+ else:
+ try:
+ with open(LOCK_FILE, 'w'):
+ mirror_repos(args.repos_list,
+ args.checkout_dir,
+ user=args.user)
+
+ if args.rescan_repos:
+ print "Re-scanning git repositories directory..."
+ rescan_git_directory(args.api_key,
+ args.api_host,
+ user=args.user)
+ finally:
+ os.unlink(LOCK_FILE)
diff --git a/scripts/rhodecode-setup b/scripts/rhodecode-setup
index 23e9e8e..8cd7cd7 100755
--- a/scripts/rhodecode-setup
+++ b/scripts/rhodecode-setup
@@ -16,7 +16,7 @@ REQUIRED_PACKAGES = ["python-pip", "python-webob", "python-bcrypt",
"python-formencode", "python-pylons", "python-dev",
"python-pastescript", "python-psycopg2", "python-nose",
"build-essential", "python-amqplib", "python-anyjson",
- "python-mailer", "apache2", "git"]
+ "python-mailer", "apache2", "git", "python-ldap"]
# Packages required for celery integration.
CELERY_REQUIRED_PACKAGES = ["rabbitmq-server"]
diff --git a/scripts/update-repos b/scripts/update-repos
index 73a6539..154e0fa 100644
--- a/scripts/update-repos
+++ b/scripts/update-repos
@@ -6,9 +6,14 @@ import os
import re
import subprocess
+from tempfile import gettempdir
+
# Match a directory that ends with .git, but not only .git
GIT_DIRECTORY_ENDS = re.compile(".+\.git$")
+# Name for a lock file.
+LOCK_FILE_NAME = "update-repos.lock"
+LOCK_FILE = os.path.join(gettempdir(), LOCK_FILE_NAME)
def args_parser():
@@ -22,10 +27,8 @@ def args_parser():
return parser
-if __name__ == '__main__':
- parser = args_parser()
- args = parser.parse_args()
-
+def fetch_updates():
+ """Traverse the file system and fetc updates."""
for root, dirs, files in os.walk(os.path.abspath(args.repos_dir)):
if GIT_DIRECTORY_ENDS.match(root):
if files:
@@ -49,3 +52,17 @@ if __name__ == '__main__':
# git repositories always have a HEAD file, or it means they
# are empty.
continue
+
+
+if __name__ == '__main__':
+ parser = args_parser()
+ args = parser.parse_args()
+
+ if os.path.exists(LOCK_FILE):
+ print "Another process is still running: cannot acquire lock."
+ else:
+ try:
+ with open(LOCK_FILE, 'w'):
+ fetch_updates()
+ finally:
+ os.unlink(LOCK_FILE)