path: root/scripts/rhodecode-setup

#!/usr/bin/env python

# Copyright (C) 2013 Linaro Ltd.

import argparse
import os
import subprocess
import sys


DESCRIPTION = "Install and setup RhodeCode Linaro instance."
REQUIRED_PACKAGES = ["python-pip", "python-webob", "python-bcrypt",
                     "python-mock", "python-babel", "python-dateutil",
                     "python-markdown", "python-webhelpers", "python-docutils",
                     "python-formencode", "python-pylons", "python-dev",
                     "build-essential", "apache2"]
# Packages required for celery integration.
CELERY_REQUIRED_PACKAGES = ["python-celery", "rabbitmq-server"]

# Packages for the backend to use.
SQLITE_DB = "sqlite"
POSTGRESQL_DB = ["postgresql-9.1", "pgadmin3", "postgresql-server-dev-9.1"]
# The name of the DB for PostgreSQL.
DB_NAME = "rhodecode"

# Packages to be installed via PIP.
PIP_PACKAGES = {
    "psycopg2": None,
    "celery": "2.2.10",
    "dulwich": "0.8.7",
    "beaker": "1.6.4",
    "sqlalchemy": "0.7.9",
    "mako": "0.7.3",
    "pygments": "1.5",
    "whoosh": "2.4.0",
    "simplejson": "2.5.2",
    "waitress": "0.8.1",
}

# RhodeCode source to clone and which branch to use.
RHODECODE_SOURCE_GIT_URL = \
    "http://git.linaro.org/git-ro/infrastructure/rhodecode.git"
RHODECODE_SOURCE_BRANCH = "linaro"

# The name of the RhodeCode user to use by default. This will be used to
# install the RhodeCode code and to run RhodeCode.
RHODECODE_DEFAULT_USER = "rhodecode"

# Default parameters for RabbitMQ setup.
RABBITMQ_DEFAULT_USER = "rhodecode"
RABBITMQ_DEFAULT_VHOST = "rhodecode-vhost"

# Default user for RhodeCode admin.
RHODECODE_ADMIN_USER = "admin"

# Upstart config files.
RHODECODE_UPSTART_CONF = "rhodecode-upstart.conf"
CELERY_UPSTART_CONF = "celeryd-upstart.conf"


def cli_args():
    """Sets up the cli argument parser."""
    parser = argparse.ArgumentParser(description=DESCRIPTION)
    parser.add_argument("--rhodecode-config",
                        help="Path to the config INI file.",
                        required=True)
    parser.add_argument("--rhodecode-data-dir",
                        default="/opt/rhodecode",
                        help="The directory where to store RhodeCode data "
                             "and cache.")
    parser.add_argument("--rhodecode-admin-usr",
                        default=RHODECODE_ADMIN_USER,
                        help="The name of the admin user for RhodeCode.")
    parser.add_argument("--rhodecode-admin-pwd",
                        help="The password for RhodeCode admin user.",
                        required=True)
    parser.add_argument("--rhodecode-admin-email",
                        required=True,
                        help="The email address for RhodeCode admin user.")
    parser.add_argument("--rhodecode-usr",
                        default=RHODECODE_DEFAULT_USER,
                        help="The name for the RhodeCode sytem user and "
                             "group.")
    parser.add_argument("--rhodecode-git-url",
                        default=RHODECODE_SOURCE_GIT_URL,
                        help="The URL where to clone RhodeCode source "
                             "code from.")
    parser.add_argument("--rhodecode-checkout-dir",
                        default="rhodecode",
                        help="The destination directory where to clone "
                             "RhodeCode code into.")
    parser.add_argument("--rhodecode-branch",
                        default=RHODECODE_SOURCE_BRANCH,
                        help="The branch to use from the source code.")
    parser.add_argument("--assume-yes",
                        action="store_true",
                        help="Try to automate the process, assuming yes to "
                        "all questions.")
    parser.add_argument("--development",
                        action="store_true",
                        help="If the instance is a development instance.")
    parser.add_argument("--no-celery",
                        action="store_true",
                        help="If Celery integration should be disabled.")
    parser.add_argument("--rabbitmq-usr",
                        default=RABBITMQ_DEFAULT_USER,
                        help="The name to use for the RabbitMQ session.")
    parser.add_argument("--rabbitmq-pwd",
                        help="The password for the RabbitMQ user.",
                        required=True)
    parser.add_argument("--rabbitmq-vhost",
                        default=RABBITMQ_DEFAULT_VHOST,
                        help="The name of the RabbitMQ vhost for the user.")
    parser.add_argument("--dbname",
                        default=DB_NAME,
                        help="The name to use for the database.")
    parser.add_argument("--repos-dir",
                        default="/opt/git_repos",
                        help="Where the git repositories will be stored.")
    return parser.parse_args()


def check_cli_args(args):
    """Performs checks on the command line args passed.

    Logic to handle the various cases for the cli args.
    :param args: The args passed on the command line.
    """
    if not args.rhodecode_config:
        print ("It is necessary to specify the path to RhodeCode "
               "configuration file.")
        sys.exit(1)
    if not args.rhodecode_admin_pwd:
        print ("It is necessary to specify the administration password for "
               "RhodeCode")
        sys.exit(1)

    if not args.no_celery:
        if (args.rabbitmq_usr == RABBITMQ_DEFAULT_USER or \
              args.rabbitmq_vhost == RABBITMQ_DEFAULT_VHOST):
            print ("Warning: default values for --rabbitmq-user and/or "
                   "--rabbitmq-vhost are being used.")

        if not args.rabbitmq_pwd:
            print ("To setup RhodeCode to correctly use Celery, it is "
                   "necessary to specify also the RabbitMQ user password.")
            sys.exit(1)


def setup_user(user, home_dir, group):
    """Creates the necessary user and its associated group.

    :param user: The name of the user and group to create.
    """
    # Add the user.
    cmd_args = ["adduser", "--system", "--home", home_dir, user]
    execute_command(cmd_args)

    # Add the user group.
    cmd_args = ["addgroup", "--system", group]
    execute_command(cmd_args)

    # Add the user to its group.
    cmd_args = ["adduser", user, group]
    execute_command(cmd_args)


def setup_directories(data_dir, repos_dir, rhodecode_usr):
    if not os.path.exists(data_dir):
        cmd_args = ["mkdir", "-p", data_dir]
        execute_command(cmd_args)

    if not os.path.exists(repos_dir):
        cmd_args = ["mkdir", "-p", repos_dir]
        execute_command(cmd_args)

    # Set ownership of the directories to the RhodeCode user.
    set_owners(data_dir, rhodecode_usr)

    set_owners(repos_dir, rhodecode_usr)


def execute_command(cmd_args,
                    work_dir=os.getcwd(),
                    with_sudo=True,
                    input_str=None):
    """Runs the command passed.

    :param cmd_args: List of command and options to run.
    :type list
    :param word_dir: Where the command must be run. Defaults to current dir.
    :type str
    :param with_sudo: If the command must be executed as root. Defaults to
    True.
    :type bool
    :param input_str: String to pass to the process as input, in order to
    automate as much as possible.
    :type str
    """
    if not isinstance(cmd_args, list):
        cmd_args = list(cmd_args)
    if with_sudo:
        cmd_args.insert(0, "sudo")
    if input_str:
        process = subprocess.Popen(cmd_args,
                                   cwd=work_dir,
                                   stderr=subprocess.PIPE,
                                   stdin=subprocess.PIPE)
        p_out, p_err = process.communicate(input=input_str)
    else:
        process = subprocess.Popen(cmd_args,
                                   cwd=work_dir,
                                   stderr=subprocess.PIPE)
        p_out, p_err = process.communicate()
    if process.returncode != 0:
        print "Error executing the following command: %s" % " ".join(cmd_args)
        sys.exit(1)


def update_repositories():
    """Performs an 'apt-get update' on the system."""
    cmd_args = ["apt-get", "update"]
    execute_command(cmd_args)


def add_launchpad_ppa(ppa_string, assume_yes=False):
    """Adds a PPA repository from Launchpad.

    :param ppa_string: The PPA string as found in Launchpad.
    :type str
    """
    input_str = None
    cmd_args = ["apt-add-repository", ppa_string]
    if assume_yes:
        input_str = "\n"
    execute_command(cmd_args, input_str=input_str)


def install_packages(packages, assume_yes=False):
    """Installs a new package in the system.

    :param packages: The packages to install.
    :type list
    """
    input_str = None
    if not isinstance(packages, list):
        packages = [packages]
    cmd_args = ["apt-get", "install"] + packages
    if assume_yes:
        input_str = "Y"
    execute_command(cmd_args, input_str=input_str)


def install_pip_packages(packages):
    """Installs packages from PIP.

    :param packages: The packages to install. It has to be a dictionary, with
    key the name of the package, and value the version number or None.
    :type dict
    """
    for key, value in packages.iteritems():
        cmd_args = ["pip", "install", "-I"]
        if value:
            cmd_args.append("%s==%s" % (key, value))
        else:
            cmd_args.append(key)
        cmd_args.append("--upgrade")
        execute_command(cmd_args)


def install_git_from_ppa(assume_yes=False):
    """Install git from the git maintainers Launchpad PPA.

    :param assume_yes: If operation has to be automated.
    :type bool
    """
    print "Installing git-core package from git maintainers PPA..."
    install_packages("python-software-properties", assume_yes=assume_yes)
    add_launchpad_ppa("ppa:git-core/ppa", assume_yes=assume_yes)
    update_repositories()
    install_packages("git-core", assume_yes=assume_yes)


def create_postgresql_db(db_name, postgres_usr="postgres"):
    """Creates the PostgreSQL database.

    :param db_name: The name of the database to create.
    :type str
    :param postgres_usr: The name of the postregsql user.
    :type str
    """
    print "Creating PostgreSQL database..."
    cmd_args = ["-u", postgres_usr, "createdb", db_name]
    execute_command(cmd_args)


def setup_rabbitmq_server(user, password, vhost):
    """Sets up RabbitMQ server.

    :param user: The name of the user for the RabbitMQ session.
    :type str
    :param password: The password for the RabbitMQ user.
    :type str
    :param vhost: The name of the RabbitMQ vhost to create.
    :type str
    """
    cmd_args = ["rabbitmqctl", "add_user", user, password]
    execute_command(cmd_args)

    cmd_args = ["rabbitmqctl", "add_vhost", vhost]
    execute_command(cmd_args)

    cmd_args = ["rabbitmqctl", "set_permissions", "-p", vhost, user,
                "\".*\" \".*\" \".*\""]
    execute_command(cmd_args)


def clone_rhodecode_code(url, branch, work_dir, source_co):
    """Clones RhodeCode code from git.

    Clones RhodeCode from git and checkouts the specified branch.

    :param url: The URL of the git repository.
    :type str
    :param branch: The name of the branch to checkout.
    :type str
    :param work_dir: The working directory where to clone the code.
    :type str
    :param source_co: The destination directory where the code will be cloned.
    :type str
    """
    source_path = os.path.join(work_dir, source_co)
    if not os.path.exists(source_path):
        cmd_args = ["git", "clone", url, source_co]
        execute_command(cmd_args, work_dir=work_dir)

        cmd_args = ["git", "checkout", branch]
        execute_command(cmd_args, work_dir=source_path)


def set_owners(directory, usr, group=None):
    """Sets the correct ownership on the given directory.

    :param directory: The directory to set the ownership.
    :type str
    :param usr: The name of the user to set the ownership.
    :type str
    :param group: The name of the group to set the ownership. Default to the
    usr parameter.
    :type str
    """
    if not group:
        group = usr
    chown = "%s:%s" % (usr, group)
    cmd_args = ["chown", "-R", chown, directory]
    execute_command(cmd_args)


def install_rhodecode(work_dir):
    """Installs RhodeCode on the system.

    :param work_dir: The directory containing RhodeCode setup file.
    :type str
    """
    cmd_args = ["python", "setup.py", "install"]
    execute_command(cmd_args, work_dir=work_dir, with_sudo=True)


def setup_rhodecode(rhodecode_dir, config_file, git_repos, admin_usr,
                    admin_pwd, admin_email, user=None):
    """Sets up RhodeCode instance.

    :param rhodecode_dir: The directory where RhodeCode code was checked out.
    :type str
    :param config_file: The path to the production.ini config file.
    :type str
    :param admin_usr: The RhodeCode admin user name.
    :type str
    :param admin_pwd: The RhodeCode admin user password.
    :type str
    :param admin_email: The RhodeCode admin user email.
    :type str
    :param user: The user to run the process as.
    :type str
    """
    usr_arg = "--user=%s" % admin_usr
    pwd_arg = "--password=%s" % admin_pwd
    email_arg = "--email=%s" % admin_email
    repos = "--repos=%s" % os.path.abspath(git_repos)

    if user:
        cmd_args = ["-u", user]

    cmd_args += ["paster", "setup-rhodecode", config_file]
    cmd_args.append(usr_arg)
    cmd_args.append(pwd_arg)
    cmd_args.append(email_arg)
    cmd_args.append(repos)

    execute_command(cmd_args, work_dir=rhodecode_dir, with_sudo=False)


def install_upstart_conf(no_celery):
    """Installs the upstart conf files for RhodeCode and Celery.

    :param no_celery: If Celery has to be used.
    :type bool
    """
    basedir = os.path.dirname(__file__)
    rhodecode_upstart = os.path.join(basedir, RHODECODE_UPSTART_CONF)
    celeryd_upstart = os.path.join(basedir, CELERY_UPSTART_CONF)

    cmd_args = ["cp", rhodecode_upstart, "/etc/init/rhodecode.conf"]
    execute_command(cmd_args)

    if not no_celery:
        cmd_args = ["cp", celeryd_upstart, "/etc/init/celeryd.conf"]
        execute_command(cmd_args)


def copy_file(source, dest):
    """Copies the config file to the destination directory.

    Copies RhodeCode destination file to the specified directory, setting the
    corret user and group for the copied file.

    :param source: Path to the file to copy.
    :param dest: Where to copy the source file.
    """
    cmd_args = ["cp", source, dest]
    execute_command(cmd_args)


def start_services(no_celery):
    """Starts the installed services.

    :param no_celery: If Celery has to be started or not.
    :type bool
    """
    pass


def print_install_report(args, home_dir):
    """Prints a final report with all the installation data.

    :param args: The command line arguments.
    """
    print "RhodeCode Linaro Installation Report\n"
    print "\nRhodeCode Information"
    print "\tCode cloned from: %s\n" % args.rhodecode_git_url
    print "\tInstallation Dir: %s\n" % home_dir
    print "\tData Dir: %s\n" % args.rhodecode_data_dir
    print "\tRepos Dir: %s\n" % args.repos_dir
    print "\tAdmin Username: %s\n" % args.rhodecode_admin_usr
    print "\tAdmin Password: %s\n" % args.rhodecode_admin_pwd
    print "\tAdmin Email: %s\n" % args.rhodecode_admin_email
    print "\nDatabase Information\n"
    if args.development:
        print "\tDatabase Backend: SQLite\n"
        print "\tDatabse Location: Refer to configuration file.\n"
    else:
        print "\tDatabase Backend: PostreSQL\n"
        print "\tDatabase Name: %s\n" % args.dbname
    print "\nCelery/RabbitMQ Integration\n"
    print "\tCelery activated: %s\n" % args.no_celery
    if not args.no_celery:
        print "\tRabbitMQ User: %s\n" % args.rabbitmq_usr
        print "\tRabbitMQ Password: %s\n" % args.rabbitmq_pwd
        print "\tRabbitMQ Vhost: %s\n" % args.rabbitmq_vhost

if __name__ == '__main__':
    args = cli_args()
    check_cli_args(args)

    # Setup the necessary user for RhodeCode.
    home_dir = os.path.join("/home", args.rhodecode_usr)
    setup_user(args.rhodecode_usr, home_dir, args.rhodecode_usr)

    # Create the necessary directories to store RhodeCode data and cache.
    setup_directories(args.rhodecode_data_dir,
                      args.repos_dir,
                      args.rhodecode_usr)

    # Take git-core package from Launchpad PPA of git maintainers.
    install_git_from_ppa(assume_yes=args.assume_yes)

    # Install all the required packages from the default Ubuntu sources.
    if args.development:
        REQUIRED_PACKAGES.append(SQLITE_DB)
    else:
        REQUIRED_PACKAGES += POSTGRESQL_DB

    if not args.no_celery:
        REQUIRED_PACKAGES += CELERY_REQUIRED_PACKAGES

    install_packages(REQUIRED_PACKAGES, assume_yes=args.assume_yes)

    # Install packages through PIP
    install_pip_packages(PIP_PACKAGES)

    if not args.development:
        create_postgresql_db(args.dbname)

    if not args.no_celery:
        setup_rabbitmq_server(args.rabbitmq_usr,
                              args.rabbitmq_pwd,
                              args.rabbitmq_vhost)

    clone_rhodecode_code(args.rhodecode_git_url,
                         args.rhodecode_branch,
                         home_dir,
                         args.rhodecode_checkout_dir)

    work_dir = os.path.join(home_dir, args.rhodecode_checkout_dir)
    # Need to set the correct owners of the directory.
    set_owners(work_dir, args.rhodecode_usr)
    install_rhodecode(work_dir)

    rhodecode_conf = os.path.join(home_dir,
                                  os.path.basename(args.rhodecode_config))
    copy_file(args.rhodecode_config, home_dir, args.rhodecode_usr)
    set_owners(rhodecode_conf, args.rhodecode_usr)

    setup_rhodecode(work_dir,
                    rhodecode_conf,
                    args.repos_dir,
                    args.rhodecode_admin_usr,
                    args.rhodecode_admin_pwd,
                    args.rhodecode_admin_email,
                    args.rhodecode_usr)

    install_upstart_conf(args.no_celery)

    print_install_report(args, home_dir)