Merge from milo branch.

1 files changed, 226 insertions, 79 deletions

diff --git a/scripts/rhodecode-setup b/scripts/rhodecode-setup
index 479f954..b4d0de6 100755
--- a/scripts/rhodecode-setup
+++ b/scripts/rhodecode-setup
@@ -13,29 +13,20 @@ REQUIRED_PACKAGES = ["python-pip", "python-webob", "python-bcrypt",
                      "python-mock", "python-babel", "python-dateutil",
                      "python-markdown", "python-webhelpers", "python-docutils",
                      "python-formencode", "python-pylons", "python-dev",
-                     "build-essential", "apache2"]
+                     "python-pastescript", "python-psycopg2", "python-nose",
+                     "build-essential", "python-amqplib", "python-anyjson",
+                     "python-mailer", "apache2", "git-core"]
 # Packages required for celery integration.
-CELERY_REQUIRED_PACKAGES = ["python-celery", "rabbitmq-server"]
+CELERY_REQUIRED_PACKAGES = ["rabbitmq-server"]
 
 # Packages for the backend to use.
 SQLITE_DB = "sqlite"
-POSTGRESQL_DB = ["postgresql-9.1", "pgadmin3", "postgresql-server-dev-9.1"]
+POSTGRESQL_DB = ["postgresql-9.1"]
 # The name of the DB for PostgreSQL.
 DB_NAME = "rhodecode"
 
 # Packages to be installed via PIP.
-PIP_PACKAGES = {
-    "psycopg2": None,
-    "celery": "2.2.10",
-    "dulwich": "0.8.7",
-    "beaker": "1.6.4",
-    "sqlalchemy": "0.7.9",
-    "mako": "0.7.3",
-    "pygments": "1.5",
-    "whoosh": "2.4.0",
-    "simplejson": "2.5.2",
-    "waitress": "0.8.1",
-}
+PIP_PACKAGES = {}
 
 # RhodeCode source to clone and which branch to use.
 RHODECODE_SOURCE_GIT_URL = \
@@ -47,6 +38,9 @@ RHODECODE_CO_DIR = "rhodecode"
 # install the RhodeCode code and to run RhodeCode.
 RHODECODE_DEFAULT_USER = "rhodecode"
 
+# Default RhodeCode directory for data and cache.
+RHODECODE_DATA_DIR = "/opt/rhodecode"
+
 # Default parameters for RabbitMQ setup.
 RABBITMQ_DEFAULT_USER = "rhodecode"
 RABBITMQ_DEFAULT_VHOST = "rhodecode-vhost"
@@ -59,10 +53,26 @@ RHODECODE_UPSTART_CONF = "rhodecode-upstart.conf"
 CELERY_UPSTART_CONF = "celeryd-upstart.conf"
 
 # SQL string to create the new role in PostgreSQL.
-POSTGRES_CREATE_USER = "CREATE ROLE %(role)s LOGIN CREATEDB PASSWORD '%(pwd)s'"
+POSTGRES_CREATE_ROLE = "CREATE ROLE %(role)s LOGIN CREATEDB PASSWORD '%(pwd)s'"
 
+# The name of the PostgreSQL role to create.
 POSTGRES_ROLE = "rhodecode"
 
+# Default directory where to store git repositories.
+REPOS_DIR = "/opt/git_repos"
+
+# Default Celery log directory.
+CELERY_LOG_DIR = "/var/log/celery"
+# Default RhodeCode log directory.
+RHODECODE_LOG_DIR = "/var/log/rhodecode"
+
+# Default websites directory.
+APACHE_DEFAULT_DIR = "/etc/apache2/sites-available/"
+# Default modules to enalbe in Apache
+APACHE_ENABLE_MODULES = ["proxy", "proxy_http"]
+# Apache websites to disable.
+APACHE_DISABLE_SITES = ["default"]
+
 
 def cli_args():
     """Sets up the cli argument parser."""
@@ -71,9 +81,14 @@ def cli_args():
                         help="Path to the config INI file.",
                         required=True)
     parser.add_argument("--rhodecode-data-dir",
-                        default="/opt/rhodecode",
+                        default=RHODECODE_DATA_DIR,
                         help="The directory where to store RhodeCode data "
-                             "and cache.")
+                             "and cache. Defaults to '%s'." %
+                             RHODECODE_DATA_DIR)
+    parser.add_argument("--rhodecode-log-dir",
+                        default=RHODECODE_LOG_DIR,
+                        help="The directory to store RhodeCode logs. Defaults "
+                             "to '%s'." % RHODECODE_LOG_DIR)
     parser.add_argument("--rhodecode-admin-usr",
                         default=RHODECODE_ADMIN_USER,
                         help="The name of the admin user for RhodeCode.")
@@ -90,12 +105,11 @@ def cli_args():
     parser.add_argument("--rhodecode-git-url",
                         default=RHODECODE_SOURCE_GIT_URL,
                         help="The URL where to clone RhodeCode source "
-                             "code from. Defaults to '%s'" %
-                             RHODECODE_SOURCE_GIT_URL)
+                             "code from.")
     parser.add_argument("--rhodecode-checkout-dir",
                         default=RHODECODE_CO_DIR,
                         help="The destination directory where to clone "
-                             "RhodeCode code into. Defaults to %s" %
+                             "RhodeCode code into. Defaults to '%s'." %
                              RHODECODE_CO_DIR)
     parser.add_argument("--rhodecode-branch",
                         default=RHODECODE_SOURCE_BRANCH,
@@ -110,21 +124,29 @@ def cli_args():
     parser.add_argument("--no-celery",
                         action="store_true",
                         help="If Celery integration should be disabled.")
+    parser.add_argument("--celery-log-dir",
+                        default=CELERY_LOG_DIR,
+                        help="The Celery log directory. Defaults to '%s'." %
+                             CELERY_LOG_DIR)
     parser.add_argument("--rabbitmq-usr",
                         default=RABBITMQ_DEFAULT_USER,
-                        help="The name to use for the RabbitMQ session.")
+                        help="The name to use for the RabbitMQ session. "
+                             "Defaults to '%s'." % RABBITMQ_DEFAULT_USER)
     parser.add_argument("--rabbitmq-pwd",
                         help="The password for the RabbitMQ user.",
                         required=True)
     parser.add_argument("--rabbitmq-vhost",
                         default=RABBITMQ_DEFAULT_VHOST,
-                        help="The name of the RabbitMQ vhost for the user.")
+                        help="The name of the RabbitMQ vhost for the user. "
+                             "Defaults to '%s'." % RABBITMQ_DEFAULT_VHOST)
     parser.add_argument("--dbname",
                         default=DB_NAME,
-                        help="The name to use for the database.")
+                        help="The name to use for the database. Defauls to "
+                             "'%s'." % DB_NAME)
     parser.add_argument("--repos-dir",
-                        default="/opt/git_repos",
-                        help="Where the git repositories will be stored.")
+                        default=REPOS_DIR,
+                        help="Where the git repositories will be stored. "
+                             "Defaults to '%s'." % REPOS_DIR)
     parser.add_argument("--postgres-role",
                         default=POSTGRES_ROLE,
                         help="The PostgreSQL role to create. Defaults "
@@ -135,6 +157,13 @@ def cli_args():
     parser.add_argument("--update",
                         action="store_true",
                         help="Updates to the latest tagged version.")
+    parser.add_argument("--apache-conf",
+                        required=True,
+                        help="Path to the Apache RhodeCode configuration.")
+    parser.add_argument("--apache-website-dir",
+                        default=APACHE_DEFAULT_DIR,
+                        help="Path the the Apache sites directory. Defaults "
+                             "to '%s'." % APACHE_DEFAULT_DIR)
     return parser.parse_args()
 
 
@@ -170,41 +199,34 @@ def setup_user(user, home_dir, group):
 
     :param user: The name of the user and group to create.
     """
-    # Add the user.
-    cmd_args = ["adduser", "--system", "--home", home_dir, user]
-    execute_command(cmd_args)
-
     # Add the user group.
     cmd_args = ["addgroup", "--system", group]
     execute_command(cmd_args)
 
-    # Add the user to its group.
-    cmd_args = ["adduser", user, group]
+    # Add the user.
+    cmd_args = ["adduser", "--system", "--home", home_dir, "--ingroup",
+                group, user]
     execute_command(cmd_args)
 
 
-def setup_directories(data_dir, repos_dir, rhodecode_usr):
-    if not os.path.exists(data_dir):
-        cmd_args = ["mkdir", "-p", data_dir]
-        execute_command(cmd_args)
-
-    if not os.path.exists(repos_dir):
-        cmd_args = ["mkdir", "-p", repos_dir]
-        execute_command(cmd_args)
+def setup_directories(directories, rhodecode_usr):
+    """Sets up the correct necessary directories."""
+    if not isinstance(directories, list):
+        directories = [directories]
 
-    # Set ownership of the directories to the RhodeCode user.
-    set_owners(data_dir, rhodecode_usr)
-
-    set_owners(repos_dir, rhodecode_usr)
+    for directory in directories:
+        if not os.path.exists(directory):
+            cmd_args = ["mkdir", "-p", directory]
+            execute_command(cmd_args)
+        # Set ownership of the directories to the RhodeCode user.
+        set_owners(directory, rhodecode_usr)
 
 
 def execute_command(cmd_args,
                     work_dir=os.getcwd(),
                     with_sudo=True,
                     input_str=None,
-                    stderr=subprocess.PIPE,
-                    stdin=subprocess.PIPE,
-                    stdout=None):
+                    continue_on_error=False):
     """Runs the command passed.
 
     :param cmd_args: List of command and options to run.
@@ -217,6 +239,9 @@ def execute_command(cmd_args,
     :param input_str: String to pass to the process as input, in order to
     automate as much as possible.
     :type str
+    :param continue_on_error: If an error arise, continue instead of exiting.
+    Default to False.
+    :type bool
     """
     if not isinstance(cmd_args, list):
         cmd_args = list(cmd_args)
@@ -225,6 +250,7 @@ def execute_command(cmd_args,
     if input_str:
         process = subprocess.Popen(cmd_args,
                                    cwd=work_dir,
+<<<<<<< HEAD
                                    stderr=stderr,
                                    stdin=stdin,
                                    stdout=stdout)
@@ -238,6 +264,18 @@ def execute_command(cmd_args,
         p_out, p_err = process.communicate()
         return p_out, p_err
     if process.returncode != 0:
+=======
+                                   stdin=subprocess.PIPE)
+        process.communicate(input=input_str)
+    else:
+        process = subprocess.Popen(cmd_args,
+                                   cwd=work_dir)
+        process.communicate()
+
+    if continue_on_error:
+        return
+    elif process.returncode != 0:
+>>>>>>> milo
         print "Error executing the following command: %s" % " ".join(cmd_args)
         sys.exit(1)
 
@@ -276,20 +314,27 @@ def install_packages(packages, assume_yes=False):
     execute_command(cmd_args, input_str=input_str)
 
 
-def install_pip_packages(packages):
+def install_pip_packages(packages, user=None):
     """Installs packages from PIP.
 
     :param packages: The packages to install. It has to be a dictionary, with
     key the name of the package, and value the version number or None.
     :type dict
+    :param user: The user to install the PIP package as. This will append
+    --user to the command line.
+    :type str
     """
     for key, value in packages.iteritems():
-        cmd_args = ["pip", "install", "-q", "-I"]
+        print "Installing package %s with PIP..." % key
+        cmd_args = ["pip", "install", "-I"]
         if value:
             cmd_args.append("%s==%s" % (key, value))
         else:
             cmd_args.append(key)
-        cmd_args.append("--upgrade")
+        cmd_args.append("-U")
+        if user:
+            cmd_args = ["-u", user] + cmd_args
+            cmd_args.append("--user")
         execute_command(cmd_args)
 
 
@@ -316,10 +361,9 @@ def create_postgresql_role(role, pwd, postgres_usr="postgres"):
     :param postgres_usr: The superuser for PostreSQL. Defaults to 'postgres'.
     :type str
     """
-    postgres_commamd = POSTGRES_CREATE_USER % dict(role=str(role),
+    postgres_commamd = POSTGRES_CREATE_ROLE % dict(role=str(role),
                                                    pwd=str(pwd))
-    cmd_args = ["-u", postgres_usr, "psql", "-c",
-                '"%s"' % str(postgres_commamd)]
+    cmd_args = ["-u", postgres_usr, "psql", "-c", str(postgres_commamd)]
     execute_command(cmd_args)
 
 
@@ -353,7 +397,7 @@ def setup_rabbitmq_server(user, password, vhost):
     execute_command(cmd_args)
 
     cmd_args = ["rabbitmqctl", "set_permissions", "-p", vhost, user,
-                '".*"', '".*"', '".*"']
+                ".*", ".*", ".*"]
     execute_command(cmd_args)
 
 
@@ -379,6 +423,9 @@ def clone_rhodecode_code(url, branch, work_dir, source_co):
         cmd_args = ["git", "checkout", branch]
         execute_command(cmd_args, work_dir=source_path)
 
+        cmd_args = ["git", "pull", "origin", branch]
+        execute_command(cmd_args, work_dir=source_path)
+
 
 def set_owners(directory, usr, group=None):
     """Sets the correct ownership on the given directory.
@@ -398,14 +445,33 @@ def set_owners(directory, usr, group=None):
     execute_command(cmd_args)
 
 
-def install_rhodecode(work_dir):
+def set_permissions(file_or_dir, permissions):
+    """Sets the permissions on the given path.
+
+    :param file_or_dir: A list of paths to set the permissions.
+    :type list
+    :param permissions: String containing the permissions as passed to the
+    'chmod' command.
+    """
+    if not isinstance(file_or_dir, list):
+        file_or_dir = [file_or_dir]
+    for element in file_or_dir:
+        cmd_args = ["chmod", "-R", permissions, element]
+        execute_command(cmd_args)
+
+
+def install_rhodecode(work_dir, user=None):
     """Installs RhodeCode on the system.
 
     :param work_dir: The directory containing RhodeCode setup file.
     :type str
     """
     cmd_args = ["python", "setup.py", "install"]
-    execute_command(cmd_args, work_dir=work_dir, with_sudo=True)
+
+    if user:
+        cmd_args = ["-u", user, "-H"] + cmd_args
+        cmd_args.append("--user")
+    execute_command(cmd_args, work_dir=work_dir)
 
 def reinstall_rhodecode(work_dir):
     """Re-installs RhodeCode on the system.
@@ -432,7 +498,7 @@ def update_rhodecode(work_dir):
 
 
 def setup_rhodecode(rhodecode_dir, config_file, git_repos, admin_usr,
-                    admin_pwd, admin_email, user=None):
+                    admin_pwd, admin_email, user=None, assume_yes=False):
     """Sets up RhodeCode instance.
 
     :param rhodecode_dir: The directory where RhodeCode code was checked out.
@@ -447,6 +513,8 @@ def setup_rhodecode(rhodecode_dir, config_file, git_repos, admin_usr,
     :type str
     :param user: The user to run the process as.
     :type str
+    :param assume_yes: Tries to automate the process.
+    :type bool
     """
     usr_arg = "--user=%s" % admin_usr
     pwd_arg = "--password=%s" % admin_pwd
@@ -454,7 +522,7 @@ def setup_rhodecode(rhodecode_dir, config_file, git_repos, admin_usr,
     repos = "--repos=%s" % os.path.abspath(git_repos)
 
     if user:
-        cmd_args = ["-u", user]
+        cmd_args = ["-u", user, "-H"]
 
     cmd_args += ["paster", "setup-rhodecode", config_file]
     cmd_args.append(usr_arg)
@@ -462,7 +530,10 @@ def setup_rhodecode(rhodecode_dir, config_file, git_repos, admin_usr,
     cmd_args.append(email_arg)
     cmd_args.append(repos)
 
-    execute_command(cmd_args, work_dir=rhodecode_dir, stderr=None)
+    input_str = None
+    if assume_yes:
+        input_str = "y"
+    execute_command(cmd_args, work_dir=rhodecode_dir, input_str=input_str)
 
 
 def install_upstart_conf(no_celery):
@@ -484,10 +555,7 @@ def install_upstart_conf(no_celery):
 
 
 def copy_file(source, dest):
-    """Copies the config file to the destination directory.
-
-    Copies RhodeCode destination file to the specified directory, setting the
-    corret user and group for the copied file.
+    """Copies the a file to the defined destination.
 
     :param source: Path to the file to copy.
     :param dest: Where to copy the source file.
@@ -496,13 +564,64 @@ def copy_file(source, dest):
     execute_command(cmd_args)
 
 
-def start_services(no_celery):
-    """Starts the installed services.
+def start_service(service):
+    """Starts a service.
 
-    :param no_celery: If Celery has to be started or not.
-    :type bool
+    Start a service by using the 'service' command.
+
+    :param service: The service to start.
+    :type str
     """
-    pass
+    cmd_args = ["service", service, "start"]
+    execute_command(cmd_args, continue_on_error=True)
+
+
+def reload_apache_service():
+    """Reloads Apache configuration."""
+    cmd_args = ["service", "apache2", "reload"]
+    execute_command(cmd_args)
+
+
+def disable_apache_sites(sites):
+    """Disables Apache sites.
+
+    :param sites: The list of sites to disable.
+    :type list
+    """
+    if not isinstance(sites, list):
+        sites = [sites]
+
+    for site in sites:
+        cmd_args = ["a2dissite", site]
+        execute_command(cmd_args)
+
+
+def enable_apache_sites(sites):
+    """Enables Apache sites.
+
+    :param sites: The list of sites to enable.
+    :type list
+    """
+    if not isinstance(sites, list):
+        sites = [sites]
+
+    for site in sites:
+        cmd_args = ["a2ensite", site]
+        execute_command(cmd_args)
+
+
+def enable_apache_modules(modules):
+    """Enables Apache modules.
+
+    :param modules: List of modules to enable.
+    :type list
+    """
+    if not isinstance(modules, list):
+        modules = [modules]
+
+    for module in modules:
+        cmd_args = ["a2enmod", module]
+        execute_command(cmd_args)
 
 
 def print_install_report(args, home_dir, config_file):
@@ -511,15 +630,17 @@ def print_install_report(args, home_dir, config_file):
     :param args: The command line arguments.
     """
     print "\n\nRhodeCode Linaro Installation Report\n"
-    print "\nRhodeCode Information"
+
+    print "\nRhodeCode Information\n"
     print "\tCode cloned from: %s" % args.rhodecode_git_url
-    print "\tInstallation Dir: %s" % home_dir
+    print "\tInstallation Directory: %s" % home_dir
     print "\tConfiguration file: %s" % config_file
-    print "\tData Dir: %s" % args.rhodecode_data_dir
-    print "\tRepos Dir: %s" % args.repos_dir
+    print "\tData Directory: %s" % args.rhodecode_data_dir
+    print "\tRepos Directory: %s" % args.repos_dir
     print "\tAdmin Username: %s" % args.rhodecode_admin_usr
     print "\tAdmin Password: %s" % args.rhodecode_admin_pwd
     print "\tAdmin Email: %s" % args.rhodecode_admin_email
+
     print "\nDatabase Information\n"
     if args.development:
         print "\tDatabase Backend: SQLite"
@@ -527,6 +648,9 @@ def print_install_report(args, home_dir, config_file):
     else:
         print "\tDatabase Backend: PostreSQL"
         print "\tDatabase Name: %s" % args.dbname
+        print "\tDatabase Role: %s" % args.postgres_role
+        print "\tDatabase Role Password: %s" % args.postgres_role_pwd
+
     print "\nCelery/RabbitMQ Integration\n"
     print "\tCelery activated: %s" % (not args.no_celery)
     if not args.no_celery:
@@ -534,6 +658,7 @@ def print_install_report(args, home_dir, config_file):
         print "\tRabbitMQ Password: %s" % args.rabbitmq_pwd
         print "\tRabbitMQ Vhost: %s" % args.rabbitmq_vhost
 
+
 if __name__ == '__main__':
     args = cli_args()
     check_cli_args(args)
@@ -543,9 +668,16 @@ if __name__ == '__main__':
     setup_user(args.rhodecode_usr, home_dir, args.rhodecode_usr)
 
     # Create the necessary directories to store RhodeCode data and cache.
-    setup_directories(args.rhodecode_data_dir,
-                      args.repos_dir,
-                      args.rhodecode_usr)
+    directories = [args.rhodecode_data_dir, args.repos_dir,
+                   args.rhodecode_log_dir]
+    if not args.no_celery:
+        directories.append(args.celery_log_dir)
+
+    setup_directories(directories, args.rhodecode_usr)
+
+    # Set the correct permissions to the directories, at least the group
+    # should be able to write.
+    set_permissions(directories, "g+w")
 
     # Take git-core package from Launchpad PPA of git maintainers.
     install_git_from_ppa(assume_yes=args.assume_yes)
@@ -562,11 +694,11 @@ if __name__ == '__main__':
     install_packages(REQUIRED_PACKAGES, assume_yes=args.assume_yes)
 
     # Install packages through PIP
-    install_pip_packages(PIP_PACKAGES)
+    # install_pip_packages(PIP_PACKAGES)
 
     if not args.development:
         create_postgresql_role(args.postgres_role, args.postgres_role_pwd)
-        create_postgresql_db(args.dbname)
+        create_postgresql_db(args.dbname, args.postgres_role)
 
     if not args.no_celery:
         setup_rabbitmq_server(args.rabbitmq_usr,
@@ -581,7 +713,7 @@ if __name__ == '__main__':
     work_dir = os.path.join(home_dir, args.rhodecode_checkout_dir)
     # Need to set the correct owners of the directory.
     set_owners(work_dir, args.rhodecode_usr)
-    install_rhodecode(work_dir)
+    install_rhodecode(work_dir, args.rhodecode_usr)
 
     rhodecode_conf = os.path.join(home_dir,
                                   os.path.basename(args.rhodecode_config))
@@ -594,8 +726,23 @@ if __name__ == '__main__':
                     args.rhodecode_admin_usr,
                     args.rhodecode_admin_pwd,
                     args.rhodecode_admin_email,
-                    args.rhodecode_usr)
+                    user=args.rhodecode_usr,
+                    assume_yes=args.assume_yes)
 
+    # Apache configurations.
+    apache_conf = os.path.abspath(args.apache_conf)
+    copy_file(apache_conf, args.apache_website_dir)
+
+    disable_apache_sites(APACHE_DISABLE_SITES)
+    enable_apache_modules(APACHE_ENABLE_MODULES)
+    enable_apache_sites(os.path.basename(apache_conf))
+    reload_apache_service()
+
+    # Install upstart scripts for Celery and RhodeCode
     install_upstart_conf(args.no_celery)
+    start_service("rhodecode")
+
+    if not args.no_celery:
+        start_service("celeryd")
 
     print_install_report(args, home_dir, rhodecode_conf)