diff options
| -rw-r--r-- | modules/llvm.py | 86 | ||||
| -rw-r--r-- | scripts/llvm.py | 25 | ||||
| -rw-r--r-- | tests/cli/llvmtestcase.py | 29 | ||||
| -rw-r--r-- | tests/cli/testllvmprojects.py | 4 | ||||
| -rw-r--r-- | tests/cli/testllvmpush.py | 166 | ||||
| -rw-r--r-- | tests/unittests/testllvmsourceconfig.py | 85 | ||||
| -rw-r--r-- | tests/unittests/testpush.py | 200 |
7 files changed, 557 insertions, 38 deletions
diff --git a/modules/llvm.py b/modules/llvm.py index 34cd02c..2e7c44f 100644 --- a/modules/llvm.py +++ b/modules/llvm.py @@ -1,4 +1,5 @@ import os +import re from linaropy.git.worktree import Worktree @@ -146,6 +147,24 @@ class LLVMSourceConfig(object): for subproj in remove: self.__remove_subproject(subproj) + def for_each_enabled(self, action): + """Perform the given action for each enabled subproject including LLVM. + + The action must be a callable object receiving the path to the + subproject's directory as its only parameter. + + If the action throws an exception, it will be rethrown as a + RuntimeError. Note that this does not have transactional behaviour. + """ + for subproj in self.get_enabled_subprojects(): + try: + action(self.__get_subproj_cmake_path(subproj)) + except Exception as exc: + raise RuntimeError("Error while processing {}".format(subproj)) from exc + + # Visit LLVM last, in case getting the enabled subprojects errors out. + action(self.llvmSourceTree.repodir) + def __get_subproj_object(self, subprojName): """Get the LLVMSubproject object corresponding to subprojName.""" if not subprojName in list(self.subprojs.keys()): @@ -160,18 +179,22 @@ class LLVMSourceConfig(object): def __is_enabled(self, subprojName): """ Check if subproj is enabled in this configuration. A subproj is - considered to be enabled if it is a worktree on the correct branch. + considered to be enabled if it is a worktree on the correct branch. If + a directory for the subproject exists but does not satisfy those + conditions, an EnvironmentError is thrown. """ - try: - # If this succeeds, the subproject has already been added. - existing = Worktree(self.proj, - self.__get_subproj_cmake_path(subprojName)) - except EnvironmentError: - # If it's not a worktree (for whatever reason), it's not enabled. + subprojPath = self.__get_subproj_cmake_path(subprojName) + + if not os.path.isdir(subprojPath): return False - # If it is a worktree, but on the wrong branch, it is not enabled. - return existing.getbranch() == self.llvmSourceTree.getbranch() + existing = Worktree(self.proj, subprojPath) + + if existing.getbranch() != self.llvmSourceTree.getbranch(): + raise EnvironmentError("{} is on branch {}, but should be on {}".format( + subprojName, existing.getbranch(), self.llvmSourceTree.getbranch())) + + return True # TODO: add_subproject, remove_subproject and is_enabled should live in # another object (AddRemoveStrategy?) that would know what we want to add @@ -222,3 +245,48 @@ class LLVMSourceConfig(object): worktree = Worktree(self.proj, path) worktree.clean(False) + + +# FIXME: repo.pushToBranch doesn't work, because it doesn't handle remote +# branches properly. Furthermore, there's no support for getting the remote URL, +# so we need to resort to raw git commands. We may also consider moving the +# functionality for parsing info out of the remote URL into the repo object. +from sh import git +from linaropy.cd import cd + + +def get_user_from_remote(url): + """Get the username used as part of the remote URL, or None. + + The remote URLs that we expect to see look like $protocol://$user@$location. + If they look any different, we won't be able to parse them. + """ + pattern = re.compile("(.*://)?(?P<user>.*)@(.*)\n?") + match = pattern.match(str(url)) + if match is None: + return None + return match.group('user') + + +def push_current_branch(proj, pathToRepo): + """Push the current branch to origin. + + It will be pushed into linaro-local/$user/$branch, where $branch is the + current branch and $user is the username used as part of the remote URL. + """ + repo = Worktree(proj, pathToRepo) + + with cd(repo.repodir): + remote = git("remote", "get-url", "--push", "origin").strip() + user = get_user_from_remote(remote) + if not user: + raise EnvironmentError("Couldn't parse user from {}.".format(remote)) + + local_branch = repo.getbranch() + remote_branch = "linaro-local/{}/{}".format(user, local_branch) + if not repo.is_valid_branch_name(remote_branch): + raise EnvironmentError( + "{} is not a valid branch name.".format(remote_branch)) + + with cd(repo.repodir): + git("push", "-u", "origin", "+{}:{}".format(local_branch, remote_branch)) diff --git a/scripts/llvm.py b/scripts/llvm.py index 829c720..2252ab2 100644 --- a/scripts/llvm.py +++ b/scripts/llvm.py @@ -3,11 +3,12 @@ import os from sys import exit -from modules.llvm import LLVMSubproject, LLVMSourceConfig +from modules.llvm import LLVMSubproject, LLVMSourceConfig, push_current_branch from linaropy.git.clone import Clone from linaropy.proj import Proj from argparse import Action, ArgumentParser, RawTextHelpFormatter +from functools import partial def die(message, config_to_dump=None): @@ -64,6 +65,22 @@ def projects(args): dump_config(config) + +def push_branch(args): + """Push current branch to origin.""" + + proj = Proj() + + llvm_worktree_root = get_worktree_root(args.env) + config = LLVMSourceConfig(proj, llvm_worktree_root) + + llvmBranch = Clone(proj, llvm_worktree_root).getbranch() + + try: + config.for_each_enabled(partial(push_current_branch, proj)) + except RuntimeError as exc: + die("Failed to push branch because: " + str(exc) + str(exc.__cause__)) + ########################################################################## # Command line parsing # ########################################################################## @@ -106,5 +123,11 @@ projs.add_argument( metavar='subproject', help="Unlink given subprojects.") +# Subcommand for pushing the current branch to origin +push = subcommands.add_parser( + "push", + help="Push current branch to origin linaro-local/<user>/<branch>, for all linked subprojects.") +push.set_defaults(run_command=push_branch) + args = options.parse_args() args.run_command(args) diff --git a/tests/cli/llvmtestcase.py b/tests/cli/llvmtestcase.py index b2fc765..9b4ac3e 100644 --- a/tests/cli/llvmtestcase.py +++ b/tests/cli/llvmtestcase.py @@ -34,19 +34,28 @@ class LLVMTestCase(unittest.TestCase): script = os.path.join("scripts", "llvm.py") @classmethod - def create_dummy_commit(cls): - filename = "filethatshouldntexist" + def create_dummy_commit(cls, commitMessage="Dummy commit"): + filename = "filethatshouldntexist" + str(uuid4()) cls.run_quietly(["touch", filename]) - cls.run_quietly(["git", "add", filename]) - cls.run_quietly(["git", "commit", "-m", "Dummy commit"]) + try: + cls.run_quietly(["git", "add", filename]) + cls.run_quietly(["git", "commit", "-m", commitMessage]) + except subprocess.CalledProcessError as exc: + print("Command {} exited with error code {}:\n{}".format( + exc.cmd, exc.returncode, exc.output)) @classmethod - def create_dummy_repo(cls, repopath): - if not os.path.isdir(repopath): - os.makedirs(repopath) + def create_dummy_repo(cls, repopath, originpath=None): + if originpath is not None: + cls.run_quietly(["git", "clone", originpath, repopath]) + else: + if not os.path.isdir(repopath): + os.makedirs(repopath) + + with cd(repopath): + cls.run_quietly(["git", "init"]) with cd(repopath): - cls.run_quietly(["git", "init"]) cls.create_dummy_commit() @classmethod @@ -55,10 +64,6 @@ class LLVMTestCase(unittest.TestCase): cls.run_quietly(["git", "worktree", "add", worktreepath, "-b", branch]) - @classmethod - def get_subproj_repo(cls, subproj): - return os.path.join(cls.repos, subproj) - @staticmethod def run_with_output(*args, **kwargs): """Helper for running a command and capturing stdout and stderr""" diff --git a/tests/cli/testllvmprojects.py b/tests/cli/testllvmprojects.py index 394f3ac..30f7ff2 100644 --- a/tests/cli/testllvmprojects.py +++ b/tests/cli/testllvmprojects.py @@ -24,6 +24,10 @@ class Testllvmprojs(LLVMTestCase): return cls.command_with_defaults("projects", *args, **kwargs) @classmethod + def get_subproj_repo(cls, subproj): + return os.path.join(cls.repos, subproj) + + @classmethod def setUpClass(cls): """Create the file structure and environment that llvmprojs expects""" cls.llvm_root = mkdtemp() diff --git a/tests/cli/testllvmpush.py b/tests/cli/testllvmpush.py new file mode 100644 index 0000000..769aa18 --- /dev/null +++ b/tests/cli/testllvmpush.py @@ -0,0 +1,166 @@ +"""Command line interface tests for llvm.py push. + +Note that although this uses the unittest framework, it does *not* contain unit +tests. + +""" + +import shutil +import os +import subprocess +import unittest + +from tempfile import mkdtemp +from uuid import uuid4 + +from linaropy.cd import cd +from linaropy.git.clone import Clone +from llvmtestcase import LLVMTestCase, debug + + +class Testllvmpush(LLVMTestCase): + + @classmethod + def llvm_push(cls, *args, **kwargs): + return cls.command_with_defaults("push", *args, **kwargs) + + @classmethod + def get_origin_path(cls, subproj): + return os.path.join(cls.origins, subproj) + + @classmethod + def get_clone_path(cls, subproj): + return os.path.join(cls.repos, subproj) + + @classmethod + def setUpClass(cls): + """Create the file structure and environment for testing llvm push.""" + cls.all_repos = ("llvm", "clang", "compiler-rt", "lld", "lldb", + "libcxx", "libcxxabi", "libunwind", "test-suite") + + cls.origins = mkdtemp() + cls.repos = mkdtemp() + cls.llvm_root = mkdtemp() + + cls.user = "llvm-developer" + + # Create dummy repos - one origin that we will push to, and one clone + # that we will create worktrees from + for reponame in cls.all_repos: + origin = cls.get_origin_path(reponame) + cls.create_dummy_repo(origin) + + clone = cls.get_clone_path(reponame) + cls.create_dummy_repo( + clone, "file://{}@{}".format(cls.user, origin)) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.origins) + shutil.rmtree(cls.repos) + + @classmethod + def setUp(cls): + cls.env = os.path.join(cls.llvm_root, "env" + str(uuid4())) + cls.llvm_src = os.path.join(cls.env, "llvm") + + # Create LLVM worktree + cls.branch = "br" + str(uuid4()) + cls.add_worktree(cls.get_clone_path("llvm"), cls.llvm_src, + cls.branch) + + @classmethod + def tearDown(cls): + # Clean up the directories where we might have added subprojects. + # This isn't 100% clean, because we don't clean up the repos between + # tests (so any branches will remain), but it's good enough for the + # current tests. + for subprojdir in (os.path.join(cls.llvm_src, "projects"), + os.path.join(cls.llvm_src, "tools")): + if os.path.isdir(subprojdir): + shutil.rmtree(subprojdir) + os.makedirs(subprojdir) + + # Run prune on the original repos, to remove any dangling worktrees. + for reponame in cls.all_repos: + repopath = cls.get_clone_path(reponame) + with cd(repopath): + cls.run_quietly(["git", "worktree", "prune"]) + + def test_push(self): + with cd(self.llvm_src): + self.create_dummy_commit("Test llvm push") + + enabled = ["clang", "lld"] + + for subproj in enabled: + worktreePath = os.path.join(self.llvm_src, "tools", subproj) + self.add_worktree(self.get_clone_path(subproj), + worktreePath, + self.branch) + with cd(worktreePath): + self.create_dummy_commit("Test {} push".format(subproj)) + + self.run_quietly(self.llvm_push()) + remote_branch = "linaro-local/{}/{}".format(self.user, self.branch) + + for subproj in self.all_repos: + origin = self.get_origin_path(subproj) + + with cd(origin): + if subproj == "llvm" or subproj in enabled: + output = self.run_with_output(["git", "log", "--oneline", + "-1", remote_branch]) + + self.assertRegex( + output, ".*Test {} push.*".format(subproj)) + else: + with self.assertRaises(subprocess.CalledProcessError) as context: + output = self.run_with_output( + ["git", "log", "--oneline", "-1", remote_branch]) + + self.assertRegex( + str(context.exception.output), + "(.*\n)*.*unknown revision or path not in the working tree(.*\n)*") + + def test_push_mismatched_branches(self): + with cd(self.llvm_src): + self.create_dummy_commit("Test llvm push") + + enabled = ["clang", "lld"] + + for subproj in enabled: + branch = self.branch + + # Move only lld to a different branch + if subproj == "lld": + branch = branch + "-different" + + worktreePath = os.path.join(self.llvm_src, "tools", subproj) + self.add_worktree(self.get_clone_path(subproj), + worktreePath, + branch) + with cd(worktreePath): + self.create_dummy_commit("Test {} push".format(subproj)) + + with self.assertRaises(subprocess.CalledProcessError) as context: + output = self.run_with_output(self.llvm_push()) + + # Check that we error out because lld is on a different branch + self.assertRegex(str(context.exception.output), + "(.*\n)*.*lld is on branch {}, but should be on {}".format( + self.branch + "-different", self.branch)) + + # Check that we haven't pushed llvm or clang either, even if they're on + # the same branch + remote_branch = "linaro-local/{}/{}".format(self.user, self.branch) + + for subproj in ["llvm", "clang"]: + with cd(self.get_origin_path(subproj)): + with self.assertRaises(subprocess.CalledProcessError) as context: + self.run_with_output(["git", "log", "--oneline", "-1", + remote_branch]) + + self.assertRegex( + str(context.exception.output), + "(.*\n)*.*unknown revision or path not in the working tree(.*\n)*") diff --git a/tests/unittests/testllvmsourceconfig.py b/tests/unittests/testllvmsourceconfig.py index 3ad93b1..8a60065 100644 --- a/tests/unittests/testllvmsourceconfig.py +++ b/tests/unittests/testllvmsourceconfig.py @@ -3,6 +3,7 @@ import unittest import uuid from sh import git +from unittest.mock import MagicMock, call from linaropy.cd import cd from linaropy.proj import Proj @@ -130,13 +131,12 @@ class TestLLVMSourceConfig(unittest.TestCase): os.makedirs(path) config = LLVMSourceConfig(self.proj, sourcePath) - enabled = config.get_enabled_subprojects() + with self.assertRaises(EnvironmentError) as context: + config.get_enabled_subprojects() - # Check that if it's not a worktree, it's not enabled. - self.assertEqual( - enabled, - [], - "Detected unexpected projects %s" % str(enabled) + self.assertRegex( + str(context.exception), + ".*compiler-rt is not a worktree.*" ) def test_detect_enabled_wrong_branch(self): @@ -153,14 +153,12 @@ class TestLLVMSourceConfig(unittest.TestCase): self.assertTrue(os.path.isdir(path), "Failed to create worktree") config = LLVMSourceConfig(self.proj, sourcePath) - enabled = config.get_enabled_subprojects() + with self.assertRaises(EnvironmentError) as context: + config.get_enabled_subprojects() - # Check that if it's a worktree on the wrong branch, it's not enabled. - self.assertEqual( - enabled, - [], - "Detected unexpected projects %s" % str(enabled) - ) + self.assertRegex(str(context.exception), + "compiler-rt is on branch {}, but should be on {}".format(branch, + self.temporaryLLVMbranch)) def test_add_invalid_subproject(self): config = LLVMSourceConfig(self.proj, self.temporaryLLVM.repodir) @@ -284,7 +282,7 @@ class TestLLVMSourceConfig(unittest.TestCase): config.update({ "lldb" : self.__get_subproj_repo("lldb")}) self.assertRegex(str(context.exception), - "{} already exists but is not a valid subproject directory.*" + "{} is not a worktree.*" .format(existingPath)) # If we got this far, we're probably ok, but let's be pedantic and check @@ -314,8 +312,8 @@ class TestLLVMSourceConfig(unittest.TestCase): config.update({ "lldb" : self.__get_subproj_repo("lldb")}) self.assertRegex(str(context.exception), - "{} already exists but is not a valid subproject directory.*" - .format(existingPath)) + "lldb is on branch {}, but should be on {}.*" + .format(branch, self.temporaryLLVMbranch)) # If we got this far, we're probably ok, but let's be pedantic and check # that the subproject is still there @@ -568,5 +566,60 @@ class TestLLVMSourceConfig(unittest.TestCase): # TODO: test that CMake gets our layout + def test_for_each_enabled(self): + sourcePath = self.temporaryLLVM.repodir + + config = LLVMSourceConfig(self.proj, sourcePath) + + logPath = unittest.mock.MagicMock() + config.for_each_enabled(logPath) + + logPath.assert_called_with(self.temporaryLLVM.repodir) + + subprojs = LLVMSubproject.get_all_subprojects() + enabled = ["clang", "compiler-rt", "lld", "lldb", "libunwind"] + calls = [] + + for subproj in enabled: + path = subprojs[subproj].get_cmake_path(sourcePath) + worktree = Worktree.create( + self.proj, + self.__get_subproj_repo(subproj), + path, + self.temporaryLLVMbranch) + calls.append(call(path)) # Expect our mock to be called with path + + logPath = unittest.mock.MagicMock() + config.for_each_enabled(logPath) + + logPath.assert_has_calls(calls, any_order=True) + + def _throw(self, projPath): + if "lld" in projPath: + raise ValueError("An error has been!!1") + + def test_for_each_enabled_error(self): + """Test that we rethrow exceptions correctly.""" + sourcePath = self.temporaryLLVM.repodir + + config = LLVMSourceConfig(self.proj, sourcePath) + subprojs = LLVMSubproject.get_all_subprojects() + enabled = ["clang", "compiler-rt", "lld", "lldb", "libunwind"] + + for subproj in enabled: + path = subprojs[subproj].get_cmake_path(sourcePath) + worktree = Worktree.create( + self.proj, + self.__get_subproj_repo(subproj), + path, + self.temporaryLLVMbranch) + + with self.assertRaises(RuntimeError) as context: + config.for_each_enabled(self._throw) + self.assertRegex(str(context.exception), + "(.*\n?)*Error while processing lld(.*\n)*") + self.assertRegex(str(context.exception.__cause__), + "(.*\n?)*An error has been!!1(.*\n)*") + if __name__ == "__main__": unittest.main() diff --git a/tests/unittests/testpush.py b/tests/unittests/testpush.py new file mode 100644 index 0000000..0046e2b --- /dev/null +++ b/tests/unittests/testpush.py @@ -0,0 +1,200 @@ +import os +import unittest +import uuid + +from sh import git + +from linaropy.cd import cd +from linaropy.proj import Proj +from linaropy.git.clone import Clone +from linaropy.git.worktree import Worktree + +from modules.llvm import get_user_from_remote, push_current_branch + + +class TestGetUser(unittest.TestCase): + testdirprefix = "GetUserUT" + + def test_full_remote(self): + user = get_user_from_remote("https://user1@remote.url:1234") + self.assertEqual(user, "user1") + + user = get_user_from_remote("ssh://user2@remote.url:1234") + self.assertEqual(user, "user2") + + def test_no_protocol_remote(self): + user = get_user_from_remote("user.name@remote.url:1234") + self.assertEqual(user, "user.name") + + def test_no_user(self): + user = get_user_from_remote("https://remote.url:1234:") + self.assertEqual(user, None) + + +class TestPush(unittest.TestCase): + testdirprefix = "PushUT" + + def __create_dummy_commit( + self, + message="Branches without commits confuse git"): + filename = "file" + str(uuid.uuid4()) + open(filename, "a").close() + git("add", filename) + git("commit", "-m", message) + + def __get_last_commit_message(self, branch=""): + return str(git("rev-list", "-1", "--oneline", branch)).strip() + + def __create_dummy_repo(self, path): + if not os.path.exists(path): + os.makedirs(path) + + with cd(path): + git("init") + self.__create_dummy_commit() + + def setUp(self): + # We're going to create a hierarchy with an origin repo, a clone repo + # and a worktree of the clone. When pushing through the worktree, we + # expect to see things in origin. + self.proj = Proj(prefix=self.testdirprefix) + self.user = "llvm-developer" + + path = os.path.join(self.proj.projdir, "origin") + self.__create_dummy_repo(path) + self.origin = Clone(self.proj, path) + + path = os.path.join(self.proj.projdir, "clone") + git("clone", "file://" + self.user + "@" + self.origin.repodir, path) + self.clone = Clone(self.proj, path) + + self.worktreeBranch = "a-branch" + self.worktree = Worktree.create( + self.proj, self.clone, os.path.join( + self.proj.projdir, "worktree"), self.worktreeBranch) + + def tearDown(self): + self.proj.cleanup() + + def test_push_new_branch(self): + """Test that we can push a new branch to origin.""" + + with cd(self.worktree.repodir): + self.__create_dummy_commit("This should make it to origin") + + push_current_branch(self.proj, self.worktree.repodir) + + with cd(self.origin.repodir): + self.assertRegex( + self.__get_last_commit_message( + "linaro-local/{}/{}".format( + self.user, + self.worktreeBranch)), + ".*This should make it to origin") + + def test_push_existing_branch(self): + """Test that we can push to a branch that already exists in origin.""" + + remoteBranch = "linaro-local/{}/{}".format( + self.user, self.worktreeBranch) + + with cd(self.worktree.repodir): + self.__create_dummy_commit("This already exists in origin") + git("push", "origin", "{}:{}".format( + self.worktreeBranch, remoteBranch)) + + self.__create_dummy_commit("This should make it to origin too") + + push_current_branch(self.proj, self.worktree.repodir) + + with cd(self.origin.repodir): + self.assertRegex(self.__get_last_commit_message(remoteBranch), + ".*This should make it to origin too") + + def test_push_squashed_update(self): + """ + Test that we can push again after squashing some fixes into a commit + that has already been pushed to origin. This isn't a nice thing to do, + but we need to support it because Gerrit requires squashes. + """ + remoteBranch = "linaro-local/{}/{}".format(self.user, + self.worktreeBranch) + with cd(self.worktree.repodir): + self.__create_dummy_commit("First version of the patch") + + push_current_branch(self.proj, self.worktree.repodir) + + with cd(self.origin.repodir): + self.assertRegex( + self.__get_last_commit_message(remoteBranch), + ".*First version of the patch") + + with cd(self.worktree.repodir): + git("commit", "--amend", "-m", "Second version of the patch") + + push_current_branch(self.proj, self.worktree.repodir) + + with cd(self.origin.repodir): + self.assertRegex( + self.__get_last_commit_message(remoteBranch), + "/*Second version of the patch") + + def test_push_rebased_branch(self): + """ Test that we can push again with new updates after a rebase.""" + remoteBranch = "linaro-local/{}/{}".format(self.user, + self.worktreeBranch) + with cd(self.worktree.repodir): + self.__create_dummy_commit("First change") + + push_current_branch(self.proj, self.worktree.repodir) + + with cd(self.origin.repodir): + self.assertRegex( + self.__get_last_commit_message(remoteBranch), + ".*First change") + + with cd(self.origin.repodir): + self.__create_dummy_commit("Master moving forward") + + with cd(self.worktree.repodir): + self.__create_dummy_commit("Second change") + git("fetch", "origin", "master") + git("rebase", "origin/master") + + push_current_branch(self.proj, self.worktree.repodir) + + with cd(self.origin.repodir): + self.assertRegex(self.__get_last_commit_message(remoteBranch), + ".*Second change") + + def test_push_no_user(self): + """Test that we error out when we can't parse a user in the remote name.""" + remote = "file://" + self.origin.repodir + with cd(self.worktree.repodir): + git("remote", "set-url", "origin", remote) + self.__create_dummy_commit("Make it look like a real branch") + + with self.assertRaises(EnvironmentError) as context: + push_current_branch(self.proj, self.worktree.repodir) + + self.assertEqual(str(context.exception), + "Couldn't parse user from {}.".format(remote)) + + def test_push_invalid_user(self): + """Test that we error out when the value of LLVM_GITUSER wouldn't look good in a branch name (e.g. if it contains spaces).""" + + badUser = "LLVM Developer" + with cd(self.worktree.repodir): + git("remote", "set-url", "origin", + "file://{}@{}".format(badUser, self.origin.repodir)) + self.__create_dummy_commit("Make it look like a real branch") + + with self.assertRaises(EnvironmentError) as context: + push_current_branch(self.proj, self.worktree.repodir) + + self.assertEqual(str(context.exception), + "linaro-local/{}/{} is not a valid branch name.".format(badUser, + self.worktreeBranch)) + +if __name__ == "__main__": + unittest.main() |