[llvmprojs] Rewrite llvm-projs in python
This is the first step in moving all our scripts to python. For now we keep the
current content of the repo, but we'll start adding a new directory hierarchy
for the python stuff:
* modules: should contain most of the python code, organized as a package that
can be imported by the scripts
* scripts: the scripts themselves, which should be parsing the command line
and calling stuff from 'modules' to do the actual work; can be
broken down into the same categories we had before (helpers, bisect
etc), or we could just have one big pile
* tests: should contain unittests (for the stuff in modules) and command line
interface tests (for the scripts)
The code is heavily using functionality from the tcwg-release-tools repo (which
probably needs to be renamed / reorganized), so you should have that in your
PYTHONPATH when trying to run any of the scripts. To run the tests, just invoke
check.sh.
One of the important changes is that we'll be using python's argparse module to
parse command line flags, which means we'll have to stick to a more traditional
interface for the scripts. In particular, we can't have short options like "+c"
anymore. This isn't much of a problem, because we will keep the bash scripts as
they are and just make them invoke a tool written in python (scripts/llvm.py) to
do the work. The tool will have subcommands for any functionality that we want,
for instance the new interface for adding/removing subprojects is:
llvm.py projects [-a subproject subproject ... subproject]
[-r subproject ... subproject]
The -a and -r options (followed by any number of subprojects) can be used to
selectively enable/disable things. You have to pass the full name of the
subproject (e.g. llvmprojs.py -a clang lld -r libcxx) for it to work. This is
invoked by the llvm-projs bash script, which has the old, more convenient
interface.
For now the bash scripts will live in the same directories as they used to, but
after the transition is complete we will want to move them to the scripts
directory.
Note that we're also eliding any dashes in the names of the scripts, in keeping
with Python best practices for module names (i.e. using valid Python identifiers
as names).
Change-Id: I9ec08632dbb17464673240d6f6881a90f45d5371
diff --git a/tests/unittests/testllvmsourceconfig.py b/tests/unittests/testllvmsourceconfig.py
new file mode 100644
index 0000000..1add13a
--- /dev/null
+++ b/tests/unittests/testllvmsourceconfig.py
@@ -0,0 +1,572 @@
+import os
+import unittest
+import uuid
+
+from sh import git
+
+from linaropy.cd import cd
+from linaropy.proj import Proj
+from linaropy.git.clone import Clone
+from linaropy.git.worktree import Worktree
+
+from modules.llvm import LLVMSourceConfig, LLVMSubproject
+
+
+class TestLLVMSourceConfig(unittest.TestCase):
+ testdirprefix = "SourceConfigUT"
+
+ def __create_dummy_commit(self):
+ filename = "file" + str(uuid.uuid4())
+ open(filename, "a").close()
+ git("add", filename)
+ git("commit", "-m", "Branches without commits confuse git")
+
+ def __create_dummy_repo(self, path):
+ if not os.path.exists(path):
+ os.makedirs(path)
+
+ with cd(path):
+ git("init")
+ self.__create_dummy_commit()
+
+ def __get_subproj_repo_path(self, subproj):
+ return os.path.join(self.originalLLVM.repodir, "..", subproj + "-repo")
+
+ def __get_subproj_repo(self, subproj):
+ return Clone(self.proj, self.__get_subproj_repo_path(subproj))
+
+ def setUp(self):
+ # We're going to create a hierarchy with [llvm|clang|whatever]-repo
+ # containing dummy repos, and llvm-copy containing a worktree of
+ # llvm-repo
+ self.proj = Proj(prefix=TestLLVMSourceConfig.testdirprefix)
+ path = os.path.join(self.proj.projdir, "llvm-repo")
+ self.__create_dummy_repo(path)
+ self.originalLLVM = Clone(self.proj, path)
+
+ subprojs = LLVMSubproject.get_all_subprojects()
+ for subproj in subprojs.keys():
+ repo = self.__get_subproj_repo_path(subproj)
+ self.__create_dummy_repo(repo)
+
+ self.temporaryLLVMbranch = "a-branch"
+ self.temporaryLLVM = Worktree.create(
+ self.proj, self.originalLLVM, os.path.join(
+ self.proj.projdir, "llvm-copy"), self.temporaryLLVMbranch)
+
+ def tearDown(self):
+ self.proj.cleanup()
+
+ def test_detect_enabled_all(self):
+ subprojs = LLVMSubproject.get_all_subprojects()
+ sourcePath = self.temporaryLLVM.repodir
+
+ for subproj in subprojs:
+ path = subprojs[subproj].get_cmake_path(sourcePath)
+ worktree = Worktree.create(
+ self.proj,
+ self.__get_subproj_repo(subproj),
+ path,
+ self.temporaryLLVMbranch)
+ self.assertTrue(os.path.isdir(path), "Failed to create worktree")
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+ enabled = config.get_enabled_subprojects()
+
+ self.assertEqual(set(subprojs), set(enabled),
+ "Expected %s but detected only %s" %
+ (str(set(subprojs)), str(enabled)))
+
+ def test_detect_enabled_none(self):
+ sourcePath = self.temporaryLLVM.repodir
+
+ path = os.path.join(sourcePath, "unrelated")
+ os.makedirs(path)
+
+ path = os.path.join(sourcePath, "wrong", "place", "for", "lld")
+ os.makedirs(path)
+
+ path = os.path.join(sourcePath, "projects", "clang")
+ os.makedirs(path)
+
+ path = os.path.join(sourcePath, "tools", "libcxx")
+ os.makedirs(path)
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+ enabled = config.get_enabled_subprojects()
+
+ self.assertEqual(
+ enabled,
+ [],
+ "Detected unexpected projects %s" % str(enabled)
+ )
+
+ def test_detect_enabled_some(self):
+ sourcePath = self.temporaryLLVM.repodir
+ subprojs = LLVMSubproject.get_all_subprojects()
+
+ for subproj in ["lld", "libcxxabi", "clang"]:
+ path = subprojs[subproj].get_cmake_path(sourcePath)
+ worktree = Worktree.create(
+ self.proj,
+ self.__get_subproj_repo(subproj),
+ path,
+ self.temporaryLLVMbranch)
+ self.assertTrue(os.path.isdir(path), "Failed to create worktree")
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+ enabled = config.get_enabled_subprojects()
+
+ self.assertTrue("lld" in enabled, "Failed to detect lld")
+ self.assertTrue("clang" in enabled, "Failed to detect clang")
+ self.assertTrue("libcxxabi" in enabled,
+ "Failed to detect libcxxabi")
+
+ def test_detect_enabled_not_worktree(self):
+ sourcePath = self.temporaryLLVM.repodir
+ subprojs = LLVMSubproject.get_all_subprojects()
+
+ path = subprojs["compiler-rt"].get_cmake_path(sourcePath)
+ os.makedirs(path)
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+ enabled = config.get_enabled_subprojects()
+
+ # Check that if it's not a worktree, it's not enabled.
+ self.assertEqual(
+ enabled,
+ [],
+ "Detected unexpected projects %s" % str(enabled)
+ )
+
+ def test_detect_enabled_wrong_branch(self):
+ sourcePath = self.temporaryLLVM.repodir
+ subprojs = LLVMSubproject.get_all_subprojects()
+
+ path = subprojs["compiler-rt"].get_cmake_path(sourcePath)
+ branch = "different-than-" + self.temporaryLLVMbranch
+ worktree = Worktree.create(
+ self.proj,
+ self.__get_subproj_repo("compiler-rt"),
+ path,
+ branch)
+ self.assertTrue(os.path.isdir(path), "Failed to create worktree")
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+ enabled = config.get_enabled_subprojects()
+
+ # Check that if it's a worktree on the wrong branch, it's not enabled.
+ self.assertEqual(
+ enabled,
+ [],
+ "Detected unexpected projects %s" % str(enabled)
+ )
+
+ def test_add_invalid_subproject(self):
+ config = LLVMSourceConfig(self.proj, self.temporaryLLVM.repodir)
+ subproj = "not-an-llvm-subproject"
+ subprojPath = self.originalLLVM.repodir # Dummy path
+
+ with self.assertRaises(ValueError) as context:
+ config.update({subproj : Clone(self.proj, subprojPath)})
+
+ self.assertRegexpMatches(str(context.exception),
+ "Unknown llvm subproject %s" % subproj)
+
+ def test_add_each_subproject(self):
+ sourcePath = self.temporaryLLVM.repodir
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+
+ subprojs = LLVMSubproject.get_all_subprojects()
+ for subproj in subprojs:
+ expectedPath = subprojs[subproj].get_cmake_path(sourcePath)
+ config.update({subproj : self.__get_subproj_repo(subproj)})
+ self.assertTrue(os.path.isdir(expectedPath),
+ "Failed to add subproject %s" % subproj)
+
+ def test_add_all_subprojects(self):
+ sourcePath = self.temporaryLLVM.repodir
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+
+ to_add = {}
+ subprojs = LLVMSubproject.get_all_subprojects()
+
+ for subproj in subprojs:
+ to_add[subproj] = self.__get_subproj_repo(subproj)
+
+ config.update(to_add)
+
+ for subproj in subprojs:
+ expectedPath = subprojs[subproj].get_cmake_path(sourcePath)
+ self.assertTrue(os.path.isdir(expectedPath),
+ "Failed to add subproject %s" % subproj)
+
+ def test_add_some_subprojects(self):
+ sourcePath = self.temporaryLLVM.repodir
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+
+ to_add = {}
+ to_add["clang"] = self.__get_subproj_repo("clang")
+ to_add["compiler-rt"] = self.__get_subproj_repo("compiler-rt")
+
+ config.update(to_add)
+
+ subprojs = LLVMSubproject.get_all_subprojects()
+ self.assertTrue(
+ os.path.isdir(subprojs["clang"].get_cmake_path(sourcePath)),
+ "Failed to add subproject clang")
+ self.assertTrue(
+ os.path.isdir(subprojs["compiler-rt"].get_cmake_path(sourcePath)),
+ "Failed to add subproject compiler-rt")
+
+ def test_add_existing_subprojects(self):
+ sourcePath = self.temporaryLLVM.repodir
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+
+ subprojs = LLVMSubproject.get_all_subprojects()
+ existingPath = subprojs["lldb"].get_cmake_path(sourcePath)
+ Worktree.create(
+ self.proj,
+ self.__get_subproj_repo("lldb"),
+ existingPath,
+ self.temporaryLLVMbranch)
+
+ config.update({ "lldb" : self.__get_subproj_repo("lldb")})
+
+ # If we got this far, we're probably ok, but let's be pedantic and check
+ # that the subproject is still there
+ self.assertTrue(os.path.isdir(existingPath),
+ "Existing subproject vanished")
+
+ def test_add_subproject_existing_branch(self):
+ """
+ Test that we can add a subproject that already has the branch that LLVM
+ is on. This can happen for instance if we have added and then removed
+ the subproject and now we're trying to add it again.
+ """
+ sourcePath = self.temporaryLLVM.repodir
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+
+ clangRepo = self.__get_subproj_repo("clang")
+ with cd(clangRepo.repodir):
+ # Make sure that the branch that LLVM is on already exists in the
+ # clang repo as well.
+ git("checkout", "-b", self.temporaryLLVMbranch)
+ self.__create_dummy_commit()
+ git("checkout", "master")
+
+ config.update( { "clang" : clangRepo })
+
+ subprojs = LLVMSubproject.get_all_subprojects()
+ path = subprojs["clang"].get_cmake_path(sourcePath)
+ self.assertTrue(os.path.isdir(path), "Failed to add subproject")
+
+ def test_add_subproject_not_a_worktree(self):
+ """
+ Test that we can't update a config to include a subproject that exists
+ but is not a worktree.
+ """
+ sourcePath = self.temporaryLLVM.repodir
+ branch = "different-than-" + self.temporaryLLVMbranch
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+
+ subprojs = LLVMSubproject.get_all_subprojects()
+ existingPath = subprojs["lldb"].get_cmake_path(sourcePath)
+ os.makedirs(existingPath)
+
+ with self.assertRaises(EnvironmentError) as context:
+ config.update({ "lldb" : self.__get_subproj_repo("lldb")})
+
+ self.assertRegexpMatches(str(context.exception),
+ "{} already exists but is not a valid subproject directory.*"
+ .format(existingPath))
+
+ # If we got this far, we're probably ok, but let's be pedantic and check
+ # that the subproject is still there
+ self.assertTrue(os.path.isdir(existingPath),
+ "Existing subproject vanished")
+
+ def test_add_subproject_wrong_branch(self):
+ """
+ Test that we can't update a config to include a subproject that exists
+ but is on the wrong branch.
+ """
+ sourcePath = self.temporaryLLVM.repodir
+ branch = "different-than-" + self.temporaryLLVMbranch
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+
+ subprojs = LLVMSubproject.get_all_subprojects()
+ existingPath = subprojs["lldb"].get_cmake_path(sourcePath)
+ Worktree.create(
+ self.proj,
+ self.__get_subproj_repo("lldb"),
+ existingPath,
+ branch)
+
+ with self.assertRaises(EnvironmentError) as context:
+ config.update({ "lldb" : self.__get_subproj_repo("lldb")})
+
+ self.assertRegexpMatches(str(context.exception),
+ "{} already exists but is not a valid subproject directory.*"
+ .format(existingPath))
+
+ # If we got this far, we're probably ok, but let's be pedantic and check
+ # that the subproject is still there
+ self.assertTrue(os.path.isdir(existingPath),
+ "Existing subproject vanished")
+
+ def test_remove_subproject(self):
+ sourcePath = self.temporaryLLVM.repodir
+
+ subprojs = LLVMSubproject.get_all_subprojects()
+
+ lldPath = subprojs["lld"].get_cmake_path(sourcePath)
+ lldWorktree = Worktree.create(
+ self.proj,
+ self.__get_subproj_repo("lld"),
+ lldPath,
+ self.temporaryLLVMbranch)
+
+ clangPath = subprojs["clang"].get_cmake_path(sourcePath)
+ clangWorktree = Worktree.create(
+ self.proj,
+ self.__get_subproj_repo("clang"),
+ clangPath,
+ self.temporaryLLVMbranch)
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+ config.update(remove=["lld"])
+
+ self.assertFalse(os.path.isdir(lldPath), "Failed to remove subproject")
+ self.assertTrue(os.path.isdir(clangPath), "Removed sibling subproject")
+
+ def test_remove_some_subprojects(self):
+ sourcePath = self.temporaryLLVM.repodir
+
+ subprojs = LLVMSubproject.get_all_subprojects()
+
+ for subproj in ["clang", "compiler-rt", "lld", "lldb", "libunwind"]:
+ path = subprojs[subproj].get_cmake_path(sourcePath)
+ worktree = Worktree.create(
+ self.proj,
+ self.__get_subproj_repo(subproj),
+ path,
+ self.temporaryLLVMbranch)
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+ config.update(remove=["compiler-rt", "lld"])
+
+ for subproj in ["compiler-rt", "lld"]:
+ path = subprojs[subproj].get_cmake_path(sourcePath)
+ self.assertFalse(
+ os.path.isdir(path),
+ "Failed to remove subproject")
+
+ for subproj in ["clang", "lldb", "libunwind"]:
+ path = subprojs[subproj].get_cmake_path(sourcePath)
+ self.assertTrue(os.path.isdir(path), "Removed sibling subproject")
+
+ def test_remove_all_subprojects(self):
+ sourcePath = self.temporaryLLVM.repodir
+
+ subprojs = LLVMSubproject.get_all_subprojects()
+
+ for subproj in subprojs.keys():
+ path = subprojs[subproj].get_cmake_path(sourcePath)
+ worktree = Worktree.create(
+ self.proj,
+ self.__get_subproj_repo(subproj),
+ path,
+ self.temporaryLLVMbranch)
+ self.assertTrue(os.path.isdir(path), "Failed to create worktree")
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+ config.update(remove=subprojs.keys())
+
+ for subproj in subprojs.keys():
+ path = subprojs[subproj].get_cmake_path(sourcePath)
+ self.assertFalse(
+ os.path.isdir(path),
+ "Failed to remove subproject")
+
+ def test_remove_each_subproject(self):
+ sourcePath = self.temporaryLLVM.repodir
+
+ subprojs = LLVMSubproject.get_all_subprojects()
+
+ for subproj in subprojs.keys():
+ path = subprojs[subproj].get_cmake_path(sourcePath)
+ worktree = Worktree.create(
+ self.proj,
+ self.__get_subproj_repo(subproj),
+ path,
+ self.temporaryLLVMbranch)
+ self.assertTrue(os.path.isdir(path), "Failed to create worktree")
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+
+ for subproj in subprojs.keys():
+ config.update(remove=[subproj])
+
+ for subproj in subprojs.keys():
+ path = subprojs[subproj].get_cmake_path(sourcePath)
+ self.assertFalse(
+ os.path.isdir(path),
+ "Failed to remove subproject")
+
+ def test_remove_duplicates(self):
+ sourcePath = self.temporaryLLVM.repodir
+
+ subprojs = LLVMSubproject.get_all_subprojects()
+
+ for subproj in ["clang", "compiler-rt", "lld", "lldb", "libunwind"]:
+ path = subprojs[subproj].get_cmake_path(sourcePath)
+ worktree = Worktree.create(
+ self.proj,
+ self.__get_subproj_repo(subproj),
+ path,
+ self.temporaryLLVMbranch)
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+ config.update(remove=["compiler-rt", "lld", "lld", "compiler-rt"])
+
+ for subproj in ["compiler-rt", "lld"]:
+ path = subprojs[subproj].get_cmake_path(sourcePath)
+ self.assertFalse(
+ os.path.isdir(path),
+ "Failed to remove subproject")
+
+ for subproj in ["clang", "lldb", "libunwind"]:
+ path = subprojs[subproj].get_cmake_path(sourcePath)
+ self.assertTrue(os.path.isdir(path), "Removed sibling subproject")
+
+ def test_remove_invalid_subproject(self):
+ sourcePath = self.temporaryLLVM.repodir
+ config = LLVMSourceConfig(self.proj, sourcePath)
+
+ subproj = "not-an-llvm-subproject"
+
+ with self.assertRaises(ValueError) as context:
+ config.update(remove=[subproj])
+
+ self.assertRegexpMatches(str(context.exception),
+ "Unknown llvm subproject %s" % subproj)
+
+ def test_remove_inexistent_subproject(self):
+ sourcePath = self.temporaryLLVM.repodir
+
+ subprojs = LLVMSubproject.get_all_subprojects()
+
+ lldPath = subprojs["lld"].get_cmake_path(sourcePath)
+ lldWorktree = Worktree.create(
+ self.proj,
+ self.__get_subproj_repo("lld"),
+ lldPath,
+ self.temporaryLLVMbranch)
+
+ clangPath = subprojs["clang"].get_cmake_path(sourcePath)
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+ config.update(remove=["clang"])
+
+ self.assertFalse(
+ os.path.isdir(clangPath),
+ "Failed to remove subproject")
+ self.assertTrue(os.path.isdir(lldPath), "Removed sibling subproject")
+
+ def test_add_after_remove(self):
+ sourcePath = self.temporaryLLVM.repodir
+
+ subprojs = LLVMSubproject.get_all_subprojects()
+ lldbRepo = self.__get_subproj_repo("lldb")
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+
+ config.update({"lldb" : lldbRepo})
+ self.assertTrue(
+ os.path.isdir(subprojs["lldb"].get_cmake_path(sourcePath)),
+ "Failed to add lldb")
+
+ config.update(remove=["lldb"])
+ self.assertFalse(
+ os.path.isdir(subprojs["lldb"].get_cmake_path(sourcePath)),
+ "Failed to remove lldb")
+
+ config.update({"lldb" : lldbRepo })
+ self.assertTrue(
+ os.path.isdir(subprojs["lldb"].get_cmake_path(sourcePath)),
+ "Failed to add lldb")
+
+ def test_mixed_adds_removes(self):
+ sourcePath = self.temporaryLLVM.repodir
+
+ subprojs = LLVMSubproject.get_all_subprojects()
+
+ for subproj in ["clang", "compiler-rt", "lld", "lldb", "libunwind"]:
+ path = subprojs[subproj].get_cmake_path(sourcePath)
+ worktree = Worktree.create(
+ self.proj,
+ self.__get_subproj_repo(subproj),
+ path,
+ self.temporaryLLVMbranch)
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+ config.update({
+ "libcxx" : self.__get_subproj_repo("libcxx"),
+ "libcxxabi" : self.__get_subproj_repo("libcxxabi")},
+ ["compiler-rt", "lld"])
+
+ for subproj in ["libcxx", "libcxxabi"]:
+ path = subprojs[subproj].get_cmake_path(sourcePath)
+ self.assertTrue(os.path.isdir(path), "Failed to add subproject")
+
+ for subproj in ["compiler-rt", "lld"]:
+ path = subprojs[subproj].get_cmake_path(sourcePath)
+ self.assertFalse(
+ os.path.isdir(path),
+ "Failed to remove subproject")
+
+ for subproj in ["clang", "lldb", "libunwind"]:
+ path = subprojs[subproj].get_cmake_path(sourcePath)
+ self.assertTrue(os.path.isdir(path), "Removed sibling subproject")
+
+ def test_simultaneous_add_remove(self):
+ sourcePath = self.temporaryLLVM.repodir
+ subprojs = LLVMSubproject.get_all_subprojects()
+
+ clangRepo = self.__get_subproj_repo("clang")
+ lldRepo = self.__get_subproj_repo("lld")
+ libunwindRepo = self.__get_subproj_repo("libunwind")
+
+ config = LLVMSourceConfig(self.proj, sourcePath)
+ with self.assertRaises(ValueError) as context:
+ config.update(
+ { "clang" : clangRepo, "lld" : lldRepo, "libunwind" :
+ libunwindRepo}, ["libcxx", "lld", "libcxxabi"])
+
+ self.assertEqual(str(context.exception),
+ "Can't add and remove lld at the same time")
+
+ # Make sure we didn't add any of the others either
+ for subproj in ["clang", "libunwind"]:
+ path = subprojs[subproj].get_cmake_path(sourcePath)
+ self.assertFalse(
+ os.path.isdir(path),
+ "Incorrectly added subproject")
+
+ # TODO: test with a different dictionary than the default one (not
+ # necessarily containing subprojects - it can contain "potato", "banana" and
+ # "gazpacho" for all we care); in fact, it would probably be best to move the
+ # existing tests to that...
+
+ # TODO: test that CMake gets our layout
+
+if __name__ == "__main__":
+ unittest.main()