blob: e5fca1d0b59fde08639acd39f216aa6e917f32d0 [file] [log] [blame]
Galina Kistanova0035b5d2016-10-07 20:47:51 +00001from buildbot.steps.slave import RemoveDirectory
Galina Kistanova8358cdd2017-03-10 01:06:35 +00002from buildbot.process.properties import WithProperties, Property
3from buildbot.steps.shell import SetProperty
Galina Kistanova0035b5d2016-10-07 20:47:51 +00004
5from zorg.buildbot.commands.CmakeCommand import CmakeCommand
6from zorg.buildbot.commands.NinjaCommand import NinjaCommand
Galina Kistanova0035b5d2016-10-07 20:47:51 +00007
8from zorg.buildbot.conditions.FileConditions import FileDoesNotExist
9from zorg.buildbot.process.factory import LLVMBuildFactory
10
Galina Kistanova8358cdd2017-03-10 01:06:35 +000011import zorg.buildbot.builders.Util as builders_util
12
Galina Kistanova9124b832016-12-22 00:46:41 +000013def getLLVMBuildFactoryAndSVNSteps(
Galina Kistanova0035b5d2016-10-07 20:47:51 +000014 depends_on_projects = None,
15 llvm_srcdir = None,
16 obj_dir = None,
17 install_dir = None,
Galina Kistanova9124b832016-12-22 00:46:41 +000018 cleanBuildRequested = None,
Galina Kistanova0035b5d2016-10-07 20:47:51 +000019 env = None,
20 **kwargs):
21
22 # Set defaults
23 if not depends_on_projects:
24 depends_on_projects=['llvm', 'clang']
25
Galina Kistanova9124b832016-12-22 00:46:41 +000026 if cleanBuildRequested is None:
27 cleanBuildRequested = lambda step: step.build.getProperty("clean")
Galina Kistanova0035b5d2016-10-07 20:47:51 +000028
29 f = LLVMBuildFactory(
30 depends_on_projects=depends_on_projects,
Galina Kistanova9124b832016-12-22 00:46:41 +000031 llvm_srcdir=llvm_srcdir or "llvm",
32 obj_dir=obj_dir or "build",
Galina Kistanova0035b5d2016-10-07 20:47:51 +000033 install_dir=install_dir,
34 cleanBuildRequested=cleanBuildRequested,
Galina Kistanova9124b832016-12-22 00:46:41 +000035 **kwargs) # Pass through all the extra arguments.
Galina Kistanova0035b5d2016-10-07 20:47:51 +000036
37 # Do a clean checkout if requested.
Galina Kistanova9124b832016-12-22 00:46:41 +000038 # TODO: Some Windows slaves do not handle RemoveDirectory command well.
39 # So, consider running "rmdir /S /Q <dir>" if the build runs on Windows.
Galina Kistanova0035b5d2016-10-07 20:47:51 +000040 f.addStep(RemoveDirectory(name='clean-src-dir',
41 dir=f.llvm_srcdir,
42 haltOnFailure=False,
43 flunkOnFailure=False,
44 doStepIf=cleanBuildRequested,
45 ))
46
47 # Get the source code.
48 f.addSVNSteps()
49
Galina Kistanova9124b832016-12-22 00:46:41 +000050 return f
Galina Kistanova0035b5d2016-10-07 20:47:51 +000051
Galina Kistanova9124b832016-12-22 00:46:41 +000052def addCmakeSteps(
53 f,
54 cleanBuildRequested,
55 obj_dir,
56 install_dir = None,
57 extra_configure_args = None,
58 env = None,
59 stage_name = None,
60 **kwargs):
Galina Kistanova0035b5d2016-10-07 20:47:51 +000061
62 # Make a local copy of the configure args, as we are going to modify that.
63 if extra_configure_args:
64 cmake_args = extra_configure_args[:]
65 else:
66 cmake_args = list()
67
Galina Kistanova9124b832016-12-22 00:46:41 +000068 # This is an incremental build, unless otherwise has been requested.
69 # Remove obj and install dirs for a clean build.
70 # TODO: Some Windows slaves do not handle RemoveDirectory command well.
71 # So, consider running "rmdir /S /Q <dir>" if the build runs on Windows.
72 f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir,
73 dir=obj_dir,
74 haltOnFailure=False,
75 flunkOnFailure=False,
76 doStepIf=cleanBuildRequested,
77 ))
78
79 if install_dir:
80 install_dir_rel = LLVMBuildFactory.pathRelativeToBuild(
81 install_dir,
82 obj_dir)
83 CmakeCommand.applyRequiredOptions(cmake_args, [
84 ('-DCMAKE_INSTALL_PREFIX=', install_dir_rel),
85 ])
86
87 f.addStep(RemoveDirectory(name='clean-%s-dir' % install_dir,
88 dir=install_dir,
89 haltOnFailure=False,
90 flunkOnFailure=False,
91 doStepIf=cleanBuildRequested,
92 ))
93
94 # Reconcile the cmake options for this build.
95
Galina Kistanova0035b5d2016-10-07 20:47:51 +000096 # Set proper defaults.
97 CmakeCommand.applyDefaultOptions(cmake_args, [
98 ('-DCMAKE_BUILD_TYPE=', 'Release'),
99 ('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
100 ('-DLLVM_BUILD_TESTS=', 'ON'),
101 ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
102 ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
103 ])
104
105 # Create configuration files with cmake, unless this has been already done
106 # for an incremental build.
Galina Kistanova9124b832016-12-22 00:46:41 +0000107 if stage_name:
108 step_name = "cmake-configure-%s" % stage_name
109 else:
Galina Kistanova40b9ed52017-03-10 07:56:38 +0000110 stage_name = ""
Galina Kistanova9124b832016-12-22 00:46:41 +0000111 step_name = "cmake-configure"
112
113 src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
114
115 f.addStep(CmakeCommand(name=step_name,
116 description=["Cmake", "configure", stage_name],
Galina Kistanova9975c382017-01-03 04:10:02 +0000117 haltOnFailure=True,
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000118 options=cmake_args,
119 path=src_dir,
120 env=env,
121 workdir=obj_dir,
122 doStepIf=FileDoesNotExist("CMakeCache.txt"),
123 **kwargs # Pass through all the extra arguments.
124 ))
125
Galina Kistanova9124b832016-12-22 00:46:41 +0000126def addNinjaSteps(
127 f,
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000128 obj_dir = None,
Galina Kistanova9124b832016-12-22 00:46:41 +0000129 env = None,
130 stage_name = None,
131 **kwargs):
132
133 # Build the unified tree.
134 if stage_name:
135 step_name = "%s-" % stage_name
136 else:
Galina Kistanova40b9ed52017-03-10 07:56:38 +0000137 stage_name = ""
Galina Kistanova9124b832016-12-22 00:46:41 +0000138 step_name = ""
139
140 if obj_dir is None:
141 obj_dir = f.obj_dir
142
143 f.addStep(NinjaCommand(name="build-%sunified-tree" % step_name,
Galina Kistanova9975c382017-01-03 04:10:02 +0000144 haltOnFailure=True,
Galina Kistanova9124b832016-12-22 00:46:41 +0000145 description=["Build", stage_name, "unified", "tree"],
146 env=env,
147 workdir=obj_dir,
148 **kwargs # Pass through all the extra arguments.
149 ))
150
151 # Test just built components
152 f.addStep(NinjaCommand(name="test-%scheck-all" % step_name,
153 targets=["check-all"],
Galina Kistanova9975c382017-01-03 04:10:02 +0000154 haltOnFailure=True,
Galina Kistanova9124b832016-12-22 00:46:41 +0000155 description=["Test", "just", "built", "components"],
156 env=env,
157 workdir=obj_dir,
158 **kwargs # Pass through all the extra arguments.
159 ))
160
161 # Install just built components
162 f.addStep(NinjaCommand(name="install-%sall" % step_name,
163 targets=["install"],
Galina Kistanova9975c382017-01-03 04:10:02 +0000164 haltOnFailure=True,
Galina Kistanova9124b832016-12-22 00:46:41 +0000165 description=["Install", "just", "built", "components"],
166 env=env,
167 workdir=obj_dir,
168 **kwargs # Pass through all the extra arguments.
169 ))
170
171def getCmakeBuildFactory(
172 depends_on_projects = None,
173 llvm_srcdir = None,
174 obj_dir = None,
175 install_dir = None,
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000176 clean = False,
177 extra_configure_args = None,
178 env = None,
179 **kwargs):
180
Galina Kistanova9124b832016-12-22 00:46:41 +0000181 # Prepare environmental variables. Set here all env we want everywhere.
182 merged_env = {
183 'TERM' : 'dumb' # Be cautious and disable color output from all tools.
184 }
185 if env is not None:
186 # Overwrite pre-set items with the given ones, so user can set anything.
187 merged_env.update(env)
188
189 cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
190
191 f = getLLVMBuildFactoryAndSVNSteps(
192 depends_on_projects=depends_on_projects,
193 llvm_srcdir=llvm_srcdir,
194 obj_dir=obj_dir,
195 install_dir=install_dir,
196 cleanBuildRequested=cleanBuildRequested,
197 **kwargs) # Pass through all the extra arguments.
198
199 addCmakeSteps(
200 f,
201 cleanBuildRequested=cleanBuildRequested,
202 obj_dir=f.obj_dir,
203 install_dir=f.install_dir,
204 extra_configure_args=extra_configure_args,
205 env=env,
206 **kwargs)
207
208 return f
209
Galina Kistanova8358cdd2017-03-10 01:06:35 +0000210
Galina Kistanova9124b832016-12-22 00:46:41 +0000211def getCmakeWithNinjaBuildFactory(
212 depends_on_projects = None,
213 llvm_srcdir = None,
214 obj_dir = None,
215 install_dir = None,
216 clean = False,
217 extra_configure_args = None,
218 env = None,
219 **kwargs):
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000220
221 # Make a local copy of the configure args, as we are going to modify that.
222 if extra_configure_args:
223 cmake_args = extra_configure_args[:]
224 else:
225 cmake_args = list()
226
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000227 # Some options are required for this build no matter what.
228 CmakeCommand.applyRequiredOptions(cmake_args, [
229 ('-G', 'Ninja'),
230 ])
231
232 f = getCmakeBuildFactory(
233 depends_on_projects=depends_on_projects,
234 llvm_srcdir=llvm_srcdir,
235 obj_dir=obj_dir,
Galina Kistanova9124b832016-12-22 00:46:41 +0000236 install_dir=install_dir,
237 clean=clean,
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000238 extra_configure_args=cmake_args,
239 env=env,
Galina Kistanova9124b832016-12-22 00:46:41 +0000240 **kwargs) # Pass through all the extra arguments.
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000241
Galina Kistanova9124b832016-12-22 00:46:41 +0000242 addNinjaSteps(
243 f,
244 obj_dir=obj_dir,
245 env=env,
246 **kwargs)
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000247
248 return f
249
Galina Kistanova8358cdd2017-03-10 01:06:35 +0000250def getCmakeWithNinjaWithMSVCBuildFactory(
251 depends_on_projects = None,
252 llvm_srcdir = None,
253 obj_dir = None,
254 install_dir = None,
255 clean = False,
256 extra_configure_args = None,
257 # VS tools environment variable if using MSVC. For example,
258 # %VS140COMNTOOLS% selects the 2015 toolchain.
259 vs=None,
260 target_arch=None,
261 env = None,
262 **kwargs):
263
264 assert not env, "Can't have custom builder env vars with MSVC build"
265
266 # Make a local copy of the configure args, as we are going to modify that.
267 if extra_configure_args:
268 cmake_args = extra_configure_args[:]
269 else:
270 cmake_args = list()
271
272 # Set up VS environment, if appropriate.
273 if not vs:
274 # We build by Visual Studio 2015, unless otherwise is requested.
275 vs=r"""%VS140COMNTOOLS%"""
276
277 cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
278
279 f = getLLVMBuildFactoryAndSVNSteps(
280 depends_on_projects=depends_on_projects,
281 llvm_srcdir=llvm_srcdir,
282 obj_dir=obj_dir,
283 install_dir=install_dir,
284 cleanBuildRequested=cleanBuildRequested,
285 **kwargs) # Pass through all the extra arguments.
286
287 f.addStep(SetProperty(
288 command=builders_util.getVisualStudioEnvironment(vs, target_arch),
289 extract_fn=builders_util.extractSlaveEnvironment))
290 env = Property('slave_env')
291
292 # Some options are required for this build no matter what.
293 CmakeCommand.applyRequiredOptions(cmake_args, [
294 ('-G', 'Ninja'),
295 ])
296
297 addCmakeSteps(
298 f,
299 f.cleanBuildRequested,
300 obj_dir=f.obj_dir,
301 install_dir=f.install_dir,
302 extra_configure_args=cmake_args,
303 env=env,
304 **kwargs)
305
306 addNinjaSteps(
307 f,
308 obj_dir=obj_dir,
309 env=env,
310 **kwargs)
311
312 return f
313
Galina Kistanova9124b832016-12-22 00:46:41 +0000314def getCmakeWithNinjaMultistageBuildFactory(
315 depends_on_projects = None,
316 llvm_srcdir = None,
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000317 obj_dir = None,
Galina Kistanova9124b832016-12-22 00:46:41 +0000318 install_dir = None,
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000319 clean = False,
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000320 extra_configure_args = None,
321 env = None,
Galina Kistanova9124b832016-12-22 00:46:41 +0000322 stages=2,
323 stage_names=None,
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000324 **kwargs):
325
Galina Kistanova9124b832016-12-22 00:46:41 +0000326 # Prepare environmental variables. Set here all env we want everywhere.
327 merged_env = {
328 'TERM' : 'dumb' # Be cautious and disable color output from all tools.
329 }
330 if env is not None:
331 # Overwrite pre-set items with the given ones, so user can set anything.
332 merged_env.update(env)
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000333
334 # Make a local copy of the configure args, as we are going to modify that.
335 if extra_configure_args:
336 cmake_args = extra_configure_args[:]
337 else:
338 cmake_args = list()
339
Galina Kistanova9124b832016-12-22 00:46:41 +0000340 assert stages > 1, "It should be at least 2 stages in a multistage build."
341 if stage_names is None:
342 stage_names = list()
343 for i in range(1, stages + 1):
344 stage_names.append("stage%s" % i)
345 else:
346 assert len(stage_names) == stages, "Please specify names for none or all of the requested stages."
347
348 if obj_dir is None:
349 obj_dir = "build"
350 if install_dir is None:
351 install_dir = "install"
352
353 stage_objdirs = list()
354 stage_installdirs = list()
355 for s in stage_names:
356 stage_objdirs.append("%s/%s" % (obj_dir, s))
357 stage_installdirs.append("%s/%s" % (install_dir, s))
358
359 cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
360
361 f = getLLVMBuildFactoryAndSVNSteps(
362 depends_on_projects=depends_on_projects,
363 llvm_srcdir=llvm_srcdir,
364 obj_dir=obj_dir,
365 install_dir=install_dir,
366 cleanBuildRequested=cleanBuildRequested,
367 env=env,
368 stage_objdirs=stage_objdirs,
369 stage_installdirs=stage_installdirs,
370 stage_names=stage_names,
371 **kwargs) # Pass through all the extra arguments.
372
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000373 # Set proper defaults.
374 CmakeCommand.applyDefaultOptions(cmake_args, [
Galina Kistanova9124b832016-12-22 00:46:41 +0000375 ('-DCMAKE_BUILD_TYPE=', 'Release'),
376 ('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
377 ('-DLLVM_BUILD_TESTS=', 'ON'),
378 ('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'),
379 ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000380 ])
381
382 # Some options are required for this build no matter what.
383 CmakeCommand.applyRequiredOptions(cmake_args, [
Galina Kistanova9124b832016-12-22 00:46:41 +0000384 ('-G', 'Ninja'),
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000385 ])
386
Galina Kistanova9124b832016-12-22 00:46:41 +0000387 # The stage 1 is special, though. We use the system compiler and
Galina Kistanova9975c382017-01-03 04:10:02 +0000388 # do incremental build, unless a clean one has been requested.
Galina Kistanova9124b832016-12-22 00:46:41 +0000389 cmake_args_stage1 = cmake_args[:]
390 CmakeCommand.applyDefaultOptions(cmake_args_stage1, [
391 # Do not expect warning free build by the system toolchain.
392 ('-DLLVM_ENABLE_WERROR=', 'OFF'),
393 ])
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000394
Galina Kistanova9124b832016-12-22 00:46:41 +0000395 addCmakeSteps(
396 f,
397 f.cleanBuildRequested,
398 obj_dir=stage_objdirs[0],
399 install_dir=stage_installdirs[0],
400 extra_configure_args=cmake_args_stage1,
401 env=env,
402 stage_name=stage_names[0],
403 **kwargs)
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000404
Galina Kistanova9124b832016-12-22 00:46:41 +0000405 addNinjaSteps(
406 f,
407 obj_dir=stage_objdirs[0],
408 env=env,
409 stage_name=stage_names[0],
410 **kwargs)
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000411
Galina Kistanova9124b832016-12-22 00:46:41 +0000412 # Build the rest stage by stage, using just built compiler to compile
413 # the next stage.
414 CmakeCommand.applyDefaultOptions(cmake_args, [
Galina Kistanova9975c382017-01-03 04:10:02 +0000415 # We should be warnings free when use just built compiler.
Galina Kistanova9124b832016-12-22 00:46:41 +0000416 ('-DLLVM_ENABLE_WERROR=', 'ON'),
417 ])
418 # If we build LLD, we would link with LLD.
419 # Otherwise we link with a system linker.
420 if 'lld' in f.depends_on_projects:
421 CmakeCommand.applyDefaultOptions(cmake_args, [
422 ('-DLLVM_ENABLE_LLD=', 'ON'),
423 ])
424
425 for stage_idx in range(1, stages):
426
Galina Kistanova9975c382017-01-03 04:10:02 +0000427 # Directories to use in this stage.
Galina Kistanova9124b832016-12-22 00:46:41 +0000428 obj_dir = f.stage_objdirs[stage_idx]
429 src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
430 install_dir = LLVMBuildFactory.pathRelativeToBuild(f.stage_installdirs[stage_idx], obj_dir)
431 staged_install = f.stage_installdirs[stage_idx - 1]
432
Galina Kistanova9975c382017-01-03 04:10:02 +0000433 # Configure the compiler to use in this stage.
Galina Kistanova9124b832016-12-22 00:46:41 +0000434 cmake_args_stageN = cmake_args[:]
435 CmakeCommand.applyRequiredOptions(cmake_args_stageN, [
436 ('-DCMAKE_INSTALL_PREFIX=', install_dir),
437 ])
438 cmake_args_stageN.append(
439 WithProperties(
440 "-DCMAKE_CXX_COMPILER=%(workdir)s/" + staged_install + "/bin/clang++"
441 ))
442 cmake_args_stageN.append(
443 WithProperties(
444 "-DCMAKE_C_COMPILER=%(workdir)s/" + staged_install + "/bin/clang"
445 ))
446
447 addCmakeSteps(
448 f,
449 True, # We always do a clean build for the staged builds.
450 obj_dir=stage_objdirs[stage_idx],
451 install_dir=stage_installdirs[stage_idx],
452 extra_configure_args=cmake_args_stageN,
453 env=env,
454 stage_name=stage_names[stage_idx],
455 **kwargs)
456
457 addNinjaSteps(
458 f,
459 obj_dir=stage_objdirs[stage_idx],
460 env=env,
461 stage_name=stage_names[stage_idx],
462 **kwargs)
Galina Kistanova0035b5d2016-10-07 20:47:51 +0000463
464 return f