blob: 8850fbec8e6f30223c91c7d60f160823f716ced1 [file] [log] [blame]
Chris Matthewsf0d9e882018-01-11 23:55:23 +00001#!/usr/bin/python2.7
2"""
3Dependency manager for llvm CI builds.
4
5We have complex dependencies for some of our CI builds. This will serve
6as a system to help document and enforce them.
7
8Developer notes:
9
10- We are trying to keep package dependencies to a minimum in this project. So it
11does not require an installer. It should be able to be run as a stand alone script
12when checked out of VCS. So, don't import anything not in the Python 2.7
13standard library.
14
15"""
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20from __future__ import unicode_literals
21
22import argparse
Chris Matthewsea57dc42018-01-31 22:37:34 +000023import collections
Chris Matthewsf0d9e882018-01-11 23:55:23 +000024import json
25import platform
26import re
27import subprocess
28
29
30try:
Chris Matthewsbcc3c052018-02-01 00:33:15 +000031 # noinspection PyUnresolvedReferences
32 from typing import List, Text, Union, Dict, Type, Optional # noqa
Chris Matthewsf0d9e882018-01-11 23:55:23 +000033except ImportError as e:
Chris Matthewsf0d9e882018-01-11 23:55:23 +000034 pass # Not really needed at runtime, so okay to not have installed.
35
36
37VERSION = '0.1'
38"""We have a built in version check, so we can require specific features and fixes."""
39
40
41class Version(object):
42 """Model a version number, which can be compared to another version number.
43
44 Keeps a nice looking text version around as well for printing.
45
46 This abstraction exists to make some of the more complex comparisons easier,
47 as well as collecting and printing versions.
48
49 In the future, we might want to have some different comparison,
50 for instance, 4.0 == 4.0.0 -> True.
51
52 """
53
54 def __init__(self, text):
55 """Create a version from a . separated version string."""
56 self.text = text
57 self.numeric = [int(d) for d in text.split(".")]
58
59 def __gt__(self, other):
60 """Compare the numeric representation of the version."""
61 return self.numeric.__gt__(other.numeric)
62
63 def __lt__(self, other):
64 """Compare the numeric representation of the version."""
65 return self.numeric.__lt__(other.numeric)
66
67 def __eq__(self, other):
68 """Compare the numeric representation of the version."""
69 return self.numeric.__eq__(other.numeric)
70
71 def __le__(self, other):
72 """Compare the numeric representation of the version."""
73 return self.numeric.__le__(other.numeric)
74
75 def __ge__(self, other):
76 """Compare the numeric representation of the version."""
77 return self.numeric.__ge__(other.numeric)
78
79 def __repr__(self):
80 """Print the original text representation of the Version."""
81 return "v{}".format(self.text)
82
83
84class Dependency(object):
85 """Dependency Abstract base class."""
86
87 def __init__(self, line, str_kind):
88 """Save line information.
89
90 :param line: A parsed Line object that contains the raw dependency deceleration.
91 :param str_kind: The determined kind of the Dependency.
92 """
93 # type: (Line, Text) -> object
94 self.line = line
95 self.str_kind = str_kind
96 self.installed_version = None
97
98 def parse(self):
99 """Read the input line and prepare to verify this dependency.
100
101 Raise a MalformedDependencyError if three is something wrong.
102
103 Should return nothing, but get the dependency ready for verification.
104 """
105 raise NotImplementedError()
106
107 def verify(self):
108 # type: () -> bool
109 """Determine if this dependency met.
110
111 :returns: True when the dependency is met, otherwise False.
112 """
113 raise NotImplementedError()
114
115 def inject(self):
116 """If possible, modify the system to meet the dependency."""
117 raise NotImplementedError()
118
119 def verify_and_act(self):
120 """Parse, then verify and trigger pass or fail.
121
122 Extract that out here, so we don't duplicate the logic in the subclasses.
123 """
124 met = self.verify()
125 if met:
126 self.verify_pass()
127 else:
128 self.verify_fail()
129
130 def verify_fail(self):
131 """When dependency is not met, raise an exception.
132
133 This is the default behavior; but I want the subclasses to be able
134 to change it.
135 """
136 raise MissingDependencyError(self, self.installed_version)
137
138 def verify_pass(self):
139 """Print a nice message that the dependency is met.
140
141 I'm not sure we even want to print this, but we will for now. It might
142 be to verbose. Subclasses should override this if wanted.
143 """
144 print("Dependency met", str(self))
145
146
147class MalformedDependency(Exception):
148 """Raised when parsing a dependency directive fails.
149
150 This is situations like the regexes not matching, or part of the dependency directive missing.
151
152 Should probably record more useful stuff, but Exception.message is set. So we can print it later.
153 """
154
155 pass
156
157
158def brew_cmd(command):
Chris Matthewsbcc3c052018-02-01 00:33:15 +0000159 # type: (List[Text]) -> List[Dict[Text, object]]
Chris Matthewsf0d9e882018-01-11 23:55:23 +0000160 """Help run a brew command, and parse the output.
161
162 Brew has a json output option which we use. Run the command and parse the stdout
163 as json and return the result.
164 :param command: The brew command to execute, and parse the output of.
165 :return:
166 """
167 assert "--json=v1" in command, "Must pass JSON arg so we can parse the output."
168 out = subprocess.check_output(command)
169 brew_info = json.loads(out)
170 return brew_info
171
172
173class MissingDependencyError(Exception):
174 """Fail verification with one of these when we determine a dependency is not met.
175
176 For each dependency, we will print a useful message with the dependency line as well as the
177 reason it was not matched.
178 """
179
180 def __init__(self, dependency, installed=None):
181 # type: (Dependency, Optional[Text]) -> None
182 """Raise when a dependency is not met.
183
184 This exception can be printed as the error message.
185
186 :param dependency: The dependency that is not being met.
187 :param installed: what was found to be installed instead.
188 """
189 # type: (Dependency, Union[Text, Version]) -> None
190 super(MissingDependencyError, self).__init__()
191 self.dependency = dependency
192 self.installed = installed
193
194 def __str__(self):
195 """For now, we will just print these as our error message."""
196 return "missing dependency: {}, found {} installed, requested from {}".format(self.dependency,
197 self.installed,
198 self.dependency.line)
199
200
201def check_version(installed, operator, requested):
202 """Check that the installed version does the operator of the requested.
203
204 :param installed: The installed Version of the requirement.
205 :param operator: the text operator (==, <=, >=)
206 :param requested: The requested Version of the requirement.
207 :return: True if the requirement is satisfied.
208 """
209 # type: (Version, Text, Version) -> bool
210
211 dependency_met = False
212 if operator == "==" and installed == requested:
213 dependency_met = True
214 if operator == "<=" and installed <= requested:
215 dependency_met = True
216 if operator == ">=" and installed >= requested:
217 dependency_met = True
218 return dependency_met
219
220
221class ConMan(Dependency):
222 """Version self-check of this tool.
223
224 In case we introduce something in the future, the dep files can
225 be made to depend on a specific version of this tool. We will
226 increment the versions manually.
227
228 """
229
230 conman_re = re.compile(r'(?P<command>\w+)\s+(?P<operator>>=|<=|==)\s*(?P<version_text>[\d.-_]+)')
231 """For example: config_manager <= 0.1"""
232
233 def __init__(self, line, kind):
234 """Check that this tool is up to date."""
235 super(ConMan, self).__init__(line, kind)
236 self.command = None
237 self.operator = None
238 self.version = None
239 self.version_text = None
240 self.installed_version = None
241
242 def parse(self):
243 """Parse dependency."""
244 text = self.line.text
245 match = self.conman_re.match(text)
246 if not match:
247 raise MalformedDependency("Expression does not compile in {}: {}".format(self.__class__.__name__,
248 self.line))
249 self.__dict__.update(match.groupdict())
250
251 self.version = Version(self.version_text)
252
253 def verify(self):
254 """Verify the version of this tool."""
255 self.installed_version = Version(VERSION)
256
257 return check_version(self.installed_version, self.operator, self.version)
258
259 def inject(self):
260 """Can't really do much here."""
261 pass
262
263 def __str__(self):
264 """Show as dependency and version."""
265 return "{} {}".format(self.str_kind, self.version)
266
267
268class HostOSVersion(Dependency):
269 """Use Python's platform module to get host OS information and verify.
270
271 Wew can only verify, but not inject for host OS version.
272 """
273
274 conman_re = re.compile(r'(?P<command>\w+)\s+(?P<operator>>=|<=|==)\s*(?P<version_text>[\d.-_]+)')
275
276 def __init__(self, line, kind):
277 """Parse and Verify host OS version using Python's platform module.
278
279 :param line: Line with teh Dependencies deceleration.
280 :param kind: the dependency kind that was detected by the parser.
281 """
282 # type: (Line, Text) -> None
283 super(HostOSVersion, self).__init__(line, kind)
284 self.command = None
285 self.operator = None
286 self.version = None
287 self.version_text = None
288 self.installed_version = None
289
290 def parse(self):
291 """Parse dependency."""
292 text = self.line.text
293 match = self.conman_re.match(text)
294 if not match:
295 raise MalformedDependency("Expression does not compile in {}: {}".format(self.__class__.__name__,
296 self.line))
297 self.__dict__.update(match.groupdict())
298
299 self.version = Version(self.version_text)
300
301 def verify(self):
302 """Verify the request host OS version holds."""
303 self.installed_version = Version(platform.mac_ver()[0])
304
305 return check_version(self.installed_version, self.operator, self.version)
306
307 def inject(self):
308 """Can't change the host OS version, so not much to do here."""
309 pass
310
311 def __str__(self):
312 """For printing in error messages."""
313 return "{} {}".format(self.str_kind, self.version)
314
315
316class Brew(Dependency):
317 """Verify and Inject brew package dependencies."""
318
319 # brew <package> <operator> <version>. Operator may not have spaces around it.
320 brew_re = re.compile(r'(?P<command>\w+)\s+(?P<package>\w+)\s*(?P<operator>>=|<=|==)\s*(?P<version_text>[\d.-_]+)')
321
322 def __init__(self, line, kind):
323 # type: (Line, Text) -> None
324 """Parse and verify brew package is installed.
325
326 :param line: the Line with the deceleration of the dependency.
327 :param kind: the detected dependency kind.
328 """
329 super(Brew, self).__init__(line, kind)
330 self.command = None
331 self.operator = None
332 self.package = None
333 self.version = None
334 self.version_text = None
335 self.installed_version = None
336
337 def parse(self):
338 """Parse this dependency."""
339 text = self.line.text
340 match = self.brew_re.match(text)
341 if not match:
342 raise MalformedDependency("Expression does not compile in {}: {}".format(self.__class__.__name__,
343 self.line))
344 self.__dict__.update(match.groupdict())
345
346 self.version = Version(self.version_text)
347
348 def verify(self):
349 """Verify the packages in brew match this dependency."""
350 brew_package_config = brew_cmd(['/usr/local/bin/brew', 'info', self.package, "--json=v1"])
351 version = None
352 for brew_package in brew_package_config:
353 name = brew_package['name']
354 install_info = brew_package.get('installed')
355 for versions in install_info:
356 version = versions['version'] if versions else None
357 if name == self.package:
358 break
359 if not version:
360 # The package is not installed at all.
361 raise MissingDependencyError(self, "nothing")
362 self.installed_version = Version(version)
363 return check_version(self.installed_version, self.operator, self.version)
364
365 def inject(self):
366 """Not implemented."""
367 raise NotImplementedError()
368
369 def __str__(self):
370 """Dependency kind, package and version, for printing in error messages."""
371 return "{} {} {}".format(self.str_kind, self.package, self.version)
372
373
Chris Matthewsea57dc42018-01-31 22:37:34 +0000374class Xcode(Dependency):
375 """Verify and Inject Xcode version dependencies."""
376
377 # xcode <operator> <version>. Operator may not have spaces around it.
378 xcode_dep_re = re.compile(r'(?P<command>\w+)\s+(?P<operator>>=|<=|==)\s*(?P<version_text>[\d.-_]+)')
379
380 def __init__(self, line, kind):
381 # type: (Line, Text) -> None
382 """Parse and xcode version installed.
383
384 :param line: the Line with the deceleration of the dependency.
385 :param kind: the detected dependency kind.
386 """
387 super(Xcode, self).__init__(line, kind)
388 self.command = None
389 self.operator = None
390 self.version = None
391 self.version_text = None
392 self.installed_version = None
393
394 def parse(self):
395 """Parse this dependency."""
396 text = self.line.text
397 match = self.xcode_dep_re.match(text)
398 if not match:
399 raise MalformedDependency("Expression does not compile in {}: {}".format(self.__class__.__name__,
400 self.line))
401 self.__dict__.update(match.groupdict())
402
403 self.version = Version(self.version_text)
404
405 def verify(self):
406 """Verify the installed Xcode matches this dependency."""
407 installed_version_output = subprocess.check_output(['/usr/bin/xcrun', 'xcodebuild', "-version"])
408 installed_version_re = re.compile(r"^Xcode\s(?P<version_text>[\d/.]+)")
409 match = installed_version_re.match(installed_version_output)
410 if not match:
411 raise MissingDependencyError(self, "Did not find Xcode version in output:" + installed_version_output)
412 version = match.groupdict().get('version_text')
413 if not version:
414 # The package is not installed at all.
415 raise AssertionError("No version text found.")
416 self.installed_version = Version(version)
417 return check_version(self.installed_version, self.operator, self.version)
418
419 def inject(self):
420 """Not implemented."""
421 raise NotImplementedError()
422
423 def __str__(self):
424 """Dependency kind, package and version, for printing in error messages."""
425 return "{} {}".format(self.str_kind, self.version)
426
427
428class Sdk(Dependency):
429 """Verify and Inject Sdk version dependencies."""
430
431 # sdk <operator> <version>. Operator may not have spaces around it.
Chris Matthewsbcc3c052018-02-01 00:33:15 +0000432 sdk_dep_re = re.compile(r'(?P<command>\w+)\s+(?P<sdk>[\w/.]+)\s*'
433 r'(?P<operator>>=|<=|==)\s*(?P<version_text>[\d.-_]+)')
Chris Matthewsea57dc42018-01-31 22:37:34 +0000434
435 def __init__(self, line, kind):
436 # type: (Line, Text) -> None
437 """Parse and sdk version installed.
438
439 :param line: the Line with the deceleration of the dependency.
440 :param kind: the detected dependency kind.
441 """
442 super(Sdk, self).__init__(line, kind)
443 self.command = None
444 self.sdk = None
445 self.operator = None
446 self.version = None
447 self.version_text = None
448 self.installed_version = None
449
450 def parse(self):
451 """Parse this dependency."""
452 text = self.line.text
453 match = self.sdk_dep_re.match(text)
454 if not match:
455 raise MalformedDependency("Expression does not compile in {}: {}".format(self.__class__.__name__,
456 self.line))
457 self.__dict__.update(match.groupdict())
458
459 self.version = Version(self.version_text)
460
461 def verify(self):
462 """Verify the installed Sdk matches this dependency."""
463 installed_version_output = subprocess.check_output(['/usr/bin/xcrun', 'xcodebuild', "-showsdks"])
464 installed_version_re = re.compile(r".*-sdk\s+(?P<sdk_text>\S+)")
465
466 matches = [installed_version_re.match(l).groupdict()['sdk_text']
467 for l in installed_version_output.split('\n') if installed_version_re.match(l)]
468
469 if not matches:
470 raise MissingDependencyError(self, "Did not find Sdk version in output:" + installed_version_output)
471
472 extract_version_names = re.compile(r'(?P<pre>\D*)(?P<version_text>[\d+/.]*)(?P<post>.*)')
473
474 sdks = [extract_version_names.match(sdk_text).groupdict()
475 for sdk_text in matches if extract_version_names.match(sdk_text)]
476
477 installed_sdks = collections.defaultdict(list)
478 for sdk in sdks:
479 name = sdk['pre']
480 if sdk.get('post'):
481 name += "." + sdk.get('post')
482 if sdk.get('version_text'):
483 version = Version(sdk['version_text'].rstrip('.'))
484 else:
485 continue
486 installed_sdks[name].append(version)
487
488 if self.sdk not in installed_sdks.keys():
Chris Matthewsbcc3c052018-02-01 00:33:15 +0000489 raise MissingDependencyError(self, "{} not found in installed SDKs.".format(self.sdk))
Chris Matthewsea57dc42018-01-31 22:37:34 +0000490
491 self.installed_version = installed_sdks[self.sdk]
492
493 satisfied = [check_version(s, self.operator, self.version) for s in self.installed_version]
494 return any(satisfied)
495
496 def inject(self):
497 """Not implemented."""
498 raise NotImplementedError()
499
500 def __str__(self):
501 """Dependency kind, package and version, for printing in error messages."""
502 return "{} {}".format(self.str_kind, self.version)
503
504
Chris Matthews6daa0802018-02-01 00:33:16 +0000505class Pip(Dependency):
506 """Verify and Inject pip package dependencies."""
507
508 # pip <package> <operator> <version>. Operator may not have spaces around it.
509 pip_re = re.compile(r'(?P<command>\w+)\s+(?P<package>\w+)\s*(?P<operator>>=|<=|==)\s*(?P<version_text>[\d.-_]+)')
510
511 def __init__(self, line, kind):
512 # type: (Line, Text) -> None
513 """Parse and verify pip package is installed.
514
515 :param line: the Line with the deceleration of the dependency.
516 :param kind: the detected dependency kind.
517 """
518 super(Pip, self).__init__(line, kind)
519 self.command = None
520 self.operator = None
521 self.package = None
522 self.version = None
523 self.version_text = None
524 self.installed_version = None
525
526 def parse(self):
527 """Parse this dependency."""
528 text = self.line.text
529 match = self.pip_re.match(text)
530 if not match:
531 raise MalformedDependency("Expression does not compile in {}: {}".format(self.__class__.__name__,
532 self.line))
533 self.__dict__.update(match.groupdict())
534
535 self.version = Version(self.version_text)
536
537 def verify(self):
538 """Verify the packages in pip match this dependency."""
539 try:
540 pip_package_config = json.loads(subprocess.check_output(["/usr/bin/env",
541 "python", "-m", "pip", "list", "--format=json"]))
542 except (subprocess.CalledProcessError, OSError):
543 raise MissingDependencyError(self, "Cannot find pip")
544
545 installed = {p['name']: p['version'] for p in pip_package_config} # type: Dict[Text, Text]
546
547 package = installed.get(self.package)
548
549 if not package:
550 # The package is not installed at all.
551 raise MissingDependencyError(self, "not in package list")
552 self.installed_version = Version(package)
553 return check_version(self.installed_version, self.operator, self.version)
554
555 def inject(self):
556 """Not implemented."""
557 raise NotImplementedError()
558
559 def __str__(self):
560 """Dependency kind, package and version, for printing in error messages."""
561 return "{} {} {}".format(self.str_kind, self.package, self.version)
562
563
Chris Matthewsf0d9e882018-01-11 23:55:23 +0000564dependencies_implementations = {'brew': Brew,
565 'os_version': HostOSVersion,
Chris Matthewsea57dc42018-01-31 22:37:34 +0000566 'config_manager': ConMan,
567 'xcode': Xcode,
568 'sdk': Sdk,
Chris Matthews6daa0802018-02-01 00:33:16 +0000569 'pip': Pip,
Chris Matthewsea57dc42018-01-31 22:37:34 +0000570 }
Chris Matthewsf0d9e882018-01-11 23:55:23 +0000571
572
573def dependency_factory(line):
574 """Given a line, create a concrete dependency for it.
575
576 :param line: The line with the dependency info
577 :return: Some subclass of Dependency, based on what was in the line.
578 """
579 # type: Text -> Dependency
580 kind = line.text.split()[0]
581 try:
582 return dependencies_implementations[kind](line, kind)
583 except KeyError:
584 raise MalformedDependency("Don't know about {} kind of dependency.".format(kind))
585
586
587class Line(object):
588 """A preprocessed line. Understands file and line number as well as comments."""
589
590 def __init__(self, filename, line_number, text, comment):
591 # type: (Text, int, Text, Text) -> None
592 """Raw Line information, split into the dependency deceleration and comment.
593
594 :param filename: the input filename.
595 :param line_number: the line number in the input file.
596 :param text: Non-comment part of the line.
597 :param comment: Text from the comment part of the line if any.
598 """
599 self.filename = filename
600 self.line_number = line_number
601 self.text = text
602 self.comment = comment
603
604 def __repr__(self):
605 """Reconstruct the line for pretty printing."""
606 return "{}:{}: {}{}".format(self.filename,
607 self.line_number,
608 self.text,
609 " # " + self.comment if self.comment else "")
610
611
612# For stripping comments out of lines.
613comment_re = re.compile(r'(?P<main_text>.*)#(?P<comment>.*)')
614
615
616# noinspection PyUnresolvedReferences
617def _parse_dep_file(lines, filename):
618 # type: (List[Text], Text) -> List[Line]
619 process_lines = []
620 for num, text in enumerate(lines):
621 if "#" in text:
622 bits = comment_re.match(text)
623 main_text = bits.groupdict().get('main_text')
624 comment = bits.groupdict().get('comment')
625 else:
626 main_text = text
627 comment = None
628 if main_text:
629 main_text = main_text.strip()
630 if comment:
631 comment = comment.strip()
632 process_lines.append(Line(filename, num, main_text, comment))
633
634 return process_lines
635
636
637def parse_dependencies(file_names):
638 """Program logic: read files, verify dependencies.
639
640 For each input file, read lines and create dependencies. Verify each dependency.
641
642 :param file_names: files to read dependencies from.
643 :return: The list of dependencies, each verified.
644 """
645 # type: (List[Text]) -> List[Type[Dependency]]
646 preprocessed_lines = []
647 for file_name in file_names:
648 with open(file_name, 'r') as f:
649 lines = f.readlines()
650 preprocessed_lines.extend(_parse_dep_file(lines, file_name))
651
652 dependencies = [dependency_factory(l) for l in preprocessed_lines if l.text]
653 [d.parse() for d in dependencies]
654 for d in dependencies:
655 try:
656 met = d.verify()
657 if met:
658 d.verify_pass()
659 else:
660 d.verify_fail()
661
662 except MissingDependencyError as exec_info:
663 print("Error:", exec_info)
664
665 return dependencies
666
667
668def main():
669 """Parse arguments and trigger dependency verification."""
670 parser = argparse.ArgumentParser(description='Verify and install dependencies.')
671 parser.add_argument('command', help="What to do.")
672
673 parser.add_argument('dependencies', nargs='+', help="Path to dependency files.")
674
675 args = parser.parse_args()
676
677 parse_dependencies(args.dependencies)
678
679 return True
680
681
682if __name__ == '__main__':
683 main()