#!/usr/bin/env python # utils/build-script - The ultimate tool for building Swift -*- python -*- # # This source file is part of the Swift.org open source project # # Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors # Licensed under Apache License v2.0 with Runtime Library Exception # # See http://swift.org/LICENSE.txt for license information # See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors from __future__ import print_function import argparse import multiprocessing import os import shutil import sys import textwrap # FIXME: Instead of modifying the system path in order to enable imports from # other directories, all Python modules related to the build script # should be moved to the `swift_build_support` module. # For additional information, see: https://bugs.swift.org/browse/SR-237. sys.path.append(os.path.dirname(__file__)) from SwiftBuildSupport import ( HOME, SWIFT_BUILD_ROOT, SWIFT_SOURCE_ROOT, check_call, get_all_preset_names, get_preset_options, print_with_argv0, quote_shell_command, ) sys.path.append(os.path.join(os.path.dirname(__file__), 'swift_build_support')) import swift_build_support.clang import swift_build_support.cmake from swift_build_support.migration import migrate_impl_args # Main entry point for the preset mode. def main_preset(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description="""Builds Swift using a preset.""") parser.add_argument("--preset-file", help="load presets from the specified file", metavar="PATH", action="append", dest="preset_file_names", default=[]) parser.add_argument("--preset", help="use the specified option preset", metavar="NAME") parser.add_argument("--show-presets", help="list all presets and exit", action="store_true") parser.add_argument("--distcc", help="use distcc", action="store_true") parser.add_argument("preset_substitutions_raw", help="'name=value' pairs that are substituted in the preset", nargs="*", metavar="SUBSTITUTION") args = parser.parse_args() if len(args.preset_file_names) == 0: args.preset_file_names = [ os.path.join(HOME, ".swift-build-presets"), os.path.join( SWIFT_SOURCE_ROOT, "swift", "utils", "build-presets.ini") ] if args.show_presets: for name in sorted(get_all_preset_names(args.preset_file_names), key=str.lower): print(name) return 0 if not args.preset: print_with_argv0("Missing --preset option") return 1 args.preset_substitutions = {} for arg in args.preset_substitutions_raw: name, value = arg.split("=", 1) args.preset_substitutions[name] = value preset_args = get_preset_options( args.preset_substitutions, args.preset_file_names, args.preset) build_script_args = [ sys.argv[0] ] build_script_args += preset_args if args.distcc: build_script_args += [ "--distcc" ] print_with_argv0( "using preset '" + args.preset + "', which expands to " + quote_shell_command(build_script_args)) check_call(build_script_args) return 0 # Main entry point for the normal mode. def main_normal(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=""" Use this tool to build, test, and prepare binary distribution archives of Swift and related tools. Builds Swift (and, optionally, LLDB), incrementally, optionally testing it thereafter. Different build configurations are maintained in parallel.""", epilog=""" Using option presets: --preset-file=PATH load presets from the specified file --preset=NAME use the specified option preset The preset mode is mutually exclusive with other options. It is not possible to add ad-hoc customizations to a preset. This is a deliberate design decision. (Rationale: a preset is a certain important set of options that we want to keep in a centralized location. If you need to customize it, you should create another preset in a centralized location, rather than scattering the knowledge about the build across the system.) Presets support substitutions for controlled customizations. Substitutions are defined in the preset file. Values for substitutions are supplied using the name=value syntax on the command line. Any arguments passed after "--" are forwarded directly to Swift's 'build-script-impl'. See that script's help for details. Environment variables --------------------- This script respects a few environment variables, should you choose to set them: SWIFT_SOURCE_ROOT: a directory containing the source for LLVM, Clang, Swift. If this script is located in a Swift source directory, the location of SWIFT_SOURCE_ROOT will be inferred if the variable is not set. 'build-script' expects the sources to be laid out in the following way: $SWIFT_SOURCE_ROOT/llvm /clang /swift /lldb (optional) /llbuild (optional) /swiftpm (optional, requires llbuild) /swift-corelibs-xctest (optional) /swift-corelibs-foundation (optional) SWIFT_BUILD_ROOT: a directory in which to create out-of-tree builds. Defaults to "$SWIFT_SOURCE_ROOT/build/". Preparing to run this script ---------------------------- See README.md for instructions on cloning Swift subprojects. If you intend to use the -l, -L, --lldb, or --lldb-debug options. That's it; you're ready to go! Examples -------- Given the above layout of sources, the simplest invocation of 'build-script' is just: [~/src/s]$ ./swift/utils/build-script This builds LLVM, Clang, Swift and Swift standard library in debug mode. All builds are incremental. To incrementally build changed files, repeat the same 'build-script' command. Typical uses of 'build-script' ------------------------------ To build everything with optimization without debug information: [~/src/s]$ ./swift/utils/build-script -R To run tests, add '-t': [~/src/s]$ ./swift/utils/build-script -R -t To run normal tests and validation tests, add '-T': [~/src/s]$ ./swift/utils/build-script -R -T To build LLVM+Clang with optimization without debug information, and a debuggable Swift compiler: [~/src/s]$ ./swift/utils/build-script -R --debug-swift To build a debuggable Swift standard library: [~/src/s]$ ./swift/utils/build-script -R --debug-swift-stdlib iOS build targets are always configured and present, but are not built by default. To build the standard library for OS X, iOS simulator and iOS device: [~/src/s]$ ./swift/utils/build-script -R -i To run OS X and iOS tests that don't require a device: [~/src/s]$ ./swift/utils/build-script -R -i -t To use 'make' instead of 'ninja', use '-m': [~/src/s]$ ./swift/utils/build-script -m -R To create Xcode projects that can build Swift, use '-x': [~/src/s]$ ./swift/utils/build-script -x -R The Xcode IDE may have performance issues with this configuration. To create slimmer Xcode projects that can edit Swift compiler source, but cannot build, use '-X' instead: [~/src/s]$ ./swift/utils/build-script -X -R Preset mode in build-script --------------------------- All buildbots and automated environments use 'build-script' in *preset mode*. In preset mode, the command line only specifies the preset name and allows limited customization (extra output paths). The actual options come from the selected preset in 'utils/build-presets.ini'. For example, to build like the incremental buildbot, run: [~/src/s]$ ./swift/utils/build-script --preset=buildbot_incremental To build with AddressSanitizer: [~/src/s]$ ./swift/utils/build-script --preset=asan To build a root for Xcode XYZ, '/tmp/xcode-xyz-root.tar.gz': [~/src/s]$ ./swift/utils/build-script --preset=buildbot_BNI_internal_XYZ \\ install_destdir="/tmp/install" install_symroot="/tmp/symroot" installable_package="/tmp/xcode-xyz-root.tar.gz" If you have your own favorite set of options, you can create your own, local, preset. For example, let's create a preset called 'ds' (which stands for Debug Swift): $ cat > ~/.swift-build-presets [preset: ds] release debug-swift debug-swift-stdlib test build-subdir=ds To use it, specify the '--preset=' argument: [~/src/s]$ ./swift/utils/build-script --preset=ds ./swift/utils/build-script: using preset 'ds', which expands to ./swift/utils/build-script --release --debug-swift --debug-swift-stdlib --test --build-subdir=ds -- ... Philosophy ---------- While you can invoke CMake directly to build Swift, this tool will save you time by taking away the mechanical parts of the process, providing you controls for the important options. For all automated build environments, this tool is regarded as *the* *only* way to build Swift. This is not a technical limitation of the Swift build system. It is a policy decision aimed at making the builds uniform across all environments and easily reproducible by engineers who are not familiar with the details of the setups of other systems or automated environments.""") projects_group = parser.add_argument_group( title="Options to select projects") projects_group.add_argument("-l", "--lldb", help="build LLDB", action="store_true", dest="build_lldb") projects_group.add_argument("-b", "--llbuild", help="build llbuild", action="store_true", dest="build_llbuild") projects_group.add_argument("-p", "--swiftpm", help="build swiftpm", action="store_true", dest="build_swiftpm") projects_group.add_argument("--xctest", help="build xctest", action="store_true", dest="build_xctest") projects_group.add_argument("--foundation", help="build foundation", action="store_true", dest="build_foundation") extra_actions_group = parser.add_argument_group( title="Extra actions to perform before or in addition to building") extra_actions_group.add_argument("-c", "--clean", help="do a clean build", action="store_true") extra_actions_group.add_argument("--export-compile-commands", help="generate compilation databases in addition to building", action="store_true") build_variant_group = parser.add_mutually_exclusive_group(required=False) build_variant_group.add_argument("-d", "--debug", help=""" build the Debug variant of everything (LLVM, Clang, Swift host tools, target Swift standard libraries, LLDB (if enabled) (default)""", action="store_const", const="Debug", dest="build_variant") build_variant_group.add_argument("-r", "--release-debuginfo", help=""" build the RelWithDebInfo variant of everything (default is Debug)""", action="store_const", const="RelWithDebInfo", dest="build_variant") build_variant_group.add_argument("-R", "--release", help="build the Release variant of everything (default is Debug)", action="store_const", const="Release", dest="build_variant") build_variant_override_group = parser.add_argument_group( title="Override build variant for a specific project") build_variant_override_group.add_argument("--debug-llvm", help="build the Debug variant of LLVM", action="store_const", const="Debug", dest="llvm_build_variant") build_variant_override_group.add_argument("--debug-swift", help="build the Debug variant of Swift host tools", action="store_const", const="Debug", dest="swift_build_variant") build_variant_override_group.add_argument("--debug-swift-stdlib", help=""" build the Debug variant of the Swift standard library and SDK overlay""", action="store_const", const="Debug", dest="swift_stdlib_build_variant") build_variant_override_group.add_argument("--debug-lldb", help="build the Debug variant of LLDB", action="store_const", const="Debug", dest="lldb_build_variant") build_variant_override_group.add_argument("--debug-cmark", help="build the Debug variant of CommonMark", action="store_const", const="Debug", dest="cmark_build_variant") build_variant_override_group.add_argument("--debug-foundation", help="build the Debug variant of Foundation", action="store_const", const="Debug", dest="foundation_build_variant") assertions_group = parser.add_mutually_exclusive_group(required=False) assertions_group.add_argument("--assertions", help="enable assertions in all projects", action="store_const", const=True, dest="assertions") assertions_group.add_argument("--no-assertions", help="disable assertions in all projects", action="store_const", const=False, dest="assertions") assertions_override_group = parser.add_argument_group( title="Control assertions in a specific project") assertions_override_group.add_argument("--cmark-assertions", help="enable assertions in CommonMark", action="store_const", const=True, dest="cmark_assertions") assertions_override_group.add_argument("--llvm-assertions", help="enable assertions in LLVM", action="store_const", const=True, dest="llvm_assertions") assertions_override_group.add_argument("--no-llvm-assertions", help="disable assertions in LLVM", action="store_const", const=False, dest="llvm_assertions") assertions_override_group.add_argument("--swift-assertions", help="enable assertions in Swift", action="store_const", const=True, dest="swift_assertions") assertions_override_group.add_argument("--no-swift-assertions", help="disable assertions in Swift", action="store_const", const=False, dest="swift_assertions") assertions_override_group.add_argument("--swift-stdlib-assertions", help="enable assertions in the Swift standard library", action="store_const", const=True, dest="swift_stdlib_assertions") assertions_override_group.add_argument("--no-swift-stdlib-assertions", help="disable assertions in the Swift standard library", action="store_const", const=False, dest="swift_stdlib_assertions") assertions_override_group.add_argument("--lldb-assertions", help="enable assertions in LLDB", action="store_const", const=True, dest="lldb_assertions") assertions_override_group.add_argument("--no-lldb-assertions", help="disable assertions in LLDB", action="store_const", const=False, dest="lldb_assertions") cmake_generator_group = parser.add_argument_group( title="Select the CMake generator") cmake_generator_group.add_argument("-x", "--xcode", help="use CMake's Xcode generator (default is Ninja)", action="store_const", const="Xcode", dest="cmake_generator") cmake_generator_group.add_argument("-X", "--xcode-ide-only", help="use CMake's Xcode generator, with only IDE support (no build)", action="store_const", const="XcodeIDEOnly", dest="cmake_generator") cmake_generator_group.add_argument("-m", "--make", help="use CMake's Makefile generator (default is Ninja)", action="store_const", const="Unix Makefiles", dest="cmake_generator") cmake_generator_group.add_argument("-e", "--eclipse", help="use CMake's Eclipse generator (default is Ninja)", action="store_const", const="Eclipse CDT4 - Ninja", dest="cmake_generator") run_tests_group = parser.add_argument_group( title="Run tests") run_tests_group.add_argument("-t", "--test", help="test Swift after building", action="store_true") run_tests_group.add_argument("-T", "--validation-test", help="run the validation test suite (implies --test)", action="store_true") parser.add_argument("-o", "--test-optimized", help="run the test suite in optimized mode too (implies --test)", action="store_true") run_build_group = parser.add_argument_group( title="Run build") run_build_group.add_argument("-S", "--skip-build", help="generate build directory only without building", action="store_true") parser.add_argument("-i", "--ios", help=""" also build for iOS, but disallow tests that require an iOS device""", action="store_true") parser.add_argument("--tvos", help=""" also build for tvOS, but disallow tests that require an tvos device""", action="store_true") parser.add_argument("--watchos", help=""" also build for Apple watchos, but disallow tests that require an watchOS device""", action="store_true") parser.add_argument("--build-subdir", help=""" name of the directory under $SWIFT_BUILD_ROOT where the build products will be placed""", metavar="PATH") parser.add_argument("-j", "--jobs", help=""" the number of parallel build jobs to use""", type=int, dest="build_jobs", default=multiprocessing.cpu_count()) parser.add_argument("--darwin-xcrun-toolchain", help="the name of the toolchain to use on Darwin", default="default") parser.add_argument("--cmake", help="the path to a CMake executable that will be " "used to build Swift") parser.add_argument("--extra-swift-args", help=textwrap.dedent(""" Pass through extra flags to swift in the form of a cmake list 'module_regexp;flag'. Can be called multiple times to add multiple such module_regexp flag pairs. All semicolons in flags must be escaped with a '\'"""), action="append", dest="extra_swift_args", default=[]) parser.add_argument("build_script_impl_args", help="", nargs="*") args = parser.parse_args(migrate_impl_args(sys.argv[1:], [ '--darwin-xcrun-toolchain', '--cmake', ])) # Build cmark if any cmark-related options were specified. if (args.cmark_build_variant is not None): args.build_cmark = True # Build LLDB if any LLDB-related options were specified. if (args.lldb_build_variant is not None or args.lldb_assertions is not None): args.build_lldb = True # Set the default build variant. if args.build_variant is None: args.build_variant = "Debug" if args.cmark_build_variant is None: args.cmark_build_variant = args.build_variant # Propagate the default build variant. if args.llvm_build_variant is None: args.llvm_build_variant = args.build_variant if args.swift_build_variant is None: args.swift_build_variant = args.build_variant if args.swift_stdlib_build_variant is None: args.swift_stdlib_build_variant = args.build_variant if args.lldb_build_variant is None: args.lldb_build_variant = args.build_variant if args.foundation_build_variant is None: args.foundation_build_variant = args.build_variant # Assertions are enabled by default. if args.assertions is None: args.assertions = True # Propagate the default assertions setting. if args.cmark_assertions is None: args.cmark_assertions = args.assertions if args.llvm_assertions is None: args.llvm_assertions = args.assertions if args.swift_assertions is None: args.swift_assertions = args.assertions if args.swift_stdlib_assertions is None: args.swift_stdlib_assertions = args.assertions # Set the default CMake generator. if args.cmake_generator is None: args.cmake_generator = "Ninja" # --validation-test implies --test. if args.validation_test: args.test = True # --test-optimized implies --test. if args.test_optimized: args.test = True build_script_impl_inferred_args = [] if args.export_compile_commands: build_script_impl_inferred_args += [ "--export-compile-commands" ] if args.cmake_generator == "XcodeIDEOnly": args.cmake_generator = "Xcode" build_script_impl_inferred_args += [ "--xcode-ide-only" ] if not args.test: build_script_impl_inferred_args += [ "--skip-test-cmark", "--skip-test-lldb", "--skip-test-swift", "--skip-test-llbuild", "--skip-test-swiftpm", "--skip-test-xctest", "--skip-test-foundation", "--skip-test-ios", "--skip-test-tvos", "--skip-test-watchos", ] if not args.validation_test: build_script_impl_inferred_args += [ "--skip-test-validation" ] if not args.test_optimized: build_script_impl_inferred_args += [ "--skip-test-optimized" ] if not args.ios: build_script_impl_inferred_args += [ "--skip-build-ios", "--skip-test-ios" ] if not args.tvos: build_script_impl_inferred_args += [ "--skip-build-tvos", "--skip-test-tvos", ] if not args.watchos: build_script_impl_inferred_args += [ "--skip-build-watchos", "--skip-test-watchos", ] if not args.build_lldb: build_script_impl_inferred_args += [ "--skip-build-lldb" ] if not args.build_llbuild: build_script_impl_inferred_args += [ "--skip-build-llbuild" ] if not args.build_swiftpm: build_script_impl_inferred_args += [ "--skip-build-swiftpm" ] if not args.build_xctest: build_script_impl_inferred_args += [ "--skip-build-xctest" ] if not args.build_foundation: build_script_impl_inferred_args += [ "--skip-build-foundation" ] if args.skip_build: build_script_impl_inferred_args += [ "--skip-build" ] if args.build_subdir is None: # Create a name for the build directory. args.build_subdir = args.cmake_generator.replace(" ", "_") cmark_build_dir_label = args.cmark_build_variant if args.cmark_assertions: cmark_build_dir_label += "Assert" llvm_build_dir_label = args.llvm_build_variant if args.llvm_assertions: llvm_build_dir_label += "Assert" swift_build_dir_label = args.swift_build_variant if args.swift_assertions: swift_build_dir_label += "Assert" swift_stdlib_build_dir_label = args.swift_stdlib_build_variant if args.swift_stdlib_assertions: swift_stdlib_build_dir_label += "Assert" # FIXME: mangle LLDB build configuration into the directory name. if (llvm_build_dir_label == swift_build_dir_label and llvm_build_dir_label == swift_stdlib_build_dir_label and llvm_build_dir_label == cmark_build_dir_label): # Use a simple directory name if all projects use the same build type. args.build_subdir += "-" + llvm_build_dir_label elif (llvm_build_dir_label != swift_build_dir_label and llvm_build_dir_label == swift_stdlib_build_dir_label and llvm_build_dir_label == cmark_build_dir_label): # Swift build type differs. args.build_subdir += "-" + llvm_build_dir_label args.build_subdir += "+swift-" + swift_build_dir_label elif (llvm_build_dir_label == swift_build_dir_label and llvm_build_dir_label != swift_stdlib_build_dir_label and llvm_build_dir_label == cmark_build_dir_label): # Swift stdlib build type differs. args.build_subdir += "-" + llvm_build_dir_label args.build_subdir += "+stdlib-" + swift_stdlib_build_dir_label elif (llvm_build_dir_label == swift_build_dir_label and llvm_build_dir_label == swift_stdlib_build_dir_label and llvm_build_dir_label != cmark_build_dir_label): # cmark build type differs. args.build_subdir += "-" + llvm_build_dir_label args.build_subdir += "+cmark-" + cmark_build_dir_label else: # We don't know how to create a short name, so just mangle in all # the information. args.build_subdir += "+cmark-" + cmark_build_dir_label args.build_subdir += "+llvm-" + llvm_build_dir_label args.build_subdir += "+swift-" + swift_build_dir_label args.build_subdir += "+stdlib-" + swift_stdlib_build_dir_label build_dir = os.path.join(SWIFT_BUILD_ROOT, args.build_subdir) if args.clean and os.path.isdir(build_dir): shutil.rmtree(build_dir) host_clang = swift_build_support.clang.host_clang( xcrun_toolchain=args.darwin_xcrun_toolchain) if not host_clang: print_with_argv0( "Can't find clang. Please install clang-3.5 or a later version.") return 1 host_cmake = args.cmake if not host_cmake: host_cmake = swift_build_support.cmake.host_cmake( args.darwin_xcrun_toolchain) if not host_cmake: print_with_argv0("Can't find CMake. Please install CMake.") return 1 build_script_impl_args = [ os.path.join(SWIFT_SOURCE_ROOT, "swift", "utils", "build-script-impl"), "--build-dir", build_dir, "--host-cc", host_clang.cc, "--host-cxx", host_clang.cxx, "--darwin-xcrun-toolchain", args.darwin_xcrun_toolchain, "--cmake", host_cmake, "--cmark-build-type", args.cmark_build_variant, "--llvm-build-type", args.llvm_build_variant, "--swift-build-type", args.swift_build_variant, "--swift-stdlib-build-type", args.swift_stdlib_build_variant, "--lldb-build-type", args.lldb_build_variant, "--llvm-enable-assertions", str(args.llvm_assertions).lower(), "--swift-enable-assertions", str(args.swift_assertions).lower(), "--swift-stdlib-enable-assertions", str(args.swift_stdlib_assertions).lower(), "--cmake-generator", args.cmake_generator, "--build-jobs", str(args.build_jobs), "--workspace", SWIFT_SOURCE_ROOT ] if args.build_foundation: build_script_impl_args += [ "--foundation-build-type", args.foundation_build_variant ] build_script_impl_args += build_script_impl_inferred_args # If we have extra_swift_args, combine all of them together and then add # them as one command. if args.extra_swift_args: build_script_impl_args += ["--extra-swift-args", ";".join(args.extra_swift_args)] build_script_impl_args += args.build_script_impl_args # Unset environment variables that might affect how tools behave. for v in [ 'MAKEFLAGS', 'SDKROOT', 'MACOSX_DEPLOYMENT_TARGET', 'IPHONEOS_DEPLOYMENT_TARGET' ]: os.environ.pop(v, None) check_call(build_script_impl_args) return 0 def main(): if not SWIFT_SOURCE_ROOT: print_with_argv0("Could not infer source root directory. " + "Forgot to set $SWIFT_SOURCE_ROOT environment variable?") return 1 if not os.path.isdir(SWIFT_SOURCE_ROOT): print_with_argv0("Source root directory \'" + SWIFT_SOURCE_ROOT + "\' does not exist. " + "Forgot to set $SWIFT_SOURCE_ROOT environment variable?") return 1 # Determine if we are invoked in the preset mode and dispatch accordingly. if any([ (opt.startswith("--preset") or opt == "--show-presets") for opt in sys.argv[1:] ]): return main_preset() else: return main_normal() if __name__ == "__main__": sys.exit(main())