#!/usr/bin/env python # utils/build-script - The ultimate tool for building Swift -*- python -*- # # This source file is part of the Swift.org open source project # # Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors # Licensed under Apache License v2.0 with Runtime Library Exception # # See https://swift.org/LICENSE.txt for license information # See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors from __future__ import print_function import argparse import os import pipes import platform import sys import time from build_swift import defaults from build_swift import driver_arguments from build_swift import presets from build_swift.migration import migrate_swift_sdks from swift_build_support.swift_build_support import ( arguments, debug, diagnostics, migration, products, shell, tar, targets, workspace ) from swift_build_support.swift_build_support.SwiftBuildSupport import ( HOME, SWIFT_BUILD_ROOT, SWIFT_REPO_NAME, SWIFT_SOURCE_ROOT, ) from swift_build_support.swift_build_support.cmake import CMake from swift_build_support.swift_build_support.targets import \ StdlibDeploymentTarget from swift_build_support.swift_build_support.toolchain import host_toolchain build_script_impl = os.path.join( SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME, "utils", "build-script-impl") def exit_rejecting_arguments(message, parser=None): print(message, file=sys.stderr) if parser: parser.print_usage(sys.stderr) sys.exit(2) # 2 is the same as `argparse` error exit code. def call_without_sleeping(command, env=None, dry_run=False, echo=False): """ Execute a command during which system sleep is disabled. By default, this ignores the state of the `shell.dry_run` flag. """ # Disable system sleep, if possible. if platform.system() == 'Darwin': # Don't mutate the caller's copy of the arguments. command = ["caffeinate"] + list(command) shell.call(command, env=env, dry_run=dry_run, echo=echo) class HostSpecificConfiguration(object): """Configuration information for an individual host.""" def __init__(self, host_target, invocation): """Initialize for the given `host_target`.""" # Compute the set of deployment targets to configure/build. args = invocation.args if host_target == args.host_target: # This host is the user's desired product, so honor the requested # set of targets to configure/build. stdlib_targets_to_configure = args.stdlib_deployment_targets if "all" in args.build_stdlib_deployment_targets: stdlib_targets_to_build = set(stdlib_targets_to_configure) else: stdlib_targets_to_build = set( args.build_stdlib_deployment_targets).intersection( set(args.stdlib_deployment_targets)) else: # Otherwise, this is a host we are building as part of # cross-compiling, so we only need the target itself. stdlib_targets_to_configure = [host_target] stdlib_targets_to_build = set(stdlib_targets_to_configure) # Compute the lists of **CMake** targets for each use case (configure # vs. build vs. run) and the SDKs to configure with. self.sdks_to_configure = set() self.swift_stdlib_build_targets = [] self.swift_test_run_targets = [] self.swift_benchmark_build_targets = [] self.swift_benchmark_run_targets = [] for deployment_target_name in stdlib_targets_to_configure: # Get the target object. deployment_target = StdlibDeploymentTarget.get_target_for_name( deployment_target_name) if deployment_target is None: diagnostics.fatal("unknown target: %r" % ( deployment_target_name,)) # Add the SDK to use. deployment_platform = deployment_target.platform self.sdks_to_configure.add(deployment_platform.sdk_name) # If we aren't actually building this target (only configuring # it), do nothing else. if deployment_target_name not in stdlib_targets_to_build: continue # Compute which actions are desired. build = ( deployment_platform not in invocation.platforms_to_skip_build) test = ( deployment_platform not in invocation.platforms_to_skip_test) test_host_only = None dt_supports_benchmark = deployment_target.supports_benchmark build_benchmarks = build and dt_supports_benchmark build_external_benchmarks = all([build, dt_supports_benchmark, args.build_external_benchmarks]) # FIXME: Note, `build-script-impl` computed a property here # w.r.t. testing, but it was actually unused. # For platforms which normally require a connected device to # test, the default behavior is to run tests that only require # the host (i.e., they do not attempt to execute). if deployment_platform.is_darwin and \ deployment_platform.is_embedded and \ not deployment_platform.is_simulator: if deployment_platform not in \ invocation.platforms_to_skip_test_host: test_host_only = True test = True else: test = False name = deployment_target.name for skip_test_arch in invocation.platforms_archs_to_skip_test: if deployment_target.name == skip_test_arch.name: test = False if build: # Validation, long, and stress tests require building the full # standard library, whereas the other targets can build a # slightly smaller subset which is faster to build. if args.build_swift_stdlib_unittest_extra or \ args.validation_test or args.long_test or \ args.stress_test: self.swift_stdlib_build_targets.append( "swift-stdlib-" + name) else: self.swift_stdlib_build_targets.append( "swift-test-stdlib-" + name) if build_benchmarks: self.swift_benchmark_build_targets.append( "swift-benchmark-" + name) if args.benchmark: self.swift_benchmark_run_targets.append( "check-swift-benchmark-" + name) if build_external_benchmarks: # Add support for the external benchmarks. self.swift_benchmark_build_targets.append( "swift-benchmark-{}-external".format(name)) if args.benchmark: self.swift_benchmark_run_targets.append( "check-swift-benchmark-{}-external".format(name)) if test: if test_host_only: suffix = "-non-executable" else: suffix = "" subset_suffix = "" if args.validation_test and args.long_test and \ args.stress_test: subset_suffix = "-all" elif args.validation_test: subset_suffix = "-validation" elif args.long_test: subset_suffix = "-only_long" elif args.stress_test: subset_suffix = "-only_stress" else: subset_suffix = "" self.swift_test_run_targets.append("check-swift{}{}-{}".format( subset_suffix, suffix, name)) if args.test_optimized and not test_host_only: self.swift_test_run_targets.append( "check-swift{}-optimize-{}".format( subset_suffix, name)) if args.test_optimize_for_size and not test_host_only: self.swift_test_run_targets.append( "check-swift{}-optimize_size-{}".format( subset_suffix, name)) class BuildScriptInvocation(object): """Represent a single build script invocation.""" @staticmethod def validate_arguments(toolchain, args): if toolchain.cc is None or toolchain.cxx is None: diagnostics.fatal( "can't find clang (please install clang-3.5 or a " "later version)") if toolchain.cmake is None: diagnostics.fatal("can't find CMake (please install CMake)") if args.distcc: if toolchain.distcc is None: diagnostics.fatal( "can't find distcc (please install distcc)") if toolchain.distcc_pump is None: diagnostics.fatal( "can't find distcc-pump (please install distcc-pump)") if args.host_target is None or args.stdlib_deployment_targets is None: diagnostics.fatal("unknown operating system") if args.symbols_package: if not os.path.isabs(args.symbols_package): print( '--symbols-package must be an absolute path ' '(was \'{}\')'.format(args.symbols_package)) return 1 if not args.install_symroot: diagnostics.fatal( "--install-symroot is required when specifying " "--symbols-package.") if args.android: if args.android_ndk is None or \ args.android_api_level is None or \ args.android_icu_uc is None or \ args.android_icu_uc_include is None or \ args.android_icu_i18n is None or \ args.android_icu_i18n_include is None: diagnostics.fatal( "when building for Android, --android-ndk, " "--android-ndk-version, --android-icu-uc, " "--android-icu-uc-include, --android-icu-i18n, " "and --android-icu-i18n-include must be specified") @staticmethod def apply_default_arguments(toolchain, args): # Infer if ninja is required ninja_required = ( args.cmake_generator == 'Ninja' or args.build_foundation) if ninja_required and toolchain.ninja is None: args.build_ninja = True # Set the default stdlib-deployment-targets, if none were provided. if args.stdlib_deployment_targets is None: stdlib_targets = \ StdlibDeploymentTarget.default_stdlib_deployment_targets() args.stdlib_deployment_targets = [ target.name for target in stdlib_targets] # SwiftPM and XCTest have a dependency on Foundation. # On OS X, Foundation is built automatically using xcodebuild. # On Linux, we must ensure that it is built manually. if ((args.build_swiftpm or args.build_xctest) and platform.system() != "Darwin"): args.build_foundation = True # Foundation has a dependency on libdispatch. # On OS X, libdispatch is provided by the OS. # On Linux, we must ensure that it is built manually. if (args.build_foundation and platform.system() != "Darwin"): args.build_libdispatch = True if args.build_subdir is None: args.build_subdir = \ workspace.compute_build_subdir(args) # Add optional stdlib-deployment-targets if args.android: args.stdlib_deployment_targets.append( StdlibDeploymentTarget.Android.armv7.name) # Infer platform flags from manually-specified configure targets. # This doesn't apply to Darwin platforms, as they are # already configured. No building without the platform flag, though. android_tgts = [tgt for tgt in args.stdlib_deployment_targets if StdlibDeploymentTarget.Android.contains(tgt)] if not args.android and len(android_tgts) > 0: args.android = True args.build_android = False # --- def __init__(self, toolchain, args): self.toolchain = toolchain self.args = args self.workspace = workspace.Workspace( source_root=SWIFT_SOURCE_ROOT, build_root=os.path.join(SWIFT_BUILD_ROOT, args.build_subdir)) # Compute derived information from the arguments. # # FIXME: We should move the platform-derived arguments to be entirely # data driven, so that we can eliminate this code duplication and just # iterate over all supported platforms. self.platforms_to_skip_build = set() if not args.build_linux: self.platforms_to_skip_build.add(StdlibDeploymentTarget.Linux) if not args.build_freebsd: self.platforms_to_skip_build.add(StdlibDeploymentTarget.FreeBSD) if not args.build_cygwin: self.platforms_to_skip_build.add(StdlibDeploymentTarget.Cygwin) if not args.build_osx: self.platforms_to_skip_build.add(StdlibDeploymentTarget.OSX) if not args.build_ios_device: self.platforms_to_skip_build.add(StdlibDeploymentTarget.iOS) if not args.build_ios_simulator: self.platforms_to_skip_build.add( StdlibDeploymentTarget.iOSSimulator) if not args.build_tvos_device: self.platforms_to_skip_build.add(StdlibDeploymentTarget.AppleTV) if not args.build_tvos_simulator: self.platforms_to_skip_build.add( StdlibDeploymentTarget.AppleTVSimulator) if not args.build_watchos_device: self.platforms_to_skip_build.add(StdlibDeploymentTarget.AppleWatch) if not args.build_watchos_simulator: self.platforms_to_skip_build.add( StdlibDeploymentTarget.AppleWatchSimulator) if not args.build_android: self.platforms_to_skip_build.add(StdlibDeploymentTarget.Android) self.platforms_to_skip_test = set() self.platforms_archs_to_skip_test = set() if not args.test_linux: self.platforms_to_skip_test.add(StdlibDeploymentTarget.Linux) if not args.test_freebsd: self.platforms_to_skip_test.add(StdlibDeploymentTarget.FreeBSD) if not args.test_cygwin: self.platforms_to_skip_test.add(StdlibDeploymentTarget.Cygwin) if not args.test_osx: self.platforms_to_skip_test.add(StdlibDeploymentTarget.OSX) if not args.test_ios_host: self.platforms_to_skip_test.add(StdlibDeploymentTarget.iOS) else: exit_rejecting_arguments("error: iOS device tests are not " + "supported in open-source Swift.") if not args.test_ios_simulator: self.platforms_to_skip_test.add( StdlibDeploymentTarget.iOSSimulator) if not args.test_ios_32bit_simulator: self.platforms_archs_to_skip_test.add( StdlibDeploymentTarget.iOSSimulator.i386) if not args.test_tvos_host: self.platforms_to_skip_test.add(StdlibDeploymentTarget.AppleTV) else: exit_rejecting_arguments("error: tvOS device tests are not " + "supported in open-source Swift.") if not args.test_tvos_simulator: self.platforms_to_skip_test.add( StdlibDeploymentTarget.AppleTVSimulator) if not args.test_watchos_host: self.platforms_to_skip_test.add(StdlibDeploymentTarget.AppleWatch) else: exit_rejecting_arguments("error: watchOS device tests are not " + "supported in open-source Swift.") if not args.test_watchos_simulator: self.platforms_to_skip_test.add( StdlibDeploymentTarget.AppleWatchSimulator) if not args.test_android_host: self.platforms_to_skip_test.add(StdlibDeploymentTarget.Android) self.platforms_to_skip_test_host = set() if not args.test_ios_host: self.platforms_to_skip_test_host.add(StdlibDeploymentTarget.iOS) if not args.test_tvos_host: self.platforms_to_skip_test_host.add( StdlibDeploymentTarget.AppleTV) if not args.test_watchos_host: self.platforms_to_skip_test_host.add( StdlibDeploymentTarget.AppleWatch) def initialize_runtime_environment(self): """Change the program environment for building.""" # Set an appropriate default umask. os.umask(0o022) # Unset environment variables that might affect how tools behave. for v in [ 'MAKEFLAGS', 'SDKROOT', 'MACOSX_DEPLOYMENT_TARGET', 'IPHONEOS_DEPLOYMENT_TARGET', 'TVOS_DEPLOYMENT_TARGET', 'WATCHOS_DEPLOYMENT_TARGET']: os.environ.pop(v, None) def build_ninja(self): if not os.path.exists(self.workspace.source_dir("ninja")): diagnostics.fatal( "can't find source directory for ninja " "(tried %s)" % (self.workspace.source_dir("ninja"))) ninja_build = products.Ninja( args=self.args, toolchain=self.toolchain, source_dir=self.workspace.source_dir("ninja"), build_dir=self.workspace.build_dir("build", "ninja")) ninja_build.do_build() self.toolchain.ninja = ninja_build.ninja_bin_path def convert_to_impl_arguments(self): """convert_to_impl_arguments() -> (env, args) Convert the invocation to an environment and list of arguments suitable for invoking `build-script-impl`. """ # Create local shadows, for convenience. args = self.args toolchain = self.toolchain cmake = CMake(args=args, toolchain=self.toolchain) impl_args = [ "--workspace", self.workspace.source_root, "--build-dir", self.workspace.build_root, "--install-prefix", args.install_prefix, "--host-target", args.host_target, "--stdlib-deployment-targets", " ".join(args.stdlib_deployment_targets), "--host-cc", toolchain.cc, "--host-cxx", toolchain.cxx, "--darwin-xcrun-toolchain", args.darwin_xcrun_toolchain, "--darwin-deployment-version-osx=%s" % ( args.darwin_deployment_version_osx), "--darwin-deployment-version-ios=%s" % ( args.darwin_deployment_version_ios), "--darwin-deployment-version-tvos=%s" % ( args.darwin_deployment_version_tvos), "--darwin-deployment-version-watchos=%s" % ( args.darwin_deployment_version_watchos), "--cmake", toolchain.cmake, "--cmark-build-type", args.cmark_build_variant, "--llvm-build-type", args.llvm_build_variant, "--swift-build-type", args.swift_build_variant, "--swift-stdlib-build-type", args.swift_stdlib_build_variant, "--lldb-build-type", args.lldb_build_variant, "--lldb-build-with-xcode", args.lldb_build_with_xcode, "--foundation-build-type", args.foundation_build_variant, "--libdispatch-build-type", args.libdispatch_build_variant, "--libicu-build-type", args.libicu_build_variant, "--xctest-build-type", args.build_variant, "--swiftpm-build-type", args.build_variant, "--swiftsyntax-build-type", args.build_variant, "--skstresstester-build-type", args.build_variant, "--llbuild-build-type", args.build_variant, "--swift-enable-assertions", str(args.swift_assertions).lower(), "--swift-stdlib-enable-assertions", str( args.swift_stdlib_assertions).lower(), "--swift-analyze-code-coverage", str( args.swift_analyze_code_coverage).lower(), "--llbuild-enable-assertions", str( args.llbuild_assertions).lower(), "--lldb-assertions", str( args.lldb_assertions).lower(), "--cmake-generator", args.cmake_generator, "--build-jobs", str(args.build_jobs), "--common-cmake-options=%s" % ' '.join( pipes.quote(opt) for opt in cmake.common_options()), "--build-args=%s" % ' '.join( pipes.quote(arg) for arg in cmake.build_args()), ] # Compute any product specific cmake arguments. for product_class in self.compute_product_classes(): product_name = product_class.product_name() product_source_name = product_class.product_source_name() source_dir = self.workspace.source_dir(product_source_name) if not os.path.exists(source_dir): diagnostics.fatal( "can't find source directory for %s " "(tried %s)" % (product_name, source_dir)) product = product_class( args=args, toolchain=self.toolchain, source_dir=source_dir, # FIXME: This is incorrect since it always assumes the host # target I think? build_dir=self.workspace.build_dir( args.host_target, product_name)) cmake_opts = product.cmake_options # FIXME: We should be using pipes.quote here but we run into issues # with build-script-impl/cmake not being happy with all of the # extra "'" in the strings. To fix this easily, we really need to # just invoke cmake from build-script directly rather than futzing # with build-script-impl. This makes even more sense since there # really isn't a security issue here. impl_args += [ "--%s-cmake-options=%s" % (product_name, ' '.join(cmake_opts)) ] if args.build_stdlib_deployment_targets: impl_args += [ "--build-stdlib-deployment-targets", " ".join( args.build_stdlib_deployment_targets)] if args.cross_compile_hosts: impl_args += [ "--cross-compile-hosts", " ".join(args.cross_compile_hosts)] if args.test_paths: impl_args += ["--test-paths", " ".join(args.test_paths)] if toolchain.ninja: impl_args += ["--ninja-bin=%s" % toolchain.ninja] if args.distcc: impl_args += [ "--distcc", "--distcc-pump=%s" % toolchain.distcc_pump ] # *NOTE* We use normal cmake to pass through tsan/ubsan options. We do # NOT pass them to build-script-impl. if args.enable_asan: impl_args += ["--enable-asan"] # If we are on linux, disable leak detection when running ASAN. We # have a separate bot that checks for leaks. if platform.system() == 'Linux': os.environ['ASAN_OPTIONS'] = 'detect_leaks=0' if args.enable_ubsan: impl_args += ["--enable-ubsan"] # If we have lsan, we need to export our suppression list. The actual # passing in of the LSAN flag is done via the normal cmake method. We # do not pass the flag to build-script-impl. if args.enable_lsan: supp_file = os.path.join(SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME, "utils", "lsan_leaks_suppression_list.txt") os.environ['LSAN_OPTIONS'] = 'suppressions={}'.format(supp_file) if args.verbose_build: impl_args += ["--verbose-build"] if args.install_symroot: impl_args += [ "--install-symroot", os.path.abspath(args.install_symroot) ] if args.skip_build: impl_args += ["--skip-build-cmark", "--skip-build-llvm", "--skip-build-swift"] if not args.build_benchmarks: impl_args += ["--skip-build-benchmarks"] # Currently we do not build external benchmarks by default. if args.build_external_benchmarks: impl_args += ["--skip-build-external-benchmarks=0"] if not args.build_foundation: impl_args += ["--skip-build-foundation"] if not args.build_xctest: impl_args += ["--skip-build-xctest"] if not args.build_lldb: impl_args += ["--skip-build-lldb"] if not args.build_llbuild: impl_args += ["--skip-build-llbuild"] if not args.build_libcxx: impl_args += ["--skip-build-libcxx"] if not args.build_libdispatch: impl_args += ["--skip-build-libdispatch"] if not args.build_libicu: impl_args += ["--skip-build-libicu"] if not args.build_swiftpm: impl_args += ["--skip-build-swiftpm"] if not args.build_swiftsyntax: impl_args += ["--skip-build-swiftsyntax"] if not args.build_skstresstester: impl_args += ["--skip-build-skstresstester"] if not args.build_playgroundsupport: impl_args += ["--skip-build-playgroundsupport"] if args.build_swift_dynamic_stdlib: impl_args += ["--build-swift-dynamic-stdlib"] if args.build_swift_static_stdlib: impl_args += ["--build-swift-static-stdlib"] if args.build_swift_stdlib_unittest_extra: impl_args += ["--build-swift-stdlib-unittest-extra"] if args.build_swift_dynamic_sdk_overlay: impl_args += ["--build-swift-dynamic-sdk-overlay"] if args.build_swift_static_sdk_overlay: impl_args += ["--build-swift-static-sdk-overlay"] if not args.build_linux: impl_args += ["--skip-build-linux"] if not args.build_freebsd: impl_args += ["--skip-build-freebsd"] if not args.build_cygwin: impl_args += ["--skip-build-cygwin"] if not args.build_osx: impl_args += ["--skip-build-osx"] if not args.build_ios_device: impl_args += ["--skip-build-ios-device"] if not args.build_ios_simulator: impl_args += ["--skip-build-ios-simulator"] if not args.build_tvos_device: impl_args += ["--skip-build-tvos-device"] if not args.build_tvos_simulator: impl_args += ["--skip-build-tvos-simulator"] if not args.build_watchos_device: impl_args += ["--skip-build-watchos-device"] if not args.build_watchos_simulator: impl_args += ["--skip-build-watchos-simulator"] if not args.build_android: impl_args += ["--skip-build-android"] if not args.test and not args.long_test and not args.stress_test: impl_args += ["--skip-test-swift"] if not args.test: impl_args += ["--skip-test-cmark", "--skip-test-lldb", "--skip-test-llbuild", "--skip-test-swiftpm", "--skip-test-swiftsyntax", "--skip-test-skstresstester", "--skip-test-xctest", "--skip-test-foundation", "--skip-test-libdispatch", "--skip-test-libicu", "--skip-test-playgroundsupport"] if not args.test_linux: impl_args += ["--skip-test-linux"] if not args.test_freebsd: impl_args += ["--skip-test-freebsd"] if not args.test_cygwin: impl_args += ["--skip-test-cygwin"] if not args.test_osx: impl_args += ["--skip-test-osx"] if not args.test_ios_host: impl_args += ["--skip-test-ios-host"] if not args.test_ios_simulator: impl_args += ["--skip-test-ios-simulator"] if not args.test_ios_32bit_simulator: impl_args += ["--skip-test-ios-32bit-simulator"] if not args.test_tvos_host: impl_args += ["--skip-test-tvos-host"] if not args.test_tvos_simulator: impl_args += ["--skip-test-tvos-simulator"] if not args.test_watchos_host: impl_args += ["--skip-test-watchos-host"] if not args.test_watchos_simulator: impl_args += ["--skip-test-watchos-simulator"] if not args.test_android_host: impl_args += ["--skip-test-android-host"] if args.build_runtime_with_host_compiler: impl_args += ["--build-runtime-with-host-compiler"] if args.validation_test: impl_args += ["--validation-test"] if args.long_test: impl_args += ["--long-test"] if args.stress_test: impl_args += ["--stress-test"] if not args.benchmark: impl_args += ["--skip-test-benchmarks"] if not args.test_optimized: impl_args += ["--skip-test-optimized"] if not args.test_optimize_for_size: impl_args += ["--skip-test-optimize-for-size"] if args.android: impl_args += [ "--android-ndk", args.android_ndk, "--android-api-level", args.android_api_level, "--android-ndk-gcc-version", args.android_ndk_gcc_version, "--android-icu-uc", args.android_icu_uc, "--android-icu-uc-include", args.android_icu_uc_include, "--android-icu-i18n", args.android_icu_i18n, "--android-icu-i18n-include", args.android_icu_i18n_include, ] if args.android_deploy_device_path: impl_args += [ "--android-deploy-device-path", args.android_deploy_device_path, ] if platform.system() == 'Darwin': impl_args += [ "--toolchain-prefix", targets.darwin_toolchain_prefix( args.install_prefix), "--host-lipo", toolchain.lipo, ] if toolchain.libtool is not None: impl_args += [ "--host-libtool", toolchain.libtool, ] # If we have extra_swift_args, combine all of them together and then # add them as one command. if args.extra_swift_args: impl_args += [ "--extra-swift-args", ";".join(args.extra_swift_args)] # If we have extra_cmake_options, combine all of them together and then # add them as one command. if args.extra_cmake_options: impl_args += [ "--extra-cmake-options=%s" % ' '.join( pipes.quote(opt) for opt in args.extra_cmake_options) ] if args.lto_type is not None: impl_args += [ "--llvm-enable-lto=%s" % args.lto_type, "--swift-tools-enable-lto=%s" % args.lto_type ] if args.llvm_max_parallel_lto_link_jobs is not None: impl_args += [ "--llvm-num-parallel-lto-link-jobs=%s" % min(args.llvm_max_parallel_lto_link_jobs, args.build_jobs) ] if args.swift_tools_max_parallel_lto_link_jobs is not None: impl_args += [ "--swift-tools-num-parallel-lto-link-jobs=%s" % min(args.swift_tools_max_parallel_lto_link_jobs, args.build_jobs) ] impl_args += args.build_script_impl_args if args.dry_run: impl_args += ["--dry-run"] if args.clang_profile_instr_use: impl_args += [ "--clang-profile-instr-use=%s" % os.path.abspath(args.clang_profile_instr_use) ] if args.lit_args: impl_args += ["--llvm-lit-args=%s" % args.lit_args] if args.coverage_db: impl_args += [ "--coverage-db=%s" % os.path.abspath(args.coverage_db) ] # Compute the set of host-specific variables, which we pass through to # the build script via environment variables. host_specific_variables = self.compute_host_specific_variables() impl_env = {} for (host_target, options) in host_specific_variables.items(): for (name, value) in options.items(): # We mangle into an environment variable we can easily evaluate # from the `build-script-impl`. impl_env["HOST_VARIABLE_{}__{}".format( host_target.replace("-", "_"), name)] = value return (impl_env, impl_args) def compute_host_specific_variables(self): """compute_host_specific_variables(args) -> dict Compute the host-specific options, organized as a dictionary keyed by host of options. """ args = self.args options = {} for host_target in [args.host_target] + args.cross_compile_hosts: # Compute the host specific configuration. config = HostSpecificConfiguration(host_target, self) # Convert into `build-script-impl` style variables. options[host_target] = { "SWIFT_SDKS": " ".join(sorted( config.sdks_to_configure)), "SWIFT_STDLIB_TARGETS": " ".join( config.swift_stdlib_build_targets), "SWIFT_BENCHMARK_TARGETS": " ".join( config.swift_benchmark_build_targets), "SWIFT_RUN_BENCHMARK_TARGETS": " ".join( config.swift_benchmark_run_targets), "SWIFT_TEST_TARGETS": " ".join( config.swift_test_run_targets), } return options def compute_product_classes(self): """compute_product_classes() -> list Compute the list of all Product classes used in this build. This list is constructed in dependency order. """ # FIXME: This is a weird division (returning a list of class objects), # but it matches the existing structure of the `build-script-impl`. product_classes = [] product_classes.append(products.CMark) product_classes.append(products.LLVM) if self.args.build_libcxx: product_classes.append(products.LibCXX) if self.args.build_libicu: product_classes.append(products.LibICU) product_classes.append(products.Swift) if self.args.build_lldb: product_classes.append(products.LLDB) if self.args.build_llbuild: product_classes.append(products.LLBuild) if self.args.build_libdispatch: product_classes.append(products.LibDispatch) if self.args.build_foundation: product_classes.append(products.Foundation) if self.args.build_xctest: product_classes.append(products.XCTest) if self.args.build_swiftpm: product_classes.append(products.SwiftPM) return product_classes def execute(self): """Execute the invocation with the configured arguments.""" # Convert to a build-script-impl invocation. (impl_env, impl_args) = self.convert_to_impl_arguments() # If using the legacy implementation, delegate all behavior to # `build-script-impl`. if self.args.legacy_impl: # Execute the underlying build script implementation. call_without_sleeping([build_script_impl] + impl_args, env=impl_env, echo=True) return # Otherwise, we compute and execute the individual actions ourselves. def execute_one_impl_action(host=None, product_class=None, name=None): if host is None: assert (product_class is None and name == "merged-hosts-lipo"), "invalid action" action_name = name elif product_class is None: assert name == "package", "invalid action" action_name = "{}-{}".format(host.name, name) else: assert name is not None, "invalid action" action_name = "{}-{}-{}".format( host.name, product_class.product_name(), name) call_without_sleeping( [build_script_impl] + impl_args + [ "--only-execute", action_name], env=impl_env, echo=self.args.verbose_build) # Compute the list of hosts to operate on. all_host_names = [ self.args.host_target] + self.args.cross_compile_hosts all_hosts = [StdlibDeploymentTarget.get_target_for_name(name) for name in all_host_names] # Compute the list of product classes to operate on. # # FIXME: This should really be per-host, but the current structure # matches that of `build-script-impl`. product_classes = self.compute_product_classes() # Execute each "pass". # Build... for host_target in all_hosts: # FIXME: We should only compute these once. config = HostSpecificConfiguration(host_target.name, self) print("Building the standard library for: {}".format( " ".join(config.swift_stdlib_build_targets))) if config.swift_test_run_targets and ( self.args.test or self.args.long_test): print("Running Swift tests for: {}".format( " ".join(config.swift_test_run_targets))) if config.swift_benchmark_run_targets and self.args.benchmark: print("Running Swift benchmarks for: {}".format( " ".join(config.swift_benchmark_run_targets))) for product_class in product_classes: execute_one_impl_action(host_target, product_class, "build") # Test... for host_target in all_hosts: for product_class in product_classes: execute_one_impl_action(host_target, product_class, "test") # Install... for host_target in all_hosts: for product_class in product_classes: execute_one_impl_action(host_target, product_class, "install") # Package... for host_target in all_hosts: execute_one_impl_action(host_target, name="package") # Lipo... execute_one_impl_action(name="merged-hosts-lipo") # Provide a short delay so accidentally invoked clean builds can be canceled. def clean_delay(): sys.stdout.write('Starting clean build in ') for i in range(3, 0, -1): sys.stdout.write('\b%d' % i) sys.stdout.flush() time.sleep(1) print('\b\b\b\bnow.') # Main entry point for the preset mode. def main_preset(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description="""Builds Swift using a preset.""") parser.add_argument( "-n", "--dry-run", help="print the commands that would be executed, but do not execute " "them", action="store_true", default=False) parser.add_argument( "--preset-file", help="load presets from the specified file", metavar="PATH", action="append", dest="preset_file_names", default=[]) parser.add_argument( "--preset", help="use the specified option preset", metavar="NAME") parser.add_argument( "--show-presets", help="list all presets and exit", action=arguments.action.optional_bool) parser.add_argument( "--distcc", help="use distcc", action=arguments.action.optional_bool) parser.add_argument( "-j", "--jobs", help="the number of parallel build jobs to use", type=int, dest="build_jobs") parser.add_argument( "preset_substitutions_raw", help="'name=value' pairs that are substituted in the preset", nargs="*", metavar="SUBSTITUTION") parser.add_argument( "--expand-build-script-invocation", help="Print the expanded build-script invocation generated " "by the preset, but do not run the preset", action=arguments.action.optional_bool) args = parser.parse_args() if len(args.preset_file_names) == 0: args.preset_file_names = [ os.path.join( SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME, "utils", "build-presets.ini") ] user_presets_file = os.path.join(HOME, '.swift-build-presets') if os.path.isfile(user_presets_file): args.preset_file_names.append(user_presets_file) preset_parser = presets.PresetParser() try: preset_parser.read(args.preset_file_names) except presets.Error as e: diagnostics.fatal(e.message) if args.show_presets: for name in sorted(preset_parser.preset_names(), key=lambda name: name.lower()): print(name) return 0 if not args.preset: diagnostics.fatal("missing --preset option") args.preset_substitutions = {} for arg in args.preset_substitutions_raw: name, value = arg.split("=", 1) args.preset_substitutions[name] = value try: preset = preset_parser.get_preset(args.preset, vars=args.preset_substitutions) except presets.Error as e: diagnostics.fatal(e.message) preset_args = migrate_swift_sdks(preset.format_args()) build_script_args = [sys.argv[0]] if args.dry_run: build_script_args += ["--dry-run"] build_script_args += preset_args if args.distcc: build_script_args += ["--distcc"] if args.build_jobs: build_script_args += ["--jobs", str(args.build_jobs)] diagnostics.note('using preset "{}", which expands to \n\n{}\n'.format( args.preset, shell.quote_command(build_script_args))) if args.expand_build_script_invocation: return 0 call_without_sleeping(build_script_args) return 0 # Main entry point for the normal mode. def main_normal(): parser = driver_arguments.create_argument_parser() args = migration.parse_args(parser, sys.argv[1:]) if args.build_script_impl_args: # If we received any impl args, check if `build-script-impl` would # accept them or not before any further processing. try: migration.check_impl_args(build_script_impl, args.build_script_impl_args) except ValueError as e: exit_rejecting_arguments(e, parser) if '--check-args-only' in args.build_script_impl_args: return 0 shell.dry_run = args.dry_run # Prepare and validate toolchain if args.darwin_xcrun_toolchain is None: xcrun_toolchain = os.environ.get('TOOLCHAINS', defaults.DARWIN_XCRUN_TOOLCHAIN) diagnostics.note('Using toolchain {}'.format(xcrun_toolchain)) args.darwin_xcrun_toolchain = xcrun_toolchain toolchain = host_toolchain(xcrun_toolchain=args.darwin_xcrun_toolchain) os.environ['TOOLCHAINS'] = args.darwin_xcrun_toolchain if args.host_cc is not None: toolchain.cc = args.host_cc if args.host_cxx is not None: toolchain.cxx = args.host_cxx if args.host_lipo is not None: toolchain.lipo = args.host_lipo if args.host_libtool is not None: toolchain.libtool = args.host_libtool if args.cmake is not None: toolchain.cmake = args.cmake # Preprocess the arguments to apply defaults. BuildScriptInvocation.apply_default_arguments(toolchain, args) # Validate the arguments. BuildScriptInvocation.validate_arguments(toolchain, args) # Create the build script invocation. invocation = BuildScriptInvocation(toolchain, args) # Sanitize the runtime environment. invocation.initialize_runtime_environment() # Show SDKs, if requested. if args.show_sdks: debug.print_xcodebuild_versions() # Clean build directory if requested. if args.clean: clean_delay() shell.rmtree(invocation.workspace.build_root) # Create build directory. shell.makedirs(invocation.workspace.build_root) # Build ninja if required, which will update the toolchain. if args.build_ninja: invocation.build_ninja() # Execute the underlying build script implementation. invocation.execute() if args.symbols_package: print('--- Creating symbols package ---') print('-- Package file: {} --'.format(args.symbols_package)) if platform.system() == 'Darwin': prefix = targets.darwin_toolchain_prefix(args.install_prefix) else: prefix = args.install_prefix # As a security measure, `tar` normally strips leading '/' from paths # it is archiving. To stay safe, we change working directories, then # run `tar` without the leading '/' (we remove it ourselves to keep # `tar` from emitting a warning). with shell.pushd(args.install_symroot): tar.tar(source=prefix.lstrip('/'), destination=args.symbols_package) return 0 def main(): if not SWIFT_SOURCE_ROOT: diagnostics.fatal( "could not infer source root directory " + "(forgot to set $SWIFT_SOURCE_ROOT environment variable?)") if not os.path.isdir(SWIFT_SOURCE_ROOT): diagnostics.fatal( "source root directory \'" + SWIFT_SOURCE_ROOT + "\' does not exist " + "(forgot to set $SWIFT_SOURCE_ROOT environment variable?)") # Determine if we are invoked in the preset mode and dispatch accordingly. if any([(opt.startswith("--preset") or opt == "--show-presets") for opt in sys.argv[1:]]): return main_preset() else: return main_normal() if __name__ == "__main__": try: sys.exit(main()) except KeyboardInterrupt: sys.exit(1)