From 924917891fa06750ac048cc3b8b066ad84cdde9b Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Mon, 24 Jun 2024 14:38:22 +0900 Subject: [PATCH] [macOS] Generate universal gen_snapshots This reverts commit ac4c31ac97552681ce7405e60358f755b514b653. (#52913) This relands commit 4e33c102e56e9b3e9898c93c06373066c7eefc92. (#52885) Previously, the `gen_snapshot_arm64` and `gen_snapshot_x64` binaries used by the tool were all built for x64 architecture. As such, developers building apps with Flutter rely on Rosetta translation with every build. This refactors the gen_snapshot build rules on macOS hosts to consistently produce `gen_snapshot_arm64` and `gen_snapshot_x64` binaries with the target architecture of the build but with as universal binaries with both host architectures. Prior to this patch we emitted: * gen_snapshot_arm64 (arch: x64, target_arch: simarm64) After this patch, we emit: * artifacts_x64/gen_snapshot_arm64 (arch: x64, target_arch: simarm64) * artifacts_arm64/gen_snapshot_arm64 (arch: arm64, target_arch: arm64) * gen_snapshot_arm64 (universal binary composed of both of the above) Prior to this patch we emitted: * gen_snapshot_x64 (arch: x64, target_arch: x64) After this patch, we emit: * artifacts_x64/gen_snapshot_x64 (arch: x64, target_arch: x64) * artifacts_arm64/gen_snapshot_x64 (arch: arm64, target_arch: simx64) * gen_snapshot_x64 (universal binary composed of both of the above) Note that host builds on macOS currently default to a host architecture of x64 (can be overridden via `--force-mac-arm64`) regardless of host architecture and thus, the build itself relies on Rosetta translation when invoked on Apple Silicon arm64 hardware. This is to ensure a consistent build in CI regardless of bot architecture. See: https://github.com/flutter/engine/blob/6fa734d686888a39add026a2a98d6ec311c23efb/tools/gn#L502-L505 Issue: https://github.com/flutter/flutter/issues/101138 Issue: https://github.com/flutter/flutter/issues/69157 Related issue: https://github.com/flutter/flutter/issues/103386 --- build/archives/BUILD.gn | 4 +-- ci/licenses_golden/excluded_files | 1 + lib/snapshot/BUILD.gn | 54 ++++++++++++++++++++++++++----- sky/tools/create_macos_binary.py | 54 +++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 10 deletions(-) create mode 100755 sky/tools/create_macos_binary.py diff --git a/build/archives/BUILD.gn b/build/archives/BUILD.gn index 899a031829085..0e327b4cf763a 100644 --- a/build/archives/BUILD.gn +++ b/build/archives/BUILD.gn @@ -283,8 +283,8 @@ if (is_mac) { output = "$full_platform_name$suffix/gen_snapshot.zip" files = [ { - source = "$root_out_dir/gen_snapshot_$target_cpu" - destination = "gen_snapshot_$target_cpu" + source = "${root_out_dir}/gen_snapshot_${target_cpu}" + destination = "gen_snapshot_${target_cpu}" }, ] } diff --git a/ci/licenses_golden/excluded_files b/ci/licenses_golden/excluded_files index 37a5f894c5cac..859cf03ca6fa3 100644 --- a/ci/licenses_golden/excluded_files +++ b/ci/licenses_golden/excluded_files @@ -424,6 +424,7 @@ ../../../flutter/sky/tools/create_embedder_framework.py ../../../flutter/sky/tools/create_full_ios_framework.py ../../../flutter/sky/tools/create_ios_framework.py +../../../flutter/sky/tools/create_macos_binary.py ../../../flutter/sky/tools/create_macos_framework.py ../../../flutter/sky/tools/create_macos_gen_snapshots.py ../../../flutter/sky/tools/create_xcframework.py diff --git a/lib/snapshot/BUILD.gn b/lib/snapshot/BUILD.gn index 8f9dd526fb669..e048a33b45cb4 100644 --- a/lib/snapshot/BUILD.gn +++ b/lib/snapshot/BUILD.gn @@ -186,16 +186,54 @@ if (host_os == "mac" && target_os != "mac" && # # This target is used for builds targeting macOS. if (host_os == "mac" && target_os == "mac") { - copy("create_macos_gen_snapshots") { - # The toolchain-specific output directory. For cross-compiles, this is a - # clang-x64 or clang-arm64 subdirectory of the top-level build directory. - host_output_dir = - get_label_info("$dart_src/runtime/bin:gen_snapshot($host_toolchain)", - "root_out_dir") + template("build_mac_gen_snapshot") { + assert(defined(invoker.host_arch)) + host_cpu = invoker.host_arch - sources = [ "${host_output_dir}/gen_snapshot" ] + build_toolchain = "//build/toolchain/mac:clang_$host_cpu" + if (host_cpu == target_cpu) { + gen_snapshot_target_name = "gen_snapshot_host_targeting_host" + } else { + gen_snapshot_target_name = "gen_snapshot" + } + gen_snapshot_target = + "$dart_src/runtime/bin:$gen_snapshot_target_name($build_toolchain)" + + copy(target_name) { + # The toolchain-specific output directory. For cross-compiles, this is a + # clang-x64 or clang-arm64 subdirectory of the top-level build directory. + output_dir = get_label_info(gen_snapshot_target, "root_out_dir") + + sources = [ "${output_dir}/${gen_snapshot_target_name}" ] + outputs = + [ "${root_out_dir}/artifacts_$host_cpu/gen_snapshot_${target_cpu}" ] + deps = [ gen_snapshot_target ] + } + } + + build_mac_gen_snapshot("create_macos_gen_snapshot_arm64_${target_cpu}") { + host_arch = "arm64" + } + + build_mac_gen_snapshot("create_macos_gen_snapshot_x64_${target_cpu}") { + host_arch = "x64" + } + + action("create_macos_gen_snapshots") { + script = "//flutter/sky/tools/create_macos_binary.py" outputs = [ "${root_out_dir}/gen_snapshot_${target_cpu}" ] - deps = [ "$dart_src/runtime/bin:gen_snapshot($host_toolchain)" ] + args = [ + "--in-arm64", + rebase_path("${root_out_dir}/artifacts_arm64/gen_snapshot_${target_cpu}"), + "--in-x64", + rebase_path("${root_out_dir}/artifacts_x64/gen_snapshot_${target_cpu}"), + "--out", + rebase_path("${root_out_dir}/gen_snapshot_${target_cpu}"), + ] + deps = [ + ":create_macos_gen_snapshot_arm64_${target_cpu}", + ":create_macos_gen_snapshot_x64_${target_cpu}", + ] } } diff --git a/sky/tools/create_macos_binary.py b/sky/tools/create_macos_binary.py new file mode 100755 index 0000000000000..687b53c65581b --- /dev/null +++ b/sky/tools/create_macos_binary.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# +# Copyright 2013 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import os +import subprocess +import sys + + +def canonical_path(path): + """Returns the canonical path for the input path. + If the input path is not absolute, it is treated as relative to the engine + source tree's buildroot directory.""" + if os.path.isabs(path): + return path + buildroot_dir = os.path.abspath(os.path.join(os.path.realpath(__file__), '..', '..', '..', '..')) + return os.path.join(buildroot_dir, path) + + +def assert_file_exists(binary_path, arch): + if not os.path.isfile(binary_path): + print('Cannot find macOS %s binary at %s' % (arch, binary_path)) + sys.exit(1) + + +def create_universal_binary(in_arm64, in_x64, out): + subprocess.check_call(['lipo', in_arm64, in_x64, '-create', '-output', out]) + + +def main(): + parser = argparse.ArgumentParser( + description='Creates a universal binary from input arm64, x64 binaries' + ) + parser.add_argument('--in-arm64', type=str, required=True) + parser.add_argument('--in-x64', type=str, required=True) + parser.add_argument('--out', type=str, required=True) + args = parser.parse_args() + + in_arm64 = canonical_path(args.in_arm64) + in_x64 = canonical_path(args.in_x64) + out = canonical_path(args.out) + + assert_file_exists(in_arm64, 'arm64') + assert_file_exists(in_x64, 'x64') + create_universal_binary(in_arm64, in_x64, out) + + return 0 + + +if __name__ == '__main__': + sys.exit(main())