From 2f763b7a9317b176b83217cde486e7e0f6224f6a Mon Sep 17 00:00:00 2001 From: Zachary Anderson Date: Wed, 3 Apr 2024 02:21:15 +0000 Subject: [PATCH] [et] Prepare local_engine.json for CI, teach et to understand local build names (#51803) This is steps (2) and (3) of https://github.com/flutter/flutter/issues/145263. The next steps after this are to: 1. Fix any issues that come up when running `local_engine.json` in CI. 1. Step (4) of https://github.com/flutter/flutter/issues/145263 1. Fill in some missing builds in `local_engine.json`. --- .ci.yaml | 8 + ci/builders/local_engine.json | 421 +++++++++++++++++- tools/engine_tool/lib/src/build_utils.dart | 51 +++ .../lib/src/commands/build_command.dart | 7 +- .../engine_tool/lib/src/commands/command.dart | 15 +- .../lib/src/commands/command_runner.dart | 9 +- .../lib/src/commands/query_command.dart | 7 +- .../lib/src/commands/run_command.dart | 31 +- .../lib/src/commands/test_command.dart | 7 +- .../engine_tool/test/build_command_test.dart | 103 ++++- .../engine_tool/test/fetch_command_test.dart | 3 +- tools/engine_tool/test/fixtures.dart | 49 +- .../engine_tool/test/format_command_test.dart | 1 + tools/engine_tool/test/lint_command_test.dart | 9 +- tools/engine_tool/test/proc_utils_test.dart | 3 +- .../engine_tool/test/query_command_test.dart | 31 +- tools/engine_tool/test/run_command_test.dart | 13 +- tools/engine_tool/test/test_command_test.dart | 7 +- tools/engine_tool/test/utils.dart | 14 +- tools/engine_tool/test/worker_pool_test.dart | 5 +- tools/pkg/engine_build_configs/bin/check.dart | 6 - 21 files changed, 694 insertions(+), 106 deletions(-) diff --git a/.ci.yaml b/.ci.yaml index 2081a75d112ea..c0d98403ee2f6 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -49,6 +49,14 @@ platform_properties: # https://chromium.googlesource.com/chromium/src.git/+/HEAD/tools/android/avd/proto # You may use those names for the android_virtual_device version. targets: + - name: Linux local_engine_builds + bringup: true + enabled_branches: + - main + recipe: engine_v2/engine_v2 + properties: + config_name: local_engine + - name: Linux linux_android_emulator_tests enabled_branches: - main diff --git a/ci/builders/local_engine.json b/ci/builders/local_engine.json index 0a27c7aee9810..2f0229e374f31 100644 --- a/ci/builders/local_engine.json +++ b/ci/builders/local_engine.json @@ -1,29 +1,95 @@ { "builds": [ { + "cas_archive": false, "drone_dimensions": [ "os=Mac-13", - "os=Linux" + "device_type=none" ], + "gclient_variables": { + "use_rbe": true + }, "gn": [ "--runtime-mode", "debug", "--android", "--android-cpu=arm64", "--no-stripped", - "--no-lto" + "--no-lto", + "--xcode-symlinks", + "--rbe", + "--no-goma" ], - "name": "android_debug_arm64", + "name": "macos/android_debug_arm64", "ninja": { "config": "android_debug_arm64", "targets": [] + }, + "properties": { + "$flutter/osx_sdk": { + "sdk_version": "15a240d" + } } }, { + "cas_archive": false, + "drone_dimensions": [ + "os=Linux", + "device_type=none" + ], + "gclient_variables": { + "use_rbe": true + }, + "gn": [ + "--runtime-mode", + "debug", + "--android", + "--android-cpu=arm64", + "--no-stripped", + "--no-lto", + "--rbe", + "--no-goma" + ], + "name": "linux/android_debug_arm64", + "ninja": { + "config": "android_debug_arm64", + "targets": [] + } + }, + { + "cas_archive": false, + "drone_dimensions": [ + "os=Windows-10", + "device_type=none" + ], + "gclient_variables": { + "use_rbe": true + }, + "gn": [ + "--runtime-mode", + "debug", + "--android", + "--android-cpu=arm64", + "--no-stripped", + "--no-lto", + "--rbe", + "--no-goma" + ], + "name": "windows/android_debug_arm64", + "ninja": { + "config": "android_debug_arm64", + "targets": [] + } + }, + { + "cas_archive": false, "drone_dimensions": [ "os=Mac-13", - "os=Linux" + "device_type=none" ], + "gclient_variables": { + "use_rbe": true + }, "gn": [ "--runtime-mode", "debug", @@ -31,98 +97,411 @@ "--android", "--android-cpu=arm64", "--no-stripped", - "--no-lto" + "--no-lto", + "--xcode-symlinks", + "--rbe", + "--no-goma" ], - "name": "android_debug_unopt_arm64", + "name": "macos/android_debug_unopt_arm64", "ninja": { "config": "android_debug_unopt_arm64", "targets": [] + }, + "properties": { + "$flutter/osx_sdk": { + "sdk_version": "15a240d" + } } }, { "drone_dimensions": [ "os=Mac-13", - "os=Linux" + "device_type=none" + ], + "gn": [ + "--runtime-mode", + "profile", + "--android", + "--android-cpu=arm64", + "--no-stripped", + "--no-lto", + "--xcode-symlinks", + "--rbe", + "--no-goma" + ], + "name": "macos/android_profile_arm64", + "ninja": { + "config": "android_profile_arm64", + "targets": [] + }, + "properties": { + "$flutter/osx_sdk": { + "sdk_version": "15a240d" + } + } + }, + { + "cas_archive": false, + "drone_dimensions": [ + "os=Linux", + "device_type=none" ], + "gclient_variables": { + "use_rbe": true + }, "gn": [ "--runtime-mode", "profile", "--android", "--android-cpu=arm64", "--no-stripped", - "--no-lto" + "--no-lto", + "--rbe", + "--no-goma" ], - "name": "android_profile_arm64", + "name": "linux/android_profile_arm64", "ninja": { "config": "android_profile_arm64", "targets": [] } }, { + "cas_archive": false, + "drone_dimensions": [ + "os=Windows-10", + "device_type=none" + ], + "gclient_variables": { + "use_rbe": true + }, + "gn": [ + "--runtime-mode", + "profile", + "--android", + "--android-cpu=arm64", + "--no-stripped", + "--no-lto", + "--rbe", + "--no-goma" + ], + "name": "windows/android_profile_arm64", + "ninja": { + "config": "android_profile_arm64", + "targets": [] + } + }, + { + "cas_archive": false, "drone_dimensions": [ "os=Mac-13", - "os=Linux" + "device_type=none" + ], + "gclient_variables": { + "use_rbe": true + }, + "gn": [ + "--runtime-mode", + "release", + "--android", + "--android-cpu=arm64", + "--no-stripped", + "--no-lto", + "--xcode-symlinks", + "--rbe", + "--no-goma" + ], + "name": "macos/android_release_arm64", + "ninja": { + "config": "android_release_arm64", + "targets": [] + }, + "properties": { + "$flutter/osx_sdk": { + "sdk_version": "15a240d" + } + } + }, + { + "cas_archive": false, + "drone_dimensions": [ + "os=Linux", + "device_type=none" + ], + "gclient_variables": { + "use_rbe": true + }, + "gn": [ + "--runtime-mode", + "release", + "--android", + "--android-cpu=arm64", + "--no-stripped", + "--no-lto", + "--rbe", + "--no-goma" + ], + "name": "linux/android_release_arm64", + "ninja": { + "config": "android_release_arm64", + "targets": [] + } + }, + { + "cas_archive": false, + "drone_dimensions": [ + "os=Windows-10", + "device_type=none" ], + "gclient_variables": { + "use_rbe": true + }, "gn": [ "--runtime-mode", "release", "--android", "--android-cpu=arm64", "--no-stripped", - "--no-lto" + "--no-lto", + "--rbe", + "--no-goma" ], - "name": "android_release_arm64", + "name": "windows/android_release_arm64", "ninja": { "config": "android_release_arm64", "targets": [] } }, { + "cas_archive": false, "drone_dimensions": [ "os=Mac-13", - "os=Linux" + "device_type=none" + ], + "gclient_variables": { + "download_android_deps": false, + "use_rbe": true + }, + "gn": [ + "--runtime-mode", + "debug", + "--no-stripped", + "--no-lto", + "--xcode-symlinks", + "--rbe", + "--no-goma" + ], + "name": "macos/host_debug", + "ninja": { + "config": "host_debug", + "targets": [] + }, + "properties": { + "$flutter/osx_sdk": { + "sdk_version": "15a240d" + } + } + }, + { + "cas_archive": false, + "drone_dimensions": [ + "os=Linux", + "device_type=none" + ], + "gclient_variables": { + "download_android_deps": false, + "use_rbe": true + }, + "gn": [ + "--runtime-mode", + "debug", + "--no-stripped", + "--no-lto", + "--rbe", + "--no-goma" + ], + "name": "linux/host_debug", + "ninja": { + "config": "host_debug", + "targets": [] + } + }, + { + "cas_archive": false, + "drone_dimensions": [ + "os=Windows-10", + "device_type=none" ], + "gclient_variables": { + "download_android_deps": false, + "use_rbe": true + }, "gn": [ "--runtime-mode", "debug", "--no-stripped", - "--no-lto" + "--no-lto", + "--rbe", + "--no-goma" ], - "name": "host_debug", + "name": "windows/host_debug", "ninja": { "config": "host_debug", "targets": [] } }, { + "cas_archive": false, "drone_dimensions": [ "os=Mac-13", - "os=Linux" + "device_type=none" + ], + "gclient_variables": { + "download_android_deps": false, + "use_rbe": true + }, + "gn": [ + "--runtime-mode", + "profile", + "--no-stripped", + "--no-lto", + "--xcode-symlinks", + "--rbe", + "--no-goma" + ], + "name": "macos/host_profile", + "ninja": { + "config": "host_profile", + "targets": [] + }, + "properties": { + "$flutter/osx_sdk": { + "sdk_version": "15a240d" + } + } + }, + { + "cas_archive": false, + "drone_dimensions": [ + "os=Linux", + "device_type=none" ], + "gclient_variables": { + "download_android_deps": false, + "use_rbe": true + }, "gn": [ "--runtime-mode", "profile", "--no-stripped", - "--no-lto" + "--no-lto", + "--rbe", + "--no-goma" ], - "name": "host_profile", + "name": "linux/host_profile", "ninja": { "config": "host_profile", "targets": [] } }, { + "cas_archive": false, + "drone_dimensions": [ + "os=Windows-10", + "device_type=none" + ], + "gclient_variables": { + "download_android_deps": false, + "use_rbe": true + }, + "gn": [ + "--runtime-mode", + "profile", + "--no-stripped", + "--no-lto", + "--rbe", + "--no-goma" + ], + "name": "windows/host_profile", + "ninja": { + "config": "host_profile", + "targets": [] + } + }, + { + "cas_archive": false, "drone_dimensions": [ "os=Mac-13", - "os=Linux" + "device_type=none" + ], + "gclient_variables": { + "download_android_deps": false, + "use_rbe": true + }, + "gn": [ + "--runtime-mode", + "release", + "--no-stripped", + "--no-lto", + "--xcode-symlinks", + "--rbe", + "--no-goma" + ], + "name": "macos/host_release", + "ninja": { + "config": "host_release", + "targets": [] + }, + "properties": { + "$flutter/osx_sdk": { + "sdk_version": "15a240d" + } + } + }, + { + "cas_archive": false, + "drone_dimensions": [ + "os=Linux", + "device_type=none" + ], + "gclient_variables": { + "download_android_deps": false, + "use_rbe": true + }, + "gn": [ + "--runtime-mode", + "release", + "--no-stripped", + "--no-lto", + "--rbe", + "--no-goma", + "--xcode-symlinks" + ], + "name": "linux/host_release", + "ninja": { + "config": "host_release", + "targets": [] + } + }, + { + "cas_archive": false, + "drone_dimensions": [ + "os=Windows-10", + "device_type=none" ], + "gclient_variables": { + "download_android_deps": false, + "use_rbe": true + }, "gn": [ "--runtime-mode", "release", "--no-stripped", - "--no-lto" + "--no-lto", + "--rbe", + "--no-goma", + "--xcode-symlinks" ], - "name": "host_release", + "name": "windows/host_release", "ninja": { "config": "host_release", "targets": [] diff --git a/tools/engine_tool/lib/src/build_utils.dart b/tools/engine_tool/lib/src/build_utils.dart index 433f57ce9ab6c..b04d442092b79 100644 --- a/tools/engine_tool/lib/src/build_utils.dart +++ b/tools/engine_tool/lib/src/build_utils.dart @@ -65,6 +65,57 @@ void debugCheckBuilds(List builds) { } } +String _ciPrefix(Environment env) => 'ci${env.platform.pathSeparator}'; +String _osPrefix(Environment env) => '${env.platform.operatingSystem}/'; +const String _webTestsPrefix = 'web_tests/'; + +bool _doNotMangle(Environment env, String name) { + return name.startsWith(_ciPrefix(env)) || name.startsWith(_webTestsPrefix); +} + +/// Transform the name of a build into the name presented and accepted by the +/// CLI +/// +/// If a name starts with '$OS/', it is a local development build, and the +/// mangled name has the '$OS/' part stripped off, where $OS is the value of +/// `platform.operatingSystem` in the passed-in `environment`. +/// +/// If the name does not start with '$OS/', then it must start with 'ci/', +/// 'ci\', or 'web_tests/' in which case the name is returned unchanged. +/// +/// Examples: +/// macos/host_debug -> host_debug +/// ci/ios_release -> ci/ios_release +/// ci\host_profile -> ci\host_profile +/// web_tests/artifacts -> web_tests/artifacts +String mangleConfigName(Environment env, String name) { + final String osPrefix = _osPrefix(env); + if (name.startsWith(osPrefix)) { + return name.substring(osPrefix.length); + } + if (_doNotMangle(env, name)) { + return name; + } + throw ArgumentError( + 'name argument "$name" must start with a valid platform name or "ci"', + ); +} + +/// Transform the mangled (CLI) name of a build into its true name in the build +/// config json file. +/// +/// This does the reverse of [mangleConfigName] taking the operating system +/// name from `environment`. +/// +/// Examples: +/// host_debug -> macos/host_debug +/// ci/ios_release -> ci/ios_release +/// ci\host_profile -> ci\host_profile +/// web_tests/artifacts -> web_tests/artifacts +String demangleConfigName(Environment env, String name) { + return _doNotMangle(env, name) ? name : '${_osPrefix(env)}$name'; +} + /// Build the build target in the environment. Future runBuild(Environment environment, Build build, {List extraGnArgs = const [], diff --git a/tools/engine_tool/lib/src/commands/build_command.dart b/tools/engine_tool/lib/src/commands/build_command.dart index fc6a81dad54f8..2b5c2210fc42e 100644 --- a/tools/engine_tool/lib/src/commands/build_command.dart +++ b/tools/engine_tool/lib/src/commands/build_command.dart @@ -17,7 +17,9 @@ final class BuildCommand extends CommandBase { }) { builds = runnableBuilds(environment, configs); debugCheckBuilds(builds); - addConfigOption(argParser, runnableBuilds(environment, configs)); + addConfigOption( + environment, argParser, runnableBuilds(environment, configs), + ); argParser.addFlag( rbeFlag, defaultsTo: true, @@ -39,8 +41,9 @@ final class BuildCommand extends CommandBase { Future run() async { final String configName = argResults![configFlag] as String; final bool useRbe = argResults![rbeFlag] as bool; + final String demangledName = demangleConfigName(environment, configName); final Build? build = - builds.where((Build build) => build.name == configName).firstOrNull; + builds.where((Build build) => build.name == demangledName).firstOrNull; if (build == null) { environment.logger.error('Could not find config $configName'); return 1; diff --git a/tools/engine_tool/lib/src/commands/command.dart b/tools/engine_tool/lib/src/commands/command.dart index 3224f7654cc56..0ef132d808320 100644 --- a/tools/engine_tool/lib/src/commands/command.dart +++ b/tools/engine_tool/lib/src/commands/command.dart @@ -6,6 +6,7 @@ import 'package:args/args.dart'; import 'package:args/command_runner.dart'; import 'package:engine_build_configs/engine_build_configs.dart'; +import '../build_utils.dart'; import '../environment.dart'; import 'flags.dart'; @@ -19,18 +20,24 @@ abstract base class CommandBase extends Command { } /// Adds the -c (--config) option to the parser. -void addConfigOption(ArgParser parser, List builds, - {String defaultsTo = 'host_debug'}) { +void addConfigOption( + Environment environment, + ArgParser parser, + List builds, { + String defaultsTo = 'host_debug', +}) { parser.addOption( configFlag, abbr: 'c', defaultsTo: defaultsTo, help: 'Specify the build config to use', allowed: [ - for (final Build config in builds) config.name, + for (final Build config in builds) + mangleConfigName(environment, config.name), ], allowedHelp: { - for (final Build config in builds) config.name: config.gn.join(' '), + for (final Build config in builds) + mangleConfigName(environment, config.name): config.gn.join(' '), }, ); } diff --git a/tools/engine_tool/lib/src/commands/command_runner.dart b/tools/engine_tool/lib/src/commands/command_runner.dart index 12c1d09edb832..649ef75f28b58 100644 --- a/tools/engine_tool/lib/src/commands/command_runner.dart +++ b/tools/engine_tool/lib/src/commands/command_runner.dart @@ -2,10 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'package:args/args.dart'; import 'package:args/command_runner.dart'; import 'package:engine_build_configs/engine_build_configs.dart'; import '../environment.dart'; +import '../logger.dart'; import 'build_command.dart'; import 'fetch_command.dart'; import 'flags.dart'; @@ -61,8 +63,13 @@ final class ToolCommandRunner extends CommandRunner { @override Future run(Iterable args) async { + final ArgResults argResults = parse(args); + final bool verbose = argResults[verboseFlag]! as bool; + if (verbose) { + environment.logger.level = Logger.infoLevel; + } try { - return await runCommand(parse(args)) ?? 0; + return await runCommand(argResults) ?? 0; } on FormatException catch (e) { environment.logger.error(e); return 1; diff --git a/tools/engine_tool/lib/src/commands/query_command.dart b/tools/engine_tool/lib/src/commands/query_command.dart index 6f4c6ee367e37..4b7fa50230c5b 100644 --- a/tools/engine_tool/lib/src/commands/query_command.dart +++ b/tools/engine_tool/lib/src/commands/query_command.dart @@ -140,7 +140,9 @@ final class QueryTestsCommand extends CommandBase { }) { builds = runnableBuilds(environment, configs); debugCheckBuilds(builds); - addConfigOption(argParser, runnableBuilds(environment, configs)); + addConfigOption( + environment, argParser, runnableBuilds(environment, configs), + ); } /// Build configurations loaded from the engine from under ci/builders. @@ -158,8 +160,9 @@ final class QueryTestsCommand extends CommandBase { @override Future run() async { final String configName = argResults![configFlag] as String; + final String demangledName = demangleConfigName(environment, configName); final Build? build = - builds.where((Build build) => build.name == configName).firstOrNull; + builds.where((Build build) => build.name == demangledName).firstOrNull; if (build == null) { environment.logger.error('Could not find config $configName'); return 1; diff --git a/tools/engine_tool/lib/src/commands/run_command.dart b/tools/engine_tool/lib/src/commands/run_command.dart index 35e5657c58941..61f289cfded02 100644 --- a/tools/engine_tool/lib/src/commands/run_command.dart +++ b/tools/engine_tool/lib/src/commands/run_command.dart @@ -23,8 +23,10 @@ final class RunCommand extends CommandBase { debugCheckBuilds(builds); // We default to nothing in order to automatically detect attached devices // and select an appropriate target from them. - addConfigOption(argParser, runnableBuilds(environment, configs), - defaultsTo: ''); + addConfigOption( + environment, argParser, runnableBuilds(environment, configs), + defaultsTo: '', + ); argParser.addFlag( rbeFlag, defaultsTo: true, @@ -47,26 +49,27 @@ final class RunCommand extends CommandBase { 'See `flutter run --help` for a listing'; Build? _lookup(String configName) { - return builds.where((Build build) => build.name == configName).firstOrNull; + final String demangledName = demangleConfigName(environment, configName); + return builds.where((Build build) => build.name == demangledName).firstOrNull; } Build? _findHostBuild(Build? targetBuild) { if (targetBuild == null) { return null; } - - final String name = targetBuild.name; - if (name.startsWith('host_')) { + final String mangledName = mangleConfigName(environment, targetBuild.name); + if (mangledName.contains('host_')) { return targetBuild; } // TODO(johnmccutchan): This is brittle, it would be better if we encoded // the host config name in the target config. - if (name.contains('_debug')) { - return _lookup('host_debug'); - } else if (name.contains('_profile')) { - return _lookup('host_profile'); - } else if (name.contains('_release')) { - return _lookup('host_release'); + final String ci = mangledName.startsWith('ci') ? mangledName.substring(0, 3) : ''; + if (mangledName.contains('_debug')) { + return _lookup('${ci}host_debug'); + } else if (mangledName.contains('_profile')) { + return _lookup('${ci}host_profile'); + } else if (mangledName.contains('_release')) { + return _lookup('${ci}host_release'); } return null; } @@ -101,13 +104,13 @@ final class RunCommand extends CommandBase { Future _selectTargetConfig() async { final String configName = argResults![configFlag] as String; if (configName.isNotEmpty) { - return configName; + return demangleConfigName(environment, configName); } final String deviceId = _getDeviceId(); final RunTarget? target = await detectAndSelectRunTarget(environment, deviceId); if (target == null) { - return 'host_debug'; + return demangleConfigName(environment, 'host_debug'); } environment.logger.status( 'Building to run on "${target.name}" running ${target.targetPlatform}'); diff --git a/tools/engine_tool/lib/src/commands/test_command.dart b/tools/engine_tool/lib/src/commands/test_command.dart index 7e7d82faae0ac..4ba6d6ac62206 100644 --- a/tools/engine_tool/lib/src/commands/test_command.dart +++ b/tools/engine_tool/lib/src/commands/test_command.dart @@ -23,7 +23,9 @@ final class TestCommand extends CommandBase { }) { builds = runnableBuilds(environment, configs); debugCheckBuilds(builds); - addConfigOption(argParser, runnableBuilds(environment, configs)); + addConfigOption( + environment, argParser, runnableBuilds(environment, configs), + ); } /// List of compatible builds. @@ -40,8 +42,9 @@ final class TestCommand extends CommandBase { @override Future run() async { final String configName = argResults![configFlag] as String; + final String demangledName = demangleConfigName(environment, configName); final Build? build = - builds.where((Build build) => build.name == configName).firstOrNull; + builds.where((Build build) => build.name == demangledName).firstOrNull; if (build == null) { environment.logger.error('Could not find config $configName'); return 1; diff --git a/tools/engine_tool/test/build_command_test.dart b/tools/engine_tool/test/build_command_test.dart index bc1c2344d6a13..759e660d0f67e 100644 --- a/tools/engine_tool/test/build_command_test.dart +++ b/tools/engine_tool/test/build_command_test.dart @@ -23,19 +23,19 @@ import 'fixtures.dart' as fixtures; void main() { final BuilderConfig linuxTestConfig = BuilderConfig.fromJson( path: 'ci/builders/linux_test_config.json', - map: convert.jsonDecode(fixtures.testConfig('Linux')) + map: convert.jsonDecode(fixtures.testConfig('Linux', Platform.linux)) as Map, ); final BuilderConfig macTestConfig = BuilderConfig.fromJson( path: 'ci/builders/mac_test_config.json', - map: convert.jsonDecode(fixtures.testConfig('Mac-12')) + map: convert.jsonDecode(fixtures.testConfig('Mac-12', Platform.macOS)) as Map, ); final BuilderConfig winTestConfig = BuilderConfig.fromJson( path: 'ci/builders/win_test_config.json', - map: convert.jsonDecode(fixtures.testConfig('Windows-11')) + map: convert.jsonDecode(fixtures.testConfig('Windows-11', Platform.windows)) as Map, ); @@ -67,7 +67,8 @@ void main() { engine: engine, platform: FakePlatform( operatingSystem: Platform.linux, - resolvedExecutable: io.Platform.resolvedExecutable), + resolvedExecutable: io.Platform.resolvedExecutable, + pathSeparator: '/'), processRunner: ProcessRunner( processManager: FakeProcessManager( canRun: (Object? exe, {String? workingDirectory}) => true, @@ -101,7 +102,7 @@ void main() { try { final List result = runnableBuilds(env, configs); expect(result.length, equals(8)); - expect(result[0].name, equals('build_name')); + expect(result[0].name, equals('ci/build_name')); } finally { cleanupEnv(env); } @@ -118,7 +119,7 @@ void main() { final int result = await runner.run([ 'build', '--config', - 'build_name', + 'ci/build_name', ]); expect(result, equals(0)); expect(runHistory.length, greaterThanOrEqualTo(1)); @@ -140,7 +141,7 @@ void main() { final int result = await runner.run([ 'build', '--config', - 'build_name', + 'ci/build_name', ]); expect(result, equals(0)); expect(runHistory.length, greaterThanOrEqualTo(2)); @@ -162,7 +163,7 @@ void main() { final int result = await runner.run([ 'build', '--config', - 'build_name', + 'ci/build_name', ]); expect(result, equals(0)); expect(runHistory.length, greaterThanOrEqualTo(3)); @@ -186,7 +187,7 @@ void main() { final int result = await runner.run([ 'build', '--config', - 'build_name', + 'ci/build_name', ]); expect(result, equals(0)); expect(runHistory.length, lessThanOrEqualTo(3)); @@ -208,7 +209,7 @@ void main() { final int result = await runner.run([ 'build', '--config', - 'android_debug_rbe_arm64', + 'ci/android_debug_rbe_arm64', ]); expect(result, equals(0)); expect(runHistory[0][0], contains(path.join('tools', 'gn'))); @@ -232,7 +233,7 @@ void main() { final int result = await runner.run([ 'build', '--config', - 'android_debug_rbe_arm64', + 'ci/android_debug_rbe_arm64', '--no-rbe', ]); expect(result, equals(0)); @@ -255,7 +256,7 @@ void main() { final int result = await runner.run([ 'build', '--config', - 'android_debug_rbe_arm64', + 'ci/android_debug_rbe_arm64', ]); expect(result, equals(0)); expect(runHistory[0][0], contains(path.join('tools', 'gn'))); @@ -265,4 +266,82 @@ void main() { cleanupEnv(env); } }); + + test('mangleConfigName removes the OS and adds ci/ as needed', () { + final Logger logger = Logger.test(); + final (Environment env, _) = linuxEnv(logger); + expect(mangleConfigName(env, 'linux/build'), equals('build')); + expect(mangleConfigName(env, 'ci/build'), equals('ci/build')); + }); + + test('mangleConfigName throws when the input config name is malformed', () { + final Logger logger = Logger.test(); + final (Environment env, _) = linuxEnv(logger); + expectArgumentError(() => mangleConfigName(env, 'build')); + }); + + test('demangleConfigName adds the OS and removes ci/ as needed', () { + final Logger logger = Logger.test(); + final (Environment env, _) = linuxEnv(logger); + expect(demangleConfigName(env, 'build'), equals('linux/build')); + expect(demangleConfigName(env, 'ci/build'), equals('ci/build')); + }); + + test('local config name on the command line is correctly translated', () async { + final BuilderConfig namespaceTestConfigs = BuilderConfig.fromJson( + path: 'ci/builders/namespace_test_config.json', + map: convert.jsonDecode(fixtures.configsToTestNamespacing) + as Map, + ); + final Map configs = { + 'namespace_test_config': namespaceTestConfigs, + }; + final Logger logger = Logger.test(); + final (Environment env, List> runHistory) = linuxEnv(logger); + try { + final ToolCommandRunner runner = ToolCommandRunner( + environment: env, + configs: configs, + ); + final int result = await runner.run([ + 'build', + '--config', + 'host_debug', + ]); + expect(result, equals(0)); + expect(runHistory[1][0], contains(path.join('ninja', 'ninja'))); + expect(runHistory[1][2], contains('local_host_debug')); + } finally { + cleanupEnv(env); + } + }); + + test('ci config name on the command line is correctly translated', () async { + final BuilderConfig namespaceTestConfigs = BuilderConfig.fromJson( + path: 'ci/builders/namespace_test_config.json', + map: convert.jsonDecode(fixtures.configsToTestNamespacing) + as Map, + ); + final Map configs = { + 'namespace_test_config': namespaceTestConfigs, + }; + final Logger logger = Logger.test(); + final (Environment env, List> runHistory) = linuxEnv(logger); + try { + final ToolCommandRunner runner = ToolCommandRunner( + environment: env, + configs: configs, + ); + final int result = await runner.run([ + 'build', + '--config', + 'ci/host_debug', + ]); + expect(result, equals(0)); + expect(runHistory[1][0], contains(path.join('ninja', 'ninja'))); + expect(runHistory[1][2], contains('ci/host_debug')); + } finally { + cleanupEnv(env); + } + }); } diff --git a/tools/engine_tool/test/fetch_command_test.dart b/tools/engine_tool/test/fetch_command_test.dart index 16ccdd6ee4139..41abe4d4f8827 100644 --- a/tools/engine_tool/test/fetch_command_test.dart +++ b/tools/engine_tool/test/fetch_command_test.dart @@ -33,7 +33,8 @@ void main() { engine: engine, platform: FakePlatform( operatingSystem: Platform.linux, - resolvedExecutable: io.Platform.resolvedExecutable), + resolvedExecutable: io.Platform.resolvedExecutable, + pathSeparator: '/'), processRunner: ProcessRunner( processManager: FakeProcessManager(onStart: (List command) { runHistory.add(command); diff --git a/tools/engine_tool/test/fixtures.dart b/tools/engine_tool/test/fixtures.dart index 16096fe0ad618..02511d2459af4 100644 --- a/tools/engine_tool/test/fixtures.dart +++ b/tools/engine_tool/test/fixtures.dart @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -String testConfig(String os) => ''' +String testConfig(String osDimension, String osPlatform) => ''' { "builds": [ { @@ -16,13 +16,13 @@ String testConfig(String os) => ''' } ], "drone_dimensions": [ - "os=$os" + "os=$osDimension" ], "gclient_variables": { "variable": false }, "gn": ["--gn-arg", "--lto", "--goma", "--no-rbe"], - "name": "build_name", + "name": "ci/build_name", "ninja": { "config": "build_name", "targets": ["ninja_target"] @@ -51,10 +51,10 @@ String testConfig(String os) => ''' {}, { "drone_dimensions": [ - "os=$os" + "os=$osDimension" ], "gn": ["--gn-arg", "--lto", "--goma", "--no-rbe"], - "name": "host_debug", + "name": "$osPlatform/host_debug", "ninja": { "config": "host_debug", "targets": ["ninja_target"] @@ -62,10 +62,10 @@ String testConfig(String os) => ''' }, { "drone_dimensions": [ - "os=$os" + "os=$osDimension" ], "gn": ["--gn-arg", "--lto", "--goma", "--no-rbe"], - "name": "android_debug_arm64", + "name": "$osPlatform/android_debug_arm64", "ninja": { "config": "android_debug_arm64", "targets": ["ninja_target"] @@ -73,10 +73,10 @@ String testConfig(String os) => ''' }, { "drone_dimensions": [ - "os=$os" + "os=$osDimension" ], "gn": ["--gn-arg", "--lto", "--no-goma", "--rbe"], - "name": "android_debug_rbe_arm64", + "name": "ci/android_debug_rbe_arm64", "ninja": { "config": "android_debug_rbe_arm64", "targets": ["ninja_target"] @@ -98,7 +98,7 @@ String testConfig(String os) => ''' "name": "global test", "recipe": "engine_v2/tester_engine", "drone_dimensions": [ - "os=$os" + "os=$osDimension" ], "gclient_variables": { "variable": false @@ -122,6 +122,35 @@ String testConfig(String os) => ''' } '''; +const String configsToTestNamespacing = ''' +{ + "builds": [ + { + "drone_dimensions": [ + "os=Linux" + ], + "gn": ["--gn-arg", "--lto", "--goma", "--no-rbe"], + "name": "linux/host_debug", + "ninja": { + "config": "local_host_debug", + "targets": ["ninja_target"] + } + }, + { + "drone_dimensions": [ + "os=Linux" + ], + "gn": ["--gn-arg", "--lto", "--goma", "--no-rbe"], + "name": "ci/host_debug", + "ninja": { + "config": "ci/host_debug", + "targets": ["ninja_target"] + } + } + ] +} +'''; + String attachedDevices() => ''' [ { diff --git a/tools/engine_tool/test/format_command_test.dart b/tools/engine_tool/test/format_command_test.dart index 029d9b268b7d2..ebce685cd6fe7 100644 --- a/tools/engine_tool/test/format_command_test.dart +++ b/tools/engine_tool/test/format_command_test.dart @@ -34,6 +34,7 @@ void main() { platform: FakePlatform( resolvedExecutable: '/dart', operatingSystem: Platform.linux, + pathSeparator: '/', ), processRunner: ProcessRunner( processManager: processManager, diff --git a/tools/engine_tool/test/lint_command_test.dart b/tools/engine_tool/test/lint_command_test.dart index 659f617d510aa..46fe19e19d365 100644 --- a/tools/engine_tool/test/lint_command_test.dart +++ b/tools/engine_tool/test/lint_command_test.dart @@ -29,19 +29,19 @@ void main() { } final BuilderConfig linuxTestConfig = BuilderConfig.fromJson( path: 'ci/builders/linux_test_config.json', - map: convert.jsonDecode(fixtures.testConfig('Linux')) + map: convert.jsonDecode(fixtures.testConfig('Linux', Platform.linux)) as Map, ); final BuilderConfig macTestConfig = BuilderConfig.fromJson( path: 'ci/builders/mac_test_config.json', - map: convert.jsonDecode(fixtures.testConfig('Mac-12')) + map: convert.jsonDecode(fixtures.testConfig('Mac-12', Platform.macOS)) as Map, ); final BuilderConfig winTestConfig = BuilderConfig.fromJson( path: 'ci/builders/win_test_config.json', - map: convert.jsonDecode(fixtures.testConfig('Windows-11')) + map: convert.jsonDecode(fixtures.testConfig('Windows-11', Platform.windows)) as Map, ); @@ -60,7 +60,8 @@ void main() { engine: engine, platform: FakePlatform( operatingSystem: Platform.macOS, - resolvedExecutable: io.Platform.resolvedExecutable), + resolvedExecutable: io.Platform.resolvedExecutable, + pathSeparator: '/'), processRunner: ProcessRunner( processManager: FakeProcessManager(onStart: (List command) { runHistory.add(command); diff --git a/tools/engine_tool/test/proc_utils_test.dart b/tools/engine_tool/test/proc_utils_test.dart index fd29f7a48f40d..e1027b053e06b 100644 --- a/tools/engine_tool/test/proc_utils_test.dart +++ b/tools/engine_tool/test/proc_utils_test.dart @@ -33,7 +33,8 @@ void main() { engine: engine, platform: FakePlatform( operatingSystem: Platform.macOS, - resolvedExecutable: io.Platform.resolvedExecutable), + resolvedExecutable: io.Platform.resolvedExecutable, + pathSeparator: '/'), processRunner: ProcessRunner( processManager: FakeProcessManager(onStart: (List command) { runHistory.add(command); diff --git a/tools/engine_tool/test/query_command_test.dart b/tools/engine_tool/test/query_command_test.dart index b570ad2e28dac..f99878a9e141f 100644 --- a/tools/engine_tool/test/query_command_test.dart +++ b/tools/engine_tool/test/query_command_test.dart @@ -12,6 +12,7 @@ import 'package:engine_tool/src/commands/command_runner.dart'; import 'package:engine_tool/src/environment.dart'; import 'package:litetest/litetest.dart'; import 'package:logging/logging.dart' as log; +import 'package:platform/platform.dart'; import 'fixtures.dart' as fixtures; import 'utils.dart'; @@ -28,19 +29,19 @@ void main() { final BuilderConfig linuxTestConfig = BuilderConfig.fromJson( path: 'ci/builders/linux_test_config.json', - map: convert.jsonDecode(fixtures.testConfig('Linux')) + map: convert.jsonDecode(fixtures.testConfig('Linux', Platform.linux)) as Map, ); final BuilderConfig macTestConfig = BuilderConfig.fromJson( path: 'ci/builders/mac_test_config.json', - map: convert.jsonDecode(fixtures.testConfig('Mac-12')) + map: convert.jsonDecode(fixtures.testConfig('Mac-12', Platform.macOS)) as Map, ); final BuilderConfig winTestConfig = BuilderConfig.fromJson( path: 'ci/builders/win_test_config.json', - map: convert.jsonDecode(fixtures.testConfig('Windows-11')) + map: convert.jsonDecode(fixtures.testConfig('Windows-11', Platform.windows)) as Map, ); @@ -89,15 +90,15 @@ fml_arc_unittests 'Add --verbose to see detailed information about each builder\n', '\n', '"linux_test_config" builder:\n', - ' "build_name" config\n', - ' "host_debug" config\n', - ' "android_debug_arm64" config\n', - ' "android_debug_rbe_arm64" config\n', + ' "ci/build_name" config\n', + ' "linux/host_debug" config\n', + ' "linux/android_debug_arm64" config\n', + ' "ci/android_debug_rbe_arm64" config\n', '"linux_test_config2" builder:\n', - ' "build_name" config\n', - ' "host_debug" config\n', - ' "android_debug_arm64" config\n', - ' "android_debug_rbe_arm64" config\n', + ' "ci/build_name" config\n', + ' "linux/host_debug" config\n', + ' "linux/android_debug_arm64" config\n', + ' "ci/android_debug_rbe_arm64" config\n', ]), ); }); @@ -124,10 +125,10 @@ fml_arc_unittests 'Add --verbose to see detailed information about each builder\n', '\n', '"linux_test_config" builder:\n', - ' "build_name" config\n', - ' "host_debug" config\n', - ' "android_debug_arm64" config\n', - ' "android_debug_rbe_arm64" config\n', + ' "ci/build_name" config\n', + ' "linux/host_debug" config\n', + ' "linux/android_debug_arm64" config\n', + ' "ci/android_debug_rbe_arm64" config\n', ])); }); diff --git a/tools/engine_tool/test/run_command_test.dart b/tools/engine_tool/test/run_command_test.dart index 9c2352fdb1864..982ddedf48ba8 100644 --- a/tools/engine_tool/test/run_command_test.dart +++ b/tools/engine_tool/test/run_command_test.dart @@ -31,19 +31,19 @@ void main() { final BuilderConfig linuxTestConfig = BuilderConfig.fromJson( path: 'ci/builders/linux_test_config.json', - map: convert.jsonDecode(fixtures.testConfig('Linux')) + map: convert.jsonDecode(fixtures.testConfig('Linux', Platform.linux)) as Map, ); final BuilderConfig macTestConfig = BuilderConfig.fromJson( path: 'ci/builders/mac_test_config.json', - map: convert.jsonDecode(fixtures.testConfig('Mac-12')) + map: convert.jsonDecode(fixtures.testConfig('Mac-12', Platform.macOS)) as Map, ); final BuilderConfig winTestConfig = BuilderConfig.fromJson( path: 'ci/builders/win_test_config.json', - map: convert.jsonDecode(fixtures.testConfig('Windows-11')) + map: convert.jsonDecode(fixtures.testConfig('Windows-11', Platform.windows)) as Map, ); @@ -62,7 +62,8 @@ void main() { engine: engine, platform: FakePlatform( operatingSystem: Platform.linux, - resolvedExecutable: io.Platform.resolvedExecutable), + resolvedExecutable: io.Platform.resolvedExecutable, + pathSeparator: '/'), processRunner: ProcessRunner( processManager: FakeProcessManager(onStart: (List command) { runHistory.add(command); @@ -152,9 +153,9 @@ void main() { 'flutter', 'run', '--local-engine', - 'android_debug_arm64', + 'linux/android_debug_arm64', '--local-engine-host', - 'host_debug', + 'linux/host_debug', '-d', 'emulator' ])); diff --git a/tools/engine_tool/test/test_command_test.dart b/tools/engine_tool/test/test_command_test.dart index d7629cedbeb79..081b81adae1c2 100644 --- a/tools/engine_tool/test/test_command_test.dart +++ b/tools/engine_tool/test/test_command_test.dart @@ -11,6 +11,7 @@ import 'package:engine_repo_tools/engine_repo_tools.dart'; import 'package:engine_tool/src/commands/command_runner.dart'; import 'package:engine_tool/src/environment.dart'; import 'package:litetest/litetest.dart'; +import 'package:platform/platform.dart'; import 'fixtures.dart' as fixtures; import 'utils.dart'; @@ -27,19 +28,19 @@ void main() { final BuilderConfig linuxTestConfig = BuilderConfig.fromJson( path: 'ci/builders/linux_test_config.json', - map: convert.jsonDecode(fixtures.testConfig('Linux')) + map: convert.jsonDecode(fixtures.testConfig('Linux', Platform.linux)) as Map, ); final BuilderConfig macTestConfig = BuilderConfig.fromJson( path: 'ci/builders/mac_test_config.json', - map: convert.jsonDecode(fixtures.testConfig('Mac-12')) + map: convert.jsonDecode(fixtures.testConfig('Mac-12', Platform.macOS)) as Map, ); final BuilderConfig winTestConfig = BuilderConfig.fromJson( path: 'ci/builders/win_test_config.json', - map: convert.jsonDecode(fixtures.testConfig('Windows-11')) + map: convert.jsonDecode(fixtures.testConfig('Windows-11', Platform.windows)) as Map, ); diff --git a/tools/engine_tool/test/utils.dart b/tools/engine_tool/test/utils.dart index 02abc466e10f2..f1548ede24b1e 100644 --- a/tools/engine_tool/test/utils.dart +++ b/tools/engine_tool/test/utils.dart @@ -61,7 +61,8 @@ class TestEnvironment { engine: engine, platform: FakePlatform( operatingSystem: _operatingSystemForAbi(abi), - resolvedExecutable: io.Platform.resolvedExecutable), + resolvedExecutable: io.Platform.resolvedExecutable, + pathSeparator: _pathSeparatorForAbi(abi)), processRunner: ProcessRunner( processManager: FakeProcessManager(onStart: (List command) { final FakeProcess processResult = @@ -102,6 +103,17 @@ String _operatingSystemForAbi(ffi.Abi abi) { } } +String _pathSeparatorForAbi(ffi.Abi abi) { + switch (abi) { + case ffi.Abi.windowsArm64: + case ffi.Abi.windowsIA32: + case ffi.Abi.windowsX64: + return r'\'; + default: + return '/'; + } +} + FakeProcess _getCannedResult( List command, List cannedProcesses) { for (final CannedProcess cp in cannedProcesses) { diff --git a/tools/engine_tool/test/worker_pool_test.dart b/tools/engine_tool/test/worker_pool_test.dart index ad99d3700c78d..3b6b9aa568bf6 100644 --- a/tools/engine_tool/test/worker_pool_test.dart +++ b/tools/engine_tool/test/worker_pool_test.dart @@ -73,7 +73,10 @@ void main() { Environment( abi: ffi.Abi.macosArm64, engine: engine, - platform: FakePlatform(operatingSystem: Platform.macOS), + platform: FakePlatform( + operatingSystem: Platform.macOS, + pathSeparator: '/', + ), processRunner: ProcessRunner( processManager: FakeProcessManager(onStart: (List command) { runHistory.add(command); diff --git a/tools/pkg/engine_build_configs/bin/check.dart b/tools/pkg/engine_build_configs/bin/check.dart index 2e057e93f4597..fb890eb880a07 100644 --- a/tools/pkg/engine_build_configs/bin/check.dart +++ b/tools/pkg/engine_build_configs/bin/check.dart @@ -129,12 +129,6 @@ List checkForInvalidBuildNames(Map configs) { ? osNames : ciNames; if (!goodPrefixes.any(build.name.startsWith)) { - if (name.contains('local_engine')) { - // TODO(zanderso): Check these builds as well after local_engine is - // fixed. - // https://github.com/flutter/flutter/issues/145263 - return; - } errors.add( '${build.name} in $name must start with one of ' '{${goodPrefixes.join(', ')}}',