Skip to content

Commit

Permalink
[Android] Refactor the flutter run Android console output test (#11…
Browse files Browse the repository at this point in the history 
…5023)

* [Android] Refactor the flutter run Android console output test

* CI bump
  • Loading branch information
@loic-sharma
loic-sharma authored Dec 6, 2022
1 parent 577a88b commit 30c5751
Show file tree
Hide file tree
Showing 2 changed files with 112 additions and 154 deletions.
156 changes: 2 additions & 154 deletions dev/devicelab/bin/tasks/run_release_test.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,161 +2,9 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:convert';
import 'dart:io';

import 'package:flutter_devicelab/framework/devices.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
import 'package:flutter_devicelab/framework/utils.dart';
import 'package:path/path.dart' as path;
import 'package:flutter_devicelab/tasks/run_tests.dart';

void main() {
task(() async {
final Device device = await devices.workingDevice;
await device.unlock();
final Directory appDir = dir(path.join(flutterDirectory.path, 'dev/integration_tests/ui'));
await inDirectory(appDir, () async {
final Completer<void> ready = Completer<void>();
final List<String> stdout = <String>[];
final List<String> stderr = <String>[];

// Uninstall if the app is already installed on the device to get to a clean state.
print('uninstalling...');
final Process uninstall = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
<String>['--suppress-analytics', 'install', '--uninstall-only', '-d', device.deviceId],
)..stdout
.transform<String>(utf8.decoder)
.transform<String>(const LineSplitter())
.listen((String line) {
print('uninstall:stdout: $line');
})..stderr
.transform<String>(utf8.decoder)
.transform<String>(const LineSplitter())
.listen((String line) {
print('uninstall:stderr: $line');
stderr.add(line);
});
if (await uninstall.exitCode != 0) {
throw 'flutter install --uninstall-only failed.';
}

print('run: starting...');
final Process run = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
<String>['--suppress-analytics', 'run', '--release', '-d', device.deviceId, 'lib/main.dart'],
isBot: false, // we just want to test the output, not have any debugging info
);
int? runExitCode;
run.stdout
.transform<String>(utf8.decoder)
.transform<String>(const LineSplitter())
.listen((String line) {
print('run:stdout: $line');
if (
!line.startsWith('Building flutter tool...') &&
!line.startsWith('Running "flutter pub get" in ui...') &&
!line.startsWith('Initializing gradle...') &&
!line.contains('settings_aar.gradle') &&
!line.startsWith('Resolving dependencies...') &&
// Catch engine piped output from unrelated concurrent Flutter apps
!line.contains(RegExp(r'[A-Z]\/flutter \([0-9]+\):')) &&
// Empty lines could be due to the progress spinner breaking up.
line.length > 1
) {
stdout.add(line);
}
if (line.contains('Quit (terminate the application on the device).')) {
ready.complete();
}
});
run.stderr
.transform<String>(utf8.decoder)
.transform<String>(const LineSplitter())
// TODO(egarciad): Remove once https://github.com/flutter/flutter/issues/95131 is fixed.
.skipWhile((String line) => line.contains('Mapping new ns'))
.listen((String line) {
print('run:stderr: $line');
stderr.add(line);
});
unawaited(run.exitCode.then<void>((int exitCode) { runExitCode = exitCode; }));
await Future.any<dynamic>(<Future<dynamic>>[ ready.future, run.exitCode ]);
if (runExitCode != null) {
throw 'Failed to run test app; runner unexpected exited, with exit code $runExitCode.';
}
run.stdin.write('q');

await run.exitCode;

if (stderr.isNotEmpty) {
throw 'flutter run --release had output on standard error.';
}

_findNextMatcherInList(
stdout,
(String line) => line.startsWith('Launching lib/main.dart on ') && line.endsWith(' in release mode...'),
'Launching lib/main.dart on',
);

_findNextMatcherInList(
stdout,
(String line) => line.startsWith("Running Gradle task 'assembleRelease'..."),
"Running Gradle task 'assembleRelease'...",
);

_findNextMatcherInList(
stdout,
(String line) => line.contains('Built build/app/outputs/flutter-apk/app-release.apk (') && line.contains('MB).'),
'Built build/app/outputs/flutter-apk/app-release.apk',
);

_findNextMatcherInList(
stdout,
(String line) => line.startsWith('Installing build/app/outputs/flutter-apk/app-release.apk...'),
'Installing build/app/outputs/flutter-apk/app-release.apk...',
);

_findNextMatcherInList(
stdout,
(String line) => line.contains('Quit (terminate the application on the device).'),
'q Quit (terminate the application on the device)',
);

_findNextMatcherInList(
stdout,
(String line) => line == 'Application finished.',
'Application finished.',
);
});
return TaskResult.success(null);
});
}

void _findNextMatcherInList(
List<String> list,
bool Function(String testLine) matcher,
String errorMessageExpectedLine
) {
final List<String> copyOfListForErrorMessage = List<String>.from(list);

while (list.isNotEmpty) {
final String nextLine = list.first;
list.removeAt(0);

if (matcher(nextLine)) {
return;
}
}

throw '''
Did not find expected line
$errorMessageExpectedLine
in flutter run --release stdout
$copyOfListForErrorMessage
''';
task(createAndroidRunReleaseTest());
}
110 changes: 110 additions & 0 deletions dev/devicelab/lib/tasks/run_tests.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,24 +11,121 @@ import '../framework/framework.dart';
import '../framework/task_result.dart';
import '../framework/utils.dart';

TaskFunction createAndroidRunReleaseTest() {
return AndroidRunOutputTest(release: true);
}

TaskFunction createMacOSRunReleaseTest() {
return DesktopRunOutputTest(
// TODO(cbracken): https://github.com/flutter/flutter/issues/87508#issuecomment-1043753201
// Switch to dev/integration_tests/ui once we have CocoaPods working on M1 Macs.
'${flutterDirectory.path}/examples/hello_world',
'lib/main.dart',
release: true,
allowStderr: true,
);
}

class AndroidRunOutputTest extends RunOutputTask {
AndroidRunOutputTest({required super.release}) : super(
'${flutterDirectory.path}/dev/integration_tests/ui',
'lib/main.dart',
);

@override
Future<void> prepare(String deviceId) async {
// Uninstall if the app is already installed on the device to get to a clean state.
final List<String> stderr = <String>[];
print('uninstalling...');
final Process uninstall = await startFlutter(
'install',
options: <String>['--suppress-analytics', '--uninstall-only', '-d', deviceId],
isBot: false,
);
uninstall.stdout
.transform<String>(utf8.decoder)
.transform<String>(const LineSplitter())
.listen((String line) {
print('uninstall:stdout: $line');
});
uninstall.stderr
.transform<String>(utf8.decoder)
.transform<String>(const LineSplitter())
.listen((String line) {
print('uninstall:stderr: $line');
stderr.add(line);
});
if (await uninstall.exitCode != 0) {
throw 'flutter install --uninstall-only failed.';
}
if (stderr.isNotEmpty) {
throw 'flutter install --uninstall-only had output on standard error.';
}
}

@override
bool isExpectedStderr(String line) {
// TODO(egarciad): Remove once https://github.com/flutter/flutter/issues/95131 is fixed.
return line.contains('Mapping new ns');
}

@override
TaskResult verify(List<String> stdout, List<String> stderr) {
_findNextMatcherInList(
stdout,
(String line) => line.startsWith('Launching lib/main.dart on ') && line.endsWith(' in release mode...'),
'Launching lib/main.dart on',
);

_findNextMatcherInList(
stdout,
(String line) => line.startsWith("Running Gradle task 'assembleRelease'..."),
"Running Gradle task 'assembleRelease'...",
);

_findNextMatcherInList(
stdout,
(String line) => line.contains('Built build/app/outputs/flutter-apk/app-release.apk (') && line.contains('MB).'),
'Built build/app/outputs/flutter-apk/app-release.apk',
);

_findNextMatcherInList(
stdout,
(String line) => line.startsWith('Installing build/app/outputs/flutter-apk/app-release.apk...'),
'Installing build/app/outputs/flutter-apk/app-release.apk...',
);

_findNextMatcherInList(
stdout,
(String line) => line.contains('Quit (terminate the application on the device).'),
'q Quit (terminate the application on the device)',
);

_findNextMatcherInList(
stdout,
(String line) => line == 'Application finished.',
'Application finished.',
);

return TaskResult.success(null);
}
}

class DesktopRunOutputTest extends RunOutputTask {
DesktopRunOutputTest(
super.testDirectory,
super.testTarget, {
required super.release,
this.allowStderr = false,
}
);

/// Whether `flutter run` is expected to produce output on stderr.
final bool allowStderr;

@override
bool isExpectedStderr(String line) => allowStderr;

@override
TaskResult verify(List<String> stdout, List<String> stderr) {
_findNextMatcherInList(
Expand Down Expand Up @@ -80,6 +177,8 @@ abstract class RunOutputTask {
final List<String> stdout = <String>[];
final List<String> stderr = <String>[];

await prepare(deviceId);

final List<String> options = <String>[
testTarget,
'-d',
Expand Down Expand Up @@ -107,6 +206,7 @@ abstract class RunOutputTask {
run.stderr
.transform<String>(utf8.decoder)
.transform<String>(const LineSplitter())
.skipWhile(isExpectedStderr)
.listen((String line) {
print('run:stderr: $line');
stderr.add(line);
Expand All @@ -120,10 +220,20 @@ abstract class RunOutputTask {

await run.exitCode;

if (stderr.isNotEmpty) {
throw 'flutter run ${release ? '--release' : ''} had unexpected output on standard error.';
}

return verify(stdout, stderr);
});
}

/// Prepare the device for running the test app.
Future<void> prepare(String deviceId) => Future<void>.value();

/// Returns true if this stderr output line is expected.
bool isExpectedStderr(String line) => false;

/// Verify the output of `flutter run`.
TaskResult verify(List<String> stdout, List<String> stderr) => throw UnimplementedError('verify is not implemented');

Expand Down

0 comments on commit 30c5751

Please sign in to comment.