[instrumentation_adapter] enable Firebase Test Lab Android testing (#1866)

collinjackson · web-flow · commit f02aab8103e2 · 2019-08-27T13:47:59.000-07:00
This pull request adds Firebase Test Lab Android instrumentation test adapter to the first-party plugins repo. Using the plugin requires calling `InstrumentationAdapterFlutterBinding.ensureInitialized` and using `testWidgets` rather than `test`. Furthermore, a Java test file must be added to the androidTest folder. Examples of this will be provided in a subsequent PR. Ultimately I'd like to refactor the common functionality into engine and see the "flutter create" template include the boilerplate files for running instrumentation tests automatically, but first we need to try them out a bit with plugins and iterate on usability. So I've put the functionality into a plugin. Joint work with @digiter
diff --git a/.ci/Dockerfile b/.ci/Dockerfile
@@ -5,7 +5,6 @@ RUN yes | sdkmanager \
     "platforms;android-27" \
     "build-tools;27.0.3" \
     "extras;google;m2repository" \
-    "extras;android;m2repository" \
-    "system-images;android-21;default;armeabi-v7a"
+    "extras;android;m2repository"
 
 RUN yes | sdkmanager --licenses
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -30,11 +30,6 @@ task:
           PLUGIN_SHARDING: "--shardIndex 0 --shardCount 2"
           PLUGIN_SHARDING: "--shardIndex 1 --shardCount 2"
         MAPS_API_KEY: ENCRYPTED[596a9f6bca436694625ac50851dc5da6b4d34cba8025f7db5bc9465142e8cd44e15f69e3507787753accebfc4910d550]
-      create_device_script:
-        echo no | avdmanager -v create avd -n test -k "system-images;android-21;default;armeabi-v7a"
-      start_emulator_background_script:
-      - $ANDROID_HOME/emulator/emulator -avd test -no-audio -no-window
-      wait_for_emulator_script: adb wait-for-device
       script:
         # Unsetting CIRRUS_CHANGE_MESSAGE and CIRRUS_COMMIT_MESSAGE as they
         # might include non-ASCII characters which makes Gradle crash.
@@ -48,9 +43,6 @@ task:
         - export CIRRUS_COMMIT_MESSAGE=""
         - ./script/incremental_build.sh build-examples --apk
         - ./script/incremental_build.sh java-test  # must come after apk build
-        # TODO(jackson): Re-enable once Android emulators support Firebase
-        # https://github.com/flutter/flutter/issues/29571
-        # - ./script/incremental_build.sh drive-examples
         - export CIRRUS_CHANGE_MESSAGE=`cat /tmp/cirrus_change_message.txt`
         - export CIRRUS_COMMIT_MESSAGE=`cat /tmp/cirrus_commit_message.txt`
 
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -50,13 +50,20 @@ To run the unit tests:
 flutter test test/<name_of_plugin>_test.dart
 ```
 
-To run the integration tests:
+To run the integration tests using Flutter driver:
 
 ```
 cd example
 flutter drive test/<name_of_plugin>.dart
 ```
 
+To run integration tests as instrumentation tests on a local Android device:
+
+```
+cd example
+(cd android && ./gradlew -Ptarget=$(pwd)/../test_live/<name_of_plugin>_test.dart connectedAndroidTest)
+```
+
 ## Contributing code
 
 We gladly accept contributions via GitHub pull requests.
diff --git a/packages/instrumentation_adapter/.gitignore b/packages/instrumentation_adapter/.gitignore
@@ -0,0 +1,7 @@
+.DS_Store
+.dart_tool/
+
+.packages
+.pub/
+
+build/
diff --git a/packages/instrumentation_adapter/.metadata b/packages/instrumentation_adapter/.metadata
@@ -0,0 +1,10 @@
+# This file tracks properties of this Flutter project.
+# Used by Flutter tool to assess capabilities and perform upgrades etc.
+#
+# This file should be version controlled and should not be manually edited.
+
+version:
+  revision: 3374ee380b499d99c50ed6dfdd45510aa8318741
+  channel: master
+
+project_type: plugin
diff --git a/packages/instrumentation_adapter/CHANGELOG.md b/packages/instrumentation_adapter/CHANGELOG.md
@@ -0,0 +1,3 @@
+## 0.0.1
+
+* Initial release
diff --git a/packages/instrumentation_adapter/LICENSE b/packages/instrumentation_adapter/LICENSE
@@ -0,0 +1,27 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//    * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//    * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//    * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/packages/instrumentation_adapter/README.md b/packages/instrumentation_adapter/README.md
@@ -0,0 +1,45 @@
+# instrumentation_adapter
+
+Adapts flutter_test results as Android instrumentation tests, making them usable for Firebase Test Lab and other Android CI providers.
+
+iOS support is not available yet, but is planned in the future.
+
+## Usage
+
+Add a dependency on the `instrumentation_adapter` package in the `dev_dependencies` section of pubspec.yaml. For plugins, do this in the pubspec.yaml of the example app.
+
+Invoke `InstrumentationAdapterFlutterBinding.ensureInitialized()` at the start of a test file.
+
+```dart
+import 'package:instrumentation_adapter/instrumentation_adapter.dart';
+import '../test/package_info.dart' as test;
+
+void main() {
+  InstrumentationAdapterFlutterBinding.ensureInitialized();
+  testWidgets("failing test example", (WidgetTester tester) async {
+    expect(2 + 2, equals(5));
+  });
+}
+```
+
+Use gradle commands to build an instrumentation test for Android.
+
+```
+pushd android
+./gradlew assembleAndroidTest
+./gradlew assembleDebug -Ptarget=<path_to_test>.dart
+popd
+```
+
+Upload to Firebase Test Lab, making sure to replace <PATH_TO_KEY_FILE>, <PROJECT_NAME>, <RESULTS_BUCKET>, and <RESULTS_DIRECTORY> with your values.
+
+```
+gcloud auth activate-service-account --key-file=<PATH_TO_KEY_FILE>
+gcloud --quiet config set project <PROJECT_NAME>
+gcloud firebase test android run --type instrumentation \
+  --app build/app/outputs/apk/debug/app-debug.apk \
+  --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk\
+  --timeout 2m \
+  --results-bucket=<RESULTS_BUCKET> \
+  --results-dir=<RESULTS_DIRECTORY>
+```
diff --git a/packages/instrumentation_adapter/android/.gitignore b/packages/instrumentation_adapter/android/.gitignore
@@ -0,0 +1,8 @@
+*.iml
+.gradle
+/local.properties
+/.idea/workspace.xml
+/.idea/libraries
+.DS_Store
+/build
+/captures
diff --git a/packages/instrumentation_adapter/android/build.gradle b/packages/instrumentation_adapter/android/build.gradle
@@ -0,0 +1,41 @@
+group 'com.example.instrumentation_adapter'
+version '1.0-SNAPSHOT'
+
+buildscript {
+    repositories {
+        google()
+        jcenter()
+    }
+
+    dependencies {
+        classpath 'com.android.tools.build:gradle:3.2.1'
+    }
+}
+
+rootProject.allprojects {
+    repositories {
+        google()
+        jcenter()
+    }
+}
+
+apply plugin: 'com.android.library'
+
+android {
+    compileSdkVersion 28
+
+    defaultConfig {
+        minSdkVersion 16
+        testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
+    }
+    lintOptions {
+        disable 'InvalidPackage'
+    }
+    dependencies {
+        api 'junit:junit:4.12'
+        api 'androidx.test:core:1.0.0'
+        api 'androidx.test:runner:1.1.1'
+        api 'androidx.test:rules:1.1.1'
+        api 'androidx.test.espresso:espresso-core:3.1.1'
+    }
+}
diff --git a/packages/instrumentation_adapter/android/gradle.properties b/packages/instrumentation_adapter/android/gradle.properties
@@ -0,0 +1,2 @@
+org.gradle.jvmargs=-Xmx1536M
+
diff --git a/packages/instrumentation_adapter/android/settings.gradle b/packages/instrumentation_adapter/android/settings.gradle
@@ -0,0 +1 @@
+rootProject.name = 'instrumentation_adapter'
diff --git a/packages/instrumentation_adapter/android/src/main/AndroidManifest.xml b/packages/instrumentation_adapter/android/src/main/AndroidManifest.xml
@@ -0,0 +1,3 @@
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+  package="dev.flutter.instrumentation_adapter">
+</manifest>
diff --git a/packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/FlutterRunner.java b/packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/FlutterRunner.java
@@ -0,0 +1,53 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package dev.flutter.plugins.instrumentationadapter;
+
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import org.junit.runner.Description;
+import org.junit.runner.Runner;
+import org.junit.runner.notification.Failure;
+import org.junit.runner.notification.RunNotifier;
+
+public class FlutterRunner extends Runner {
+
+  final Class testClass;
+
+  public FlutterRunner(Class<FlutterTest> testClass) {
+    super();
+    this.testClass = testClass;
+    try {
+      testClass.newInstance().launchActivity();
+    } catch (InstantiationException | IllegalAccessException e) {
+      throw new IllegalThreadStateException("Unable to launch test");
+    }
+  }
+
+  @Override
+  public Description getDescription() {
+    return Description.createTestDescription(testClass, "Flutter Tests");
+  }
+
+  @Override
+  public void run(RunNotifier notifier) {
+    Map<String, String> results = null;
+    try {
+      results = InstrumentationAdapterPlugin.testResults.get();
+    } catch (ExecutionException | InterruptedException e) {
+      throw new IllegalThreadStateException("Unable to get test results");
+    }
+
+    for (String name : results.keySet()) {
+      Description d = Description.createTestDescription(testClass, name);
+      notifier.fireTestStarted(d);
+      String outcome = results.get(name);
+      if (outcome.equals("failed")) {
+        Exception dummyException = new Exception(outcome);
+        notifier.fireTestFailure(new Failure(d, dummyException));
+      }
+      notifier.fireTestFinished(d);
+    }
+  }
+}
diff --git a/packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/FlutterTest.java b/packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/FlutterTest.java
@@ -0,0 +1,9 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package dev.flutter.plugins.instrumentationadapter;
+
+public abstract class FlutterTest {
+  public abstract void launchActivity();
+}
diff --git a/packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/InstrumentationAdapterPlugin.java b/packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/InstrumentationAdapterPlugin.java
@@ -0,0 +1,38 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package dev.flutter.plugins.instrumentationadapter;
+
+import io.flutter.plugin.common.MethodCall;
+import io.flutter.plugin.common.MethodChannel;
+import io.flutter.plugin.common.MethodChannel.MethodCallHandler;
+import io.flutter.plugin.common.MethodChannel.Result;
+import io.flutter.plugin.common.PluginRegistry.Registrar;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+
+/** InstrumentationAdapterPlugin */
+public class InstrumentationAdapterPlugin implements MethodCallHandler {
+
+  public static CompletableFuture<Map<String, String>> testResults = new CompletableFuture<>();
+
+  private static final String CHANNEL = "dev.flutter/InstrumentationAdapterFlutterBinding";
+
+  /** Plugin registration. */
+  public static void registerWith(Registrar registrar) {
+    final MethodChannel channel = new MethodChannel(registrar.messenger(), CHANNEL);
+    channel.setMethodCallHandler(new InstrumentationAdapterPlugin());
+  }
+
+  @Override
+  public void onMethodCall(MethodCall call, Result result) {
+    if (call.method.equals("allTestsFinished")) {
+      Map<String, String> results = call.argument("results");
+      testResults.complete(results);
+      result.success(null);
+    } else {
+      result.notImplemented();
+    }
+  }
+}
diff --git a/packages/instrumentation_adapter/lib/instrumentation_adapter.dart b/packages/instrumentation_adapter/lib/instrumentation_adapter.dart
@@ -0,0 +1,48 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:flutter/foundation.dart';
+import 'package:flutter/services.dart';
+import 'package:flutter/widgets.dart';
+
+/// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results
+/// on a channel to adapt them to native instrumentation test format.
+class InstrumentationAdapterFlutterBinding
+    extends LiveTestWidgetsFlutterBinding {
+  InstrumentationAdapterFlutterBinding() {
+    // TODO(jackson): Report test results as they arrive
+    tearDownAll(() async {
+      await _channel.invokeMethod<void>(
+          'allTestsFinished', <String, dynamic>{'results': _results});
+    });
+  }
+
+  static WidgetsBinding ensureInitialized() {
+    if (WidgetsBinding.instance == null) {
+      InstrumentationAdapterFlutterBinding();
+    }
+    assert(WidgetsBinding.instance is InstrumentationAdapterFlutterBinding);
+    return WidgetsBinding.instance;
+  }
+
+  static const MethodChannel _channel =
+      MethodChannel('dev.flutter/InstrumentationAdapterFlutterBinding');
+
+  static Map<String, String> _results = <String, String>{};
+
+  @override
+  Future<void> runTest(Future<void> testBody(), VoidCallback invariantTester,
+      {String description = '', Duration timeout}) async {
+    // TODO(jackson): Report the results individually instead of all at once
+    // See https://github.com/flutter/flutter/issues/38985
+    reportTestException =
+        (FlutterErrorDetails details, String testDescription) {
+      _results[description] = 'failed';
+    };
+    await super.runTest(testBody, invariantTester,
+        description: description, timeout: timeout);
+    _results[description] ??= 'success';
+  }
+}
diff --git a/packages/instrumentation_adapter/pubspec.yaml b/packages/instrumentation_adapter/pubspec.yaml
@@ -0,0 +1,19 @@
+name: instrumentation_adapter
+description: Runs tests that use the flutter_test API as platform native instrumentation tests.
+version: 0.0.1
+author: Flutter Team <flutter-dev@googlegroups.com>
+homepage: https://github.com/flutter/plugins/tree/master/packages/instrumentation_adapter
+
+environment:
+  sdk: ">=2.1.0 <3.0.0"
+
+dependencies:
+  flutter:
+    sdk: flutter
+  flutter_test:
+    sdk: flutter
+
+flutter:
+  plugin:
+    androidPackage: dev.flutter.plugins.instrumentationadapter
+    pluginClass: InstrumentationAdapterPlugin
diff --git a/script/build_all_plugins_app.sh b/script/build_all_plugins_app.sh
@@ -12,7 +12,7 @@ check_changed_packages > /dev/null
 
 cd $REPO_DIR/examples/all_plugins
 flutter clean > /dev/null
-(cd "$REPO_DIR" && pub global run flutter_plugin_tools gen-pubspec)
+(cd "$REPO_DIR" && pub global run flutter_plugin_tools gen-pubspec --exclude instrumentation_adapter)
 
 function error() {
   echo "$@" 1>&2

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+## 0.0.1`
	`2`	`+`
	`3`	`+* Initial release`