Android emulator detection, and a few test fixes (#1150)

* Add IsRunningOnEmulator method.

* Move emulator detection to all platforms.

* Make method public.

* Fix SKIP_TEST_ON_SIMULATOR to work on Android via run-time detection.

* Only disable messaging tests on Android emulator.

* Skip phone auth test on android emu.

* Add extra wait for TestInfoConnected

* Change virtual device to Android 32, since 33 preview is flaky.

* Turn off stress tests on emulator

* Add extra retry to TestInfoConnected

* Skip a test that's flaky on Android emulator.

* Skip another test on emu

* Format code.

Author: jonsimantov

auth/integration_test/src/integration_test.cc

@@ -862,6 +862,7 @@ class PhoneListener : public firebase::auth::PhoneAuthProvider::Listener {
 TEST_F(FirebaseAuthTest, TestPhoneAuth) {
   SKIP_TEST_ON_DESKTOP;
   SKIP_TEST_ON_TVOS;
+  SKIP_TEST_ON_ANDROID_EMULATOR;
 
 #if TARGET_OS_IPHONE
   // Note: This test requires interactivity on iOS, as it displays a CAPTCHA.

database/integration_test/src/integration_test.cc

@@ -375,6 +375,9 @@ TEST_F(FirebaseDatabaseTest, TestSignIn) {
 }
 
 TEST_F(FirebaseDatabaseTest, TestCreateWorkingPath) {
+  // This test is flaky on Android emulator for unknown reasons.
+  SKIP_TEST_ON_ANDROID_EMULATOR;
+
   SignIn();
   firebase::database::DatabaseReference working_path = CreateWorkingPath();
   LogInfo("Database URL: %s", working_path.url().c_str());
@@ -1295,7 +1298,7 @@ TEST_F(FirebaseDatabaseTest, TestInfoConnected) {
   LogDebug("Disconnecting...");
   database_->GoOffline();
   // Pause a moment to give the SDK time to realize we are disconnected.
-  ProcessEvents(1000);
+  ProcessEvents(2000);
   {
     auto disconnected = info.GetValue();
     WaitForCompletion(disconnected, "GetValue 3");
@@ -1304,12 +1307,19 @@ TEST_F(FirebaseDatabaseTest, TestInfoConnected) {
   LogDebug("Reconnecting...");
   database_->GoOnline();
   // Pause a moment to give the SDK time to realize we are reconnected.
-  ProcessEvents(1000);
+  ProcessEvents(5000);
   // Force getting a value so that we reconnect to the database.
   WaitForCompletion(ref.GetValue(), "GetValue 4 [ignored]");
+  // Pause a moment to give the SDK time to realize we are reconnected.
+#if defined(ANDROID) || (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE)
+  // This is extra brittle on mobile, so give the SDK an EXTRA opportunity
+  // to notice we are reconnected.
+  ProcessEvents(2000);
+  WaitForCompletion(ref.GetValue(), "GetValue 4B [ignored]");
+#endif  // defined(ANDROID) || (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE)
   // Pause another moment to REALLY give the SDK time to realize we are
   // reconnected.
-  ProcessEvents(1000);
+  ProcessEvents(5000);
   {
     auto reconnected = info.GetValue();
     WaitForCompletion(reconnected, "GetValue 5");

gma/integration_test/src/integration_test.cc

@@ -2015,6 +2015,7 @@ TEST_F(FirebaseGmaTest, TestRewardedAdErrorBadExtrasClassName) {
 // Stress tests.  These take a while so run them near the end.
 TEST_F(FirebaseGmaTest, TestAdViewStress) {
   SKIP_TEST_ON_DESKTOP;
+  SKIP_TEST_ON_EMULATOR;
 
   // TODO(@drsanta): remove when GMA whitelists CI devices
   TEST_REQUIRES_USER_INTERACTION_ON_IOS;
@@ -2039,6 +2040,7 @@ TEST_F(FirebaseGmaTest, TestAdViewStress) {
 
 TEST_F(FirebaseGmaTest, TestInterstitialAdStress) {
   SKIP_TEST_ON_DESKTOP;
+  SKIP_TEST_ON_EMULATOR;
 
   // TODO(@drsanta): remove when GMA whitelists CI devices
   TEST_REQUIRES_USER_INTERACTION_ON_IOS;
@@ -2061,6 +2063,7 @@ TEST_F(FirebaseGmaTest, TestInterstitialAdStress) {
 
 TEST_F(FirebaseGmaTest, TestRewardedAdStress) {
   SKIP_TEST_ON_DESKTOP;
+  SKIP_TEST_ON_EMULATOR;
 
   // TODO(@drsanta): remove when GMA whitelists CI devices
   TEST_REQUIRES_USER_INTERACTION_ON_IOS;

messaging/integration_test/src/integration_test.cc

@@ -360,11 +360,7 @@ TEST_F(FirebaseMessagingTest, TestRequestPermission) {
 TEST_F(FirebaseMessagingTest, TestReceiveToken) {
   TEST_REQUIRES_USER_INTERACTION_ON_IOS;
 
-  // TODO(b/196589796) Test fails on Android emulators and causes failures in
-  // our CI. Since we don't have a good way to deterine if the runtime is an
-  // emulator or real device, we should disable the test in CI until we find
-  // the cause of problem.
-  TEST_REQUIRES_USER_INTERACTION_ON_ANDROID;
+  SKIP_TEST_ON_ANDROID_EMULATOR;
 
   EXPECT_TRUE(RequestPermission());
 
@@ -514,11 +510,7 @@ TEST_F(FirebaseMessagingTest, TestSendMessageToToken) {
   TEST_REQUIRES_USER_INTERACTION_ON_IOS;
   SKIP_TEST_ON_DESKTOP;
 
-  // TODO(b/196589796) Test fails on Android emulators and causes failures in
-  // our CI. Since we don't have a good way to deterine if the runtime is an
-  // emulator or real device, we should disable the test in CI until we find
-  // the cause of problem.
-  TEST_REQUIRES_USER_INTERACTION_ON_ANDROID;
+  SKIP_TEST_ON_ANDROID_EMULATOR;
 
   EXPECT_TRUE(RequestPermission());
   EXPECT_TRUE(WaitForToken());
@@ -551,11 +543,7 @@ TEST_F(FirebaseMessagingTest, TestSendMessageToTopic) {
   TEST_REQUIRES_USER_INTERACTION_ON_IOS;
   SKIP_TEST_ON_DESKTOP;
 
-  // TODO(b/196589796) Test fails on Android emulators and causes failures in
-  // our CI. Since we don't have a good way to deterine if the runtime is an
-  // emulator or real device, we should disable the test in CI until we find
-  // the cause of problem.
-  TEST_REQUIRES_USER_INTERACTION_ON_ANDROID;
+  SKIP_TEST_ON_ANDROID_EMULATOR;
 
   EXPECT_TRUE(RequestPermission());
   EXPECT_TRUE(WaitForToken());
@@ -605,11 +593,7 @@ TEST_F(FirebaseMessagingTest, TestChangingListener) {
   TEST_REQUIRES_USER_INTERACTION_ON_IOS;
   SKIP_TEST_ON_DESKTOP;
 
-  // TODO(b/196589796) Test fails on Android emulators and causes failures in
-  // our CI. Since we don't have a good way to deterine if the runtime is an
-  // emulator or real device, we should disable the test in CI until we find
-  // the cause of problem.
-  TEST_REQUIRES_USER_INTERACTION_ON_ANDROID;
+  SKIP_TEST_ON_ANDROID_EMULATOR;
 
   EXPECT_TRUE(RequestPermission());
   EXPECT_TRUE(WaitForToken());

scripts/gha/print_matrix_configuration.py

@@ -168,7 +168,7 @@
   "android_target": {"type": "ftl", "device": "model=blueline,version=28"},
   "android_latest": {"type": "ftl", "device": "model=oriole,version=33"},
   "emulator_ftl_target": {"type": "ftl", "device": "model=Pixel2,version=28"},
-  "emulator_ftl_latest": {"type": "ftl", "device": "model=Pixel2.arm,version=33"},
+  "emulator_ftl_latest": {"type": "ftl", "device": "model=Pixel2.arm,version=32"},
   "emulator_target": {"type": "virtual", "image":"system-images;android-30;google_apis;x86_64"},
   "emulator_latest": {"type": "virtual", "image":"system-images;android-32;google_apis;x86_64"},
   "emulator_32bit": {"type": "virtual", "image":"system-images;android-30;google_apis;x86"},

storage/integration_test/src/integration_test.cc

@@ -399,6 +399,8 @@ const std::string kSimpleTestFile =
     "culpa qui officia deserunt mollit anim id est laborum.";
 
 TEST_F(FirebaseStorageTest, TestWriteAndReadByteBuffer) {
+  SKIP_TEST_ON_ANDROID_EMULATOR;
+
   SignIn();
 
   firebase::storage::StorageReference ref =

testing/test_framework/src/android/android_firebase_test_framework.cc

@@ -222,4 +222,26 @@ bool FirebaseTest::GetPersistentString(const char* key,
   return true;
 }
 
+bool FirebaseTest::IsRunningOnEmulator() {
+  JNIEnv* env = app_framework::GetJniEnv();
+  jobject activity = app_framework::GetActivity();
+  jclass test_helper_class = app_framework::FindClass(
+      env, activity, "com/google/firebase/example/TestHelper");
+  if (env->ExceptionCheck()) {
+    env->ExceptionDescribe();
+    env->ExceptionClear();
+    return false;
+  }
+  jmethodID is_running_on_emulator =
+      env->GetStaticMethodID(test_helper_class, "isRunningOnEmulator", "()Z");
+  jboolean result =
+      env->CallStaticBooleanMethod(test_helper_class, is_running_on_emulator);
+  if (env->ExceptionCheck()) {
+    env->ExceptionDescribe();
+    env->ExceptionClear();
+    return false;
+  }
+  return result ? true : false;
+}
+
 }  // namespace firebase_test_framework

testing/test_framework/src/android/java/com/google/firebase/example/TestHelper.java

@@ -0,0 +1,31 @@
+// Copyright 2022 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.firebase.example;
+
+import android.os.Build;
+
+/**
+ * A simple class with test helper methods.
+ */
+
+public final class TestHelper {
+  private static final String TAG = "TestHelper";
+  public static boolean isRunningOnEmulator() {
+    return Build.BRAND.contains("generic") || Build.DEVICE.contains("generic")
+        || Build.PRODUCT.contains("sdk") || Build.HARDWARE.contains("goldfish")
+        || Build.MANUFACTURER.contains("Genymotion") || Build.PRODUCT.contains("vbox86p")
+        || Build.DEVICE.contains("vbox86p") || Build.HARDWARE.contains("vbox86");
+  }
+}

testing/test_framework/src/desktop/desktop_firebase_test_framework.cc

@@ -49,4 +49,9 @@ bool FirebaseTest::GetPersistentString(const char* key,
   return false;
 }
 
+bool FirebaseTest::IsRunningOnEmulator() {
+  // No emulators on desktop.
+  return false;
+}
+
 }  // namespace firebase_test_framework

testing/test_framework/src/firebase_test_framework.h

@@ -86,7 +86,8 @@ namespace firebase_test_framework {
 // SKIP_TEST_ON_LINUX
 // SKIP_TEST_ON_WINDOWS
 // SKIP_TEST_ON_MACOS
-// SKIP_TEST_ON_SIMULATOR
+// SKIP_TEST_ON_SIMULATOR / SKIP_TEST_ON_EMULATOR (identical)
+// SKIP_TEST_ON_IOS_SIMULATOR / SKIP_TEST_ON_ANDROID_EMULATOR
 //
 // Also includes a special macro SKIP_TEST_IF_USING_STLPORT if compiling for
 // Android STLPort, which does not fully support C++11.
@@ -179,24 +180,29 @@ namespace firebase_test_framework {
 #define SKIP_TEST_ON_ANDROID ((void)0)
 #endif  // defined(ANDROID)
 
-#if defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE && TARGET_OS_SIMULATOR
-#define SKIP_TEST_ON_SIMULATOR                              \
-  {                                                         \
-    app_framework::LogInfo("Skipping %s on iOS simulator.", \
-                           test_info_->name());             \
-    GTEST_SKIP();                                           \
-    return;                                                 \
-  }
-#elif defined(ANDROID) && (defined(__x86_64__) || defined(__i386__))
-#define SKIP_TEST_ON_SIMULATOR                                  \
-  {                                                             \
-    app_framework::LogInfo("Skipping %s on Android simulator.", \
-                           test_info_->name());                 \
-    GTEST_SKIP();                                               \
-    return;                                                     \
+// Android needs to determine emulator at runtime, so we can't just use #ifdef.
+#define SKIP_TEST_ON_SIMULATOR                                     \
+  {                                                                \
+    if (IsRunningOnEmulator()) {                                   \
+      app_framework::LogInfo("Skipping %s on simulator/emulator.", \
+                             test_info_->name());                  \
+      GTEST_SKIP();                                                \
+      return;                                                      \
+    }                                                              \
   }
+
+// Accept either name, simulator or emulator.
+#define SKIP_TEST_ON_EMULATOR SKIP_TEST_ON_SIMULATOR
+
+#if defined(ANDROID)
+#define SKIP_TEST_ON_ANDROID_EMULATOR SKIP_TEST_ON_EMULATOR
+#define SKIP_TEST_ON_IOS_SIMULATOR ((void)0)
+#elif defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE
+#define SKIP_TEST_ON_ANDROID_EMULATOR ((void)0)
+#define SKIP_TEST_ON_IOS_SIMULATOR SKIP_TEST_ON_SIMULATOR
 #else
-#define SKIP_TEST_ON_SIMULATOR ((void)0)
+#define SKIP_TEST_ON_IOS_SIMULATOR ((void)0)
+#define SKIP_TEST_ON_ANDROID_EMULATOR ((void)0)
 #endif
 
 #if defined(STLPORT)
@@ -322,6 +328,10 @@ class FirebaseTest : public testing::Test {
   // successful, false if something went wrong.
   static bool SetPersistentString(const char* key, const char* value);
 
+  // Return true if the app is running on simulator/emulator, false if
+  // on a real device (or on desktop).
+  static bool IsRunningOnEmulator();
+
   // Returns true if the future completed as expected, fails the test and
   // returns false otherwise.
   static bool WaitForCompletion(const firebase::FutureBase& future,

testing/test_framework/src/ios/ios_firebase_test_framework.mm

@@ -191,4 +191,13 @@ static bool SendHttpRequest(const char* method, const char* url,
   return true;
 }
 
+bool FirebaseTest::IsRunningOnEmulator() {
+  // On iOS/tvOS it's an easy compile-time definition.
+#if TARGET_OS_SIMULATOR
+  return true;
+#else
+  return false;
+#endif
+}
+
 }  // namespace firebase_test_framework