Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-23 04:41:11 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1931842 - Fix click actions in UI tests on snackbar buttons r=aaronmt,ohorvath

Browse Source 
The UI tests started to be flaky or occasionally fail after [[ https://phabricator.services.mozilla.com/D226967 | this ]] change landed.
The click on the snackbar button wasn't properly performed using UiObject 2 (By selectors)

To overcome this problem I've switched to using exclusively compose, taking use of the newly added test tag and text to perform the click action.
I've also added verifyUrl after performing the "switch" action in the search group related tests, because in some cases the pressBack() action was performed to early and the test exited to the home screen instead on the browser screen.

✅ All 27 UI tests that use the snackbar click action successfully passed 50x on Firebase.

🔍 While trying to fix the click problems, I've noticed some inconsistencies related to some "Collections" related tests
In some cases the snackbar is covered by the tabs tray, I'll investigate this problems sepparately.

Differential Revision: https://phabricator.services.mozilla.com/D229476
This commit is contained in:
AndiAJ 2024-11-21 07:52:46 +00:00
parent 9b9714a299
commit fad4e801c3
12 changed files with 71 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/helpers/TestHelper.kt
View File

@ -8,6 +8,10 @@ import android.content.Context
import android.net.Uri
import android.util.Log
import android.view.View
import androidx.compose.ui.test.hasTestTag
import androidx.compose.ui.test.hasText
import androidx.compose.ui.test.junit4.ComposeTestRule
import androidx.compose.ui.test.performClick
import androidx.test.espresso.Espresso.onView
import androidx.test.espresso.action.ViewActions.longClick
import androidx.test.espresso.intent.Intents
@ -22,7 +26,6 @@ import androidx.test.uiautomator.By.res
import androidx.test.uiautomator.UiDevice
import androidx.test.uiautomator.UiObject
import androidx.test.uiautomator.UiObject2
import androidx.test.uiautomator.UiObjectNotFoundException
import androidx.test.uiautomator.UiScrollable
import androidx.test.uiautomator.UiSelector
import androidx.test.uiautomator.Until
@ -35,7 +38,6 @@ import org.junit.Assert.assertTrue
import org.mozilla.fenix.R
import org.mozilla.fenix.compose.snackbar.SNACKBAR_BUTTON_TEST_TAG
import org.mozilla.fenix.compose.snackbar.SNACKBAR_TEST_TAG
import org.mozilla.fenix.helpers.Constants.RETRY_COUNT
import org.mozilla.fenix.helpers.Constants.TAG
import org.mozilla.fenix.helpers.MatcherHelper.assertUIObjectExists
import org.mozilla.fenix.helpers.MatcherHelper.itemContainingText
@ -115,27 +117,13 @@ object TestHelper {
)
}
fun clickSnackbarButton(expectedText: String) {
for (i in 1..RETRY_COUNT) {
Log.i(TAG, "clickSnackbarButton: Started try #$i")
try {
Log.i(TAG, "clickSnackbarButton: Waiting for $waitingTimeShort ms for the $expectedText snackbar button to exist")
waitUntilObjectIsFound(SNACKBAR_BUTTON_TEST_TAG)
assertTrue(snackbarButton!!.children.count { it.text == expectedText } == 1)
Log.i(TAG, "clickSnackbarButton: Waited for $waitingTimeShort ms for the $expectedText snackbar button to exist")
Log.i(TAG, "clickSnackbarButton: Trying to click the $expectedText and wait for $waitingTime ms for a new window")
snackbarButton!!.click()
mDevice.waitForIdle()
Log.i(TAG, "clickSnackbarButton: Clicked the $expectedText and waited for $waitingTime ms for a new window")
break
} catch (e: UiObjectNotFoundException) {
Log.i(TAG, "clickSnackbarButton: UiObjectNotFoundException caught, executing fallback methods")
if (i == RETRY_COUNT) {
throw e
}
}
}
fun clickSnackbarButton(composeTestRule: ComposeTestRule, expectedText: String) {
Log.i(TAG, "clickSnackbarButton: Waiting for compose test rule to be idle")
composeTestRule.waitForIdle()
Log.i(TAG, "clickSnackbarButton: Waited for compose test rule to be idle")
Log.i(TAG, "clickSnackbarButton: Trying to click $expectedText snackbar button")
composeTestRule.onNode(hasTestTag(SNACKBAR_BUTTON_TEST_TAG) or hasText(expectedText)).performClick()
Log.i(TAG, "clickSnackbarButton: Clicked $expectedText snackbar button")
}
fun waitUntilSnackbarGone() {

8
mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/ui/BookmarksTest.kt
View File

@ -97,7 +97,7 @@ class BookmarksTest : TestSetup() {
}.enterURLAndEnterToBrowser(defaultWebPage.url) {
}.openThreeDotMenu {
}.bookmarkPage {
clickSnackbarButton("EDIT")
clickSnackbarButton(activityTestRule, "EDIT")
}
bookmarksMenu {
verifyEditBookmarksView()
@ -314,7 +314,7 @@ class BookmarksTest : TestSetup() {
}.openThreeDotMenu(defaultWebPage.title) {
}.clickDelete {
verifyUndoDeleteSnackBarButton()
clickSnackbarButton("UNDO")
clickSnackbarButton(activityTestRule, "UNDO")
verifySnackBarHidden()
registerAndCleanupIdlingResources(
RecyclerViewIdlingResource(activityTestRule.activity.findViewById(R.id.bookmark_list), 2),
@ -447,7 +447,7 @@ class BookmarksTest : TestSetup() {
bookmarksMenu {
verifySnackBarText(expectedText = "Bookmarks deleted")
clickSnackbarButton("UNDO")
clickSnackbarButton(activityTestRule, "UNDO")
verifyBookmarkedURL(webPages[0].url.toString())
verifyBookmarkedURL(webPages[1].url.toString())
registerAndCleanupIdlingResources(
@ -740,7 +740,7 @@ class BookmarksTest : TestSetup() {
}.clickDelete {
confirmDeletion()
verifySnackBarText(expectedText = "Deleted")
clickSnackbarButton("UNDO")
clickSnackbarButton(activityTestRule, "UNDO")
verifyFolderTitle("My Folder")
}.openThreeDotMenu("My Folder") {
}.clickDelete {

14
mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/ui/CollectionTest.kt
View File

@ -71,7 +71,7 @@ class CollectionTest : TestSetup() {
composeTabDrawer(composeTestRule) {
verifySnackBarText("Collection saved!")
clickSnackbarButton("VIEW")
clickSnackbarButton(composeTestRule, "VIEW")
}
homeScreen {
@ -103,7 +103,7 @@ class CollectionTest : TestSetup() {
}.enterURLAndEnterToBrowser(webPage.url) {
}.openTabDrawer(composeTestRule) {
createCollection(webPage.title, collectionName = collectionName)
clickSnackbarButton("VIEW")
clickSnackbarButton(composeTestRule, "VIEW")
}
homeScreen {
@ -218,7 +218,7 @@ class CollectionTest : TestSetup() {
}.enterURLAndEnterToBrowser(webPage.url) {
}.openTabDrawer(composeTestRule) {
createCollection(webPage.title, collectionName = collectionName)
clickSnackbarButton("VIEW")
clickSnackbarButton(composeTestRule, "VIEW")
}
homeScreen {
@ -230,7 +230,7 @@ class CollectionTest : TestSetup() {
homeScreen {
verifySnackBarText("Collection deleted")
clickSnackbarButton("UNDO")
clickSnackbarButton(composeTestRule, "UNDO")
verifyCollectionIsDisplayed(collectionName, true)
}
@ -370,7 +370,7 @@ class CollectionTest : TestSetup() {
}
homeScreen {
verifySnackBarText("Collection deleted")
clickSnackbarButton("UNDO")
clickSnackbarButton(composeTestRule, "UNDO")
verifyCollectionIsDisplayed(collectionName)
}.expandCollection(collectionName) {
verifyTabSavedInCollection(webPage.title, true)
@ -406,7 +406,7 @@ class CollectionTest : TestSetup() {
}
homeScreen {
verifySnackBarText("Collection deleted")
clickSnackbarButton("UNDO")
clickSnackbarButton(composeTestRule, "UNDO")
verifyCollectionIsDisplayed(collectionName)
}.expandCollection(collectionName) {
verifyTabSavedInCollection(testPage.title, true)
@ -439,7 +439,7 @@ class CollectionTest : TestSetup() {
}
homeScreen {
verifySnackBarText("Collection deleted")
clickSnackbarButton("UNDO")
clickSnackbarButton(composeTestRule, "UNDO")
verifyCollectionIsDisplayed(collectionName)
}.expandCollection(collectionName) {
verifyTabSavedInCollection(testPage.title, true)

2
mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/ui/ComposeTopSitesTest.kt
View File

@ -201,7 +201,7 @@ class ComposeTopSitesTest : TestSetup() {
}.openContextMenuOnTopSitesWithTitle(defaultWebPage.title) {
verifyTopSiteContextMenuItems()
}.removeTopSite {
clickSnackbarButton("UNDO")
clickSnackbarButton(composeTestRule, "UNDO")
verifyExistingTopSiteItem(defaultWebPage.title)
}.openContextMenuOnTopSitesWithTitle(defaultWebPage.title) {
verifyTopSiteContextMenuItems()

6
mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/ui/ContextMenusTest.kt
View File

@ -71,7 +71,7 @@ class ContextMenusTest : TestSetup() {
verifyContextMenuForLocalHostLinks(genericURL.url)
clickContextMenuItem("Open link in new tab")
verifySnackBarText("New tab opened")
clickSnackbarButton("SWITCH")
clickSnackbarButton(composeTestRule, "SWITCH")
verifyUrl(genericURL.url.toString())
}.openTabDrawer(composeTestRule) {
verifyNormalBrowsingButtonIsSelected()
@ -95,7 +95,7 @@ class ContextMenusTest : TestSetup() {
verifyContextMenuForLocalHostLinks(genericURL.url)
clickContextMenuItem("Open link in private tab")
verifySnackBarText("New private tab opened")
clickSnackbarButton("SWITCH")
clickSnackbarButton(composeTestRule, "SWITCH")
verifyUrl(genericURL.url.toString())
}.openTabDrawer(composeTestRule) {
verifyPrivateBrowsingButtonIsSelected()
@ -160,7 +160,7 @@ class ContextMenusTest : TestSetup() {
verifyLinkImageContextMenuItems(imageResource.url)
clickContextMenuItem("Open image in new tab")
verifySnackBarText("New tab opened")
clickSnackbarButton("SWITCH")
clickSnackbarButton(composeTestRule, "SWITCH")
verifyUrl(imageResource.url.toString())
}
}

4
mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/ui/DownloadTest.kt
View File

@ -161,7 +161,7 @@ class DownloadTest : TestSetup() {
}.openDownloadsManager() {
verifyDownloadedFileExistsInDownloadsList(activityTestRule, "smallZip.zip")
deleteDownloadedItem(activityTestRule, "smallZip.zip")
clickSnackbarButton("UNDO")
clickSnackbarButton(activityTestRule, "UNDO")
verifyDownloadedFileExistsInDownloadsList(activityTestRule, "smallZip.zip")
deleteDownloadedItem(activityTestRule, "smallZip.zip")
verifyEmptyDownloadsList(activityTestRule)
@ -192,7 +192,7 @@ class DownloadTest : TestSetup() {
clickDownloadedItem(activityTestRule, secondDownloadedFile)
openMultiSelectMoreOptionsMenu()
clickMultiSelectRemoveButton()
clickSnackbarButton("UNDO")
clickSnackbarButton(activityTestRule, "UNDO")
verifyDownloadedFileExistsInDownloadsList(activityTestRule, firstDownloadedFile)
verifyDownloadedFileExistsInDownloadsList(activityTestRule, secondDownloadedFile)
longClickDownloadedItem(activityTestRule, firstDownloadedFile)

2
mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/ui/MainMenuTest.kt
View File

@ -232,7 +232,7 @@ class MainMenuTest : TestSetup() {
}.closeBrowserMenuToBrowser {
longClickPageObject(MatcherHelper.itemWithText("Link 2"))
clickContextMenuItem("Open link in new tab")
clickSnackbarButton("SWITCH")
clickSnackbarButton(composeTestRule, "SWITCH")
}.openThreeDotMenu {
verifyDesktopSiteModeEnabled(false)
}

60
mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/ui/SearchTest.kt
View File

@ -34,6 +34,7 @@ import org.mozilla.fenix.helpers.MockBrowserDataHelper.setCustomSearchEngine
import org.mozilla.fenix.helpers.SearchDispatcher
import org.mozilla.fenix.helpers.TestAssetHelper
import org.mozilla.fenix.helpers.TestHelper
import org.mozilla.fenix.helpers.TestHelper.clickSnackbarButton
import org.mozilla.fenix.helpers.TestHelper.exitMenu
import org.mozilla.fenix.helpers.TestHelper.verifySnackBarText
import org.mozilla.fenix.helpers.TestHelper.waitForAppWindowToBeUpdated
@ -285,6 +286,8 @@ class SearchTest : TestSetup() {
@SmokeTest
@Test
fun searchResultsOpenedInNewTabsGenerateSearchGroupsTest() {
val firstPageUrl = TestAssetHelper.getGenericAsset(searchMockServer, 1).url
val secondPageUrl = TestAssetHelper.getGenericAsset(searchMockServer, 2).url
val searchEngineName = "TestSearchEngine"
// setting our custom mockWebServer search URL
setCustomSearchEngine(searchMockServer, searchEngineName)
@ -295,13 +298,13 @@ class SearchTest : TestSetup() {
}.submitQuery(queryString) {
longClickPageObject(MatcherHelper.itemWithText("Link 1"))
clickContextMenuItem("Open link in new tab")
TestHelper.clickSnackbarButton("SWITCH")
waitForPageToLoad()
clickSnackbarButton(activityTestRule, "SWITCH")
verifyUrl(firstPageUrl.toString())
Espresso.pressBack()
longClickPageObject(MatcherHelper.itemWithText("Link 2"))
clickContextMenuItem("Open link in new tab")
TestHelper.clickSnackbarButton("SWITCH")
waitForPageToLoad()
clickSnackbarButton(activityTestRule, "SWITCH")
verifyUrl(secondPageUrl.toString())
}.openTabDrawer(activityTestRule) {
}.openThreeDotMenu {
}.closeAllTabs {
@ -326,23 +329,23 @@ class SearchTest : TestSetup() {
}.submitQuery(queryString) {
longClickPageObject(MatcherHelper.itemWithText("Link 1"))
clickContextMenuItem("Open link in new tab")
TestHelper.clickSnackbarButton("SWITCH")
waitForPageToLoad()
clickSnackbarButton(activityTestRule, "SWITCH")
verifyUrl(firstPageUrl.toString())
Espresso.pressBack()
longClickPageObject(MatcherHelper.itemWithText("Link 1"))
clickContextMenuItem("Open link in new tab")
TestHelper.clickSnackbarButton("SWITCH")
waitForPageToLoad()
clickSnackbarButton(activityTestRule, "SWITCH")
verifyUrl(firstPageUrl.toString())
Espresso.pressBack()
longClickPageObject(MatcherHelper.itemWithText("Link 2"))
clickContextMenuItem("Open link in new tab")
TestHelper.clickSnackbarButton("SWITCH")
waitForPageToLoad()
clickSnackbarButton(activityTestRule, "SWITCH")
verifyUrl(secondPageUrl.toString())
Espresso.pressBack()
longClickPageObject(MatcherHelper.itemWithText("Link 1"))
clickContextMenuItem("Open link in new tab")
TestHelper.clickSnackbarButton("SWITCH")
waitForPageToLoad()
clickSnackbarButton(activityTestRule, "SWITCH")
verifyUrl(firstPageUrl.toString())
}.openTabDrawer(activityTestRule) {
}.openThreeDotMenu {
}.closeAllTabs {
@ -426,13 +429,13 @@ class SearchTest : TestSetup() {
}.submitQuery(queryString) {
longClickPageObject(MatcherHelper.itemWithText("Link 1"))
clickContextMenuItem("Open link in new tab")
TestHelper.clickSnackbarButton("SWITCH")
waitForPageToLoad()
clickSnackbarButton(activityTestRule, "SWITCH")
verifyUrl(firstPageUrl.toString())
TestHelper.mDevice.pressBack()
longClickPageObject(MatcherHelper.itemWithText("Link 2"))
clickContextMenuItem("Open link in new tab")
TestHelper.clickSnackbarButton("SWITCH")
waitForPageToLoad()
clickSnackbarButton(activityTestRule, "SWITCH")
verifyUrl(secondPageUrl.toString())
}.openTabDrawer(activityTestRule) {
}.openThreeDotMenu {
}.closeAllTabs {
@ -457,6 +460,7 @@ class SearchTest : TestSetup() {
@Test
fun deleteSearchGroupFromHomeScreenTest() {
val firstPageUrl = TestAssetHelper.getGenericAsset(searchMockServer, 1).url
val secondPageUrl = TestAssetHelper.getGenericAsset(searchMockServer, 2).url
// setting our custom mockWebServer search URL
val searchEngineName = "TestSearchEngine"
setCustomSearchEngine(searchMockServer, searchEngineName)
@ -467,13 +471,13 @@ class SearchTest : TestSetup() {
}.submitQuery(queryString) {
longClickPageObject(MatcherHelper.itemWithText("Link 1"))
clickContextMenuItem("Open link in new tab")
TestHelper.clickSnackbarButton("SWITCH")
waitForPageToLoad()
clickSnackbarButton(activityTestRule, "SWITCH")
verifyUrl(firstPageUrl.toString())
TestHelper.mDevice.pressBack()
longClickPageObject(MatcherHelper.itemWithText("Link 2"))
clickContextMenuItem("Open link in new tab")
TestHelper.clickSnackbarButton("SWITCH")
waitForPageToLoad()
clickSnackbarButton(activityTestRule, "SWITCH")
verifyUrl(secondPageUrl.toString())
}.openTabDrawer(activityTestRule) {
}.openThreeDotMenu {
}.closeAllTabs {
@ -509,12 +513,13 @@ class SearchTest : TestSetup() {
}.submitQuery(queryString) {
longClickPageObject(MatcherHelper.itemWithText("Link 1"))
clickContextMenuItem("Open link in new tab")
TestHelper.clickSnackbarButton("SWITCH")
waitForPageToLoad()
clickSnackbarButton(activityTestRule, "SWITCH")
verifyUrl(firstPageUrl.toString())
TestHelper.mDevice.pressBack()
longClickPageObject(MatcherHelper.itemWithText("Link 2"))
clickContextMenuItem("Open link in new tab")
TestHelper.clickSnackbarButton("SWITCH")
clickSnackbarButton(activityTestRule, "SWITCH")
verifyUrl(secondPageUrl.toString())
waitForPageToLoad()
}.openTabDrawer(activityTestRule) {
}.openThreeDotMenu {
@ -545,6 +550,7 @@ class SearchTest : TestSetup() {
@Test
fun shareAPageFromHomeScreenSearchGroupTest() {
val firstPageUrl = TestAssetHelper.getGenericAsset(searchMockServer, 1).url
val secondPageUrl = TestAssetHelper.getGenericAsset(searchMockServer, 2).url
// setting our custom mockWebServer search URL
val searchEngineName = "TestSearchEngine"
setCustomSearchEngine(searchMockServer, searchEngineName)
@ -555,13 +561,13 @@ class SearchTest : TestSetup() {
}.submitQuery(queryString) {
longClickPageObject(MatcherHelper.itemWithText("Link 1"))
clickContextMenuItem("Open link in new tab")
TestHelper.clickSnackbarButton("SWITCH")
waitForPageToLoad()
clickSnackbarButton(activityTestRule, "SWITCH")
verifyUrl(firstPageUrl.toString())
TestHelper.mDevice.pressBack()
longClickPageObject(MatcherHelper.itemWithText("Link 2"))
clickContextMenuItem("Open link in new tab")
TestHelper.clickSnackbarButton("SWITCH")
waitForPageToLoad()
clickSnackbarButton(activityTestRule, "SWITCH")
verifyUrl(secondPageUrl.toString())
}.openTabDrawer(activityTestRule) {
}.openThreeDotMenu {
}.closeAllTabs {

2
mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsSearchTest.kt
View File

@ -421,7 +421,7 @@ class SettingsSearchTest : TestSetup() {
openEngineOverflowMenu(customSearchEngineTitle)
clickDeleteSearchEngine()
verifySnackBarText("Deleted $customSearchEngineTitle")
clickSnackbarButton("UNDO")
clickSnackbarButton(activityTestRule, "UNDO")
verifyEngineListContains(customSearchEngineTitle, shouldExist = true)
changeDefaultSearchEngine(customSearchEngineTitle)
openEngineOverflowMenu(customSearchEngineTitle)

4
mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/ui/TabbedBrowsingTest.kt
View File

@ -99,7 +99,7 @@ class TabbedBrowsingTest : TestSetup() {
verifyExistingOpenTabs("Test_Page_1")
closeTab()
verifySnackBarText("Tab closed")
clickSnackbarButton("UNDO")
clickSnackbarButton(composeTestRule, "UNDO")
}
browserScreen {
verifyTabCounter("1")
@ -146,7 +146,7 @@ class TabbedBrowsingTest : TestSetup() {
verifyExistingOpenTabs("Test_Page_1")
closeTab()
verifySnackBarText("Private tab closed")
clickSnackbarButton("UNDO")
clickSnackbarButton(composeTestRule, "UNDO")
}
browserScreen {
verifyTabCounter("1")

2
mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/ui/TabbedBrowsingWithNavbarTest.kt
View File

@ -46,7 +46,7 @@ class TabbedBrowsingWithNavbarTest : TestSetup() {
verifyExistingOpenTabs("Test_Page_1")
closeTab()
verifySnackBarText("Private tab closed")
clickSnackbarButton("UNDO")
clickSnackbarButton(composeTestRule, "UNDO")
}
browserScreen {
verifyTabCounter("1")

7
mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/ui/TopSitesTest.kt
View File

@ -4,6 +4,7 @@
package org.mozilla.fenix.ui
import androidx.compose.ui.test.junit4.AndroidComposeTestRule
import org.junit.Rule
import org.junit.Test
import org.mozilla.fenix.R
@ -33,7 +34,9 @@ import org.mozilla.fenix.ui.robots.navigationToolbar
class TopSitesTest : TestSetup() {
@get:Rule
val activityIntentTestRule = HomeActivityIntentTestRule.withDefaultSettingsOverrides(skipOnboarding = true)
val activityIntentTestRule = AndroidComposeTestRule(
HomeActivityIntentTestRule.withDefaultSettingsOverrides(skipOnboarding = true),
) { it.activity }
// TestRail link: https://mozilla.testrail.io/index.php?/cases/view/532598
@SmokeTest
@ -195,7 +198,7 @@ class TopSitesTest : TestSetup() {
}.openContextMenuOnTopSitesWithTitle(defaultWebPage.title) {
verifyTopSiteContextMenuItems()
}.removeTopSite {
clickSnackbarButton("UNDO")
clickSnackbarButton(activityIntentTestRule, "UNDO")
verifyExistingTopSitesTabs(defaultWebPage.title)
}.openContextMenuOnTopSitesWithTitle(defaultWebPage.title) {
verifyTopSiteContextMenuItems()