Browse Source

Adding l10n screenshot tests for Fenix (#3562)

* Adding l10n screenshot tests for Fenix

* fixing comments

* fix klint and detekt error and adding all tests

* better test name and doc added explaining tests
master
kglazko 2 years ago
committed by Colin Lee
parent
commit
166218f74a
  1. 3
      Gemfile
  2. 2
      app/build.gradle
  3. 71
      app/src/androidTest/java/org/mozilla/fenix/ui/screenshots/DefaultHomeScreenTest.kt
  4. 75
      app/src/androidTest/java/org/mozilla/fenix/ui/screenshots/ScreenshotTest.java
  5. 150
      app/src/androidTest/java/org/mozilla/fenix/ui/screenshots/ThreeDotMenuScreenShotTest.kt
  6. 10
      app/src/debug/AndroidManifest.xml
  7. 2
      architecture/build.gradle
  8. 15
      docs/l10nScreenshotsTests.md
  9. 2
      fastlane/Appfile
  10. 28
      fastlane/Screengrabfile

3
Gemfile

@ -0,0 +1,3 @@
source "https://rubygems.org"
gem "fastlane"

2
app/build.gradle

@ -376,6 +376,8 @@ dependencies {
// androidTestImplementation Deps.tools_espresso_core
androidTestImplementation Deps.uiautomator
androidTestImplementation "tools.fastlane:screengrab:1.2.0"
androidTestImplementation "br.com.concretesolutions:kappuccino:1.2.1"
androidTestImplementation Deps.espresso_core, {
exclude group: 'com.android.support', module: 'support-annotations'

71
app/src/androidTest/java/org/mozilla/fenix/ui/screenshots/DefaultHomeScreenTest.kt

@ -0,0 +1,71 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.fenix.ui.screenshots
import androidx.test.rule.ActivityTestRule
import org.junit.After
import org.junit.Rule
import org.junit.Test
import org.mozilla.fenix.helpers.HomeActivityTestRule
import org.mozilla.fenix.HomeActivity
import org.mozilla.fenix.R
import android.os.SystemClock
import tools.fastlane.screengrab.Screengrab
import tools.fastlane.screengrab.locale.LocaleTestRule
import androidx.test.espresso.Espresso.onView
import androidx.test.espresso.action.ViewActions.click
import androidx.test.espresso.action.ViewActions.swipeUp
import androidx.test.espresso.action.ViewActions.swipeDown
import androidx.test.espresso.matcher.ViewMatchers.hasFocus
import androidx.test.espresso.matcher.ViewMatchers.isDisplayed
import androidx.test.espresso.matcher.ViewMatchers.withId
import org.hamcrest.Matchers.allOf
class DefaultHomeScreenTest : ScreenshotTest() {
@Rule @JvmField
val localeTestRule = LocaleTestRule()
@get:Rule
var mActivityTestRule: ActivityTestRule<HomeActivity> = HomeActivityTestRule()
@After
fun tearDown() {
mActivityTestRule.getActivity().finishAndRemoveTask()
}
@Test
fun showDefaultHomeScreen() {
onView(allOf(withId(R.id.homeLayout), isDisplayed(), hasFocus()))
onView(allOf(withId(R.id.toolbar), isDisplayed()))
SystemClock.sleep(5000)
Screengrab.screenshot("home-screen")
onView(allOf(withId(R.id.privateBrowsingButton))).perform(click())
Screengrab.screenshot("private-browsing-menu")
onView(allOf(withId(R.id.privateBrowsingButton))).perform(click())
}
@Test
fun scrollHomeScreen() {
onView(withId(R.id.home_component)).perform(swipeUp())
Screengrab.screenshot("home-screen2")
SystemClock.sleep(3000)
onView(withId(R.id.home_component)).perform(swipeUp())
Screengrab.screenshot("home-screen3")
SystemClock.sleep(3000)
onView(withId(R.id.finish_button)).perform(click())
SystemClock.sleep(3000)
Screengrab.screenshot("finish-button")
SystemClock.sleep(3000)
onView(withId(R.id.home_component)).perform(swipeDown())
SystemClock.sleep(3000)
onView(withId(R.id.add_tab_button)).perform(click())
Screengrab.screenshot("add_tab_button")
}
}

75
app/src/androidTest/java/org/mozilla/fenix/ui/screenshots/ScreenshotTest.java

@ -0,0 +1,75 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.fenix.ui.screenshots;
import android.Manifest;
import android.app.Instrumentation;
import android.content.Context;
import androidx.annotation.StringRes;
import androidx.test.platform.app.InstrumentationRegistry;
import androidx.test.rule.GrantPermissionRule;
import androidx.test.uiautomator.UiDevice;
import android.text.format.DateUtils;
import org.junit.Before;
import org.junit.Rule;
import org.junit.rules.TestRule;
import org.junit.rules.TestWatcher;
import org.junit.runner.Description;
import tools.fastlane.screengrab.Screengrab;
import tools.fastlane.screengrab.UiAutomatorScreenshotStrategy;
/**
* Base class for tests that take screenshots.
*/
public abstract class ScreenshotTest {
final long waitingTime = DateUtils.SECOND_IN_MILLIS * 10;
private Context targetContext;
UiDevice device;
@Rule
public GrantPermissionRule permissionRule = GrantPermissionRule.grant(Manifest.permission.WRITE_EXTERNAL_STORAGE);
@Rule
public TestRule screenshotOnFailureRule = new TestWatcher() {
@Override
protected void failed(Throwable e, Description description) {
// On error take a screenshot so that we can debug it easily
Screengrab.screenshot("FAILURE-" + getScreenshotName(description));
}
private String getScreenshotName(Description description) {
return description.getClassName().replace(".", "-")
+ "_"
+ description.getMethodName().replace(".", "-");
}
};
@Before
public void setUpScreenshots() {
Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation();
targetContext = instrumentation.getTargetContext();
device = UiDevice.getInstance(instrumentation);
// Use this to switch between default strategy and HostScreencap strategy
Screengrab.setDefaultScreenshotStrategy(new UiAutomatorScreenshotStrategy());
}
String getString(@StringRes int resourceId) {
return targetContext.getString(resourceId).trim();
}
String getString(@StringRes int resourceId, Object... formatArgs) {
return targetContext.getString(resourceId, formatArgs).trim();
}
public void takeScreenshotsAfterWait(String filename, int waitingTime) throws InterruptedException {
Thread.sleep(waitingTime);
Screengrab.screenshot(filename);
}
}

150
app/src/androidTest/java/org/mozilla/fenix/ui/screenshots/ThreeDotMenuScreenShotTest.kt

@ -0,0 +1,150 @@
package org.mozilla.fenix.ui.screenshots
import android.os.SystemClock
import androidx.test.rule.ActivityTestRule
import org.junit.After
import org.junit.Rule
import org.junit.Test
import org.mozilla.fenix.HomeActivity
import org.mozilla.fenix.R
import org.mozilla.fenix.helpers.HomeActivityTestRule
import tools.fastlane.screengrab.Screengrab
import tools.fastlane.screengrab.locale.LocaleTestRule
import androidx.test.espresso.Espresso.onView
import androidx.test.espresso.action.ViewActions
import androidx.test.espresso.action.ViewActions.click
import androidx.test.espresso.action.ViewActions.swipeUp
import androidx.test.espresso.matcher.ViewMatchers.withId
import br.com.concretesolutions.kappuccino.actions.ClickActions
import org.hamcrest.Matchers.allOf
import org.mozilla.fenix.ui.robots.homeScreen
class ThreeDotMenuScreenShotTest : ScreenshotTest() {
@Rule @JvmField
val localeTestRule = LocaleTestRule()
@get:Rule
var mActivityTestRule: ActivityTestRule<HomeActivity> = HomeActivityTestRule()
@After
fun tearDown() {
mActivityTestRule.getActivity().finishAndRemoveTask()
}
@Test
fun threeDotMenu() {
onView(allOf(withId(R.id.menuButton))).perform(click())
Screengrab.screenshot("three-dot-menu")
device.pressBack()
}
@Test
fun settingsTest() {
homeScreen {
}.openThreeDotMenu {
}
settingsButton2()
Screengrab.screenshot("settings")
SystemClock.sleep(1000)
settingsAccount()
Screengrab.screenshot("settings-sync")
device.pressBack()
settingsTheme()
Screengrab.screenshot("settings-theme")
device.pressBack()
settingsSearch()
Screengrab.screenshot("settings-search")
device.pressBack()
settingsAccessibility()
Screengrab.screenshot("settings-accessibility")
device.pressBack()
settingsTp()
Screengrab.screenshot("settings-tp")
device.pressBack()
}
@Test
fun settingsAfterScrollMenusTest() {
homeScreen {
}.openThreeDotMenu {
}
settingsButton2()
SystemClock.sleep(1000)
onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
Screengrab.screenshot("settings-scroll")
SystemClock.sleep(2000)
settingsRemoveData()
Screengrab.screenshot("settings-delete-browsing-data")
device.pressBack()
SystemClock.sleep(1000)
onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
Screengrab.screenshot("settings-scroll")
SystemClock.sleep(3000)
settingsTelemetry()
Screengrab.screenshot("settings-telemetry")
device.pressBack()
}
@Test
fun settingsScrollToBottomTest() {
homeScreen {
}.openThreeDotMenu {
}
settingsButton2()
SystemClock.sleep(1000)
onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
Screengrab.screenshot("settings-scroll2")
SystemClock.sleep(1000)
onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
Screengrab.screenshot("settings-scroll3")
SystemClock.sleep(1000)
onView(withId(R.id.recycler_view)).perform(swipeUp())
Screengrab.screenshot("settings-scroll4")
}
@Test
fun libraryTest() {
homeScreen {
}.openThreeDotMenu {
}
libraryButton()
Screengrab.screenshot("library")
bookmarksButton()
Screengrab.screenshot("library-bookmarks")
device.pressBack()
historyButton()
Screengrab.screenshot("library-history")
}
}
fun settingsButton2() = ClickActions.click { text(R.string.settings) }
fun libraryButton() = ClickActions.click { text(R.string.browser_menu_your_library) }
fun bookmarksButton() = ClickActions.click { text(R.string.library_bookmarks) }
fun historyButton() = ClickActions.click { text(R.string.library_history) }
fun settingsAccount() = ClickActions.click { text(R.string.preferences_sync) }
fun settingsSearch() = ClickActions.click { text(R.string.preferences_search_engine) }
fun settingsTheme() = ClickActions.click { text(R.string.preferences_theme) }
fun settingsAccessibility() = ClickActions.click { text(R.string.preferences_accessibility) }
fun settingsTp() = ClickActions.click { text(R.string.preferences_tracking_protection) }
fun settingsRemoveData() = ClickActions.click { text(R.string.preferences_delete_browsing_data) }
fun settingsTelemetry() = ClickActions.click { text(R.string.preferences_data_choices) }

10
app/src/debug/AndroidManifest.xml

@ -4,6 +4,16 @@
xmlns:tools="http://schemas.android.com/tools">
<uses-permission android:name="android.permission.ACCESS_WIFI_STATE" />
<!-- Allows unlocking your device and activating its screen so UI tests can succeed -->
<uses-permission android:name="android.permission.DISABLE_KEYGUARD"/>
<uses-permission android:name="android.permission.WAKE_LOCK"/>
<!-- Allows for storing and retrieving screenshots -->
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
<!-- Allows changing locales -->
<uses-permission android:name="android.permission.CHANGE_CONFIGURATION" />
<application
tools:replace="android:name"

2
architecture/build.gradle

@ -12,7 +12,7 @@ android {
defaultConfig {
minSdkVersion 21
targetSdkVersion 28
testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
}
buildTypes {

15
docs/l10nScreenshotsTests.md

@ -0,0 +1,15 @@
### Screenshtos Tests
We are using [`screengrab`](https://docs.fastlane.tools/getting-started/android/screenshots/) which works with fastlane to automate the process of capturing screenshots.
All the l10n screenshots are generated through the ui tests. These particular tests run as part of the screenshots package (`app/src/androidTest/mozilla/fenix/ui/screenshots`)
### Steps to Run / Build
1. Install the gem:
`sudo gem install screengrab`
2. From command line run:
`fastlane screengrab --test_instrumentation_runner "androidx.test.runner.AndroidJUnitRunner"`
Screenshots will be saved in the root directory: `fastlane/metadata/android`
If there is a failure and screenshots are not saved, it may be necessary to create these folders manually first.
Currently screenshots are uploaded to [`gcloud`](https://storage.googleapis.com/l10n-fenix/2019-06-21/screenshots.html) once they are generated manually by running the previous command line.
In the future this process will be automated too and documented here.

2
fastlane/Appfile

@ -0,0 +1,2 @@
json_key_file("") # Path to the json secret file - Follow https://docs.fastlane.tools/actions/supply/#setup to get one
package_name("org.mozilla.fenix") # e.g. com.krausefx.app

28
fastlane/Screengrabfile

@ -0,0 +1,28 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
app_package_name('org.mozilla.fenix.debug')
use_tests_in_packages(['org.mozilla.fenix.ui.screenshots'])
app_apk_path('app/build/outputs/apk/x86/debug/app-x86-debug.apk')
tests_apk_path('app/build/outputs/apk/androidTest/x86/debug/app-x86-debug-androidTest.apk')
test_instrumentation_runner 'androidx.test.runner.AndroidJUnitRunner'
locales(['en-US', 'fr-FR', 'it-IT', 'de-DE', 'ja', 'ru', 'zh-CN', 'zh-TW', 'ko'])
# clear all previously generated screenshots in your local output directory before creating new ones
clear_previous_screenshots(true)
# Clear all previous screenshots locally. Technically not needed in automation.
# But it's easier to debug this on a local device if there are no old screenshots
# hanging around.
clear_previous_screenshots true
exit_on_test_failure false
skip_open_summary false
reinstall_app true
# For more information about all available options run
# fastlane screengrab --help
Loading…
Cancel
Save