Browse Source

Adding l10n screenshot tests for Fenix (#3562)

* Adding l10n screenshot tests for Fenix

* fixing comments

* fix klint and detekt error and adding all tests

* better test name and doc added explaining tests
master
kglazko 2 years ago
committed by Colin Lee
parent
commit
166218f74a
10 changed files with 357 additions and 1 deletions
  1. +3
    -0
      Gemfile
  2. +2
    -0
      app/build.gradle
  3. +71
    -0
      app/src/androidTest/java/org/mozilla/fenix/ui/screenshots/DefaultHomeScreenTest.kt
  4. +75
    -0
      app/src/androidTest/java/org/mozilla/fenix/ui/screenshots/ScreenshotTest.java
  5. +150
    -0
      app/src/androidTest/java/org/mozilla/fenix/ui/screenshots/ThreeDotMenuScreenShotTest.kt
  6. +10
    -0
      app/src/debug/AndroidManifest.xml
  7. +1
    -1
      architecture/build.gradle
  8. +15
    -0
      docs/l10nScreenshotsTests.md
  9. +2
    -0
      fastlane/Appfile
  10. +28
    -0
      fastlane/Screengrabfile

+ 3
- 0
Gemfile View File

@@ -0,0 +1,3 @@
source "https://rubygems.org"

gem "fastlane"

+ 2
- 0
app/build.gradle View File

@@ -376,6 +376,8 @@ dependencies {
// androidTestImplementation Deps.tools_espresso_core

androidTestImplementation Deps.uiautomator
androidTestImplementation "tools.fastlane:screengrab:1.2.0"
androidTestImplementation "br.com.concretesolutions:kappuccino:1.2.1"

androidTestImplementation Deps.espresso_core, {
exclude group: 'com.android.support', module: 'support-annotations'


+ 71
- 0
app/src/androidTest/java/org/mozilla/fenix/ui/screenshots/DefaultHomeScreenTest.kt View File

@@ -0,0 +1,71 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package org.mozilla.fenix.ui.screenshots

import androidx.test.rule.ActivityTestRule
import org.junit.After
import org.junit.Rule
import org.junit.Test
import org.mozilla.fenix.helpers.HomeActivityTestRule
import org.mozilla.fenix.HomeActivity
import org.mozilla.fenix.R
import android.os.SystemClock
import tools.fastlane.screengrab.Screengrab
import tools.fastlane.screengrab.locale.LocaleTestRule
import androidx.test.espresso.Espresso.onView
import androidx.test.espresso.action.ViewActions.click
import androidx.test.espresso.action.ViewActions.swipeUp
import androidx.test.espresso.action.ViewActions.swipeDown
import androidx.test.espresso.matcher.ViewMatchers.hasFocus
import androidx.test.espresso.matcher.ViewMatchers.isDisplayed
import androidx.test.espresso.matcher.ViewMatchers.withId
import org.hamcrest.Matchers.allOf

class DefaultHomeScreenTest : ScreenshotTest() {
@Rule @JvmField
val localeTestRule = LocaleTestRule()
@get:Rule
var mActivityTestRule: ActivityTestRule<HomeActivity> = HomeActivityTestRule()

@After
fun tearDown() {
mActivityTestRule.getActivity().finishAndRemoveTask()
}

@Test
fun showDefaultHomeScreen() {
onView(allOf(withId(R.id.homeLayout), isDisplayed(), hasFocus()))
onView(allOf(withId(R.id.toolbar), isDisplayed()))
SystemClock.sleep(5000)
Screengrab.screenshot("home-screen")

onView(allOf(withId(R.id.privateBrowsingButton))).perform(click())
Screengrab.screenshot("private-browsing-menu")
onView(allOf(withId(R.id.privateBrowsingButton))).perform(click())
}

@Test
fun scrollHomeScreen() {
onView(withId(R.id.home_component)).perform(swipeUp())
Screengrab.screenshot("home-screen2")
SystemClock.sleep(3000)

onView(withId(R.id.home_component)).perform(swipeUp())
Screengrab.screenshot("home-screen3")
SystemClock.sleep(3000)

onView(withId(R.id.finish_button)).perform(click())
SystemClock.sleep(3000)

Screengrab.screenshot("finish-button")
SystemClock.sleep(3000)

onView(withId(R.id.home_component)).perform(swipeDown())
SystemClock.sleep(3000)

onView(withId(R.id.add_tab_button)).perform(click())
Screengrab.screenshot("add_tab_button")
}
}

+ 75
- 0
app/src/androidTest/java/org/mozilla/fenix/ui/screenshots/ScreenshotTest.java View File

@@ -0,0 +1,75 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package org.mozilla.fenix.ui.screenshots;

import android.Manifest;
import android.app.Instrumentation;
import android.content.Context;
import androidx.annotation.StringRes;
import androidx.test.platform.app.InstrumentationRegistry;
import androidx.test.rule.GrantPermissionRule;
import androidx.test.uiautomator.UiDevice;
import android.text.format.DateUtils;

import org.junit.Before;
import org.junit.Rule;
import org.junit.rules.TestRule;
import org.junit.rules.TestWatcher;
import org.junit.runner.Description;

import tools.fastlane.screengrab.Screengrab;
import tools.fastlane.screengrab.UiAutomatorScreenshotStrategy;

/**
* Base class for tests that take screenshots.
*/
public abstract class ScreenshotTest {
final long waitingTime = DateUtils.SECOND_IN_MILLIS * 10;

private Context targetContext;

UiDevice device;

@Rule
public GrantPermissionRule permissionRule = GrantPermissionRule.grant(Manifest.permission.WRITE_EXTERNAL_STORAGE);

@Rule
public TestRule screenshotOnFailureRule = new TestWatcher() {
@Override
protected void failed(Throwable e, Description description) {
// On error take a screenshot so that we can debug it easily
Screengrab.screenshot("FAILURE-" + getScreenshotName(description));
}

private String getScreenshotName(Description description) {
return description.getClassName().replace(".", "-")
+ "_"
+ description.getMethodName().replace(".", "-");
}
};

@Before
public void setUpScreenshots() {
Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation();
targetContext = instrumentation.getTargetContext();
device = UiDevice.getInstance(instrumentation);

// Use this to switch between default strategy and HostScreencap strategy
Screengrab.setDefaultScreenshotStrategy(new UiAutomatorScreenshotStrategy());
}

String getString(@StringRes int resourceId) {
return targetContext.getString(resourceId).trim();
}

String getString(@StringRes int resourceId, Object... formatArgs) {
return targetContext.getString(resourceId, formatArgs).trim();
}

public void takeScreenshotsAfterWait(String filename, int waitingTime) throws InterruptedException {
Thread.sleep(waitingTime);
Screengrab.screenshot(filename);
}
}

+ 150
- 0
app/src/androidTest/java/org/mozilla/fenix/ui/screenshots/ThreeDotMenuScreenShotTest.kt View File

@@ -0,0 +1,150 @@
package org.mozilla.fenix.ui.screenshots

import android.os.SystemClock
import androidx.test.rule.ActivityTestRule

import org.junit.After
import org.junit.Rule
import org.junit.Test
import org.mozilla.fenix.HomeActivity
import org.mozilla.fenix.R
import org.mozilla.fenix.helpers.HomeActivityTestRule

import tools.fastlane.screengrab.Screengrab
import tools.fastlane.screengrab.locale.LocaleTestRule

import androidx.test.espresso.Espresso.onView
import androidx.test.espresso.action.ViewActions
import androidx.test.espresso.action.ViewActions.click
import androidx.test.espresso.action.ViewActions.swipeUp
import androidx.test.espresso.matcher.ViewMatchers.withId

import br.com.concretesolutions.kappuccino.actions.ClickActions
import org.hamcrest.Matchers.allOf
import org.mozilla.fenix.ui.robots.homeScreen

class ThreeDotMenuScreenShotTest : ScreenshotTest() {
@Rule @JvmField
val localeTestRule = LocaleTestRule()

@get:Rule
var mActivityTestRule: ActivityTestRule<HomeActivity> = HomeActivityTestRule()

@After
fun tearDown() {
mActivityTestRule.getActivity().finishAndRemoveTask()
}

@Test
fun threeDotMenu() {
onView(allOf(withId(R.id.menuButton))).perform(click())
Screengrab.screenshot("three-dot-menu")
device.pressBack()
}

@Test
fun settingsTest() {
homeScreen {
}.openThreeDotMenu {
}
settingsButton2()
Screengrab.screenshot("settings")

SystemClock.sleep(1000)
settingsAccount()
Screengrab.screenshot("settings-sync")
device.pressBack()

settingsTheme()
Screengrab.screenshot("settings-theme")
device.pressBack()

settingsSearch()
Screengrab.screenshot("settings-search")
device.pressBack()

settingsAccessibility()
Screengrab.screenshot("settings-accessibility")
device.pressBack()

settingsTp()
Screengrab.screenshot("settings-tp")
device.pressBack()
}

@Test
fun settingsAfterScrollMenusTest() {
homeScreen {
}.openThreeDotMenu {
}
settingsButton2()
SystemClock.sleep(1000)
onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
Screengrab.screenshot("settings-scroll")
SystemClock.sleep(2000)

settingsRemoveData()
Screengrab.screenshot("settings-delete-browsing-data")
device.pressBack()

SystemClock.sleep(1000)
onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
Screengrab.screenshot("settings-scroll")
SystemClock.sleep(3000)

settingsTelemetry()
Screengrab.screenshot("settings-telemetry")
device.pressBack()
}

@Test
fun settingsScrollToBottomTest() {
homeScreen {
}.openThreeDotMenu {
}
settingsButton2()
SystemClock.sleep(1000)
onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
Screengrab.screenshot("settings-scroll2")
SystemClock.sleep(1000)

onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
onView(withId(R.id.switch_widget)).perform(ViewActions.swipeUp())
Screengrab.screenshot("settings-scroll3")
SystemClock.sleep(1000)

onView(withId(R.id.recycler_view)).perform(swipeUp())
Screengrab.screenshot("settings-scroll4")
}

@Test
fun libraryTest() {
homeScreen {
}.openThreeDotMenu {
}
libraryButton()
Screengrab.screenshot("library")
bookmarksButton()
Screengrab.screenshot("library-bookmarks")
device.pressBack()
historyButton()
Screengrab.screenshot("library-history")
}
}

fun settingsButton2() = ClickActions.click { text(R.string.settings) }
fun libraryButton() = ClickActions.click { text(R.string.browser_menu_your_library) }
fun bookmarksButton() = ClickActions.click { text(R.string.library_bookmarks) }
fun historyButton() = ClickActions.click { text(R.string.library_history) }
fun settingsAccount() = ClickActions.click { text(R.string.preferences_sync) }

fun settingsSearch() = ClickActions.click { text(R.string.preferences_search_engine) }
fun settingsTheme() = ClickActions.click { text(R.string.preferences_theme) }
fun settingsAccessibility() = ClickActions.click { text(R.string.preferences_accessibility) }
fun settingsTp() = ClickActions.click { text(R.string.preferences_tracking_protection) }
fun settingsRemoveData() = ClickActions.click { text(R.string.preferences_delete_browsing_data) }
fun settingsTelemetry() = ClickActions.click { text(R.string.preferences_data_choices) }

+ 10
- 0
app/src/debug/AndroidManifest.xml View File

@@ -4,6 +4,16 @@
xmlns:tools="http://schemas.android.com/tools">

<uses-permission android:name="android.permission.ACCESS_WIFI_STATE" />
<!-- Allows unlocking your device and activating its screen so UI tests can succeed -->
<uses-permission android:name="android.permission.DISABLE_KEYGUARD"/>
<uses-permission android:name="android.permission.WAKE_LOCK"/>

<!-- Allows for storing and retrieving screenshots -->
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />

<!-- Allows changing locales -->
<uses-permission android:name="android.permission.CHANGE_CONFIGURATION" />

<application
tools:replace="android:name"


+ 1
- 1
architecture/build.gradle View File

@@ -12,7 +12,7 @@ android {
defaultConfig {
minSdkVersion 21
targetSdkVersion 28
testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
}

buildTypes {


+ 15
- 0
docs/l10nScreenshotsTests.md View File

@@ -0,0 +1,15 @@
### Screenshtos Tests
We are using [`screengrab`](https://docs.fastlane.tools/getting-started/android/screenshots/) which works with fastlane to automate the process of capturing screenshots.
All the l10n screenshots are generated through the ui tests. These particular tests run as part of the screenshots package (`app/src/androidTest/mozilla/fenix/ui/screenshots`)

### Steps to Run / Build
1. Install the gem:
`sudo gem install screengrab`

2. From command line run:
`fastlane screengrab --test_instrumentation_runner "androidx.test.runner.AndroidJUnitRunner"`

Screenshots will be saved in the root directory: `fastlane/metadata/android`
If there is a failure and screenshots are not saved, it may be necessary to create these folders manually first.
Currently screenshots are uploaded to [`gcloud`](https://storage.googleapis.com/l10n-fenix/2019-06-21/screenshots.html) once they are generated manually by running the previous command line.
In the future this process will be automated too and documented here.

+ 2
- 0
fastlane/Appfile View File

@@ -0,0 +1,2 @@
json_key_file("") # Path to the json secret file - Follow https://docs.fastlane.tools/actions/supply/#setup to get one
package_name("org.mozilla.fenix") # e.g. com.krausefx.app

+ 28
- 0
fastlane/Screengrabfile View File

@@ -0,0 +1,28 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

app_package_name('org.mozilla.fenix.debug')
use_tests_in_packages(['org.mozilla.fenix.ui.screenshots'])

app_apk_path('app/build/outputs/apk/x86/debug/app-x86-debug.apk')
tests_apk_path('app/build/outputs/apk/androidTest/x86/debug/app-x86-debug-androidTest.apk')
test_instrumentation_runner 'androidx.test.runner.AndroidJUnitRunner'

locales(['en-US', 'fr-FR', 'it-IT', 'de-DE', 'ja', 'ru', 'zh-CN', 'zh-TW', 'ko'])

# clear all previously generated screenshots in your local output directory before creating new ones
clear_previous_screenshots(true)

# Clear all previous screenshots locally. Technically not needed in automation.
# But it's easier to debug this on a local device if there are no old screenshots
# hanging around.
clear_previous_screenshots true

exit_on_test_failure false

skip_open_summary false
reinstall_app true

# For more information about all available options run
# fastlane screengrab --help

Loading…
Cancel
Save