diff --git a/mobile/android/app/build.gradle b/mobile/android/app/build.gradle index 32dc28d5f098..da792b23dc7a 100644 --- a/mobile/android/app/build.gradle +++ b/mobile/android/app/build.gradle @@ -514,51 +514,3 @@ android.applicationVariants.all { variant -> tasks["compile${productFlavor}${buildType}UnitTestSources"].dependsOn(tasks["merge${productFlavor}${buildType}Assets"]) } -apply from: "${topsrcdir}/mobile/android/gradle/jacoco_dependencies.gradle" -if (project.hasProperty('enable_code_coverage')) { - apply from: "${topsrcdir}/mobile/android/gradle/jacoco_for_junit.gradle" -} - - -// Set up code coverage for tests on emulators. -if (mozconfig.substs.MOZ_JAVA_CODE_COVERAGE) { - android { - jacoco { - version = "$jacoco_version" - } - buildTypes { - debug { - testCoverageEnabled true - } - } - } - - dependencies { - // This is required both in the instrumented application classes and the test classes, - // so `api` has to be used instead of `androidTestImplementation`. - api "org.jacoco:org.jacoco.agent:$jacoco_version:runtime" - } - - // Generate tasks to archive compiled classfiles for later use with JaCoCo report generation. - // One of these tasks is used by `mach android archive-coverage-artifacts`. - android.applicationVariants.all { variant -> - def name = variant.name - def compileTask = tasks.getByName("compile${name.capitalize()}JavaWithJavac") - task "archiveClassfiles${name.capitalize()}"(type: Zip, dependsOn: compileTask) { - description = "Archive compiled classfiles for $name in order to export them as code coverage artifacts." - def fileFilter = ['**/androidTest/**', - '**/test/**', - '**/R.class', - '**/R$*.class', - '**/BuildConfig.*', - '**/Manifest*.*', - '**/*Test*.*', - 'android/**/*.*'] - from fileTree(dir: compileTask.destinationDir, excludes: fileFilter) - destinationDir = file("${buildDir}/coverage") - // Note: This task assumes only one variant of archiveClassfiles* will be used. - // Running multiple variants of this task will overwrite the output archive. - archiveName = 'target.app_classfiles.zip' - } - } -} diff --git a/mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly b/mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly index 5cd1a81cf5bc..bcd153dead61 100644 --- a/mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly +++ b/mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly @@ -33,9 +33,6 @@ ac_add_options --target=arm-linux-androideabi ac_add_options --with-branding=mobile/android/branding/nightly -# Pull code coverage dependencies too. -ac_add_options --enable-java-coverage - export MOZILLA_OFFICIAL=1 export MOZ_TELEMETRY_REPORTING=1 export MOZ_ANDROID_MMA=1 diff --git a/mobile/android/docs/testcoverage.rst b/mobile/android/docs/testcoverage.rst index 872298ad2f71..b6c9a7a61377 100644 --- a/mobile/android/docs/testcoverage.rst +++ b/mobile/android/docs/testcoverage.rst @@ -17,33 +17,9 @@ Collecting and exporting code coverage information Relevant tools and libraries ---------------------------- -JaCoCo_ is the tool used to gather code coverage data. JaCoCo uses class file -instrumentation to record execution coverage data. It has two operating modes: - -- `online instrumentation`_: Class files are instrumented on-the-fly using a so - called Java agent. The JaCoCo agent collects execution information and dumps - it on request or when the JVM exits. This method is used for test suites - which run on the JVM (generally, `*UnitTest` Gradle tasks). -- `offline instrumentation`_: At runtime the pre-instrumented classes needs be - on the classpath instead of the original classes. In addition, - `jacocoagent.jar` must be put on the classpath. This method is used for test - suites with run on the Android emulator (generally, `*AndroidTest` Gradle - tasks, including `robocop`). - -JaCoCo is integrated with Gradle in two ways: a Gradle plugin (activated in the -``build.gradle`` file as a top-level ``apply plugin: 'jacoco'``), and an -Android-specific Gradle plugin, activated with ``android { buildTypes { debug { -testCoverageEnabled true } } }``. These two methods of activating JaCoCo have a -different syntax of choosing the JaCoCo tool version, and both should be -configured to use the same version. See the -``mobile/android/geckoview/build.gradle`` file for example usage. - grcov_ is a tool implemented in Rust that transforms code coverage reports between formats. Among others, it supports reading JaCoCo XML reports. -.. _JaCoCo: https://www.eclemma.org/jacoco/ -.. _online instrumentation: https://www.jacoco.org/jacoco/trunk/doc/agent.html -.. _offline instrumentation: https://www.jacoco.org/jacoco/trunk/doc/offline.html .. _grcov: https://github.com/mozilla/grcov/ Generating the coverage report artifacts @@ -63,71 +39,3 @@ job. It is also responsible for running ``grcov`` after the tests are finished, to convert and merge the coverage reports. .. _ActiveData: https://wiki.mozilla.org/EngineeringProductivity/Projects/ActiveData - -Code coverage for android-test -=============================== - -The `android-test` suite is a JUnit test suite that runs locally on the -host's JVM. It can be run with ``mach android test``. The test suite is -implemented as a build task, defined at -``taskcluster/ci/build/android-stuff.yml``. - -To collect code coverage from this suite, a duplicate build task is defined, -called `android-test-ccov`. It can be run with ``mach android test-ccov``. - -The mach subcommand is responsible for downloading and running `grcov`, -instead of the ``CodeCoverageMixin`` class. This is because the -`android-test-ccov` task is a build task (not a test task), so it doesn't use -mozharness to run the tests. - - -Code coverage for geckoview-junit -================================== - -The geckoview-junit_ tests are on-device Android JUnit tests written for -GeckoView_. The tests are implemented with ``mochitest``. The automation -entry point is ``testing/mozharness/scripts/android_emulator_unittest.py``, -which then calls ``testing/mochitest/runjunit.py``. This is an out-of-tree -task, so a source tree clone is unavailable. - -To generate the coverage report, we need three things: - -- The classfiles before instrumentation. These are archived as a public - artifact during build time, and downloaded on the test machine during testing - time; -- The coverage.ec file with coverage counters. This is generated while running - the tests on the emulator, then downloaded with ``adb pull``; -- ``jacoco-cli``, a command-line package JaCoCo component that takes the - classfiles and the coverage counters as input and generates XML reports as - output. - -The ``mach android archive-coverage-artifacts`` command archives the -class files and exports the ``jacoco-cli`` jar file after the build is done. -These files are later saved as public artifacts of the build. - -To enable offline instrumentation for the test suites, the mozconfig flag -``--enable-java-coverage`` should be set. When the flag is checked both during -build and test time. During test time, the flag instructs ``CodeCoverageMixin`` -to download the coverage artifacts from the build task before the tests run, -and generate and export the reports after testing is finished. The flag also -instructs the ``runjunit.py`` script to insert the arguments ``-e coverage -true`` to ``am instrument``. - -.. _GeckoView: https://wiki.mozilla.org/Mobile/GeckoView -.. _geckoview-junit: https://developer.mozilla.org/en-US/docs/Mozilla/Geckoview-Junit_Tests - - -Code coverage for Robocop UI tests -================================== - -Robocop_ provides UI-level testing for Fennec. The tests use the same -automation scripts as geckoview-junit, only differing by the fact that they are -run by ``runrobocop.py`` instead of ``runjunit.py``. - -The only notable difference from the geckoview-junit tests is that robocop -tests are run with one ``am instrument`` call per test class. This means that -one ``robocop-coverage-.ec`` file will be generated for each test -class. After the tests finish, ``jacoco-cli`` will be called on all the -resulting coverage data files. - -.. _Robocop: https://wiki.mozilla.org/Auto-tools/Projects/Robocop diff --git a/mobile/android/geckoview/build.gradle b/mobile/android/geckoview/build.gradle index 6282d0c87775..4b2004bb148f 100644 --- a/mobile/android/geckoview/build.gradle +++ b/mobile/android/geckoview/build.gradle @@ -449,74 +449,6 @@ task("generateSDKBindings", type: JavaExec) { dependsOn project(':annotations').jar } -apply from: "${topsrcdir}/mobile/android/gradle/jacoco_dependencies.gradle" -if (project.hasProperty('enable_code_coverage')) { - apply from: "${topsrcdir}/mobile/android/gradle/jacoco_for_junit.gradle" -} - -// Set up code coverage for tests on emulators. -if (mozconfig.substs.MOZ_JAVA_CODE_COVERAGE) { - android { - jacoco { - version = "$jacoco_version" - } - buildTypes { - debug { - testCoverageEnabled true - } - } - } - - configurations { - // This configuration is used for dependencies that are not needed at compilation or - // runtime, but need to be exported as artifacts of the build for usage on the testing - // machines. - coverageDependency - } - - dependencies { - // This is required both in the instrumented application classes and the test classes, - // so `api` has to be used instead of `androidTestImplementation`. - api "org.jacoco:org.jacoco.agent:$jacoco_version:runtime" - - coverageDependency ("org.jacoco:org.jacoco.cli:$jacoco_version:nodeps") { - exclude group: 'org.ow2.asm', module: '*' - } - } - - // This task is used by `mach android archive-coverage-artifacts`. - task copyCoverageDependencies(type: Copy) { - from(configurations.coverageDependency) { - include 'org.jacoco.cli-*-nodeps.jar' - rename { _ -> 'target.jacoco-cli.jar' } - } - into "$buildDir/coverage" - } - - // Generate tasks to archive compiled classfiles for later use with JaCoCo report generation. - // One of these tasks is used by `mach android archive-coverage-artifacts`. - android.libraryVariants.all { variant -> - def name = variant.name - def compileTask = tasks.getByName("compile${name.capitalize()}JavaWithJavac") - task "archiveClassfiles${name.capitalize()}"(type: Zip, dependsOn: compileTask) { - description = "Archive compiled classfiles for $name in order to export them as code coverage artifacts." - def fileFilter = ['**/androidTest/**', - '**/test/**', - '**/R.class', - '**/R$*.class', - '**/BuildConfig.*', - '**/Manifest*.*', - '**/*Test*.*', - 'android/**/*.*'] - from fileTree(dir: compileTask.destinationDir, excludes: fileFilter) - destinationDir = file("${buildDir}/coverage") - // Note: This task assumes only one variant of archiveClassfiles* will be used. - // Running multiple variants of this task will overwrite the output archive. - archiveName = 'target.geckoview_classfiles.zip' - } - } -} - apply plugin: 'org.mozilla.apilint' apiLint { diff --git a/mobile/android/gradle.configure b/mobile/android/gradle.configure index b8cda51be1ac..1020d04c0d6f 100644 --- a/mobile/android/gradle.configure +++ b/mobile/android/gradle.configure @@ -205,16 +205,7 @@ def gradle_android_test_tasks(build_config): ] -@dependable -def gradle_android_test_ccov_report_tasks(): - '''Additional gradle tasks run by |mach android test-ccov|.''' - return [ - 'app:jacocoTestReport', - 'geckoview:jacocoTestReport', - ] - set_config('GRADLE_ANDROID_TEST_TASKS', gradle_android_test_tasks) -set_config('GRADLE_ANDROID_TEST_CCOV_REPORT_TASKS', gradle_android_test_ccov_report_tasks) @depends(gradle_android_build_config) diff --git a/mobile/android/gradle/jacoco_dependencies.gradle b/mobile/android/gradle/jacoco_dependencies.gradle deleted file mode 100644 index a01d0ef80f6b..000000000000 --- a/mobile/android/gradle/jacoco_dependencies.gradle +++ /dev/null @@ -1,11 +0,0 @@ -/* -*- Mode: Groovy; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*- - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -dependencies { - testImplementation "org.jacoco:org.jacoco.agent:$jacoco_version" - testImplementation "org.jacoco:org.jacoco.ant:$jacoco_version" - testImplementation "org.jacoco:org.jacoco.core:$jacoco_version" - testImplementation "org.jacoco:org.jacoco.report:$jacoco_version" -} diff --git a/mobile/android/gradle/jacoco_for_junit.gradle b/mobile/android/gradle/jacoco_for_junit.gradle deleted file mode 100644 index 424b050e9ad5..000000000000 --- a/mobile/android/gradle/jacoco_for_junit.gradle +++ /dev/null @@ -1,41 +0,0 @@ -/* -*- Mode: Groovy; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*- - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -apply plugin: "jacoco" - -jacoco { - toolVersion = "${project.jacoco_version}" -} - -android { - testOptions { - unitTests.all { - jacoco { - includeNoLocationClasses = true - } - } - } -} - -dependencies { - jacocoAgent "org.jacoco:org.jacoco.agent:${project.jacoco_version}" - jacocoAnt "org.jacoco:org.jacoco.ant:${project.jacoco_version}" -} - -task jacocoTestReport(type: JacocoReport) { - reports { - xml.enabled = true - html.enabled = false - csv.enabled = false - } - def fileFilter = ['**/androidTest/**', '**/test/**', '**/R.class', '**/R$*.class', - '**/BuildConfig.*', '**/Manifest*.*', '**/*Test*.*', 'android/**/*.*'] - def debugTree = fileTree(dir: "${buildDir}/intermediates/classes", excludes: fileFilter) - def mainSrc = "${project.projectDir}/src/main/java" - - sourceDirectories = files([mainSrc]) - classDirectories = files([debugTree]) - executionData = fileTree("${buildDir}/jacoco") -} diff --git a/mobile/android/mach_commands.py b/mobile/android/mach_commands.py index ecb5c286d37c..2f3140e9f315 100755 --- a/mobile/android/mach_commands.py +++ b/mobile/android/mach_commands.py @@ -7,17 +7,12 @@ from __future__ import absolute_import, print_function, unicode_literals import argparse import logging import os -import shutil -import subprocess -import zipfile import json from zipfile import ZipFile import mozpack.path as mozpath -from mozfile import TemporaryDirectory - from mozbuild.base import ( MachCommandBase, MachCommandConditions as conditions, @@ -233,39 +228,6 @@ class MachCommands(MachCommandBase): return ret - @SubCommand('android', 'test-ccov', - """Run Android local unit tests in order to get a code coverage report. - See https://firefox-source-docs.mozilla.org/mobile/android/fennec/testcoverage.html""") # NOQA: E501 - @CommandArgument('args', nargs=argparse.REMAINDER) - def android_test_ccov(self, args): - enable_ccov = '-Penable_code_coverage' - - # Don't care if the tests are failing, we only want the coverage information. - self.android_test([enable_ccov]) - - self.gradle(self.substs['GRADLE_ANDROID_TEST_CCOV_REPORT_TASKS'] + - [enable_ccov] + args, verbose=True) - self._process_jacoco_reports() - return 0 - - def _process_jacoco_reports(self): - def run_grcov(grcov_path, input_path): - args = [grcov_path, input_path, '-t', 'lcov'] - return subprocess.check_output(args) - - with TemporaryDirectory() as xml_dir: - grcov = os.path.join(os.environ['MOZ_FETCHES_DIR'], 'grcov') - - report_xml_template = self.topobjdir + '/gradle/build/mobile/android/%s/reports/jacoco/jacocoTestReport/jacocoTestReport.xml' # NOQA: E501 - shutil.copy(report_xml_template % 'app', os.path.join(xml_dir, 'app.xml')) - shutil.copy(report_xml_template % 'geckoview', os.path.join(xml_dir, 'geckoview.xml')) - - # Parse output files with grcov. - grcov_output = run_grcov(grcov, xml_dir) - grcov_zip_path = os.path.join(self.topobjdir, 'code-coverage-grcov.zip') - with zipfile.ZipFile(grcov_zip_path, 'w', zipfile.ZIP_DEFLATED) as z: - z.writestr('grcov_lcov_output.info', grcov_output) - @SubCommand('android', 'lint', """Run Android lint. See https://developer.mozilla.org/en-US/docs/Mozilla/Android-specific_test_suites#android-lint""") # NOQA: E501 @@ -441,16 +403,6 @@ class MachCommands(MachCommandBase): return 0 - @SubCommand('android', 'archive-coverage-artifacts', - """Archive compiled classfiles to be used later in generating code - coverage reports. See https://firefox-source-docs.mozilla.org/mobile/android/fennec/testcoverage.html""") # NOQA: E501 - @CommandArgument('args', nargs=argparse.REMAINDER) - def android_archive_classfiles(self, args): - self.gradle(self.substs['GRADLE_ANDROID_ARCHIVE_COVERAGE_ARTIFACTS_TASKS'] + - args, verbose=True) - - return 0 - @SubCommand('android', 'archive-geckoview', """Create GeckoView archives. See http://firefox-source-docs.mozilla.org/build/buildsystem/toolchains.html#firefox-for-android-with-gradle""") # NOQA: E501 diff --git a/taskcluster/ci/build/android-stuff.yml b/taskcluster/ci/build/android-stuff.yml index 9471339fbd5d..4ab10038ba9e 100644 --- a/taskcluster/ci/build/android-stuff.yml +++ b/taskcluster/ci/build/android-stuff.yml @@ -52,48 +52,6 @@ android-test/opt: - "mobile/android/tests/background/junit4/**" - "**/*.gradle" -android-test-ccov/opt: - description: "Android armv7 unit test coverage report" - index: - product: mobile - job-name: android-test-ccov - treeherder: - platform: android-4-0-armv7-api16/opt - kind: build - tier: 1 - symbol: A(test-ccov) - run-on-projects: [mozilla-central] - worker-type: aws-provisioner-v1/gecko-{level}-b-android - worker: - docker-image: {in-tree: android-build} - env: - GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-offline" - PERFHERDER_EXTRA_OPTIONS: android-test-ccov - artifacts: - - name: public/code-coverage-grcov.zip - path: /builds/worker/workspace/build/src/obj-firefox/code-coverage-grcov.zip - type: file - max-run-time: 7200 - run: - using: mozharness - actions: [get-secrets, build] - config: - - builds/releng_base_android_64_builds.py - script: "mozharness/scripts/fx_desktop_build.py" - secrets: true - custom-build-variant-cfg: android-test-ccov - tooltool-downloads: internal - toolchains: - - android-gradle-dependencies - - android-sdk-linux - - linux64-node - fetches: - # We use a fetch instead of toolchains here, because mozharness expects - # it there (since the code is shared with tests that don't support - # `toolchains`). - toolchain: - - linux64-grcov - android-lint/opt: description: "Android lint" index: diff --git a/taskcluster/ci/build/android.yml b/taskcluster/ci/build/android.yml index 87e579cb1129..b92e9bf36adb 100644 --- a/taskcluster/ci/build/android.yml +++ b/taskcluster/ci/build/android.yml @@ -57,65 +57,6 @@ android-api-16/debug: - linux64-nasm - linux64-node -android-api-16-ccov/debug: - description: "Android 4.0 api-16+ Debug Coverage" - index: - product: mobile - job-name: android-api-16-ccov-debug - treeherder: - platform: android-4-0-armv7-api16-ccov/debug - symbol: B - worker-type: aws-provisioner-v1/gecko-{level}-b-android - worker: - docker-image: {in-tree: android-build} - max-run-time: 7200 - env: - GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-offline" - TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest" - artifacts: - - name: public/android/maven - path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/maven/ - type: directory - - name: public/build/geckoview-androidTest.apk - path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk - type: file - - name: public/build/geckoview_example.apk - path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview_example/outputs/apk/withGeckoBinaries/debug/geckoview_example-withGeckoBinaries-debug.apk - type: file - - name: public/build/target.geckoview_classfiles.zip - path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/coverage/target.geckoview_classfiles.zip - type: file - - name: public/build/target.app_classfiles.zip - path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/app/coverage/target.app_classfiles.zip - type: file - - name: public/build/target.jacoco-cli.jar - path: /builds/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/coverage/target.jacoco-cli.jar - type: file - - name: public/build - path: /builds/worker/artifacts/ - type: directory - run: - using: mozharness - actions: [get-secrets, build] - config: - - builds/releng_base_android_64_builds.py - script: "mozharness/scripts/fx_desktop_build.py" - secrets: true - custom-build-variant-cfg: api-16-debug-ccov - tooltool-downloads: internal - run-on-projects: ['mozilla-central'] - toolchains: - - android-gradle-dependencies - - android-ndk-linux - - android-sdk-linux - - linux64-clang - - linux64-rust-android - - linux64-rust-size - - linux64-cbindgen - - linux64-sccache - - linux64-nasm - - linux64-node - android-x86/opt: description: "Android 4.2 x86 Opt" index: