gecko-dev/substitute-local-geckoview.gradle
Michael Comella f6fb629fa9 Bug 1744336 - load mozconfig only once in substitute-local-geckoview. r=agi
Root cause analysis: in the changed implementation, the
substitute-local-geckoview script is called for every subproject in
android-components. Since this script calls a mach command, we're forced to wait
on the python interpretter for each a-c subproject (~113), which is very slow.

This patch addresses the problem by caching the output of loadMozconfig, which
calls the mach command, on the first run and using the cache on subsequent
invocations of the script.

Locally, with non-robust testing, I saw incremental build times decrease from
over five minutes to between 5-30 seconds.

This solution is ideal because it does not break downstream consumers. However,
it may not be the most correct solution because it's difficult to avoid writing
code that's not redundant if called multiple times. A more correct solution
would request that consumers call the script once and return a function for them
to call for each subproject.

Differential Revision: https://phabricator.services.mozilla.com/D132850
2021-12-07 17:53:25 +00:00

190 lines
8.2 KiB
Groovy

/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
// Substitute a local GeckoView AAR into a consuming Gradle project.
//
// To use this, in a consuming Gradle project's `/build.gradle` add a stanza like:
//
// ext.topsrcdir = '/absolute/path/to/mozilla-central'; apply from: "${ext.topsrcdir}/substitute-local-geckoview.gradle"
//
// The object directory will be determined using `mach environment` and will agree with `./mach
// gradle` and Android Studio. Or, specify the exact object directory with a stanza like:
//
// ext.topsrcdir = '/absolute/path/to/mozilla-central'
// ext.topobjdir = '/absolute/path/to/objdir'
// apply from: "${ext.topsrcdir}/substitute-local-geckoview.gradle"
//
// Substitution works with artifact and non-artifact builds.
//
// If you get errors about .jar files not being found, ensure that the consuming
// application is using a recent Android-Gradle plugin (say, 3.4+). There were
// issues with Jetifier, and issues with .jar vs. .aar extensions, in older
// versions.
import groovy.json.JsonSlurper
def log(message) {
logger.lifecycle("[substitute-local-geckoview] ${message}")
}
def warn(message) {
logger.warn("[substitute-local-geckoview] Warning: ${message}")
}
if (!project.ext.has('topsrcdir')) {
throw new GradleException("ext.topsrcdir must be specified to substitute for a local GeckoView")
}
/**
* Loads the mozconfig and returns any variables derived from it, avoiding side effects.
*
* This method is relatively slow because it calls mach, which starts a python interpreter, will
* becomes very slow if called for numerous subprojects. Therefore, it should only be called once
* per build.
*/
def loadMozconfig() {
apply from: "${topsrcdir}/mobile/android/gradle/mach_env.gradle"
// Cribbed from https://hg.mozilla.org/mozilla-central/file/tip/settings.gradle. When run in
// topobjdir, `mach environment` correctly finds the mozconfig corresponding to that object
// directory.
def commandLine = ["${topsrcdir}/mach", "environment", "--format", "json", "--verbose"]
def proc = commandLine.execute(
machEnv(topsrcdir),
new File(ext.has('topobjdir') ? ext.get('topobjdir') : topsrcdir))
def standardOutput = new ByteArrayOutputStream()
proc.consumeProcessOutput(standardOutput, standardOutput)
proc.waitFor()
// Only show the output if something went wrong.
if (proc.exitValue() != 0) {
throw new GradleException("Process '${commandLine}' finished with non-zero exit value ${proc.exitValue()}:\n\n${standardOutput.toString()}")
}
def slurper = new JsonSlurper()
def mozconfig = slurper.parseText(standardOutput.toString())
if (topsrcdir != mozconfig.topsrcdir) {
throw new GradleException("Specified topsrcdir ('${topsrcdir}') is not mozconfig topsrcdir ('${mozconfig.topsrcdir}')")
}
def topobjdir
if (ext.has('topobjdir')) {
topobjdir = ext.topobjdir
} else {
topobjdir = mozconfig.topobjdir
log("Found topobjdir ${topobjdir} from topsrcdir ${topsrcdir}")
}
if (mozconfig.substs.MOZ_BUILD_APP != 'mobile/android') {
throw new GradleException("Building with Gradle is only supported for GeckoView, i.e., MOZ_BUILD_APP == 'mobile/android'.")
}
log("Will substitute GeckoView (geckoview-{nightly,beta}) with local GeckoView (geckoview-default) from ${topobjdir}/gradle/build/mobile/android/geckoview/maven")
if (!mozconfig.substs.COMPILE_ENVIRONMENT) {
log("To update the local GeckoView, run `./mach gradle geckoview:publishWithGeckoBinariesDebugPublicationToMavenRepository` in ${topsrcdir}")
} else {
log("To update the local GeckoView, run `./mach build binaries && ./mach gradle geckoview:publishWithGeckoBinariesDebugPublicationToMavenRepository` in ${topsrcdir}")
}
return [mozconfig, topobjdir]
}
// This script is expected to be called for every subproject in the build (in ac, this is over 100)
// but loadMozconfig should only be called once per build (see the javadoc) so we store the output
// of that call as a global variable and re-use it when this script is called again.
def LOAD_MOZCONFIG_CACHE = "substitute-local-geckoview-mozconfig-cache"
if (!rootProject.ext.has(LOAD_MOZCONFIG_CACHE)) {
rootProject.ext.set(LOAD_MOZCONFIG_CACHE, loadMozconfig())
}
def (mozconfig, topobjdir) = rootProject.ext.get(LOAD_MOZCONFIG_CACHE)
repositories {
maven {
name "Local GeckoView Maven repository"
url "${topobjdir}/gradle/build/mobile/android/geckoview/maven"
}
}
configurations.all { config ->
// Like `geckoview-nightly` for a multi-architecture fat AAR or
// `geckoview-nightly-armeabi-v7a` for an architecture-specific AAR.
def geckoviewModules = [
'geckoview-nightly',
'geckoview-nightly-armeabi-v7a',
'geckoview-nightly-arm64-v8a',
'geckoview-nightly-x86',
'geckoview-nightly-x86_64',
'geckoview-beta',
'geckoview-beta-armeabi-v7a',
'geckoview-beta-arm64-v8a',
'geckoview-beta-x86',
'geckoview-beta-x86_64',
]
def geckoviewOmniModules = [
'geckoview-nightly-omni',
'geckoview-nightly-omni-armeabi-v7a',
'geckoview-nightly-omni-arm64-v8a',
'geckoview-nightly-omni-x86',
'geckoview-nightly-omni-x86_64',
'geckoview-beta-omni',
'geckoview-beta-omni-armeabi-v7a',
'geckoview-beta-omni-arm64-v8a',
'geckoview-beta-omni-x86',
'geckoview-beta-omni-x86_64',
]
if (config.isCanBeResolved()) {
config.resolutionStrategy { strategy ->
dependencySubstitution {
all { dependency ->
// We could restrict based on target architecture, but there doesn't seem to
// be much advantage to doing so right now.
if (!(dependency.requested instanceof ModuleComponentSelector)) {
// We can only substitute for a module: we're never going to substitute
// for a project.
return
}
def group = dependency.requested.group
def module = dependency.requested.module
if (group == 'org.mozilla.geckoview'
&& (geckoviewModules.contains(module) || geckoviewOmniModules.contains(module))) {
def name = ''
def isLite = mozconfig.substs.MOZ_ANDROID_GECKOVIEW_LITE
if (isLite) {
name = 'geckoview-default'
} else {
name = 'geckoview-default-omni'
}
if (geckoviewModules.contains(module) && !isLite) {
warn("Substituting a geckoview omni build into a lite dependency. Add ac_add_options --enable-geckoview-lite to ${mozconfig.mozconfig.path} to fix this.")
} else if (geckoviewOmniModules.contains(module) && isLite) {
// Substituting lite into omni is unlikely to work at
// all so we just error out here.
throw new GradleException("Substituting a geckoview lite build into an omni dependency. Remove ac_add_options --enable-geckoview-lite in ${mozconfig.mozconfig.path} to fix this.")
}
log("Substituting ${group}:${dependency.requested.module} with local GeckoView ${group}:${name} in ${config}")
dependency.useTarget([group: group, name: name, version: '+'])
// We substitute with a dynamic version ('+'). It seems that Gradle
// discovers the underlying AAR is out of date correctly based on file
// timestamp already, but let's try to avoid some class of cache
// invalidation error while we're here.
strategy.cacheDynamicVersionsFor 0, 'seconds'
strategy.cacheChangingModulesFor 0, 'seconds'
}
}
}
}
}
}