Magic-Mirror/Apktool
1
0
Fork 0
mirror of https://github.com/iBotPeaches/Apktool.git synced 2024-11-23 12:39:43 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Feature: Parallel Building (#3476)

Browse Source 
* perf: process smali code in parallel

Note: backsmali can't be properly multithreaded because of the synchronized methods inside

* perf: start backsmali concurrently with a resources decompiler

* perf: speed up apk building by skipping temp archive creation

Now we're not compressing the same data twice

* refactor: extract duplicated code

* refactor: rename methods and inline some comments
This commit is contained in:
Cregrant 2023-12-26 18:20:26 +07:00 committed by GitHub
parent 0741664808
commit 81aae6936a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 227 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

155
brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java
View File

@ -42,17 +42,19 @@ import javax.xml.transform.TransformerException;
import java.io.*;
import java.nio.file.Files;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Logger;
import java.util.zip.CRC32;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
public class ApkBuilder {
private final static Logger LOGGER = Logger.getLogger(ApkBuilder.class.getName());
private final AtomicReference<AndrolibException> mBuildError = new AtomicReference<>(null);
private final Config mConfig;
private final ExtFile mApkDir;
private BackgroundWorker mWorker;
private ApkInfo mApkInfo;
private int mMinSdkVersion = 0;
@ -78,7 +80,8 @@ public class ApkBuilder {
public void build(File outFile) throws BrutException {
LOGGER.info("Using Apktool " + ApktoolProperties.getVersion());
try {
mWorker = new BackgroundWorker();
mApkInfo = ApkInfo.load(mApkDir);
if (mApkInfo.getSdkInfo() != null && mApkInfo.getSdkInfo().get("minSdkVersion") != null) {
@ -96,17 +99,17 @@ public class ApkBuilder {
File manifest = new File(mApkDir, "AndroidManifest.xml");
File manifestOriginal = new File(mApkDir, "AndroidManifest.xml.orig");
buildSources();
buildNonDefaultSources();
buildManifestFile(manifest, manifestOriginal);
scheduleBuildDexFiles();
backupManifestFile(manifest, manifestOriginal);
buildResources();
buildLibs();
buildCopyOriginalFiles();
buildApk(outFile);
copyLibs();
copyOriginalFilesIfEnabled();
mWorker.waitForFinish();
if (mBuildError.get() != null) {
throw mBuildError.get();
}
// we must go after the Apk is built, and copy the files in via Zip
// this is because Aapt won't add files it doesn't know (ex unknown files)
buildUnknownFiles(outFile);
buildApk(outFile);
// we copied the AndroidManifest.xml to AndroidManifest.xml.orig so we can edit it
// lets restore the unedited one, to not change the original
@ -120,9 +123,12 @@ public class ApkBuilder {
}
}
LOGGER.info("Built apk into: " + outFile.getPath());
} finally {
mWorker.shutdownNow();
}
}
private void buildManifestFile(File manifest, File manifestOriginal) throws AndrolibException {
private void backupManifestFile(File manifest, File manifestOriginal) throws AndrolibException {
// If we decoded in "raw", we cannot patch AndroidManifest
if (new File(mApkDir, "resources.arsc").exists()) {
return;
@ -141,24 +147,17 @@ public class ApkBuilder {
}
}
private void buildSources() throws AndrolibException {
if (!buildSourcesRaw("classes.dex") && !buildSourcesSmali("smali", "classes.dex")) {
LOGGER.warning("Could not find sources");
}
}
private void buildNonDefaultSources() throws AndrolibException {
private void scheduleBuildDexFiles() throws AndrolibException {
try {
mWorker.submit(() -> scheduleDexBuild("classes.dex", "smali"));
// loop through any smali_ directories for multi-dex apks
Map<String, Directory> dirs = mApkDir.getDirectory().getDirs();
for (Map.Entry<String, Directory> directory : dirs.entrySet()) {
String name = directory.getKey();
if (name.startsWith("smali_")) {
String filename = name.substring(name.indexOf("_") + 1) + ".dex";
if (!buildSourcesRaw(filename) && !buildSourcesSmali(name, filename)) {
LOGGER.warning("Could not find sources");
}
mWorker.submit(() -> scheduleDexBuild(filename, name));
}
}
@ -177,6 +176,19 @@ public class ApkBuilder {
}
}
private void scheduleDexBuild(String filename, String smali) {
try {
if (mBuildError.get() != null) {
return;
}
if (!buildSourcesRaw(filename) && !buildSourcesSmali(smali, filename)) {
LOGGER.warning("Could not find sources");
}
} catch (AndrolibException e) {
mBuildError.compareAndSet(null, e);
}
}
private boolean buildSourcesRaw(String filename) throws AndrolibException {
File working = new File(mApkDir, filename);
if (!working.exists()) {
@ -214,6 +226,7 @@ public class ApkBuilder {
}
private void buildResources() throws BrutException {
// create res folder, manifest file and resources.arsc
if (!buildResourcesRaw() && !buildResourcesFull() && !buildManifest()) {
LOGGER.warning("Could not find resources");
}
@ -375,7 +388,7 @@ public class ApkBuilder {
}
}
private void buildLibs() throws AndrolibException {
private void copyLibs() throws AndrolibException {
buildLibrary("lib");
buildLibrary("libs");
buildLibrary("kotlin");
@ -401,7 +414,7 @@ public class ApkBuilder {
}
}
private void buildCopyOriginalFiles() throws AndrolibException {
private void copyOriginalFilesIfEnabled() throws AndrolibException {
if (mConfig.copyOriginalFiles) {
File originalDir = new File(mApkDir, "original");
if (originalDir.exists()) {
@ -427,49 +440,34 @@ public class ApkBuilder {
}
}
private void buildUnknownFiles(File outFile) throws AndrolibException {
private void buildApk(File outApk) throws AndrolibException {
LOGGER.info("Building apk file...");
if (outApk.exists()) {
//noinspection ResultOfMethodCallIgnored
outApk.delete();
} else {
File outDir = outApk.getParentFile();
if (outDir != null && !outDir.exists()) {
//noinspection ResultOfMethodCallIgnored
outDir.mkdirs();
}
}
File assetDir = new File(mApkDir, "assets");
if (!assetDir.exists()) {
assetDir = null;
}
try (ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(outApk.toPath()))) {
// zip all AAPT-generated files
ZipUtils.zipFoldersPreserveStream(new File(mApkDir, APK_DIRNAME), zipOutputStream, assetDir, mApkInfo.doNotCompress);
// we must copy some files manually
// this is because Aapt won't add files it doesn't know (ex unknown files)
if (mApkInfo.unknownFiles != null) {
LOGGER.info("Copying unknown files/dir...");
Map<String, String> files = mApkInfo.unknownFiles;
File tempFile = new File(outFile.getParent(), outFile.getName() + ".apktool_temp");
boolean renamed = outFile.renameTo(tempFile);
if (!renamed) {
throw new AndrolibException("Unable to rename temporary file");
copyUnknownFiles(zipOutputStream, mApkInfo.unknownFiles);
}
try (
ZipFile inputFile = new ZipFile(tempFile);
ZipOutputStream actualOutput = new ZipOutputStream(Files.newOutputStream(outFile.toPath()))
) {
copyExistingFiles(inputFile, actualOutput);
copyUnknownFiles(actualOutput, files);
} catch (IOException | BrutException ex) {
throw new AndrolibException(ex);
}
// Remove our temporary file.
//noinspection ResultOfMethodCallIgnored
tempFile.delete();
}
}
private void copyExistingFiles(ZipFile inputFile, ZipOutputStream outputFile) throws IOException {
// First, copy the contents from the existing outFile:
Enumeration<? extends ZipEntry> entries = inputFile.entries();
while (entries.hasMoreElements()) {
ZipEntry entry = new ZipEntry(entries.nextElement());
// We can't reuse the compressed size because it depends on compression sizes.
entry.setCompressedSize(-1);
outputFile.putNextEntry(entry);
// No need to create directory entries in the final apk
if (!entry.isDirectory()) {
BrutIO.copy(inputFile, outputFile, entry);
}
outputFile.closeEntry();
} catch (IOException | BrutException e) {
throw new AndrolibException(e);
}
}
@ -513,33 +511,6 @@ public class ApkBuilder {
}
}
private void buildApk(File outApk) throws AndrolibException {
LOGGER.info("Building apk file...");
if (outApk.exists()) {
//noinspection ResultOfMethodCallIgnored
outApk.delete();
} else {
File outDir = outApk.getParentFile();
if (outDir != null && !outDir.exists()) {
//noinspection ResultOfMethodCallIgnored
outDir.mkdirs();
}
}
File assetDir = new File(mApkDir, "assets");
if (!assetDir.exists()) {
assetDir = null;
}
zipPackage(outApk, new File(mApkDir, APK_DIRNAME), assetDir);
}
private void zipPackage(File apkFile, File rawDir, File assetDir) throws AndrolibException {
try {
ZipUtils.zipFolders(rawDir, apkFile, assetDir, mApkInfo.doNotCompress);
} catch (IOException | BrutException ex) {
throw new AndrolibException(ex);
}
}
private File[] getIncludeFiles() throws AndrolibException {
UsesFramework usesFramework = mApkInfo.usesFramework;
if (usesFramework == null) {

103
brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java
View File

@ -32,15 +32,18 @@ import org.apache.commons.io.FilenameUtils;
import java.io.*;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Logger;
import java.util.regex.Pattern;
public class ApkDecoder {
private final static Logger LOGGER = Logger.getLogger(ApkDecoder.class.getName());
private final AtomicReference<RuntimeException> mBuildError = new AtomicReference<>(null);
private final Config mConfig;
private final ApkInfo mApkInfo;
private int mMinSdkVersion = 0;
private volatile int mMinSdkVersion = 0;
private BackgroundWorker mWorker;
private final static String SMALI_DIRNAME = "smali";
private final static String UNK_DIRNAME = "unknown";
@ -75,6 +78,7 @@ public class ApkDecoder {
public ApkInfo decode(File outDir) throws AndrolibException, IOException, DirectoryException {
ExtFile apkFile = mApkInfo.getApkFile();
try {
mWorker = new BackgroundWorker();
if (!mConfig.forceDelete && outDir.exists()) {
throw new OutDirExistsException();
}
@ -93,6 +97,44 @@ public class ApkDecoder {
LOGGER.info("Using Apktool " + ApktoolProperties.getVersion() + " on " + mApkInfo.apkFileName);
if (mApkInfo.hasSources()) {
switch (mConfig.decodeSources) {
case Config.DECODE_SOURCES_NONE:
copySourcesRaw(outDir, "classes.dex");
break;
case Config.DECODE_SOURCES_SMALI:
case Config.DECODE_SOURCES_SMALI_ONLY_MAIN_CLASSES:
scheduleDecodeSourcesSmali(outDir, "classes.dex");
break;
}
}
if (mApkInfo.hasMultipleSources()) {
// foreach unknown dex file in root, lets disassemble it
Set<String> files = apkFile.getDirectory().getFiles(true);
for (String file : files) {
if (file.endsWith(".dex")) {
if (!file.equalsIgnoreCase("classes.dex")) {
switch(mConfig.decodeSources) {
case Config.DECODE_SOURCES_NONE:
copySourcesRaw(outDir, file);
break;
case Config.DECODE_SOURCES_SMALI:
scheduleDecodeSourcesSmali(outDir, file);
break;
case Config.DECODE_SOURCES_SMALI_ONLY_MAIN_CLASSES:
if (file.startsWith("classes") && file.endsWith(".dex")) {
scheduleDecodeSourcesSmali(outDir, file);
} else {
copySourcesRaw(outDir, file);
}
break;
}
}
}
}
}
ResourcesDecoder resourcesDecoder = new ResourcesDecoder(mConfig, mApkInfo);
if (mApkInfo.hasResources()) {
@ -117,42 +159,13 @@ public class ApkDecoder {
}
resourcesDecoder.updateApkInfo(outDir);
if (mApkInfo.hasSources()) {
switch (mConfig.decodeSources) {
case Config.DECODE_SOURCES_NONE:
copySourcesRaw(outDir, "classes.dex");
break;
case Config.DECODE_SOURCES_SMALI:
case Config.DECODE_SOURCES_SMALI_ONLY_MAIN_CLASSES:
decodeSourcesSmali(outDir, "classes.dex");
break;
}
}
if (mApkInfo.hasMultipleSources()) {
// foreach unknown dex file in root, lets disassemble it
Set<String> files = apkFile.getDirectory().getFiles(true);
for (String file : files) {
if (file.endsWith(".dex")) {
if (!file.equalsIgnoreCase("classes.dex")) {
switch(mConfig.decodeSources) {
case Config.DECODE_SOURCES_NONE:
copySourcesRaw(outDir, file);
break;
case Config.DECODE_SOURCES_SMALI:
decodeSourcesSmali(outDir, file);
break;
case Config.DECODE_SOURCES_SMALI_ONLY_MAIN_CLASSES:
if (file.startsWith("classes") && file.endsWith(".dex")) {
decodeSourcesSmali(outDir, file);
} else {
copySourcesRaw(outDir, file);
}
break;
}
}
}
}
copyRawFiles(outDir);
copyUnknownFiles(outDir);
recordUncompressedFiles(resourcesDecoder.getResFileMapping());
copyOriginalFiles(outDir);
mWorker.waitForFinish();
if (mBuildError.get() != null) {
throw mBuildError.get();
}
// In case we have no resources. We should store the minSdk we pulled from the source opcode api level
@ -160,14 +173,11 @@ public class ApkDecoder {
mApkInfo.setSdkInfoField("minSdkVersion", Integer.toString(mMinSdkVersion));
}
copyRawFiles(outDir);
copyUnknownFiles(outDir);
recordUncompressedFiles(resourcesDecoder.getResFileMapping());
copyOriginalFiles(outDir);
writeApkInfo(outDir);
return mApkInfo;
} finally {
mWorker.shutdownNow();
try {
apkFile.close();
} catch (IOException ignored) {}
@ -205,6 +215,17 @@ public class ApkDecoder {
}
}
private void scheduleDecodeSourcesSmali(File outDir, String filename) {
Runnable r = () -> {
try {
decodeSourcesSmali(outDir, filename);
} catch (AndrolibException e) {
mBuildError.compareAndSet(null, new RuntimeException(e));
}
};
mWorker.submit(r);
}
private void decodeSourcesSmali(File outDir, String filename) throws AndrolibException {
try {
File smaliDir;

65
brut.apktool/apktool-lib/src/main/java/brut/androlib/BackgroundWorker.java Normal file
View File

@ -0,0 +1,65 @@
package brut.androlib;
import java.util.ArrayList;
import java.util.concurrent.*;
public class BackgroundWorker {
private static final int THREADS_COUNT = Runtime.getRuntime().availableProcessors();
private final ArrayList<Future<?>> mWorkerFutures = new ArrayList<>();
private final ExecutorService mExecutor;
private volatile boolean mSubmitAllowed = true;
public BackgroundWorker() {
this(THREADS_COUNT);
}
public BackgroundWorker(int threads) {
mExecutor = Executors.newFixedThreadPool(threads);
}
public void waitForFinish() {
checkState();
mSubmitAllowed = false;
for (Future<?> future : mWorkerFutures) {
try {
future.get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
}
mWorkerFutures.clear();
mSubmitAllowed = true;
}
public void clearFutures() {
mWorkerFutures.clear();
}
private void checkState() {
if (!mSubmitAllowed) {
throw new IllegalStateException("BackgroundWorker is not ready");
}
}
public void shutdownNow() {
mSubmitAllowed = false;
mExecutor.shutdownNow();
}
public ExecutorService getExecutor() {
return mExecutor;
}
public void submit(Runnable task) {
checkState();
mWorkerFutures.add(mExecutor.submit(task));
}
public <T> Future<T> submit(Callable<T> task) {
checkState();
Future<T> future = mExecutor.submit(task);
mWorkerFutures.add(future);
return future;
}
}

10
brut.j.dir/src/main/java/brut/directory/ZipUtils.java
View File

@ -39,15 +39,21 @@ public class ZipUtils {
public static void zipFolders(final File folder, final File zip, final File assets, final Collection<String> doNotCompress)
throws BrutException, IOException {
mDoNotCompress = doNotCompress;
ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(zip.toPath()));
zipFoldersPreserveStream(folder, zipOutputStream, assets, doNotCompress);
zipOutputStream.close();
}
public static void zipFoldersPreserveStream(final File folder, final ZipOutputStream zipOutputStream, final File assets, final Collection<String> doNotCompress)
throws BrutException, IOException {
mDoNotCompress = doNotCompress;
zipFolders(folder, zipOutputStream);
// We manually set the assets because we need to retain the folder structure
if (assets != null) {
processFolder(assets, zipOutputStream, assets.getPath().length() - 6);
}
zipOutputStream.close();
}
private static void zipFolders(final File folder, final ZipOutputStream outputStream)