From cb6b6dfda813c377b4406b12a8b78fbb79afdda5 Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Thu, 29 Mar 2018 15:40:31 -0400 Subject: [PATCH 1/7] options: deprecate "doNotCompress", introduce "noCompressAssets" --- .../apktool-lib/src/main/java/brut/androlib/ApkOptions.java | 3 +++ .../apktool-lib/src/main/java/brut/androlib/meta/MetaInfo.java | 3 +++ 2 files changed, 6 insertions(+) diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkOptions.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkOptions.java index 0a81ef07b4..8eec92054d 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkOptions.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkOptions.java @@ -28,7 +28,10 @@ public class ApkOptions { public boolean isFramework = false; public boolean resourcesAreCompressed = false; public boolean useAapt2 = false; + + @Deprecated public Collection doNotCompress; + public Collection noCompressAssets; public String frameworkFolderLocation = null; public String frameworkTag = null; diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/meta/MetaInfo.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/meta/MetaInfo.java index 7da11d4d17..9921090edd 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/meta/MetaInfo.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/meta/MetaInfo.java @@ -37,7 +37,10 @@ public class MetaInfo { public boolean sharedLibrary; public boolean sparseResources; public Map unknownFiles; + + @Deprecated public Collection doNotCompress; + public Collection noCompressAssets; private static Yaml getYaml() { DumperOptions options = new DumperOptions(); From bd9c9c7cec67337518f0f6e986be8634f2a452bb Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Thu, 29 Mar 2018 15:41:02 -0400 Subject: [PATCH 2/7] decode: only store full relative path to uncompressed assets --- .../src/main/java/brut/androlib/Androlib.java | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java index cddc3631c0..13f446634e 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java @@ -41,8 +41,6 @@ import java.util.zip.ZipOutputStream; import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang3.StringUtils; /** * @author Ryszard Wiśniewski @@ -161,28 +159,15 @@ public void decodeRawFiles(ExtFile apkFile, File outDir, short decodeAssetMode) } } - public void recordUncompressedFiles(ExtFile apkFile, Collection uncompressedFilesOrExts) throws AndrolibException { + public void recordUncompressedFiles(ExtFile apkFile, Collection uncompressedFiles) throws AndrolibException { try { Directory unk = apkFile.getDirectory(); Set files = unk.getFiles(true); - String ext; for (String file : files) { if (isAPKFileNames(file) && !NO_COMPRESS_PATTERN.matcher(file).find()) { - if (unk.getCompressionLevel(file) == 0) { - - if (StringUtils.countMatches(file, ".") > 1) { - ext = file; - } else { - ext = FilenameUtils.getExtension(file); - if (ext.isEmpty()) { - ext = file; - } - } - - if (! uncompressedFilesOrExts.contains(ext)) { - uncompressedFilesOrExts.add(ext); - } + if (unk.getCompressionLevel(file) == 0 && !uncompressedFiles.contains(file)) { + uncompressedFiles.add(file); } } } From 653472d006f25d9933fe60a9978cfd65f196a723 Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Thu, 29 Mar 2018 15:41:23 -0400 Subject: [PATCH 3/7] tests: update test to use new "noCompressAssets" property --- .../brut/androlib/decode/DoubleExtensionUnknownFileTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brut.apktool/apktool-lib/src/test/java/brut/androlib/decode/DoubleExtensionUnknownFileTest.java b/brut.apktool/apktool-lib/src/test/java/brut/androlib/decode/DoubleExtensionUnknownFileTest.java index 9290b05bb2..a08d5f5b11 100644 --- a/brut.apktool/apktool-lib/src/test/java/brut/androlib/decode/DoubleExtensionUnknownFileTest.java +++ b/brut.apktool/apktool-lib/src/test/java/brut/androlib/decode/DoubleExtensionUnknownFileTest.java @@ -62,7 +62,7 @@ public void multipleExtensionUnknownFileTest() throws BrutException, IOException apkDecoder.decode(); MetaInfo metaInfo = new Androlib().readMetaFile(decodedApk); - for (String string : metaInfo.doNotCompress) { + for (String string : metaInfo.noCompressAssets) { if (StringUtils.countMatches(string, ".") > 1) { assertTrue(string.equalsIgnoreCase("assets/bin/Data/sharedassets1.assets.split0")); } From d6e3d013e7117d9c41ddb21db1e42859dfec4da2 Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Thu, 29 Mar 2018 15:41:41 -0400 Subject: [PATCH 4/7] tests: refactor tests to use assertEquals instead of hacky assertTrue --- .../apktool-lib/src/test/java/brut/androlib/BaseTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/brut.apktool/apktool-lib/src/test/java/brut/androlib/BaseTest.java b/brut.apktool/apktool-lib/src/test/java/brut/androlib/BaseTest.java index d24bd9d6c7..5c9d7a8c70 100644 --- a/brut.apktool/apktool-lib/src/test/java/brut/androlib/BaseTest.java +++ b/brut.apktool/apktool-lib/src/test/java/brut/androlib/BaseTest.java @@ -31,6 +31,7 @@ import java.util.Set; import java.util.logging.Logger; +import static junit.framework.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual; @@ -45,11 +46,11 @@ protected void compareUnknownFiles() throws BrutException { Map controlFiles = control.unknownFiles; Map testFiles = test.unknownFiles; - assertTrue(controlFiles.size() == testFiles.size()); + assertEquals(controlFiles.size(), testFiles.size()); // Make sure that the compression methods are still the same for (Map.Entry controlEntry : controlFiles.entrySet()) { - assertTrue(controlEntry.getValue().equals(testFiles.get(controlEntry.getKey()))); + assertEquals(controlEntry.getValue(), testFiles.get(controlEntry.getKey())); } } From 14eedde165623e0f28bad036b3ce8af883a0a635 Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Thu, 29 Mar 2018 15:41:58 -0400 Subject: [PATCH 5/7] decode: store uncompressed assets to new property --- .../apktool-lib/src/main/java/brut/androlib/ApkDecoder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java index 4bf5b001c8..5188beb7ac 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java @@ -418,7 +418,7 @@ private void putUnknownInfo(MetaInfo meta) throws AndrolibException { private void putFileCompressionInfo(MetaInfo meta) throws AndrolibException { if (mUncompressedFiles != null && !mUncompressedFiles.isEmpty()) { - meta.doNotCompress = mUncompressedFiles; + meta.noCompressAssets = mUncompressedFiles; } } From e35f66b0e084252a96a800298fd98e3a184ae0e4 Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Thu, 29 Mar 2018 15:43:06 -0400 Subject: [PATCH 6/7] build: introduce "buildUnknownAndUncompressedAssets" - processes assets that are stored (uncompressed) - additionally process unknown files - modifies application after build process - prevents "too long" execution command - fixes #1272 --- .../src/main/java/brut/androlib/Androlib.java | 112 +++++++++++++----- 1 file changed, 80 insertions(+), 32 deletions(-) diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java index 13f446634e..9e91fbf8c4 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java @@ -19,7 +19,6 @@ import brut.androlib.meta.MetaInfo; import brut.androlib.meta.UsesFramework; import brut.androlib.res.AndrolibResources; -import brut.androlib.res.data.ResConfigFlags; import brut.androlib.res.data.ResPackage; import brut.androlib.res.data.ResTable; import brut.androlib.res.data.ResUnknownFiles; @@ -262,6 +261,7 @@ public void build(ExtFile appDir, File outFile) apkOptions.isFramework = meta.isFrameworkApk; apkOptions.resourcesAreCompressed = meta.compressionType; apkOptions.doNotCompress = meta.doNotCompress; + apkOptions.noCompressAssets = meta.noCompressAssets; mAndRes.setSdkInfo(meta.sdkInfo); mAndRes.setPackageId(meta.packageInfo); @@ -293,8 +293,8 @@ public void build(ExtFile appDir, File outFile) buildApk(appDir, outFile); // we must go after the Apk is built, and copy the files in via Zip - // this is because Aapt won't add files it doesn't know (ex unknown files) - buildUnknownFiles(appDir, outFile, meta); + // this is because Aapt won't add files it doesn't know (ex unknown files & uncompressed assets) + buildUnknownAndUncompressedAssets(appDir, outFile, meta); // we copied the AndroidManifest.xml to AndroidManifest.xml.orig so we can edit it // lets restore the unedited one, to not change the original @@ -588,38 +588,46 @@ public void buildCopyOriginalFiles(File appDir) } } - public void buildUnknownFiles(File appDir, File outFile, MetaInfo meta) + public void buildUnknownAndUncompressedAssets(File appDir, File outFile, MetaInfo meta) throws AndrolibException { + if (meta.unknownFiles == null && meta.noCompressAssets == null) { + return; + } + if (meta.unknownFiles != null) { LOGGER.info("Copying unknown files/dir..."); + } - Map files = meta.unknownFiles; - File tempFile = new File(outFile.getParent(), outFile.getName() + ".apktool_temp"); - boolean renamed = outFile.renameTo(tempFile); - if (!renamed) { - throw new AndrolibException("Unable to rename temporary file"); - } - - try ( - ZipFile inputFile = new ZipFile(tempFile); - ZipOutputStream actualOutput = new ZipOutputStream(new FileOutputStream(outFile)) - ) { - copyExistingFiles(inputFile, actualOutput); - copyUnknownFiles(appDir, actualOutput, files); - } catch (IOException | BrutException ex) { - throw new AndrolibException(ex); - } + File tempFile = new File(outFile.getParent(), outFile.getName() + ".apktool_temp"); + boolean renamed = outFile.renameTo(tempFile); + if (!renamed) { + throw new AndrolibException("Unable to rename temporary file"); + } - // Remove our temporary file. - tempFile.delete(); + try ( + ZipFile inputFile = new ZipFile(tempFile); + ZipOutputStream actualOutput = new ZipOutputStream(new FileOutputStream(outFile)) + ) { + copyExistingFiles(inputFile, actualOutput, meta.noCompressAssets); + copyUncompressedAssetFiles(appDir, actualOutput, meta.noCompressAssets); + copyUnknownFiles(appDir, actualOutput, meta.unknownFiles); + } catch (IOException | BrutException ex) { + throw new AndrolibException(ex); } + + // Remove our temporary file. + tempFile.delete(); } - private void copyExistingFiles(ZipFile inputFile, ZipOutputStream outputFile) throws IOException { + private void copyExistingFiles(ZipFile inputFile, ZipOutputStream outputFile, Collection assets) + throws IOException { // First, copy the contents from the existing outFile: Enumeration entries = inputFile.entries(); while (entries.hasMoreElements()) { ZipEntry entry = new ZipEntry(entries.nextElement()); + if (assets != null && assets.contains(entry.getName())) { + continue; + } // We can't reuse the compressed size because it depends on compression sizes. entry.setCompressedSize(-1); @@ -636,25 +644,25 @@ private void copyExistingFiles(ZipFile inputFile, ZipOutputStream outputFile) th private void copyUnknownFiles(File appDir, ZipOutputStream outputFile, Map files) throws BrutException, IOException { + File unknownFileDir = new File(appDir, UNK_DIRNAME); + if (files == null || files.isEmpty()) { + return; + } // loop through unknown files for (Map.Entry unknownFileInfo : files.entrySet()) { - File inputFile = new File(unknownFileDir, BrutIO.sanitizeUnknownFile(unknownFileDir, unknownFileInfo.getKey())); + String cleanedPath = BrutIO.sanitizeUnknownFile(unknownFileDir, unknownFileInfo.getKey()); + File inputFile = new File(unknownFileDir, cleanedPath); if (inputFile.isDirectory()) { continue; } - ZipEntry newEntry = new ZipEntry(unknownFileInfo.getKey()); + ZipEntry newEntry = new ZipEntry(cleanedPath); int method = Integer.parseInt(unknownFileInfo.getValue()); LOGGER.fine(String.format("Copying unknown file %s with method %d", unknownFileInfo.getKey(), method)); if (method == ZipEntry.STORED) { - newEntry.setMethod(ZipEntry.STORED); - newEntry.setSize(inputFile.length()); - newEntry.setCompressedSize(-1); - BufferedInputStream unknownFile = new BufferedInputStream(new FileInputStream(inputFile)); - CRC32 crc = BrutIO.calculateCrc(unknownFile); - newEntry.setCrc(crc.getValue()); + newEntry = getStoredZipEntry(cleanedPath, inputFile); } else { newEntry.setMethod(ZipEntry.DEFLATED); } @@ -665,6 +673,45 @@ private void copyUnknownFiles(File appDir, ZipOutputStream outputFile, Map files) + throws BrutException, IOException { + + if (files == null || files.isEmpty()) { + return; + } + + File assetFileDir = new File(appDir, ASSET_DIRNAME); + + for (String asset : files) { + String cleanedPath = BrutIO.sanitizeUnknownFile(assetFileDir, asset); + + File inputFile = new File(appDir, cleanedPath); + if (inputFile.isDirectory()) { + continue; + } + + LOGGER.fine(String.format("Copying uncompressed asset: %s", asset)); + ZipEntry newEntry = getStoredZipEntry(cleanedPath, inputFile); + outputFile.putNextEntry(newEntry); + BrutIO.copy(inputFile, outputFile); + outputFile.closeEntry(); + } + } + + private ZipEntry getStoredZipEntry(String cleanedPath, File inputFile) + throws IOException { + + ZipEntry newEntry = new ZipEntry(cleanedPath); + newEntry.setMethod(ZipEntry.STORED); + newEntry.setSize(inputFile.length()); + newEntry.setCompressedSize(-1); + BufferedInputStream unknownFile = new BufferedInputStream(new FileInputStream(inputFile)); + CRC32 crc = BrutIO.calculateCrc(unknownFile); + newEntry.setCrc(crc.getValue()); + + return newEntry; + } + public void buildApk(File appDir, File outApk) throws AndrolibException { LOGGER.info("Building apk file..."); if (outApk.exists()) { @@ -762,6 +809,7 @@ public void close() throws IOException { private final static String SMALI_DIRNAME = "smali"; private final static String APK_DIRNAME = "build/apk"; private final static String UNK_DIRNAME = "unknown"; + private final static String ASSET_DIRNAME = "assets"; private final static String[] APK_RESOURCES_FILENAMES = new String[] { "resources.arsc", "AndroidManifest.xml", "res" }; private final static String[] APK_RESOURCES_WITHOUT_RES_FILENAMES = new String[] { @@ -776,6 +824,6 @@ public void close() throws IOException { // Taken from AOSP's frameworks/base/tools/aapt/Package.cpp private final static Pattern NO_COMPRESS_PATTERN = Pattern.compile("\\.(" + "jpg|jpeg|png|gif|wav|mp2|mp3|ogg|aac|mpg|mpeg|mid|midi|smf|jet|rtttl|imy|xmf|mp4|" + - "m4a|m4v|3gp|3gpp|3g2|3gpp2|amr|awb|wma|wmv|webm|mkv)$"); + "m4a|m4v|3gp|3gpp|3g2|3gpp2|amr|awb|wma|wmv|webm|mkv|arsc)$"); } From ca25a2694408ab7989fd107db7f8ff300fa3f779 Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Fri, 6 Apr 2018 10:31:28 -0400 Subject: [PATCH 7/7] fix: normalize paths for uncompressed assets & unknown files --- .../src/main/java/brut/androlib/Androlib.java | 10 ++++++---- .../java/brut/androlib/res/data/ResUnknownFiles.java | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java index 9e91fbf8c4..ccf2cdb458 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/Androlib.java @@ -652,7 +652,8 @@ private void copyUnknownFiles(File appDir, ZipOutputStream outputFile, Map unknownFileInfo : files.entrySet()) { - String cleanedPath = BrutIO.sanitizeUnknownFile(unknownFileDir, unknownFileInfo.getKey()); + String normalizedPath = BrutIO.normalizePath(unknownFileInfo.getKey()); + String cleanedPath = BrutIO.sanitizeUnknownFile(unknownFileDir, normalizedPath); File inputFile = new File(unknownFileDir, cleanedPath); if (inputFile.isDirectory()) { continue; @@ -683,15 +684,16 @@ private void copyUncompressedAssetFiles(File appDir, ZipOutputStream outputFile, File assetFileDir = new File(appDir, ASSET_DIRNAME); for (String asset : files) { - String cleanedPath = BrutIO.sanitizeUnknownFile(assetFileDir, asset); + String normalizedPath = BrutIO.normalizePath(asset); + String cleanedPath = BrutIO.sanitizeUnknownFile(assetFileDir, normalizedPath); File inputFile = new File(appDir, cleanedPath); if (inputFile.isDirectory()) { continue; } - LOGGER.fine(String.format("Copying uncompressed asset: %s", asset)); - ZipEntry newEntry = getStoredZipEntry(cleanedPath, inputFile); + LOGGER.fine(String.format("Copying uncompressed asset: %s", normalizedPath)); + ZipEntry newEntry = getStoredZipEntry(normalizedPath, inputFile); outputFile.putNextEntry(newEntry); BrutIO.copy(inputFile, outputFile); outputFile.closeEntry(); diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/data/ResUnknownFiles.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/data/ResUnknownFiles.java index 595c4343c0..a73362aaf8 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/data/ResUnknownFiles.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/data/ResUnknownFiles.java @@ -16,6 +16,7 @@ */ package brut.androlib.res.data; +import brut.util.BrutIO; import java.util.LinkedHashMap; import java.util.Map; @@ -27,7 +28,7 @@ public class ResUnknownFiles { private final Map mUnknownFiles = new LinkedHashMap<>(); public void addUnknownFileInfo(String file, String value) { - mUnknownFiles.put(file, value); + mUnknownFiles.put(BrutIO.normalizePath(file), value); } public Map getUnknownFiles() {