Introduce our own implementation of a ZipOutputStream.
Summary:
We currenty depend on the command line zip command in order to create zip files. This means that we cannot
currently build an APK on Windows, which is obviously suboptimal. In the ideal world, we'd use something
that ships with the JRE, but it turns out that the two obvious candidates fail. Our requirements are:
1) Can support changing the deflation level per-file.
2) Can support writing the same file more than once to the zip, overwriting existing entries.
3) As 2, but appending the new entry.
ZipOutputStream meets 1, but 2 or 3. ZipFileSystem supports 2, but not 1 or 3. The classes in this
diff allow us to support all three options.
Test Plan: buck test --all
diff --git a/build.xml b/build.xml
index 74ff45e..df6d3e4 100644
--- a/build.xml
+++ b/build.xml
@@ -243,10 +243,11 @@
windowtitle="Buck"
failonerror="true"
>
- <fileset dir="${src.dir}" />
+ <fileset dir="${src.dir}"/>
<fileset dir="${test.dir}">
<exclude name="**/testdata/**" />
<exclude name="**/BUCK" />
+ <exclude name="**/*.properties"/>
</fileset>
<link href="http://docs.oracle.com/javase/7/docs/api/" />
<link href="http://docs.guava-libraries.googlecode.com/git-history/v15.0/javadoc/" />
diff --git a/src/com/facebook/buck/android/BUCK b/src/com/facebook/buck/android/BUCK
index c9de1c9..d02403c 100644
--- a/src/com/facebook/buck/android/BUCK
+++ b/src/com/facebook/buck/android/BUCK
@@ -135,6 +135,7 @@
'//src/com/facebook/buck/util:util',
'//src/com/facebook/buck/util/environment:environment',
'//src/com/facebook/buck/zip:steps',
+ '//src/com/facebook/buck/zip:stream',
'//third-party/java/aosp/src/com/android:aosp',
],
visibility = ['PUBLIC'],
diff --git a/src/com/facebook/buck/android/ProGuardObfuscateStep.java b/src/com/facebook/buck/android/ProGuardObfuscateStep.java
index d0c323d..8b44d29 100644
--- a/src/com/facebook/buck/android/ProGuardObfuscateStep.java
+++ b/src/com/facebook/buck/android/ProGuardObfuscateStep.java
@@ -21,6 +21,8 @@
import com.facebook.buck.util.AndroidPlatformTarget;
import com.facebook.buck.util.Functions;
import com.facebook.buck.util.HumanReadableException;
+import com.facebook.buck.zip.CustomZipOutputStream;
+import com.facebook.buck.zip.ZipOutputStreams;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Objects;
@@ -32,12 +34,10 @@
import com.google.common.io.Files;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Map;
import java.util.Set;
import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
public final class ProGuardObfuscateStep extends ShellStep {
@@ -160,7 +160,7 @@
@VisibleForTesting
static void createEmptyZip(File file) throws IOException {
Files.createParentDirs(file);
- ZipOutputStream out = new ZipOutputStream(new FileOutputStream(file));
+ CustomZipOutputStream out = ZipOutputStreams.newOutputStream(file);
// Sun's java 6 runtime doesn't allow us to create a truly empty zip, but this should be enough
// to pass through dx/split-zip without any issue.
// ...and Sun's java 7 runtime doesn't let us use an empty string for the zip entry name.
diff --git a/src/com/facebook/buck/android/SmartDexingStep.java b/src/com/facebook/buck/android/SmartDexingStep.java
index 5b8ba13..153e5b7 100644
--- a/src/com/facebook/buck/android/SmartDexingStep.java
+++ b/src/com/facebook/buck/android/SmartDexingStep.java
@@ -73,7 +73,7 @@
private final DexStore dexStore;
private ListeningExecutorService dxExecutor;
- /** Lazily initialized. See {@link InputResolver#createOutputToInputs()}. */
+ /** Lazily initialized. See {@link InputResolver#createOutputToInputs(DexStore)}. */
private Multimap<File, File> outputToInputs;
/**
@@ -433,8 +433,7 @@
tempDexJarOutput,
repackedJar,
ImmutableSet.of("classes.dex"),
- ZipStep.MIN_COMPRESSION_LEVEL,
- /* workingDirectory */ null
+ ZipStep.MIN_COMPRESSION_LEVEL
));
steps.add(new RmStep(tempDexJarOutput, true));
steps.add(new XzStep(repackedJar));
diff --git a/src/com/facebook/buck/dalvik/BUCK b/src/com/facebook/buck/dalvik/BUCK
index cbb4a2d..dd62b8d 100644
--- a/src/com/facebook/buck/dalvik/BUCK
+++ b/src/com/facebook/buck/dalvik/BUCK
@@ -43,6 +43,7 @@
'//src/com/facebook/buck/util:exceptions',
'//src/com/facebook/buck/util:io',
'//src/com/facebook/buck/util:util',
+ '//src/com/facebook/buck/zip:stream',
],
visibility = [
'PUBLIC',
diff --git a/src/com/facebook/buck/java/BUCK b/src/com/facebook/buck/java/BUCK
index 7e317ad..d1e3e48 100644
--- a/src/com/facebook/buck/java/BUCK
+++ b/src/com/facebook/buck/java/BUCK
@@ -93,6 +93,7 @@
'//src/com/facebook/buck/util:exceptions',
'//src/com/facebook/buck/util:io',
'//src/com/facebook/buck/util:util',
+ '//src/com/facebook/buck/zip:stream',
],
visibility = ['PUBLIC'],
)
diff --git a/src/com/facebook/buck/java/JarDirectoryStep.java b/src/com/facebook/buck/java/JarDirectoryStep.java
index 78af27e..fa1d9f2 100644
--- a/src/com/facebook/buck/java/JarDirectoryStep.java
+++ b/src/com/facebook/buck/java/JarDirectoryStep.java
@@ -16,28 +16,29 @@
package com.facebook.buck.java;
+import static com.facebook.buck.zip.ZipOutputStreams.HandleDuplicates.APPEND_TO_ZIP;
+
import com.facebook.buck.event.BuckEventBus;
import com.facebook.buck.event.LogEvent;
import com.facebook.buck.step.ExecutionContext;
import com.facebook.buck.step.Step;
import com.facebook.buck.util.DirectoryTraversal;
import com.facebook.buck.util.ProjectFilesystem;
+import com.facebook.buck.zip.CustomZipOutputStream;
+import com.facebook.buck.zip.ZipOutputStreams;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.common.io.ByteStreams;
-import com.google.common.io.Closeables;
import com.google.common.io.Closer;
import com.google.common.io.Files;
-import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Enumeration;
@@ -46,7 +47,6 @@
import java.util.jar.Attributes;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
-import java.util.jar.JarOutputStream;
import java.util.jar.Manifest;
import java.util.logging.Level;
import java.util.zip.ZipEntry;
@@ -136,23 +136,16 @@
// Write the manifest, as appropriate.
ProjectFilesystem filesystem = context.getProjectFilesystem();
if (manifestFile != null) {
- FileInputStream manifestStream = new FileInputStream(
- filesystem.getFileForRelativePath(manifestFile));
- boolean readSuccessfully = false;
- try {
+ try (FileInputStream manifestStream = new FileInputStream(
+ filesystem.getFileForRelativePath(manifestFile))) {
manifest.read(manifestStream);
- readSuccessfully = true;
- } finally {
- Closeables.close(manifestStream, !readSuccessfully);
}
-
- } else {
- manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
}
- try (JarOutputStream outputFile = new JarOutputStream(
- new BufferedOutputStream(new FileOutputStream(
- filesystem.getFileForRelativePath(pathToOutputFile))))) {
+ manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+
+ try (CustomZipOutputStream outputFile = ZipOutputStreams.newOutputStream(
+ filesystem.getFileForRelativePath(pathToOutputFile), APPEND_TO_ZIP)) {
Set<String> alreadyAddedEntries = Sets.newHashSet();
ProjectFilesystem projectFilesystem = context.getProjectFilesystem();
@@ -192,7 +185,7 @@
* @param alreadyAddedEntries is used to avoid duplicate entries.
*/
private void copyZipEntriesToJar(File file,
- final JarOutputStream jar,
+ final CustomZipOutputStream jar,
Manifest manifest,
Set<String> alreadyAddedEntries,
BuckEventBus eventBus) throws IOException {
@@ -207,11 +200,16 @@
continue;
}
- // The same directory entry cannot be added more than once.
- if (!alreadyAddedEntries.add(entryName)) {
+ // We're in the process of merging a bunch of different jar files. These typically contain
+ // just ".class" files and the manifest, but they can also include things like license files
+ // from third party libraries and config files. We should include those license files within
+ // the jar we're creating. Extracting them is left as an exercise for the consumer of the jar.
+ // Because we don't know which files are important, the only ones we skip are duplicate class
+ // files.
+ if (!isDuplicateAllowed(entryName) && !alreadyAddedEntries.add(entryName)) {
// Duplicate entries. Skip.
eventBus.post(LogEvent.create(
- Level.FINE, "Duplicate found when adding file to jar: %s", entryName));
+ Level.INFO, "Duplicate found when adding file to jar: %s", entryName));
continue;
}
@@ -241,7 +239,7 @@
* @param jar is the file being written.
*/
private void addFilesInDirectoryToJar(File directory,
- final JarOutputStream jar,
+ final CustomZipOutputStream jar,
final Set<String> alreadyAddedEntries,
final BuckEventBus eventBus) throws IOException {
new DirectoryTraversal(directory) {
@@ -252,18 +250,16 @@
String entryName = entry.getName();
entry.setTime(file.lastModified());
try {
- if (alreadyAddedEntries.contains(entryName)) {
+ // We expect there to be many duplicate entries for things like directories. Creating
+ // those repeatedly would be lame, so don't do that.
+ if (!isDuplicateAllowed(entryName) && !alreadyAddedEntries.add(entryName)) {
eventBus.post(LogEvent.create(
- Level.FINE, "Duplicate found when adding directory to jar: %s", relativePath));
+ Level.INFO, "Duplicate found when adding directory to jar: %s", relativePath));
return;
}
jar.putNextEntry(entry);
Files.copy(file, jar);
jar.closeEntry();
-
- if (entryName.endsWith("/")) {
- alreadyAddedEntries.add(entryName);
- }
} catch (IOException e) {
Throwables.propagate(e);
}
@@ -298,4 +294,7 @@
}
}
+ private boolean isDuplicateAllowed(String name) {
+ return !name.endsWith(".class") && !name.endsWith("/");
+ }
}
diff --git a/src/com/facebook/buck/util/BUCK b/src/com/facebook/buck/util/BUCK
index 38fed85..d06ef41 100644
--- a/src/com/facebook/buck/util/BUCK
+++ b/src/com/facebook/buck/util/BUCK
@@ -44,6 +44,7 @@
'//lib:guava',
'//lib:jsr305',
'//src/com/facebook/buck/util/environment:environment',
+ '//src/com/facebook/buck/zip:stream',
],
visibility = [ 'PUBLIC' ],
)
@@ -72,6 +73,7 @@
':io',
'//lib:guava',
'//lib:jsr305',
+ '//src/com/facebook/buck/zip:stream',
],
visibility = [
'PUBLIC',
diff --git a/src/com/facebook/buck/util/ProjectFilesystem.java b/src/com/facebook/buck/util/ProjectFilesystem.java
index 1188610..6fbbe90 100644
--- a/src/com/facebook/buck/util/ProjectFilesystem.java
+++ b/src/com/facebook/buck/util/ProjectFilesystem.java
@@ -17,6 +17,8 @@
package com.facebook.buck.util;
import com.facebook.buck.util.environment.Platform;
+import com.facebook.buck.zip.CustomZipOutputStream;
+import com.facebook.buck.zip.ZipOutputStreams;
import com.google.common.base.Charsets;
import com.google.common.base.Function;
import com.google.common.base.Optional;
@@ -26,9 +28,7 @@
import com.google.common.io.ByteStreams;
import com.google.common.io.Files;
-import java.io.BufferedOutputStream;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.FileVisitor;
@@ -40,7 +40,6 @@
import java.util.List;
import java.util.Properties;
import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
/**
* An injectable service for interacting with the filesystem.
@@ -299,9 +298,7 @@
*/
public void createZip(Iterable<Path> pathsToIncludeInZip, File out) throws IOException {
Preconditions.checkState(!Iterables.isEmpty(pathsToIncludeInZip));
- try (ZipOutputStream zip = new ZipOutputStream(
- new BufferedOutputStream(
- new FileOutputStream(out)))) {
+ try (CustomZipOutputStream zip = ZipOutputStreams.newOutputStream(out)) {
for (Path path : pathsToIncludeInZip) {
ZipEntry entry = new ZipEntry(path.toString());
zip.putNextEntry(entry);
diff --git a/src/com/facebook/buck/zip/AppendingZipOutputStream.java b/src/com/facebook/buck/zip/AppendingZipOutputStream.java
new file mode 100644
index 0000000..8a3fc6f
--- /dev/null
+++ b/src/com/facebook/buck/zip/AppendingZipOutputStream.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2013-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.zip;
+
+import com.facebook.buck.timing.Clock;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.List;
+import java.util.Set;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipException;
+
+/**
+ * A drop-in replacement for (@link java.util.zip.ZipOutStream} that supports the ability to set a
+ * compression level and allows multiple entries with the same name.
+ *
+ * <a href="https://users.cs.jmu.edu/buchhofp/forensics/formats/pkzip.html">
+ * https://users.cs.jmu.edu/buchhofp/forensics/formats/pkzip.html
+ * </a>
+ * <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">
+ * http://www.pkware.com/documents/casestudies/APPNOTE.TXT
+ * </a>
+ */
+class AppendingZipOutputStream extends CustomZipOutputStream {
+
+ private final boolean throwExceptionsOnDuplicate;
+ private final Clock clock;
+ private long currentOffset = 0;
+ private List<EntryAccounting> entries = Lists.newLinkedList();
+ private EntryAccounting currentEntry = null;
+
+ private Set<String> seenNames = Sets.newHashSet();
+
+ public AppendingZipOutputStream(Clock clock,
+ OutputStream stream,
+ boolean throwExceptionsOnDuplicate) {
+ super(stream);
+ this.clock = Preconditions.checkNotNull(clock);
+ this.throwExceptionsOnDuplicate = throwExceptionsOnDuplicate;
+ }
+
+ @Override
+ protected void actuallyWrite(byte[] b, int off, int len) throws IOException {
+ currentOffset += currentEntry.write(delegate, b, off, len);
+ }
+
+ @Override
+ protected void actuallyPutNextEntry(ZipEntry entry) throws IOException {
+ if (throwExceptionsOnDuplicate && !seenNames.add(entry.getName())) {
+ // Same exception as ZipOutputStream.
+ throw new ZipException("duplicate entry: " + entry.getName());
+ }
+
+ currentEntry = new EntryAccounting(clock, entry, currentOffset);
+ entries.add(currentEntry);
+
+ currentOffset += currentEntry.writeLocalFileHeader(delegate);
+ }
+
+ @Override
+ protected void actuallyCloseEntry() throws IOException {
+ if (currentEntry == null) {
+ return; // no-op
+ }
+
+ currentOffset += currentEntry.close(delegate);
+
+ currentEntry = null;
+ }
+
+ @Override
+ protected void actuallyClose() throws IOException {
+ closeEntry();
+
+ new CentralDirectory().writeCentralDirectory(delegate, currentOffset, entries);
+
+ delegate.close();
+ }
+}
diff --git a/src/com/facebook/buck/zip/BUCK b/src/com/facebook/buck/zip/BUCK
index af7d40c..fc71611 100644
--- a/src/com/facebook/buck/zip/BUCK
+++ b/src/com/facebook/buck/zip/BUCK
@@ -1,16 +1,38 @@
java_library(
+ name = 'stream',
+ srcs = [
+ 'AppendingZipOutputStream.java',
+ 'ByteIo.java',
+ 'CentralDirectory.java',
+ 'CustomZipOutputStream.java',
+ 'CustomZipEntry.java',
+ 'EntryAccounting.java',
+ 'OverwritingZipOutputStream.java',
+ 'ZipOutputStreams.java',
+ ],
+ deps = [
+ '//lib:guava',
+ '//src/com/facebook/buck/timing:timing',
+ '//src/com/facebook/buck/util:exceptions',
+ ],
+ visibility = ['PUBLIC'],
+)
+
+java_library(
name = 'steps',
srcs = glob(['*Step.java']),
deps = [
+ ':stream',
'//lib:guava',
'//lib:jsr305',
- '//src/com/facebook/buck/shell:steps',
+ '//src/com/facebook/buck/event:event',
'//src/com/facebook/buck/step:step',
'//src/com/facebook/buck/step/fs:fs',
+ '//src/com/facebook/buck/util:exceptions',
'//src/com/facebook/buck/util:io',
],
visibility = [
'//src/com/facebook/buck/android/...',
'//test/com/facebook/buck/zip:zip',
],
-)
\ No newline at end of file
+)
diff --git a/src/com/facebook/buck/zip/ByteIo.java b/src/com/facebook/buck/zip/ByteIo.java
new file mode 100644
index 0000000..71bd6be
--- /dev/null
+++ b/src/com/facebook/buck/zip/ByteIo.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2013-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.zip;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+class ByteIo {
+ private ByteIo() {
+ // utility class.
+ }
+
+ public static long writeShort(OutputStream out, int value) throws IOException {
+ out.write((value & 0xff));
+ out.write(((value >>> 8) & 0xff));
+ return 2;
+ }
+
+ protected static long writeInt(OutputStream out, long value) throws IOException {
+ out.write((int) (value & 0xff));
+ out.write((int) ((value >>> 8) & 0xff));
+ out.write((int) ((value >>> 16) & 0xff));
+ out.write((int) ((value >>> 24) & 0xff));
+ return 4;
+ }
+}
diff --git a/src/com/facebook/buck/zip/CentralDirectory.java b/src/com/facebook/buck/zip/CentralDirectory.java
new file mode 100644
index 0000000..4c99859
--- /dev/null
+++ b/src/com/facebook/buck/zip/CentralDirectory.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2013-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.zip;
+
+import com.google.common.base.Charsets;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.zip.ZipEntry;
+
+/**
+ * Each zip file has a "central directory" at the end of the archive, which provides the indexes
+ * required for fast random access to the contents of the zip. This class models that.
+ * <p>
+ * The central directory consists of a series of "file headers", describing each entry in the zip,
+ * and a "end of central directory" signature containing book keeping information.
+ */
+class CentralDirectory {
+
+ /**
+ * Write the entire central directory, including the file headers and the end of central directory
+ * signature.
+ *
+ * @param out The stream to output to.
+ * @param startOffset The number of bytes offset within the zip file that this starts at.
+ * @param entries The entries that are contained within the zip.
+ * @throws IOException Should something go awry.
+ */
+ public void writeCentralDirectory(
+ OutputStream out,
+ long startOffset,
+ Iterable<EntryAccounting> entries) throws IOException {
+
+ int entryCount = 0;
+ long size = 0;
+ for (EntryAccounting entry : entries) {
+ entryCount++;
+ size += writeCentralDirectoryFileHeader(out, entry);
+ }
+
+ // End of central directory
+
+ ByteIo.writeInt(out, ZipEntry.ENDSIG);
+
+ ByteIo.writeShort(out, 0); // Number of this disk (with end of central directory)
+ ByteIo.writeShort(out, 0); // Number of disk on which central directory starts.
+ ByteIo.writeShort(out, entryCount); // Number of central directory entries in this disk.
+ ByteIo.writeShort(out, entryCount); // Number of central directory entries.
+ ByteIo.writeInt(out, size); // Size of the central directory in bytes.
+ ByteIo.writeInt(out, startOffset); // Offset of the start of the central directory.
+ ByteIo.writeShort(out, 0); // Size of the comment (we don't have one)
+ }
+
+ /**
+ * Each entry requires a description of that entry to be contained in the central directory.
+ */
+ private long writeCentralDirectoryFileHeader(
+ OutputStream out,
+ EntryAccounting entry) throws IOException {
+ long size = 0;
+ size += ByteIo.writeInt(out, ZipEntry.CENSIG);
+ size += ByteIo.writeShort(out, entry.getRequiredExtractVersion()); // version made by.
+ size += ByteIo.writeShort(out, entry.getRequiredExtractVersion()); // version to extract with.
+ size += ByteIo.writeShort(out, entry.getFlags());
+ size += ByteIo.writeShort(out, entry.getCompressionMethod()); // Compression.
+ size += ByteIo.writeInt(out, entry.getTime()); // Modification time.
+ size += ByteIo.writeInt(out, entry.getCrc());
+ size += ByteIo.writeInt(out, entry.getCompressedSize());
+ size += ByteIo.writeInt(out, entry.getSize());
+
+ byte[] nameBytes = entry.getName().getBytes(Charsets.UTF_8);
+ size += ByteIo.writeShort(out, nameBytes.length); // Length of name.
+ size += ByteIo.writeShort(out, 0); // Length of extra data.
+ size += ByteIo.writeShort(out, 0); // Length of file comment.
+ size += ByteIo.writeShort(out, 0); // Disk on which file starts.
+ size += ByteIo.writeShort(out, 0); // internal file attributes (unknown)
+ size += ByteIo.writeInt(out, 0); // external file attributes (unknown)
+ size += ByteIo.writeInt(out, entry.getOffset()); // Offset of local file header.
+ out.write(nameBytes);
+ size += nameBytes.length;
+
+ return size;
+ }
+}
diff --git a/src/com/facebook/buck/zip/CustomZipEntry.java b/src/com/facebook/buck/zip/CustomZipEntry.java
new file mode 100644
index 0000000..cf324dc
--- /dev/null
+++ b/src/com/facebook/buck/zip/CustomZipEntry.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2013-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.zip;
+
+import static java.util.zip.Deflater.BEST_COMPRESSION;
+import static java.util.zip.Deflater.NO_COMPRESSION;
+
+import com.google.common.base.Preconditions;
+
+import java.util.zip.Deflater;
+import java.util.zip.ZipEntry;
+
+public class CustomZipEntry extends ZipEntry {
+
+ private int compressionLevel = Deflater.DEFAULT_COMPRESSION;
+
+ public CustomZipEntry(ZipEntry other) {
+ super(other);
+ }
+
+ public CustomZipEntry(String name) {
+ super(name);
+ }
+
+ public void setCompressionLevel(int compressionLevel) {
+ Preconditions.checkArgument(
+ compressionLevel >= NO_COMPRESSION && compressionLevel <= BEST_COMPRESSION);
+ this.compressionLevel = compressionLevel;
+ }
+
+ public int getCompressionLevel() {
+ return compressionLevel;
+ }
+}
diff --git a/src/com/facebook/buck/zip/CustomZipOutputStream.java b/src/com/facebook/buck/zip/CustomZipOutputStream.java
new file mode 100644
index 0000000..a4101b0
--- /dev/null
+++ b/src/com/facebook/buck/zip/CustomZipOutputStream.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2013-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.zip;
+
+import com.google.common.base.Preconditions;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipException;
+
+/**
+ * An implementation of an {@link OutputStream} that will zip output. Note that, just as with
+ * {@link java.util.zip.ZipOutputStream}, no implementation of this is thread-safe.
+ */
+public abstract class CustomZipOutputStream extends OutputStream {
+
+ protected final OutputStream delegate;
+ private State state;
+ private boolean entryOpen;
+
+ protected CustomZipOutputStream(OutputStream out) {
+ this.delegate = Preconditions.checkNotNull(out);
+ this.state = State.CLEAN;
+ }
+
+ public final void putNextEntry(ZipEntry entry) throws IOException {
+ Preconditions.checkState(state != State.CLOSED, "Stream has been closed.");
+ Preconditions.checkNotNull(entry);
+
+ state = State.OPEN;
+ closeEntry();
+ actuallyPutNextEntry(entry);
+ entryOpen = true;
+ }
+
+ /**
+ * Called by {@link #putNextEntry(ZipEntry)} and used by subclasses to put the next entry into the
+ * zip file. It is guaranteed that the {@code entry} won't be null and the stream will be open. It
+ * is also guaranteed that there's no current entry open.
+ *
+ * @param entry The {@link ZipEntry} to write.
+ */
+ protected abstract void actuallyPutNextEntry(ZipEntry entry) throws IOException;
+
+ public final void closeEntry() throws IOException {
+ Preconditions.checkState(state != State.CLOSED, "Stream has been closed");
+ if (!entryOpen) {
+ return; // As ZipOutputStream does.
+ }
+
+ entryOpen = false;
+ actuallyCloseEntry();
+ }
+
+ /**
+ * Called by {@link #close()} and used by subclasses to close the delegate stream. This method
+ * will be called at most once in the lifecycle of the CustomZipOutputStream.
+ */
+ protected abstract void actuallyCloseEntry() throws IOException;
+
+ @Override
+ public final void write(byte[] b, int off, int len) throws IOException {
+ Preconditions.checkState(state != State.CLOSED, "Stream has been closed.");
+ if (!entryOpen) {
+ // Same exception as Java's ZipOutputStream.
+ throw new ZipException("no current ZIP entry");
+ }
+
+ actuallyWrite(b, off, len);
+ }
+
+ /**
+ * Called by {@link #write(byte[], int, int)} only once it is known that the stream has not been
+ * closed, and that a {@link ZipEntry} has already been put on the stream and not closed.
+ */
+ protected abstract void actuallyWrite(byte b[], int off, int len) throws IOException;
+
+ // javadocs taken from OutputStream and amended to make it clear what we're doing here.
+ /**
+ * Writes the specified byte to this output stream. Specifically one byte is written to the
+ * output stream. The byte to be written is the eight low-order bits of the argument
+ * <code>b</code>. The 24 high-order bits of <code>b</code> are ignored.
+ *
+ * @param b the <code>byte</code>.
+ * @exception IOException if an I/O error occurs. In particular,
+ * an <code>IOException</code> may be thrown if the
+ * output stream has been closed.
+ */
+ @Override
+ public void write(int b) throws IOException {
+ byte[] buf = new byte[1];
+ buf[0] = (byte)(b & 0xff);
+ write(buf, 0, 1);
+ }
+
+ @Override
+ public final void close() throws IOException {
+ if (state == State.CLOSED) {
+ return; // no-op to call close again.
+ }
+
+ try {
+ actuallyClose();
+ } finally {
+ state = State.CLOSED;
+ }
+ }
+
+ protected abstract void actuallyClose() throws IOException;
+
+ /**
+ * State of a {@link com.facebook.buck.zip.CustomZipOutputStream}. Certain operations are only
+ * available when the stream is in a particular state.
+ */
+ private static enum State {
+ CLEAN, // Open but no data written.
+ OPEN, // Open and data written.
+ CLOSED, // Just as it says on the tin.
+ }
+}
diff --git a/src/com/facebook/buck/zip/EntryAccounting.java b/src/com/facebook/buck/zip/EntryAccounting.java
new file mode 100644
index 0000000..16ddd5b
--- /dev/null
+++ b/src/com/facebook/buck/zip/EntryAccounting.java
@@ -0,0 +1,293 @@
+/*
+ * Copyright 2013-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.zip;
+
+import com.facebook.buck.timing.Clock;
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.zip.Deflater;
+import java.util.zip.ZipEntry;
+
+/**
+ * A wrapper containing the {@link ZipEntry} and additional book keeping information required to
+ * write the entry to a zip file.
+ */
+class EntryAccounting {
+ private static final int DATA_DESCRIPTOR_FLAG = 1 << 3;
+ private static final int UTF8_NAMES_FLAG = 1 << 11;
+ private static final int ARBITRARY_SIZE = 1024;
+ private static final long DOS_EPOCH_START = (1 << 21) | (1 << 16);
+
+ private final ZipEntry entry;
+ private final Method method;
+ private Hasher crc = Hashing.crc32().newHasher();
+ private long offset;
+ /*
+ * General purpose bit flag:
+ * Bit 00: encrypted file
+ * Bit 01: compression option
+ * Bit 02: compression option
+ * Bit 03: data descriptor
+ * Bit 04: enhanced deflation
+ * Bit 05: compressed patched data
+ * Bit 06: strong encryption
+ * Bit 07-10: unused
+ * Bit 11: language encoding
+ * Bit 12: reserved
+ * Bit 13: mask header values
+ * Bit 14-15: reserved
+ *
+ * The important one is bit 3: the data descriptor.
+ * Defaults to indicate that names are stored as UTF8.
+ */
+ private int flags = UTF8_NAMES_FLAG;
+ private final Deflater deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, true);
+ private final byte[] buffer = new byte[ARBITRARY_SIZE];
+
+ public EntryAccounting(Clock clock, ZipEntry entry, long currentOffset) {
+ this.entry = Preconditions.checkNotNull(entry);
+ this.method = Method.detect(entry.getMethod());
+ Preconditions.checkNotNull(clock);
+ this.offset = currentOffset;
+
+ if (entry.getTime() == -1) {
+ entry.setTime(clock.currentTimeMillis());
+ }
+
+ if (method == Method.DEFLATE) {
+ flags |= DATA_DESCRIPTOR_FLAG;
+ }
+
+ if (entry instanceof CustomZipEntry) {
+ deflater.setLevel(((CustomZipEntry) entry).getCompressionLevel());
+ }
+ }
+
+ public void updateCrc(byte[] b, int off, int len) {
+ crc = crc.putBytes(b, off, len);
+ }
+
+ /**
+ * @return The time of the entry in DOS format.
+ */
+ public long getTime() {
+ // It'd be nice to use a Calendar for this, but (and here's the fun bit), that's a Really Bad
+ // Idea since the calendar's internal time representation keeps ticking once set. Instead, do
+ // this long way.
+
+ Calendar instance = Calendar.getInstance();
+ instance.setTimeInMillis(entry.getTime());
+
+ int year = instance.get(Calendar.YEAR);
+
+ // The DOS epoch begins in 1980. If the year is before that, then default to the start of the
+ // epoch (the 1st day of the 1st month)
+ if (year < 1980) {
+ return DOS_EPOCH_START;
+ }
+ return (year - 1980) << 25 |
+ (instance.get(Calendar.MONTH) + 1) << 21 |
+ instance.get(Calendar.DAY_OF_MONTH) << 16 |
+ instance.get(Calendar.HOUR_OF_DAY) << 11 |
+ instance.get(Calendar.MINUTE) << 5 |
+ instance.get(Calendar.SECOND) >> 1;
+ }
+
+ private boolean isDeflated() {
+ return method == Method.DEFLATE;
+ }
+
+ public String getName() {
+ return entry.getName();
+ }
+
+ public long getSize() {
+ return entry.getSize();
+ }
+
+ public long getCompressedSize() {
+ return entry.getCompressedSize();
+ }
+
+ public int getFlags() {
+ return flags;
+ }
+
+ public long getOffset() {
+ return offset;
+ }
+
+ public void setOffset(long offset) {
+ this.offset = offset;
+ }
+
+ public long getCrc() {
+ return entry.getCrc();
+ }
+
+ public void calculateCrc() {
+ entry.setCrc(crc.hash().padToLong());
+ }
+
+ public int getCompressionMethod() {
+ return method.compressionMethod;
+ }
+
+ public int getRequiredExtractVersion() {
+ return method.getRequiredExtractVersion();
+ }
+
+ public long writeLocalFileHeader(OutputStream out) throws IOException {
+ try (ByteArrayOutputStream stream = new ByteArrayOutputStream()) {
+ ByteIo.writeInt(stream, ZipEntry.LOCSIG);
+
+ ByteIo.writeShort(stream, getRequiredExtractVersion());
+ ByteIo.writeShort(stream, flags);
+ ByteIo.writeShort(stream, getCompressionMethod());
+ ByteIo.writeInt(stream, getTime());
+
+ // In deflate mode, we don't know the size or CRC of the data.
+ if (isDeflated()) {
+ ByteIo.writeInt(stream, 0);
+ ByteIo.writeInt(stream, 0);
+ ByteIo.writeInt(stream, 0);
+ } else {
+ ByteIo.writeInt(stream, entry.getCrc());
+ ByteIo.writeInt(stream, entry.getSize());
+ ByteIo.writeInt(stream, entry.getSize());
+ }
+
+ byte[] nameBytes = entry.getName().getBytes(Charsets.UTF_8);
+ ByteIo.writeShort(stream, nameBytes.length);
+ ByteIo.writeShort(stream, 0);
+ stream.write(nameBytes);
+
+ byte[] bytes = stream.toByteArray();
+ out.write(bytes);
+ return bytes.length;
+ }
+ }
+
+ private byte[] close() throws IOException {
+ if (!isDeflated()) {
+ return new byte[0];
+ }
+
+ try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+ ByteIo.writeInt(out, ZipEntry.EXTSIG);
+ ByteIo.writeInt(out, getCrc());
+ ByteIo.writeInt(out, getCompressedSize());
+ ByteIo.writeInt(out, getSize());
+
+ return out.toByteArray();
+ }
+ }
+
+ private int deflate(OutputStream out) throws IOException {
+ int written = deflater.deflate(buffer, 0, buffer.length);
+ if (written > 0) {
+ out.write(Arrays.copyOf(buffer, written));
+ }
+ return written;
+ }
+
+ public long write(OutputStream out, byte[] b, int off, int len) throws IOException {
+ updateCrc(b, off, len);
+
+ if (!isDeflated()) {
+ out.write(b, off, len);
+ return len;
+ }
+
+ if (len == 0) {
+ return 0;
+ }
+
+ Preconditions.checkState(!deflater.finished());
+ deflater.setInput(b, off, len);
+
+ while (!deflater.needsInput()) {
+ deflate(out);
+ }
+ return 0; // We calculate how many bytes we write when closing deflated entries.
+ }
+
+ public long close(OutputStream out) throws IOException {
+ if (!isDeflated()) {
+ // Nothing left to do.
+ return 0;
+ }
+
+ deflater.finish();
+ while (!deflater.finished()) {
+ deflate(out);
+ }
+ entry.setSize(deflater.getBytesRead());
+ entry.setCompressedSize(deflater.getBytesWritten());
+ calculateCrc();
+
+ deflater.end();
+
+ byte[] closeBytes = close();
+ out.write(closeBytes);
+
+ return entry.getCompressedSize() + closeBytes.length;
+ }
+
+
+ private static enum Method {
+ DEFLATE(ZipEntry.DEFLATED, 20, 8),
+ STORE(ZipEntry.STORED, 10, 0),
+ ;
+
+ private final int zipEntryMethod;
+ private final int requiredVersion;
+ private final int compressionMethod;
+
+ private Method(int zipEntryMethod, int requiredVersion, int compressionMethod) {
+ this.zipEntryMethod = zipEntryMethod;
+ this.requiredVersion = requiredVersion;
+ this.compressionMethod = compressionMethod;
+ }
+
+ public int getRequiredExtractVersion() {
+ return requiredVersion;
+ }
+
+ public static Method detect(int fromZipMethod) {
+ if (fromZipMethod == -1) {
+ return DEFLATE;
+ }
+
+ for (Method value : values()) {
+ if (value.zipEntryMethod == fromZipMethod) {
+ return value;
+ }
+ }
+
+ throw new IllegalArgumentException("Cannot determine zip method from: " + fromZipMethod);
+ }
+ }
+}
diff --git a/src/com/facebook/buck/zip/OverwritingZipOutputStream.java b/src/com/facebook/buck/zip/OverwritingZipOutputStream.java
new file mode 100644
index 0000000..ab8da9b
--- /dev/null
+++ b/src/com/facebook/buck/zip/OverwritingZipOutputStream.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright 2013-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.zip;
+
+import com.facebook.buck.timing.Clock;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import com.google.common.hash.Hashing;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipException;
+
+/**
+ * An implementation of an {@link OutputStream} for zip files that allows newer entries to overwrite
+ * or refresh previously written entries.
+ * <p>
+ * This class works by spooling the bytes of each entry to a temporary holding file named after the
+ * name of the {@link ZipEntry} being stored. Once the stream is closed, these files are spooled
+ * off disk and written to the OutputStream given to the constructor.
+ */
+public class OverwritingZipOutputStream extends CustomZipOutputStream {
+ // Attempt to maintain ordering of files that are added.
+ private final Map<File, EntryAccounting> entries = Maps.newLinkedHashMap();
+ private final File scratchDir;
+ private final Clock clock;
+ private EntryAccounting currentEntry;
+ /** Place-holder for bytes. */
+ private OutputStream currentOutput;
+
+ public OverwritingZipOutputStream(Clock clock, OutputStream out) {
+ super(out);
+ this.clock = Preconditions.checkNotNull(clock);
+
+ try {
+ scratchDir = Files.createTempDirectory("overwritingzip").toFile();
+ // Reading the source, it seems like the temp dir isn't scheduled for deletion. We will delete
+ // the directory when we close the stream, but if that method is never called, we'd leave
+ // cruft on the FS. It's not foolproof, but try and avoid that.
+ scratchDir.deleteOnExit();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ protected void actuallyPutNextEntry(ZipEntry entry) throws IOException {
+ // We calculate the actual offset when closing the stream, so 0 is fine.
+ currentEntry = new EntryAccounting(clock, entry, /* currentOffset */ 0);
+
+ long md5 = Hashing.md5().hashUnencodedChars(entry.getName()).asLong();
+ String name = String.valueOf(md5);
+
+ File file = new File(scratchDir, name);
+ entries.put(file, currentEntry);
+ if (file.exists() && !file.delete()) {
+ throw new ZipException("Unable to delete existing file: " + entry.getName());
+ }
+ currentOutput = new BufferedOutputStream(new FileOutputStream(file));
+ }
+
+ @Override
+ protected void actuallyCloseEntry() throws IOException {
+ // We'll close the entry once we have the ultimate output stream and know the entry's location
+ // within the generated zip.
+ currentOutput.close();
+ currentOutput = null;
+ currentEntry = null;
+ }
+
+ @Override
+ protected void actuallyWrite(byte[] b, int off, int len) throws IOException {
+ currentEntry.write(currentOutput, b, off, len);
+ }
+
+ @Override
+ protected void actuallyClose() throws IOException {
+ long currentOffset = 0;
+
+ for (Map.Entry<File, EntryAccounting> mapEntry : entries.entrySet()) {
+ EntryAccounting entry = mapEntry.getValue();
+ entry.setOffset(currentOffset);
+ currentOffset += entry.writeLocalFileHeader(delegate);
+ currentOffset += Files.copy(mapEntry.getKey().toPath(), delegate);
+ currentOffset += entry.close(delegate);
+ }
+
+ new CentralDirectory().writeCentralDirectory(delegate, currentOffset, entries.values());
+
+ delegate.close();
+
+ // Ideally we'd just do this, but that introduces some nasty circular references. *sigh* Instead
+ // we'll do this the tedious way by hand.
+ // MoreFiles.deleteRecursively(scratchDir.toPath());
+
+ SimpleFileVisitor<Path> visitor = new SimpleFileVisitor<Path>() {
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+ Files.delete(file);
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+ if (exc == null) {
+ Files.delete(dir);
+ return FileVisitResult.CONTINUE;
+ }
+ throw exc;
+ }
+ };
+ Files.walkFileTree(scratchDir.toPath(), visitor);
+ }
+}
diff --git a/src/com/facebook/buck/zip/RepackZipEntriesStep.java b/src/com/facebook/buck/zip/RepackZipEntriesStep.java
index 0ba5e7e..eb9850d 100644
--- a/src/com/facebook/buck/zip/RepackZipEntriesStep.java
+++ b/src/com/facebook/buck/zip/RepackZipEntriesStep.java
@@ -16,17 +16,22 @@
package com.facebook.buck.zip;
-import com.facebook.buck.step.CompositeStep;
+import com.facebook.buck.event.LogEvent;
+import com.facebook.buck.step.ExecutionContext;
import com.facebook.buck.step.Step;
-import com.facebook.buck.step.fs.CopyStep;
-import com.google.common.collect.ImmutableList;
+import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableSet;
-import com.google.common.io.Files;
+import com.google.common.io.ByteStreams;
+import java.io.BufferedInputStream;
import java.io.File;
-import java.util.List;
-
-import javax.annotation.Nullable;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.logging.Level;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
/**
* A command that creates a copy of a ZIP archive, making sure that certain user-specified entries
@@ -34,60 +39,12 @@
*
* Can be used, for instance, to force the resources.arsc file in an Android .apk to be compressed.
*/
-public class RepackZipEntriesStep extends CompositeStep {
+public class RepackZipEntriesStep implements Step {
- private static List<Step> createSubCommands(
- String inputFile,
- String outputFile,
- ImmutableSet<String> entries,
- int compressionLevel,
- @Nullable File workingDirectory) {
-
- if (workingDirectory == null) {
- workingDirectory = Files.createTempDir();
- workingDirectory.deleteOnExit();
- }
-
- // Extract the entries we want to repack.
- UnzipStep unzip = new UnzipStep(
- inputFile,
- workingDirectory.getAbsolutePath(),
- true,
- entries);
-
- // Initialize destination archive with copy of source archive.
- CopyStep cp = new CopyStep(inputFile, outputFile);
-
- // Finally, update the entries in the destination archive, using compressionLevel.
- ZipStep zip = new ZipStep(
- ZipStep.Mode.ADD,
- new File(outputFile).getAbsolutePath(),
- entries,
- false /* junkPaths */,
- compressionLevel,
- workingDirectory.getAbsoluteFile());
-
- return ImmutableList.of(unzip, cp, zip);
- }
-
- /**
- * Creates a {@link RepackZipEntriesStep}.
- * @param inputFile input archive
- * @param outputFile destination archive
- * @param entries files to repack (e.g. {@code ImmutableSet.of("resources.arsc")})
- * @param compressionLevel 0 to 9
- * @param workingDirectory where to extract entries before repacking. A temporary directory will
- * be created if this is {@code null}.
- */
- public RepackZipEntriesStep(
- String inputFile,
- String outputFile,
- ImmutableSet<String> entries,
- int compressionLevel,
- @Nullable File workingDirectory) {
- super(createSubCommands(inputFile, outputFile, entries, compressionLevel, workingDirectory));
- }
-
+ private final String inputFile;
+ private final String outputFile;
+ private final ImmutableSet<String> entries;
+ private final int compressionLevel;
/**
* Creates a {@link RepackZipEntriesStep}. A temporary directory will be created and used
@@ -100,7 +57,65 @@
String inputFile,
String outputFile,
ImmutableSet<String> entries) {
- this(inputFile, outputFile, entries, ZipStep.MAX_COMPRESSION_LEVEL, null);
+ this(inputFile, outputFile, entries, ZipStep.MAX_COMPRESSION_LEVEL);
}
+ /**
+ * Creates a {@link RepackZipEntriesStep}.
+ * @param inputFile input archive
+ * @param outputFile destination archive
+ * @param entries files to repack (e.g. {@code ImmutableSet.of("resources.arsc")})
+ * @param compressionLevel 0 to 9
+ */
+ public RepackZipEntriesStep(
+ String inputFile,
+ String outputFile,
+ ImmutableSet<String> entries,
+ int compressionLevel) {
+ this.inputFile = inputFile;
+ this.outputFile = outputFile;
+ this.entries = entries;
+ this.compressionLevel = compressionLevel;
+ }
+
+ @Override
+ public int execute(ExecutionContext context) {
+ try (
+ ZipInputStream in = new ZipInputStream(new BufferedInputStream(new FileInputStream(inputFile)));
+ CustomZipOutputStream out = ZipOutputStreams.newOutputStream(new File(outputFile));
+ ) {
+ for (ZipEntry entry = in.getNextEntry(); entry != null; entry = in.getNextEntry()) {
+ CustomZipEntry customEntry = new CustomZipEntry(entry);
+ if (entries.contains(customEntry.getName())) {
+ customEntry.setCompressionLevel(compressionLevel);
+ }
+ out.putNextEntry(customEntry);
+ ByteStreams.copy(in, out);
+ out.closeEntry();
+ }
+
+ return 0;
+ } catch (IOException e) {
+ context.getBuckEventBus().post(LogEvent.create(
+ Level.SEVERE, "Unable to repack zip: %s", Throwables.getStackTraceAsString(e)));
+ return 1;
+ }
+ }
+
+ @Override
+ public String getShortName() {
+ return "repack zip";
+ }
+
+ @Override
+ public String getDescription(ExecutionContext context) {
+ // We don't actually want to create a temp directory. Since this is for cut-and-paste joy only
+ // currentTimeMillis is sufficiently random.
+ Path temp = Paths.get(System.getProperty("java.io.tmpdir"), "repack" + System.currentTimeMillis());
+
+ return new StringBuilder("cd ").append(temp.toString()).append(" && ")
+ .append("unzip ").append(inputFile).append(" && ")
+ .append("zip -r -").append(compressionLevel).append(" ").append(outputFile).append(" *")
+ .toString();
+ }
}
diff --git a/src/com/facebook/buck/zip/ZipDirectoryWithMaxDeflateStep.java b/src/com/facebook/buck/zip/ZipDirectoryWithMaxDeflateStep.java
index 4d5a6ab..1aa9ffa 100644
--- a/src/com/facebook/buck/zip/ZipDirectoryWithMaxDeflateStep.java
+++ b/src/com/facebook/buck/zip/ZipDirectoryWithMaxDeflateStep.java
@@ -34,7 +34,6 @@
import java.util.Map;
import java.util.Set;
import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
/**
* Command to zip up a directory while respecting a file size limit to be deflated.
@@ -88,8 +87,8 @@
ImmutableMap<File, ZipEntry> zipEntries = zipEntriesBuilder.build();
if (!zipEntries.isEmpty()) {
- ZipOutputStream outputStream = closer.register(
- new ZipOutputStream(new BufferedOutputStream(
+ CustomZipOutputStream outputStream = closer.register(
+ ZipOutputStreams.newOutputStream(new BufferedOutputStream(
new FileOutputStream(outputZipPath))));
for (Map.Entry<File, ZipEntry> zipEntry : zipEntries.entrySet()) {
diff --git a/src/com/facebook/buck/zip/ZipOutputStreams.java b/src/com/facebook/buck/zip/ZipOutputStreams.java
new file mode 100644
index 0000000..20f1863
--- /dev/null
+++ b/src/com/facebook/buck/zip/ZipOutputStreams.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2013-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.zip;
+
+import com.facebook.buck.timing.Clock;
+import com.facebook.buck.timing.DefaultClock;
+import com.facebook.buck.util.HumanReadableException;
+import com.google.common.base.Preconditions;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.OutputStream;
+
+public class ZipOutputStreams {
+
+ private ZipOutputStreams() {
+ // factory class
+ }
+
+ /**
+ * Create a new {@link CustomZipOutputStream} that outputs to the given {@code zipFile}. Note that
+ * the parent directory of the {@code zipFile} must exist already. The returned stream will throw
+ * an exception should duplicate entries be added.
+ *
+ * @param zipFile The file to write to.
+ */
+ public static CustomZipOutputStream newOutputStream(File zipFile) {
+ Preconditions.checkNotNull(zipFile);
+ try {
+ return newOutputStream(new BufferedOutputStream(new FileOutputStream(zipFile)));
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Create a new {@link CustomZipOutputStream} that will by default act in the same way as
+ * {@link java.util.zip.ZipOutputStream}, notably by throwing an exception if duplicate entries
+ * are added.
+ *
+ * @param out The output stream to write to.
+ */
+ public static CustomZipOutputStream newOutputStream(OutputStream out) {
+ return newOutputStream(out, HandleDuplicates.THROW_EXCEPTION);
+ }
+
+ /**
+ * Create a new {@link CustomZipOutputStream} that handles duplicate entries in the way dictated
+ * by {@code mode}.
+ *
+ * @param zipFile The file to write to.
+ * @param mode How to handle duplicate entries.
+ */
+ public static CustomZipOutputStream newOutputStream(File zipFile, HandleDuplicates mode)
+ throws FileNotFoundException {
+ Preconditions.checkNotNull(zipFile);
+
+ return newOutputStream(new BufferedOutputStream(new FileOutputStream(zipFile)), mode);
+ }
+
+ /**
+ * Create a new {@link CustomZipOutputStream} that handles duplicate entries in the way dictated
+ * by {@code mode}.
+ *
+ * @param out The output stream to write to.
+ * @param mode How to handle duplicate entries.
+ */
+ public static CustomZipOutputStream newOutputStream(OutputStream out, HandleDuplicates mode) {
+ Clock clock = new DefaultClock();
+
+ switch (mode) {
+ case APPEND_TO_ZIP:
+ case THROW_EXCEPTION:
+ return new AppendingZipOutputStream(clock,
+ out,
+ mode == HandleDuplicates.THROW_EXCEPTION);
+
+ case OVERWRITE_EXISTING:
+ return new OverwritingZipOutputStream(clock, out);
+
+ default:
+ throw new HumanReadableException(
+ "Unable to determine which zip output mode to use: %s", mode);
+ }
+ }
+
+ public static enum HandleDuplicates {
+ /** Duplicate entries are simply appended to the zip. */
+ APPEND_TO_ZIP,
+ /** An exception should be thrown if a duplicate entry is added to a zip. */
+ THROW_EXCEPTION,
+ /** A duplicate entry overwrites an existing entry with the same name. */
+ OVERWRITE_EXISTING;
+ }
+}
diff --git a/src/com/facebook/buck/zip/ZipStep.java b/src/com/facebook/buck/zip/ZipStep.java
index bdfc9bf..bdd3591 100644
--- a/src/com/facebook/buck/zip/ZipStep.java
+++ b/src/com/facebook/buck/zip/ZipStep.java
@@ -16,64 +16,37 @@
package com.facebook.buck.zip;
-import com.facebook.buck.shell.ShellStep;
+import static com.facebook.buck.zip.ZipOutputStreams.HandleDuplicates.OVERWRITE_EXISTING;
+import static java.util.logging.Level.SEVERE;
+
+import com.facebook.buck.event.LogEvent;
import com.facebook.buck.step.ExecutionContext;
-import com.facebook.buck.util.Verbosity;
+import com.facebook.buck.step.Step;
+import com.facebook.buck.util.DirectoryTraversal;
import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
+import java.io.BufferedOutputStream;
import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
import java.util.Set;
/**
- * A {@link com.facebook.buck.step.Step} that creates or updates a ZIP archive using {@code zip}.
- *
- * @see <a href="http://www.info-zip.org/mans/zip.html">ZIP</a>
+ * A {@link com.facebook.buck.step.Step} that creates a ZIP archive..
*/
-public class ZipStep extends ShellStep {
+public class ZipStep implements Step {
public static final int MIN_COMPRESSION_LEVEL = 0;
public static final int DEFAULT_COMPRESSION_LEVEL = 6;
public static final int MAX_COMPRESSION_LEVEL = 9;
- /**
- * These map to {@code zip} 'command modes' (as seen in man page).
- * @see <a href="http://www.info-zip.org/mans/zip.html">ZIP</a>
- */
- public static enum Mode {
- /**
- * Update existing entries and add new files.
- * If the archive does not exist create it. This is the default mode.
- */
- ADD(""),
- /**
- * Update existing entries if newer on the file system and add new files.
- * If the archive does not exist issue warning then create a new archive.
- */
- UPDATE("-u"),
- /**
- * Update existing entries of an archive if newer on the file system.
- * Does not add new files to the archive.
- */
- FRESHEN("-f"),
- /**
- * Select entries in an existing archive and delete them.
- */
- DELETE("-d");
-
- public final String arg;
-
- private Mode(String arg) {
- this.arg = arg;
- }
- }
-
- private final Mode mode;
private final String absolutePathToZipFile;
private final ImmutableSet<String> paths;
private final boolean junkPaths;
private final int compressionLevel;
+ private final File baseDir;
/**
@@ -84,103 +57,114 @@
* an archive containing just the file. If you were in {@code /} and added
* {@code dir/file.txt}, you would get an archive containing the file within a directory.
*
- * @param mode one of {@link ZipStep.Mode#ADD}, {@link ZipStep.Mode#UPDATE UPDATE},
- * {@link ZipStep.Mode#FRESHEN FRESHEN} or {@link ZipStep.Mode#DELETE DELETE}, as in the
- * {@code zip} command.
- * @param absolutePathToZipFile path to archive to create or update
+ *
+ * @param absolutePathToZipFile path to archive to create.
* @param paths a set of files and/or directories to work on. The entire working directory is
* assumed if this set is empty.
* @param junkPaths if {@code true}, the relative paths of added archive entries are discarded,
* i.e. they are all placed in the root of the archive.
* @param compressionLevel between 0 (store) and 9.
- * @param workingDirectory working directory for {@code zip} command.
+ * @param baseDir working directory for {@code zip} command.
* If {@code null}, project directory root is used instead.
- *
- * @see <a href="http://www.info-zip.org/mans/zip.html">ZIP</a>
*/
public ZipStep(
- Mode mode,
String absolutePathToZipFile,
Set<String> paths,
boolean junkPaths,
int compressionLevel,
- File workingDirectory) {
- super(workingDirectory);
+ File baseDir) {
Preconditions.checkArgument(compressionLevel >= MIN_COMPRESSION_LEVEL &&
compressionLevel <= MAX_COMPRESSION_LEVEL, "compressionLevel out of bounds.");
- this.mode = mode;
this.absolutePathToZipFile = Preconditions.checkNotNull(absolutePathToZipFile);
this.paths = ImmutableSet.copyOf(Preconditions.checkNotNull(paths));
this.junkPaths = junkPaths;
this.compressionLevel = compressionLevel;
+ this.baseDir = Preconditions.checkNotNull(baseDir);
}
- /**
- * Create a {@link ZipStep} that adds an entire directory to an archive. The files directly in
- * the directory will appear in the root of the archive. Default compression level is assumed.
- *
- * @param zipFile file to archive to create or update
- * @param directoryToAdd directory to add to the archive
- *
- * @see #ZipStep(Mode, String, Set, boolean, int, File)
- */
- public ZipStep(File zipFile, File directoryToAdd) {
- this(
- Mode.ADD,
- zipFile.getAbsolutePath(),
- ImmutableSet.<String>of() /* pathsToAdd */,
- false /* junkPaths */,
- DEFAULT_COMPRESSION_LEVEL /* compressionLevel */,
- Preconditions.checkNotNull(directoryToAdd));
+
+ @Override
+ public int execute(ExecutionContext context) {
+ File original = new File(absolutePathToZipFile);
+ if (original.exists()) {
+ context.getBuckEventBus().post(
+ LogEvent.create(SEVERE, "Attempting to overwrite an existing zip: %s", original)
+ );
+ return 1;
+ }
+
+ try (
+ BufferedOutputStream baseOut = new BufferedOutputStream(new FileOutputStream(absolutePathToZipFile));
+ CustomZipOutputStream out = ZipOutputStreams.newOutputStream(baseOut, OVERWRITE_EXISTING);
+ ) {
+ DirectoryTraversal traversal = new DirectoryTraversal(baseDir) {
+
+ @Override
+ public void visit(File file, String relativePath) throws IOException {
+ if (!paths.isEmpty() && !paths.contains(relativePath)) {
+ return;
+ }
+
+ String name = junkPaths ? file.getName() : relativePath;
+ if (file.isDirectory()) {
+ // Lame.
+ name += "/";
+ }
+
+ CustomZipEntry entry = new CustomZipEntry(name);
+ entry.setTime(file.lastModified());
+ entry.setSize(file.length());
+ entry.setCompressionLevel(compressionLevel);
+
+ out.putNextEntry(entry);
+ Files.copy(file.toPath(), out);
+ out.closeEntry();
+ }
+ };
+
+ traversal.traverse();
+
+ return 0;
+ } catch (IOException e) {
+ return 1;
+ }
+
}
@Override
- protected ImmutableList<String> getShellCommandInternal(ExecutionContext context) {
- ImmutableList.Builder<String> args = ImmutableList.builder();
- args.add("zip");
-
- // What to do?
- if (mode != Mode.ADD) {
- args.add(mode.arg);
- }
-
- Verbosity verbosity = context.getVerbosity();
- if (!verbosity.shouldUseVerbosityFlagIfAvailable()) {
- if (verbosity.shouldPrintStandardInformation()) {
- args.add("-q");
- } else {
- args.add("-qq");
- }
- }
+ public String getDescription(ExecutionContext context) {
+ StringBuilder args = new StringBuilder("zip ");
// Don't add extra fields, neither do the Android tools.
- args.add("-X");
+ args.append("-X ");
// recurse
- args.add("-r");
+ args.append("-r ");
// compression level
- args.add("-" + compressionLevel);
+ args.append("-").append(compressionLevel).append(" ");
// unk paths
if (junkPaths) {
- args.add("-j");
+ args.append("-j ");
}
// destination archive
- args.add(absolutePathToZipFile);
+ args.append(absolutePathToZipFile).append(" ");
// files to add to archive
if (paths.isEmpty()) {
// Add the contents of workingDirectory to archive.
- args.add("-i*");
- args.add(".");
+ args.append("-i* ");
+ args.append(". ");
} else {
// Add specified paths, relative to workingDirectory.
- args.addAll(paths);
+ for (String path : paths) {
+ args.append(path).append(" ");
+ }
}
- return args.build();
+ return args.toString();
}
@Override
diff --git a/test/com/facebook/buck/android/AndroidTransitiveDependencyGraphTest.java b/test/com/facebook/buck/android/AndroidTransitiveDependencyGraphTest.java
index c7a2947..28461f2 100644
--- a/test/com/facebook/buck/android/AndroidTransitiveDependencyGraphTest.java
+++ b/test/com/facebook/buck/android/AndroidTransitiveDependencyGraphTest.java
@@ -115,7 +115,7 @@
assertEquals(
"Because guava was passed to no_dx, it should not be treated as a third-party JAR whose " +
"resources need to be extracted and repacked in the APK. If this is done, then code in " +
- "the guava-10.0.1.dex.1.jar in the APK's assets/ folder may try to load the resource " +
+ "the guava-10.0.1.dex.1.jar in the APK's assets/ tmp may try to load the resource " +
"from the APK as a ZipFileEntry rather than as a resource within guava-10.0.1.dex.1.jar. " +
"Loading a resource in this way could take substantially longer. Specifically, this was " +
"observed to take over one second longer to load the resource in fb4a. Because the " +
diff --git a/test/com/facebook/buck/cli/TestCommandTest.java b/test/com/facebook/buck/cli/TestCommandTest.java
index 7cf38bb..14f6bac 100644
--- a/test/com/facebook/buck/cli/TestCommandTest.java
+++ b/test/com/facebook/buck/cli/TestCommandTest.java
@@ -84,7 +84,7 @@
}
/**
- * If the source paths specified are all generated files, then our path to source folder
+ * If the source paths specified are all generated files, then our path to source tmp
* should be absent.
*/
@Test
@@ -117,7 +117,7 @@
/**
* If the source paths specified are all for non-generated files then we should return
- * the correct source folder corresponding to a non-generated source path.
+ * the correct source tmp corresponding to a non-generated source path.
*/
@Test
public void testNonGeneratedSourceFile() {
@@ -157,15 +157,15 @@
ImmutableSet<String> result = TestCommand.getPathToSourceFolders(
javaLibraryRule, Optional.of(defaultJavaPackageFinder), projectFilesystem);
- assertEquals("All non-generated source files are under one source folder.",
+ assertEquals("All non-generated source files are under one source tmp.",
ImmutableSet.of("package/src/"), result);
verify(mocks);
}
/**
- * If the source paths specified are from the new unified source folder then we should return
- * the correct source folder corresponding to the unified source path.
+ * If the source paths specified are from the new unified source tmp then we should return
+ * the correct source tmp corresponding to the unified source path.
*/
@Test
public void testUnifiedSourceFile() {
@@ -190,7 +190,7 @@
ImmutableSet<String> result = TestCommand.getPathToSourceFolders(
javaLibraryRule, Optional.of(defaultJavaPackageFinder), projectFilesystem);
- assertEquals("All non-generated source files are under one source folder.",
+ assertEquals("All non-generated source files are under one source tmp.",
ImmutableSet.of("java/"), result);
verify(mocks);
@@ -198,7 +198,7 @@
/**
* If the source paths specified contains one source path to a non-generated file then
- * we should return the correct source folder corresponding to that non-generated source path.
+ * we should return the correct source tmp corresponding to that non-generated source path.
* Especially when the generated file comes first in the ordered set.
*/
@Test
diff --git a/test/com/facebook/buck/command/ProjectTest.java b/test/com/facebook/buck/command/ProjectTest.java
index caf4504..d37cca2 100644
--- a/test/com/facebook/buck/command/ProjectTest.java
+++ b/test/com/facebook/buck/command/ProjectTest.java
@@ -623,7 +623,7 @@
assertEquals(1, projectWithModules1.modules.size());
Module moduleNoJavaSource = projectWithModules1.modules.get(0);
assertListEquals(
- "Only source folder should be gen/ when setSrcRoots(null) is specified.",
+ "Only source tmp should be gen/ when setSrcRoots(null) is specified.",
ImmutableList.of(SourceFolder.GEN),
moduleNoJavaSource.sourceFolders);
@@ -650,7 +650,7 @@
assertEquals(1, projectWithModules2.modules.size());
Module moduleWithPackagePrefix = projectWithModules2.modules.get(0);
assertListEquals(
- "The current directory should be a source folder with a package prefix " +
+ "The current directory should be a source tmp with a package prefix " +
"as well as the gen/ directory.",
ImmutableList.of(
new SourceFolder("file://$MODULE_DIR$", false /* isTestSource */, "com.example.base"),
diff --git a/test/com/facebook/buck/testutil/Zip.java b/test/com/facebook/buck/testutil/Zip.java
index f8d3a8e..3acef16 100644
--- a/test/com/facebook/buck/testutil/Zip.java
+++ b/test/com/facebook/buck/testutil/Zip.java
@@ -104,6 +104,12 @@
return contents.build();
}
+ public byte[] readFully(String fileName) throws IOException {
+ Path resolved = root.resolve(fileName);
+
+ return Files.readAllBytes(resolved);
+ }
+
@Override
public void close() throws IOException {
fs.close();
diff --git a/test/com/facebook/buck/testutil/integration/DebuggableTemporaryFolder.java b/test/com/facebook/buck/testutil/integration/DebuggableTemporaryFolder.java
index 28b54a2..ab4cd2a 100644
--- a/test/com/facebook/buck/testutil/integration/DebuggableTemporaryFolder.java
+++ b/test/com/facebook/buck/testutil/integration/DebuggableTemporaryFolder.java
@@ -21,7 +21,7 @@
import org.junit.rules.TemporaryFolder;
/**
- * Subclass of {@link TemporaryFolder} that optionally keeps the contents of the folder around after
+ * Subclass of {@link TemporaryFolder} that optionally keeps the contents of the tmp around after
* the test has finished. This is often useful when debugging a failed integration test.
* <p>
* Here is an example of how to create a {@link DebuggableTemporaryFolder} that will not delete
diff --git a/test/com/facebook/buck/zip/BUCK b/test/com/facebook/buck/zip/BUCK
index b6db6fd..cc38637 100644
--- a/test/com/facebook/buck/zip/BUCK
+++ b/test/com/facebook/buck/zip/BUCK
@@ -1,16 +1,28 @@
java_test(
name = 'zip',
srcs = glob(['*.java']),
+ resources = [
+ # The sample bytes are a class file. We use the ".properties" extension so that IJ will copy
+ # the file to the output dir when compiling, allowing us to test in the IDE.
+ 'sample-bytes.properties',
+ ],
deps = [
'//lib:easymock',
'//lib:guava',
+ '//lib:hamcrest-core',
+ '//lib:hamcrest-library',
'//lib:junit',
- '//src/com/facebook/buck/shell:steps',
'//src/com/facebook/buck/step:step',
'//src/com/facebook/buck/step/fs:fs',
'//src/com/facebook/buck/util:io',
+ '//src/com/facebook/buck/util/environment:environment',
+ '//src/com/facebook/buck/zip:stream',
'//src/com/facebook/buck/zip:steps',
'//test/com/facebook/buck/step:testutil',
'//test/com/facebook/buck/testutil:testutil',
],
-)
\ No newline at end of file
+ source_under_test = [
+ '//src/com/facebook/buck/zip:steps',
+ '//src/com/facebook/buck/zip:stream',
+ ],
+)
diff --git a/test/com/facebook/buck/zip/RepackZipEntriesStepTest.java b/test/com/facebook/buck/zip/RepackZipEntriesStepTest.java
index 0ac37ae..262c39e 100644
--- a/test/com/facebook/buck/zip/RepackZipEntriesStepTest.java
+++ b/test/com/facebook/buck/zip/RepackZipEntriesStepTest.java
@@ -16,86 +16,87 @@
package com.facebook.buck.zip;
-import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.verify;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertTrue;
-import com.facebook.buck.step.ExecutionContext;
-import com.facebook.buck.step.Step;
-import com.facebook.buck.step.fs.CopyStep;
-import com.facebook.buck.util.Verbosity;
-import com.google.common.base.Joiner;
-import com.google.common.collect.ImmutableList;
+import com.facebook.buck.step.TestExecutionContext;
import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterators;
+import com.google.common.io.Resources;
+import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import java.io.File;
-import java.util.Iterator;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
public class RepackZipEntriesStepTest {
+ @Rule public TemporaryFolder tmp = new TemporaryFolder();
+ private File parent;
+ private File zipFile;
+
+ @Before
+ public void buildSampleZipFile() throws IOException {
+ parent = tmp.newFolder("foo");
+ zipFile = new File(parent, "example.zip");
+
+ // Turns out that the zip filesystem generates slightly different output from the output stream.
+ // Since we've modeled our outputstreams after the zip output stream, be compatible with that.
+ try (ZipOutputStream stream = new ZipOutputStream(new FileOutputStream(zipFile))) {
+ ZipEntry entry = new ZipEntry("file");
+ stream.putNextEntry(entry);
+ String packageName = getClass().getPackage().getName().replace(".", "/");
+ URL sample = Resources.getResource(packageName + "/sample-bytes.properties");
+ stream.write(Resources.toByteArray(sample));
+ }
+ }
+
@Test
- public void testProvidesAppropriateSubCommands() {
- final String inApk = "source.apk";
- final String outApk = "dest.apk";
- final int compressionLevel = 8;
- final ImmutableSet<String> entries = ImmutableSet.of("resources.arsc");
- File dir = new File("/tmp/mydir");
+ public void shouldLeaveZipAloneIfEntriesToCompressIsEmpty() throws IOException {
+ File out = new File(parent, "output.zip");
+ RepackZipEntriesStep step = new RepackZipEntriesStep(
+ zipFile.getAbsolutePath(),
+ out.getAbsolutePath(),
+ ImmutableSet.<String>of());
+ step.execute(TestExecutionContext.newInstance());
- ExecutionContext context = createMock(ExecutionContext.class);
- expect(context.getVerbosity()).andReturn(Verbosity.ALL).times(2);
- replay(context);
+ byte[] expected = Files.readAllBytes(zipFile.toPath());
+ byte[] actual = Files.readAllBytes(out.toPath());
+ assertArrayEquals(expected, actual);
+ }
- String unzipExpected = Joiner.on(" ").join(new ImmutableList.Builder<String>()
- .add("unzip")
- .add("-o")
- .add("-d").add(dir.getAbsolutePath())
- .add(inApk)
- .addAll(entries)
- .build());
+ @Test
+ public void repackWithHigherCompressionResultsInFewerBytes() throws IOException {
+ File out = new File(parent, "output.zip");
+ RepackZipEntriesStep step = new RepackZipEntriesStep(
+ zipFile.getAbsolutePath(),
+ out.getAbsolutePath(),
+ ImmutableSet.of("file"));
+ step.execute(TestExecutionContext.newInstance());
- ImmutableList<String> zipExpected = new ImmutableList.Builder<String>()
- .add("zip")
- .add("-X")
- .add("-r")
- .add("-"+compressionLevel)
- .add(new File(outApk).getAbsolutePath())
- .addAll(entries)
- .build();
+ assertTrue(out.length() < zipFile.length());
+ }
- RepackZipEntriesStep command = new RepackZipEntriesStep(
- inApk,
- outApk,
- entries,
- compressionLevel,
- dir);
+ @Test
+ public void justStoringEntriesLeadsToMoreBytesInOuputZip() throws IOException {
+ File out = new File(parent, "output.zip");
+ RepackZipEntriesStep step = new RepackZipEntriesStep(
+ zipFile.getAbsolutePath(),
+ out.getAbsolutePath(),
+ ImmutableSet.of("file"),
+ ZipStep.MIN_COMPRESSION_LEVEL);
+ step.execute(TestExecutionContext.newInstance());
- // Go over the subcommands.
- Iterator<Step> iter = Iterators.filter(command.iterator(), Step.class);
+ byte[] expected = Files.readAllBytes(zipFile.toPath());
+ byte[] actual = Files.readAllBytes(out.toPath());
- // First entries are unzipped.
- UnzipStep unzipStep = (UnzipStep) iter.next();
- assertEquals(unzipExpected, unzipStep.getDescription(context));
-
- // A copy of the archive would be created.
- CopyStep copyStep = (CopyStep) iter.next();
-
- assertEquals(inApk, copyStep.getSource());
- assertEquals(outApk, copyStep.getDestination());
- assertFalse(copyStep.isRecursive());
-
- // And then the entries would be zipped back in.
- ZipStep zipStep = (ZipStep) iter.next();
- assertEquals(zipExpected, zipStep.getShellCommand(context));
-
- //ShellStep zipCommand = iter.next();
- assertEquals(zipStep.getWorkingDirectory().getAbsolutePath(), dir.getAbsolutePath());
-
- verify(context);
+ assertTrue(expected.length < actual.length);
}
}
diff --git a/test/com/facebook/buck/zip/UnzipStepTest.java b/test/com/facebook/buck/zip/UnzipStepTest.java
index d0d39c9..0e00a13 100644
--- a/test/com/facebook/buck/zip/UnzipStepTest.java
+++ b/test/com/facebook/buck/zip/UnzipStepTest.java
@@ -41,7 +41,7 @@
@Before
public void setUp() throws Exception {
- zipFile = new File(tmpFolder.getRoot(), "folder.zip");
+ zipFile = new File(tmpFolder.getRoot(), "tmp.zip");
try (Zip zip = new Zip(zipFile, true)) {
zip.add("1.bin", DUMMY_FILE_CONTENTS);
zip.add("subdir/2.bin", DUMMY_FILE_CONTENTS);
diff --git a/test/com/facebook/buck/zip/ZipOutputStreamTest.java b/test/com/facebook/buck/zip/ZipOutputStreamTest.java
new file mode 100644
index 0000000..06453d1
--- /dev/null
+++ b/test/com/facebook/buck/zip/ZipOutputStreamTest.java
@@ -0,0 +1,372 @@
+/*
+ * Copyright 2013-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.zip;
+
+import static com.facebook.buck.zip.ZipOutputStreams.HandleDuplicates.APPEND_TO_ZIP;
+import static com.facebook.buck.zip.ZipOutputStreams.HandleDuplicates.OVERWRITE_EXISTING;
+import static java.util.Calendar.SEPTEMBER;
+import static java.util.zip.Deflater.BEST_COMPRESSION;
+import static java.util.zip.Deflater.NO_COMPRESSION;
+import static org.hamcrest.Matchers.lessThan;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import com.facebook.buck.testutil.Zip;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.hash.Hashing;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.CharStreams;
+import com.google.common.io.Resources;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.nio.file.Files;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipException;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
+
+public class ZipOutputStreamTest {
+
+ private File output;
+
+ @Before
+ public void createZipFileDestination() throws IOException {
+ output = File.createTempFile("example", ".zip");
+ }
+
+ @Test
+ public void shouldBeAbleToCreateEmptyArchive() throws IOException {
+ CustomZipOutputStream ignored = ZipOutputStreams.newOutputStream(output);
+ ignored.close();
+
+ try (Zip zip = new Zip(output, /* forWriting */ false)) {
+ assertTrue(zip.getFileNames().isEmpty());
+ }
+ }
+
+ @Test
+ public void shouldBeAbleToCreateEmptyArchiveWhenOverwriting() throws IOException {
+ CustomZipOutputStream ignored = ZipOutputStreams.newOutputStream(output, OVERWRITE_EXISTING);
+ ignored.close();
+
+ try (Zip zip = new Zip(output, false)) {
+ assertTrue(zip.getFileNames().isEmpty());
+ }
+ }
+
+
+ @Test(expected = ZipException.class)
+ public void mustThrowAnExceptionIfNoZipEntryIsOpenWhenWritingData() throws IOException {
+ try (
+ CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output);
+ ) {
+ // Note: we have not opened a zip entry.
+ out.write("cheese".getBytes());
+ }
+ }
+
+ @Test(expected = ZipException.class)
+ public void mustThrowAnExceptionIfNoZipEntryIsOpenWhenWritingDataWhenOverwriting()
+ throws IOException {
+ try (
+ CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output, OVERWRITE_EXISTING);
+ ) {
+ // Note: we have not opened a zip entry
+ out.write("cheese".getBytes());
+ }
+ }
+
+
+ @Test
+ public void shouldBeAbleToAddAZeroLengthFile() throws IOException {
+ File reference = File.createTempFile("reference", ".zip");
+
+ try (
+ CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output);
+ ZipOutputStream ref = new ZipOutputStream(new FileOutputStream(reference))
+ ) {
+ ZipEntry entry = new ZipEntry("example.txt");
+ entry.setTime(System.currentTimeMillis());
+ out.putNextEntry(entry);
+ ref.putNextEntry(entry);
+ }
+
+ byte[] seen = Files.readAllBytes(output.toPath());
+ byte[] expected = Files.readAllBytes(reference.toPath());
+
+ assertArrayEquals(expected, seen);
+ }
+
+ @Test
+ public void shouldBeAbleToAddAZeroLengthFileWhenOverwriting() throws IOException {
+ File reference = File.createTempFile("reference", ".zip");
+
+ try (
+ CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output, OVERWRITE_EXISTING);
+ ZipOutputStream ref = new ZipOutputStream(new FileOutputStream(reference))
+ ) {
+ ZipEntry entry = new ZipEntry("example.txt");
+ entry.setTime(System.currentTimeMillis());
+ out.putNextEntry(entry);
+ ref.putNextEntry(entry);
+ }
+
+ byte[] seen = Files.readAllBytes(output.toPath());
+ byte[] expected = Files.readAllBytes(reference.toPath());
+
+ assertArrayEquals(expected, seen);
+ }
+
+
+ @Test
+ public void shouldBeAbleToAddTwoZeroLengthFiles() throws IOException {
+ File reference = File.createTempFile("reference", ".zip");
+
+ try (
+ CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output);
+ ZipOutputStream ref = new ZipOutputStream(new FileOutputStream(reference))
+ ) {
+ ZipEntry entry = new ZipEntry("example.txt");
+ entry.setTime(System.currentTimeMillis());
+ out.putNextEntry(entry);
+ ref.putNextEntry(entry);
+
+ ZipEntry entry2 = new ZipEntry("example2.txt");
+ entry2.setTime(System.currentTimeMillis());
+ out.putNextEntry(entry2);
+ ref.putNextEntry(entry2);
+ }
+
+ byte[] seen = Files.readAllBytes(output.toPath());
+ byte[] expected = Files.readAllBytes(reference.toPath());
+
+ assertArrayEquals(expected, seen);
+ }
+
+ @Test
+ public void shouldBeAbleToAddASingleNonZeroLengthFile() throws IOException {
+ File reference = File.createTempFile("reference", ".zip");
+
+ try (
+ CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output);
+ ZipOutputStream ref = new ZipOutputStream(new FileOutputStream(reference))
+ ) {
+ byte[] bytes = "cheese".getBytes();
+ ZipEntry entry = new ZipEntry("example.txt");
+ entry.setTime(System.currentTimeMillis());
+ out.putNextEntry(entry);
+ ref.putNextEntry(entry);
+ out.write(bytes);
+ ref.write(bytes);
+ }
+
+ byte[] seen = Files.readAllBytes(output.toPath());
+ byte[] expected = Files.readAllBytes(reference.toPath());
+
+ assertArrayEquals(expected, seen);
+ }
+
+ @Test(expected = ZipException.class)
+ public void writingTheSameFileMoreThanOnceIsNormallyAnError() throws IOException {
+ try (
+ CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output)
+ ) {
+ ZipEntry entry = new ZipEntry("example.txt");
+ out.putNextEntry(entry);
+ out.putNextEntry(entry);
+ }
+ }
+
+ @Test
+ public void shouldBeAbleToSimplyStoreInputFilesWithoutCompressing() throws IOException {
+ File reference = File.createTempFile("reference", ".zip");
+
+ try (
+ CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output);
+ ZipOutputStream ref = new ZipOutputStream(new FileOutputStream(reference))
+ ) {
+ byte[] bytes = "cheese".getBytes();
+ ZipEntry entry = new ZipEntry("example.txt");
+ entry.setMethod(ZipEntry.STORED);
+ entry.setTime(System.currentTimeMillis());
+ entry.setSize(bytes.length);
+ entry.setCrc(Hashing.crc32().hashBytes(bytes).padToLong());
+ out.putNextEntry(entry);
+ ref.putNextEntry(entry);
+ out.write(bytes);
+ ref.write(bytes);
+ }
+
+ byte[] seen = Files.readAllBytes(output.toPath());
+ byte[] expected = Files.readAllBytes(reference.toPath());
+
+ assertArrayEquals(expected, seen);
+ }
+
+ @Test
+ public void writingTheSameFileMoreThanOnceWhenInAppendModeWritesItTwiceToTheZip()
+ throws IOException {
+ try (
+ CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output, APPEND_TO_ZIP)
+ ) {
+ ZipEntry entry = new ZipEntry("example.txt");
+ out.putNextEntry(entry);
+ out.write("cheese".getBytes());
+ out.putNextEntry(entry);
+ out.write("cake".getBytes());
+ }
+
+ List<String> names = Lists.newArrayList();
+ try (ZipInputStream in = new ZipInputStream(new FileInputStream(output))) {
+ for (ZipEntry entry = in.getNextEntry(); entry != null; entry = in.getNextEntry()) {
+ names.add(entry.getName());
+ }
+ }
+
+ assertEquals(ImmutableList.of("example.txt", "example.txt"), names);
+ }
+
+ @Test
+ public void writingTheSameFileMoreThanOnceWhenInOverwriteModeWritesItOnceToTheZip()
+ throws IOException {
+ try (
+ CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output, OVERWRITE_EXISTING)
+ ) {
+ ZipEntry entry = new ZipEntry("example.txt");
+ out.putNextEntry(entry);
+ out.write("cheese".getBytes());
+ out.putNextEntry(entry);
+ out.write("cake".getBytes());
+ }
+
+ List<String> names = Lists.newArrayList();
+ try (ZipInputStream in = new ZipInputStream(new FileInputStream(output))) {
+ for (ZipEntry entry = in.getNextEntry(); entry != null; entry = in.getNextEntry()) {
+ assertEquals("example.txt", entry.getName());
+ names.add(entry.getName());
+ String out = CharStreams.toString(new InputStreamReader(in));
+ assertEquals("cake", out);
+ }
+ }
+
+ assertEquals(1, names.size());
+ }
+
+ @Test
+ public void shouldSetTimestampOfEntries() throws IOException {
+ Calendar cal = Calendar.getInstance();
+ cal.set(1999, SEPTEMBER, 10);
+ long old = getTimeRoundedToSeconds(cal.getTime());
+
+ long now = getTimeRoundedToSeconds(new Date());
+
+ try (CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output)) {
+ ZipEntry oldAndValid = new ZipEntry("oldAndValid");
+ oldAndValid.setTime(old);
+ out.putNextEntry(oldAndValid);
+
+ ZipEntry current = new ZipEntry("current");
+ current.setTime(now);
+ out.putNextEntry(current);
+ }
+
+ try (ZipInputStream in = new ZipInputStream(new FileInputStream(output))) {
+ ZipEntry entry = in.getNextEntry();
+ assertEquals("oldAndValid", entry.getName());
+ assertEquals(old, entry.getTime());
+
+ entry = in.getNextEntry();
+ assertEquals("current", entry.getName());
+ assertEquals(now, entry.getTime());
+ }
+ }
+
+ private long getTimeRoundedToSeconds(Date date) {
+ long time = date.getTime();
+
+ // Work in seconds.
+ time = time / 1000;
+
+ // the dos time function is only correct to 2 seconds.
+ // http://msdn.microsoft.com/en-us/library/ms724247%28v=vs.85%29.aspx
+ if (time % 2 == 1) {
+ time += 1;
+ }
+
+ // Back to milliseconds
+ time *= 1000;
+
+ return time;
+ }
+
+ @Test
+ public void compressionCanBeSetOnAPerFileBasisAndIsHonoured() throws IOException {
+ // Create some input that can be compressed.
+ String packageName = getClass().getPackage().getName().replace(".", "/");
+ URL sample = Resources.getResource(packageName + "/sample-bytes.properties");
+ byte[] input = Resources.toByteArray(sample);
+
+ try (CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output)) {
+ CustomZipEntry entry = new CustomZipEntry("default");
+ // Don't set the compression level. Should be the default.
+ out.putNextEntry(entry);
+ out.write(input);
+
+ entry = new CustomZipEntry("stored");
+ entry.setCompressionLevel(NO_COMPRESSION);
+ out.putNextEntry(entry);
+ out.write("stored".getBytes());
+
+ entry = new CustomZipEntry("best");
+ entry.setCompressionLevel(BEST_COMPRESSION);
+ out.putNextEntry(entry);
+ out.write(input);
+ }
+
+ try (ZipInputStream in = new ZipInputStream(new FileInputStream(output))) {
+ ZipEntry entry = in.getNextEntry();
+ assertEquals("default", entry.getName());
+ ByteStreams.copy(in, ByteStreams.nullOutputStream());
+ long defaultCompressedSize = entry.getCompressedSize();
+ assertNotEquals(entry.getCompressedSize(), entry.getSize());
+
+ entry = in.getNextEntry();
+ assertEquals("stored", entry.getName());
+ assertEquals(entry.getCompressedSize(), entry.getSize());
+
+ entry = in.getNextEntry();
+ assertEquals("best", entry.getName());
+ ByteStreams.copy(in, ByteStreams.nullOutputStream());
+ assertThat(entry.getCompressedSize(), lessThan(defaultCompressedSize));
+ }
+ }
+}
diff --git a/test/com/facebook/buck/zip/ZipStepTest.java b/test/com/facebook/buck/zip/ZipStepTest.java
index 5d5fd0e..e6ad4ef 100644
--- a/test/com/facebook/buck/zip/ZipStepTest.java
+++ b/test/com/facebook/buck/zip/ZipStepTest.java
@@ -16,99 +16,164 @@
package com.facebook.buck.zip;
-import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.verify;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
-import com.facebook.buck.step.ExecutionContext;
-import com.facebook.buck.testutil.MoreAsserts;
-import com.facebook.buck.util.Verbosity;
-import com.google.common.collect.ImmutableList;
+import com.facebook.buck.step.TestExecutionContext;
+import com.facebook.buck.testutil.Zip;
+import com.facebook.buck.util.environment.Platform;
import com.google.common.collect.ImmutableSet;
+import com.google.common.io.Files;
+import org.junit.Assume;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import java.io.File;
-import java.util.List;
-import java.util.Set;
+import java.io.IOException;
+import java.nio.file.Path;
public class ZipStepTest {
- private final String zipFile = new File("/path/to/file.zip").getAbsolutePath();
+ @Rule public TemporaryFolder tmp = new TemporaryFolder();
@Test
- public void testGetShellCommandInternalOnZipCommandWithSpecifiedDirectory() {
- final int compressionLevel = ZipStep.DEFAULT_COMPRESSION_LEVEL;
+ public void shouldCreateANewZipFileFromScratch() throws IOException {
+ File parent = tmp.newFolder("zipstep");
+ File out = new File(parent, "output.zip");
- final File directory = new File("/some/other/path");
+ File toZip = tmp.newFolder("zipdir");
+ Files.touch(new File(toZip, "file1.txt"));
+ Files.touch(new File(toZip, "file2.txt"));
+ Files.touch(new File(toZip, "file3.txt"));
- ExecutionContext context = createMock(ExecutionContext.class);
- // This will trigger having the -q argument.
- expect(context.getVerbosity()).andReturn(Verbosity.STANDARD_INFORMATION);
- replay(context);
+ ZipStep step = new ZipStep(out.getAbsolutePath(),
+ ImmutableSet.<String>of(),
+ false,
+ ZipStep.DEFAULT_COMPRESSION_LEVEL,
+ toZip);
+ step.execute(TestExecutionContext.newInstance());
- List<String> expectedShellCommand = new ImmutableList.Builder<String>()
- .add("zip")
- .add("-q")
- .add("-X")
- .add("-r")
- .add("-" + compressionLevel)
- .add(zipFile)
- .add("-i*")
- .add(".")
- .build();
-
- ZipStep command = new ZipStep(new File(zipFile), directory);
-
- // Assert that the command has been constructed with the right arguments.
- MoreAsserts.assertListEquals(expectedShellCommand,
- command.getShellCommand(context));
-
- // Assert that the desired directory is saved with the ZipCommand as a working directory.
- assertEquals(directory, command.getWorkingDirectory());
-
- verify(context);
+ try (Zip zip = new Zip(out, false)) {
+ assertEquals(ImmutableSet.of("file1.txt", "file2.txt", "file3.txt"), zip.getFileNames());
+ }
}
@Test
- public void testGetShellCommandInternalOnZipCommandWithSpecifiedPaths() {
- final Set<String> paths = ImmutableSet.of("a/path", "another.path");
- final int compressionLevel = 7;
- final File workingDirectory = new File("/some/other/path");
+ public void willOnlyIncludeEntriesInThePathsArgumentIfAnyAreSet() throws IOException {
+ File parent = tmp.newFolder("zipstep");
+ File out = new File(parent, "output.zip");
- ExecutionContext context = createMock(ExecutionContext.class);
- expect(context.getVerbosity()).andReturn(Verbosity.SILENT);
- replay(context);
+ File toZip = tmp.newFolder("zipdir");
+ Files.touch(new File(toZip, "file1.txt"));
+ Files.touch(new File(toZip, "file2.txt"));
+ Files.touch(new File(toZip, "file3.txt"));
- List<String> expectedShellCommand = new ImmutableList.Builder<String>()
- .add("zip")
- .add("-u")
- .add("-qq")
- .add("-X")
- .add("-r")
- .add("-" + compressionLevel)
- .add("-j")
- .add(zipFile)
- .addAll(paths)
- .build();
+ ZipStep step = new ZipStep(out.getAbsolutePath(),
+ ImmutableSet.of("file2.txt"),
+ false,
+ ZipStep.DEFAULT_COMPRESSION_LEVEL,
+ toZip);
+ step.execute(TestExecutionContext.newInstance());
- ZipStep command = new ZipStep(
- ZipStep.Mode.UPDATE,
- zipFile,
- paths,
+ try (Zip zip = new Zip(out, false)) {
+ assertEquals(ImmutableSet.of("file2.txt"), zip.getFileNames());
+ }
+ }
+
+ @Test
+ public void willRecurseIntoSubdirectories() throws IOException {
+ File parent = tmp.newFolder("zipstep");
+ File out = new File(parent, "output.zip");
+
+ File toZip = tmp.newFolder("zipdir");
+ Files.touch(new File(toZip, "file1.txt"));
+ assertTrue(new File(toZip, "child").mkdir());
+ Files.touch(new File(toZip, "child/file2.txt"));
+
+ ZipStep step = new ZipStep(out.getAbsolutePath(),
+ ImmutableSet.<String>of(),
+ false,
+ ZipStep.DEFAULT_COMPRESSION_LEVEL,
+ toZip);
+ step.execute(TestExecutionContext.newInstance());
+
+ try (Zip zip = new Zip(out, false)) {
+ assertEquals(ImmutableSet.of("file1.txt", "child/file2.txt"), zip.getFileNames());
+ }
+ }
+
+ @Test
+ public void mustIncludeTheContentsOfFilesThatAreSymlinked() throws IOException {
+ // Symlinks on Windows are _hard_. Let's go shopping.
+ Assume.assumeTrue(Platform.detect() != Platform.WINDOWS);
+
+ File parent = tmp.newFolder("zipstep");
+ File out = new File(parent, "output.zip");
+ File target = new File(parent, "target");
+ Files.write("example content", target, UTF_8);
+
+ File toZip = tmp.newFolder("zipdir");
+ Path path = toZip.toPath().resolve("file.txt");
+ java.nio.file.Files.createSymbolicLink(path, target.toPath());
+
+ ZipStep step = new ZipStep(out.getAbsolutePath(),
+ ImmutableSet.<String>of(),
+ false,
+ ZipStep.DEFAULT_COMPRESSION_LEVEL,
+ toZip);
+ step.execute(TestExecutionContext.newInstance());
+
+ try (Zip zip = new Zip(out, false)) {
+ assertEquals(ImmutableSet.of("file.txt"), zip.getFileNames());
+ byte[] contents = zip.readFully("file.txt");
+
+ assertArrayEquals("example content".getBytes(), contents);
+ }
+ }
+
+ @Test
+ public void overwritingAnExistingZipFileIsAnError() throws IOException {
+ File parent = tmp.newFolder();
+ File out = new File(parent, "output.zip");
+
+ try (Zip zip = new Zip(out, true)) {
+ zip.add("file1.txt", "");
+ }
+
+ File toZip = tmp.newFolder();
+
+ ZipStep step = new ZipStep(out.getAbsolutePath(),
+ ImmutableSet.<String>of(),
+ false,
+ ZipStep.DEFAULT_COMPRESSION_LEVEL,
+ toZip);
+ int result = step.execute(TestExecutionContext.newInstance());
+
+ assertEquals(1, result);
+ }
+
+ @Test
+ public void shouldBeAbleToJunkPaths() throws IOException {
+ File parent = tmp.newFolder();
+ File out = new File(parent, "output.zip");
+
+ File toZip = tmp.newFolder();
+ assertTrue(new File(toZip, "child").mkdir());
+ Files.touch(new File(toZip, "child/file1.txt"));
+
+ ZipStep step = new ZipStep(out.getAbsolutePath(),
+ ImmutableSet.<String>of(),
true,
- compressionLevel,
- workingDirectory);
+ ZipStep.DEFAULT_COMPRESSION_LEVEL,
+ toZip);
+ step.execute(TestExecutionContext.newInstance());
- // Assert that the command has been constructed with the right arguments.
- MoreAsserts.assertListEquals(expectedShellCommand,
- command.getShellCommand(context));
-
- // Assert that the desired working directory is saved with the command.
- assertEquals(workingDirectory, command.getWorkingDirectory());
-
- verify(context);
+ try (Zip zip = new Zip(out, false)) {
+ assertEquals(ImmutableSet.of("file1.txt"), zip.getFileNames());
+ }
}
}
diff --git a/test/com/facebook/buck/zip/sample-bytes.properties b/test/com/facebook/buck/zip/sample-bytes.properties
new file mode 100644
index 0000000..5505f7c
--- /dev/null
+++ b/test/com/facebook/buck/zip/sample-bytes.properties
Binary files differ
