Limit how big buck-cache can grow.
Summary: This has bugged me for a while.
diff --git a/src/com/facebook/buck/cli/BUCK b/src/com/facebook/buck/cli/BUCK
index c74794e..b7e56aa 100644
--- a/src/com/facebook/buck/cli/BUCK
+++ b/src/com/facebook/buck/cli/BUCK
@@ -57,6 +57,7 @@
'//src/com/facebook/buck/util:io',
'//src/com/facebook/buck/util:network',
'//src/com/facebook/buck/util:util',
+ '//src/com/facebook/buck/util/unit:unit',
'//src/com/facebook/buck/util/environment:environment',
'//src/com/facebook/buck/timing:timing',
'//third-party/java/astyanax:astyanax-cassandra',
diff --git a/src/com/facebook/buck/cli/BuckConfig.java b/src/com/facebook/buck/cli/BuckConfig.java
index 5990222..b6c4edc 100644
--- a/src/com/facebook/buck/cli/BuckConfig.java
+++ b/src/com/facebook/buck/cli/BuckConfig.java
@@ -35,6 +35,7 @@
import com.facebook.buck.util.MorePaths;
import com.facebook.buck.util.ProjectFilesystem;
import com.facebook.buck.util.environment.Platform;
+import com.facebook.buck.util.unit.SizeUnit;
import com.google.common.annotations.Beta;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
@@ -327,11 +328,11 @@
// Normalize paths in order to eliminate trailing '/' characters and whatnot.
return ImmutableSet.<String>builder().addAll(Iterables.transform(builder.build(),
new Function<String, String>() {
- @Override
- public String apply(String path) {
- return MorePaths.newPathInstance(path).toString();
- }
- })).build();
+ @Override
+ public String apply(String path) {
+ return MorePaths.newPathInstance(path).toString();
+ }
+ })).build();
}
public ImmutableSet<Pattern> getTempFilePatterns() {
@@ -515,7 +516,9 @@
for (String mode : modes) {
switch (ArtifactCacheNames.valueOf(mode)) {
case dir:
- builder.add(createDirArtifactCache());
+ ArtifactCache dirArtifactCache = createDirArtifactCache();
+ buckEventBus.register(dirArtifactCache);
+ builder.add(dirArtifactCache);
break;
case cassandra:
ArtifactCache cassandraArtifactCache = createCassandraArtifactCache(buckEventBus);
@@ -551,11 +554,20 @@
return projectFilesystem.getPathRelativizer().apply(cacheDir);
}
+ public Optional<Long> getCacheDirMaxSizeBytes() {
+ return getValue("cache", "dir_max_size").transform(new Function<String, Long>() {
+ @Override
+ public Long apply(String input) {
+ return SizeUnit.parseBytes(input);
+ }
+ });
+ }
+
private ArtifactCache createDirArtifactCache() {
Path cacheDir = getCacheDir();
File dir = cacheDir.toFile();
try {
- return new DirArtifactCache(dir);
+ return new DirArtifactCache(dir, getCacheDirMaxSizeBytes());
} catch (IOException e) {
throw new HumanReadableException("Failure initializing artifact cache directory: %s", dir);
}
diff --git a/src/com/facebook/buck/rules/BUCK b/src/com/facebook/buck/rules/BUCK
index ce4dc87..d9ee695 100644
--- a/src/com/facebook/buck/rules/BUCK
+++ b/src/com/facebook/buck/rules/BUCK
@@ -133,6 +133,7 @@
'//src/com/facebook/buck/step/fs:fs',
'//src/com/facebook/buck/test:test',
'//src/com/facebook/buck/util:constants',
+ '//src/com/facebook/buck/util/collect:collect',
'//src/com/facebook/buck/util:exceptions',
'//src/com/facebook/buck/util:io',
'//src/com/facebook/buck/util:network',
diff --git a/src/com/facebook/buck/rules/DirArtifactCache.java b/src/com/facebook/buck/rules/DirArtifactCache.java
index e244e25..db4f867 100644
--- a/src/com/facebook/buck/rules/DirArtifactCache.java
+++ b/src/com/facebook/buck/rules/DirArtifactCache.java
@@ -18,22 +18,63 @@
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+import com.facebook.buck.util.collect.ArrayIterable;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.eventbus.Subscribe;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileTime;
+import java.util.Arrays;
+import java.util.Comparator;
import java.util.logging.Level;
import java.util.logging.Logger;
public class DirArtifactCache implements ArtifactCache {
+ private class FileAccessedEntry {
+ public final File file;
+ public final FileTime lastAccessTime;
+
+ public File getFile() {
+ return file;
+ }
+
+ public FileTime getLastAccessTime() {
+ return lastAccessTime;
+ }
+
+ private FileAccessedEntry(File file, FileTime lastAccessTime) {
+ this.file = file;
+ this.lastAccessTime = lastAccessTime;
+ }
+ }
+
+
private final static Logger logger = Logger.getLogger(DirArtifactCache.class.getName());
- private final File cacheDir;
+ /**
+ * Sorts by the lastAccessTime in descending order (more recently accessed files are first).
+ */
+ private final static Comparator<FileAccessedEntry> SORT_BY_LAST_ACCESSED_TIME_DESC =
+ new Comparator<FileAccessedEntry>() {
+ @Override
+ public int compare(FileAccessedEntry a, FileAccessedEntry b) {
+ return b.getLastAccessTime().compareTo(a.getLastAccessTime());
+ }
+ };
- public DirArtifactCache(File cacheDir) throws IOException {
+ private final File cacheDir;
+ private final Optional<Long> maxCacheSizeBytes;
+
+ public DirArtifactCache(File cacheDir, Optional<Long> maxCacheSizeBytes) throws IOException {
this.cacheDir = Preconditions.checkNotNull(cacheDir);
+ this.maxCacheSizeBytes = Preconditions.checkNotNull(maxCacheSizeBytes);
Files.createDirectories(cacheDir.toPath());
}
@@ -92,4 +133,59 @@
public boolean isStoreSupported() {
return true;
}
+
+ @Subscribe
+ public synchronized void buildFinished(BuildEvent.Finished finished) {
+ deleteOldFiles();
+ }
+
+ /**
+ * Deletes files that haven't been accessed recently from the directory cache.
+ */
+ @VisibleForTesting
+ void deleteOldFiles() {
+ if (!maxCacheSizeBytes.isPresent()) {
+ return;
+ }
+ for (FileAccessedEntry fileAccessedEntry : findFilesToDelete()) {
+ try {
+ Files.deleteIfExists(fileAccessedEntry.getFile().toPath());
+ } catch (IOException e) {
+ // Eat any IOExceptions while attempting to clean up the cache directory. If the file is
+ // now in use, we no longer want to delete it.
+ continue;
+ }
+ }
+ }
+
+ private Iterable<FileAccessedEntry> findFilesToDelete() {
+ Preconditions.checkState(maxCacheSizeBytes.isPresent());
+ long maxSizeBytes = maxCacheSizeBytes.get();
+
+ File[] artifacts = cacheDir.listFiles();
+ FileAccessedEntry[] fileAccessedEntries = new FileAccessedEntry[artifacts.length];
+ for (int i = 0; i < artifacts.length; ++i) {
+ FileTime lastAccess;
+ try {
+ lastAccess =
+ Files.readAttributes(artifacts[i].toPath(), BasicFileAttributes.class).lastAccessTime();
+ } catch (IOException e) {
+ lastAccess = FileTime.fromMillis(artifacts[i].lastModified());
+ }
+ fileAccessedEntries[i] = new FileAccessedEntry(artifacts[i], lastAccess);
+ }
+ Arrays.sort(fileAccessedEntries, SORT_BY_LAST_ACCESSED_TIME_DESC);
+
+ // Finds the first N from the list ordered by last access time who's combined size is less than
+ // maxCacheSizeBytes.
+ long currentSizeBytes = 0;
+ for (int i = 0; i < fileAccessedEntries.length; ++i) {
+ FileAccessedEntry file = fileAccessedEntries[i];
+ currentSizeBytes += file.getFile().length();
+ if (currentSizeBytes > maxSizeBytes) {
+ return ArrayIterable.of(fileAccessedEntries, i, fileAccessedEntries.length);
+ }
+ }
+ return ImmutableList.<FileAccessedEntry>of();
+ }
}
diff --git a/src/com/facebook/buck/util/unit/BUCK b/src/com/facebook/buck/util/unit/BUCK
new file mode 100644
index 0000000..9628149
--- /dev/null
+++ b/src/com/facebook/buck/util/unit/BUCK
@@ -0,0 +1,10 @@
+java_library(
+ name = 'unit',
+ srcs = glob(['*.java']),
+ deps = [
+ '//lib:guava',
+ ],
+ visibility = [
+ 'PUBLIC',
+ ],
+)
diff --git a/test/com/facebook/buck/rules/DirArtifactCacheTest.java b/test/com/facebook/buck/rules/DirArtifactCacheTest.java
index d191d93..46d1ad3 100644
--- a/test/com/facebook/buck/rules/DirArtifactCacheTest.java
+++ b/test/com/facebook/buck/rules/DirArtifactCacheTest.java
@@ -22,6 +22,8 @@
import com.facebook.buck.util.NullFileHashCache;
import com.google.common.base.Charsets;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableSet;
import com.google.common.io.Files;
import org.junit.Rule;
@@ -30,6 +32,7 @@
import java.io.File;
import java.io.IOException;
+import java.nio.file.attribute.FileTime;
public class DirArtifactCacheTest {
@Rule public TemporaryFolder tmpDir = new TemporaryFolder();
@@ -38,7 +41,7 @@
public void testCacheCreation() throws IOException {
File cacheDir = tmpDir.newFolder();
- new DirArtifactCache(cacheDir);
+ new DirArtifactCache(cacheDir, /* maxCacheSizeBytes */ Optional.of(0L));
}
@Test
@@ -46,11 +49,13 @@
File cacheDir = tmpDir.newFolder();
File fileX = tmpDir.newFile("x");
- DirArtifactCache dirArtifactCache = new DirArtifactCache(cacheDir);
+ DirArtifactCache dirArtifactCache = new DirArtifactCache(cacheDir,
+ /* maxCacheSizeBytes */ Optional.of(0L));
Files.write("x", fileX, Charsets.UTF_8);
InputRule inputRuleX = new InputRuleForTest(fileX);
- RuleKey ruleKeyX = RuleKey.builder(inputRuleX, new NullFileHashCache()).build().getTotalRuleKey();
+ RuleKey ruleKeyX = RuleKey.builder(inputRuleX,
+ new NullFileHashCache()).build().getTotalRuleKey();
assertEquals(CacheResult.MISS, dirArtifactCache.fetch(ruleKeyX, fileX));
}
@@ -60,7 +65,8 @@
File cacheDir = tmpDir.newFolder();
File fileX = tmpDir.newFile("x");
- DirArtifactCache dirArtifactCache = new DirArtifactCache(cacheDir);
+ DirArtifactCache dirArtifactCache = new DirArtifactCache(cacheDir,
+ /* maxCacheSizeBytes */ Optional.<Long>absent());
Files.write("x", fileX, Charsets.UTF_8);
InputRule inputRuleX = new InputRuleForTest(fileX);
@@ -83,7 +89,8 @@
File cacheDir = tmpDir.newFolder();
File fileX = tmpDir.newFile("x");
- DirArtifactCache dirArtifactCache = new DirArtifactCache(cacheDir);
+ DirArtifactCache dirArtifactCache = new DirArtifactCache(cacheDir,
+ /* maxCacheSizeBytes */ Optional.of(0L));
Files.write("x", fileX, Charsets.UTF_8);
InputRule inputRuleX = new InputRuleForTest(fileX);
@@ -103,7 +110,8 @@
File fileY = tmpDir.newFile("y");
File fileZ = tmpDir.newFile("z");
- DirArtifactCache dirArtifactCache = new DirArtifactCache(cacheDir);
+ DirArtifactCache dirArtifactCache = new DirArtifactCache(cacheDir,
+ /* maxCacheSizeBytes */ Optional.of(0L));
Files.write("x", fileX, Charsets.UTF_8);
Files.write("y", fileY, Charsets.UTF_8);
@@ -140,6 +148,82 @@
assertEquals(inputRuleX, new InputRuleForTest(fileX));
assertEquals(inputRuleY, new InputRuleForTest(fileY));
assertEquals(inputRuleZ, new InputRuleForTest(fileZ));
+
+ assertEquals(3, cacheDir.listFiles().length);
+
+ dirArtifactCache.deleteOldFiles();
+
+ assertEquals(0, cacheDir.listFiles().length);
+ }
+
+ @Test
+ public void testDeleteNothing() throws IOException {
+ File cacheDir = tmpDir.newFolder();
+ File fileX = new File(cacheDir, "x");
+ File fileY = new File(cacheDir, "y");
+ File fileZ = new File(cacheDir, "z");
+
+ DirArtifactCache dirArtifactCache = new DirArtifactCache(tmpDir.getRoot(),
+ /* maxCacheSizeBytes */ Optional.of(1024L));
+
+ Files.write("x", fileX, Charsets.UTF_8);
+ Files.write("y", fileY, Charsets.UTF_8);
+ Files.write("z", fileZ, Charsets.UTF_8);
+
+ assertEquals(3, cacheDir.listFiles().length);
+
+ dirArtifactCache.deleteOldFiles();
+
+ assertEquals(3, cacheDir.listFiles().length);
+ }
+
+ @Test
+ public void testDeleteNothingAbsentLimit() throws IOException {
+ File cacheDir = tmpDir.newFolder();
+ File fileX = new File(cacheDir, "x");
+ File fileY = new File(cacheDir, "y");
+ File fileZ = new File(cacheDir, "z");
+
+ DirArtifactCache dirArtifactCache = new DirArtifactCache(tmpDir.getRoot(),
+ /* maxCacheSizeBytes */ Optional.<Long>absent());
+
+ Files.write("x", fileX, Charsets.UTF_8);
+ Files.write("y", fileY, Charsets.UTF_8);
+ Files.write("z", fileZ, Charsets.UTF_8);
+
+ assertEquals(3, cacheDir.listFiles().length);
+
+ dirArtifactCache.deleteOldFiles();
+
+ assertEquals(3, cacheDir.listFiles().length);
+ }
+
+ @Test
+ public void testDeleteSome() throws IOException {
+ File cacheDir = tmpDir.newFolder();
+ File fileW = new File(cacheDir, "w");
+ File fileX = new File(cacheDir, "x");
+ File fileY = new File(cacheDir, "y");
+ File fileZ = new File(cacheDir, "z");
+
+ DirArtifactCache dirArtifactCache = new DirArtifactCache(cacheDir,
+ /* maxCacheSizeBytes */ Optional.of(2L));
+
+ Files.write("w", fileW, Charsets.UTF_8);
+ Files.write("x", fileX, Charsets.UTF_8);
+ Files.write("y", fileY, Charsets.UTF_8);
+ Files.write("z", fileZ, Charsets.UTF_8);
+
+ java.nio.file.Files.setAttribute(fileW.toPath(), "lastAccessTime", FileTime.fromMillis(9000));
+ java.nio.file.Files.setAttribute(fileX.toPath(), "lastAccessTime", FileTime.fromMillis(0));
+ java.nio.file.Files.setAttribute(fileY.toPath(), "lastAccessTime", FileTime.fromMillis(1000));
+ java.nio.file.Files.setAttribute(fileZ.toPath(), "lastAccessTime", FileTime.fromMillis(2000));
+
+ assertEquals(4, cacheDir.listFiles().length);
+
+ dirArtifactCache.deleteOldFiles();
+
+ assertEquals(ImmutableSet.of(fileZ, fileW), ImmutableSet.copyOf(cacheDir.listFiles()));
}
private static class InputRuleForTest extends InputRule {
diff --git a/test/com/facebook/buck/util/unit/BUCK b/test/com/facebook/buck/util/unit/BUCK
new file mode 100644
index 0000000..dde5d3b
--- /dev/null
+++ b/test/com/facebook/buck/util/unit/BUCK
@@ -0,0 +1,12 @@
+java_test(
+ name = 'unit',
+ srcs = glob(['*.java']),
+ source_under_test = [
+ '//src/com/facebook/buck/util/unit:unit',
+ ],
+ deps = [
+ '//lib:guava',
+ '//lib:junit',
+ '//src/com/facebook/buck/util/unit:unit',
+ ],
+)
