Speed up bloom filter build with OS page cache warmup Loading bloom filters at startup requires a full table scan of every H2 cache file. Without warm OS page caches these reads are I/O-bound, making startup significantly slower.This change performs a full read of the H2 file before building the Bloom filters, ensuring that the OS page cache is populated. On a production instance with ~84M disk cache entries combined, this reduced the load up time from ~22 mins to ~16 mins. Release-Notes: Add file warming for faster H2 bloom filter building. Change-Id: I7cac53f9b375ab0f07fc97243926660fb1e7da5b
diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt index d5b8fe8..27ee3ec 100644 --- a/Documentation/config-gerrit.txt +++ b/Documentation/config-gerrit.txt
@@ -925,6 +925,19 @@ + Default is 1. +[[cache.preWarmForBloomFilter]]cache.preWarmForBloomFilter:: ++ +When enabled, each persistent cache reads its H2 database file into the OS +page cache before building its BloomFilter. The BloomFilter build requires +a full table scan of all keys which involves reading close to the entire file +in a scattered fashion. Doing such a read tends to not perform well I/O wise as +it generally cannot take advantage of operating system level file system +readahead. Reading the file sequentially up front tends to take advantage of +readahead to fully populate the file system caches which then can make scattered +reads much faster. ++ +Default is true. + [[cache.openFiles]]cache.openFiles:: + The number of file descriptors to add to the limit set by the Gerrit daemon.
diff --git a/java/com/google/gerrit/server/cache/h2/H2CacheFactory.java b/java/com/google/gerrit/server/cache/h2/H2CacheFactory.java index a0534ca..000b03d 100644 --- a/java/com/google/gerrit/server/cache/h2/H2CacheFactory.java +++ b/java/com/google/gerrit/server/cache/h2/H2CacheFactory.java
@@ -88,6 +88,7 @@ private final Schedule schedule; private final AtomicBoolean isDiskCacheReadOnly; @Nullable private final ExecutorService startupExecutor; + private final boolean preWarmForBloomFilter; @Inject H2CacheFactory( @@ -104,6 +105,7 @@ h2CacheSize = cfg.getLong("cache", null, "h2CacheSize", -1); h2AutoServer = cfg.getBoolean("cache", null, "h2AutoServer", false); pruneOnStartup = cfg.getBoolean("cachePruning", null, "pruneOnStartup", true); + preWarmForBloomFilter = cfg.getBoolean("cache", null, "preWarmForBloomFilter", true); caches = new ArrayList<>(); schedule = ScheduleConfig.createSchedule(cfg, "cachePruning") @@ -224,14 +226,10 @@ } private <V, K> SqlStore<K, V> newSqlStore(PersistentCacheDef<K, V> def, long maxSize) { + String cacheName = def.name() + "-v" + COMPATIBILITY_VERSION; StringBuilder url = new StringBuilder(); url.append("jdbc:h2:file:") - .append( - cacheDir - .resolve(def.name() + "-v" + COMPATIBILITY_VERSION) - .toAbsolutePath() - .toString() - .replace(";", "\\;")); + .append(cacheDir.resolve(cacheName).toAbsolutePath().toString().replace(";", "\\;")); if (h2CacheSize >= 0) { url.append(";CACHE_SIZE="); // H2 CACHE_SIZE is always given in KB @@ -269,7 +267,9 @@ refreshAfterWrite, options.contains(CacheOptions.BUILD_BLOOM_FILTER), options.contains(CacheOptions.TRACK_LAST_ACCESS), - isDiskCacheReadOnly); + isDiskCacheReadOnly, + preWarmForBloomFilter, + cacheDir.resolve(cacheName + ".mv.db")); } private boolean has(String name, String var) {
diff --git a/java/com/google/gerrit/server/cache/h2/H2CacheImpl.java b/java/com/google/gerrit/server/cache/h2/H2CacheImpl.java index 7dada27..83a3eee 100644 --- a/java/com/google/gerrit/server/cache/h2/H2CacheImpl.java +++ b/java/com/google/gerrit/server/cache/h2/H2CacheImpl.java
@@ -14,6 +14,7 @@ package com.google.gerrit.server.cache.h2; +import com.google.common.base.Stopwatch; import com.google.common.base.Throwables; import com.google.common.cache.AbstractLoadingCache; import com.google.common.cache.Cache; @@ -37,7 +38,10 @@ import com.google.gerrit.util.concurrent.ConcurrentBloomFilter; import com.google.inject.TypeLiteral; import java.io.IOException; +import java.io.InputStream; import java.io.InvalidClassException; +import java.nio.file.Files; +import java.nio.file.Path; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -348,6 +352,8 @@ private boolean trackLastAccess; private final AtomicBoolean isDiskCacheReadOnly; private volatile boolean ensuredSchemaCreation; + private final Path cacheFilePath; + private final boolean preWarmForBloomFilter; SqlStore( String jdbcUrl, @@ -361,7 +367,9 @@ @Nullable Duration refreshAfterWrite, boolean buildBloomFilter, boolean trackLastAccess, - AtomicBoolean isDiskCacheReadOnly) { + AtomicBoolean isDiskCacheReadOnly, + boolean preWarmForBloomFilter, + Path cacheFilePath) { this.url = jdbcUrl; this.keyType = createKeyType(keyType, keySerializer); this.valueSerializer = valueSerializer; @@ -372,6 +380,8 @@ this.buildBloomFilter = buildBloomFilter; this.trackLastAccess = trackLastAccess; this.isDiskCacheReadOnly = isDiskCacheReadOnly; + this.cacheFilePath = cacheFilePath; + this.preWarmForBloomFilter = preWarmForBloomFilter; int cores = Runtime.getRuntime().availableProcessors(); int keep = Math.min(cores, 16); @@ -424,6 +434,23 @@ } } + void warmupOsPageCache() { + if (!Files.exists(cacheFilePath)) { + return; + } + logger.atFine().log("Warming OS page cache for %s", cacheFilePath.getFileName()); + Stopwatch sw = Stopwatch.createStarted(); + byte[] buf = new byte[65536]; + try (InputStream in = Files.newInputStream(cacheFilePath)) { + while (in.read(buf) != -1) {} + } catch (IOException e) { + logger.atWarning().log( + "Failed to warm OS page cache for %s: %s", cacheFilePath.getFileName(), e.getMessage()); + } + logger.atFine().log( + "Finished warming OS page cache for %s after %s", cacheFilePath.getFileName(), sw); + } + void open() { bloomFilter.initIfNeeded(); } @@ -461,6 +488,9 @@ } private void buildBloomFilter() { + if (preWarmForBloomFilter) { + warmupOsPageCache(); + } SqlHandle c = null; try (TraceTimer ignored = TraceContext.newTimer("Build bloom filter", Metadata.empty())) { c = acquire();
diff --git a/javatests/com/google/gerrit/server/cache/h2/H2CacheTest.java b/javatests/com/google/gerrit/server/cache/h2/H2CacheTest.java index cf8bdda..b4228b6 100644 --- a/javatests/com/google/gerrit/server/cache/h2/H2CacheTest.java +++ b/javatests/com/google/gerrit/server/cache/h2/H2CacheTest.java
@@ -75,7 +75,9 @@ refreshAfterWrite, true, true, - new AtomicBoolean(false)); + new AtomicBoolean(false), + false, + null); } @Test