Merge branch 'stable-5.4' into stable-5.5
* stable-5.4:
Fix string format parameter for invalidRefAdvertisementLine
WindowCache: add metric for cached bytes per repository
pgm daemon: fallback to user and system config if no config specified
WindowCache: add option to use strong refs to reference ByteWindows
Replace usage of ArrayIndexOutOfBoundsException in treewalk
Add config constants for WindowCache configuration options
Change-Id: I76a62da98182f0c504b1ea8b7d37cecdf4eea7e0
diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Daemon.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Daemon.java
index 4382663..8b86c16 100644
--- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Daemon.java
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Daemon.java
@@ -44,6 +44,7 @@
package org.eclipse.jgit.pgm;
import java.io.File;
+import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
import java.text.MessageFormat;
@@ -51,12 +52,14 @@
import java.util.List;
import java.util.concurrent.Executors;
+import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.internal.ketch.KetchLeader;
import org.eclipse.jgit.internal.ketch.KetchLeaderCache;
import org.eclipse.jgit.internal.ketch.KetchPreReceive;
import org.eclipse.jgit.internal.ketch.KetchSystem;
import org.eclipse.jgit.internal.ketch.KetchText;
import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.lib.StoredConfig;
import org.eclipse.jgit.pgm.internal.CLIText;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.storage.file.WindowCacheConfig;
@@ -68,6 +71,7 @@
import org.eclipse.jgit.transport.resolver.ReceivePackFactory;
import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException;
import org.eclipse.jgit.util.FS;
+import org.eclipse.jgit.util.SystemReader;
import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.Option;
@@ -120,19 +124,20 @@
@Override
protected void run() throws Exception {
PackConfig packConfig = new PackConfig();
-
- if (configFile != null) {
+ StoredConfig cfg;
+ if (configFile == null) {
+ cfg = getUserConfig();
+ } else {
if (!configFile.exists()) {
throw die(MessageFormat.format(
CLIText.get().configFileNotFound, //
configFile.getAbsolutePath()));
}
-
- FileBasedConfig cfg = new FileBasedConfig(configFile, FS.DETECTED);
- cfg.load();
- new WindowCacheConfig().fromConfig(cfg).install();
- packConfig.fromConfig(cfg);
+ cfg = new FileBasedConfig(configFile, FS.DETECTED);
}
+ cfg.load();
+ new WindowCacheConfig().fromConfig(cfg).install();
+ packConfig.fromConfig(cfg);
int threads = packConfig.getThreads();
if (threads <= 0)
@@ -172,6 +177,16 @@
outw.println(MessageFormat.format(CLIText.get().listeningOn, d.getAddress()));
}
+ private StoredConfig getUserConfig() throws IOException {
+ StoredConfig userConfig = null;
+ try {
+ userConfig = SystemReader.getInstance().getUserConfig();
+ } catch (ConfigInvalidException e) {
+ throw die(e.getMessage());
+ }
+ return userConfig;
+ }
+
private static DaemonService service(
final org.eclipse.jgit.transport.Daemon d,
final String n) {
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/WindowCacheGetTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/WindowCacheGetTest.java
index f1a18b0..a173532 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/WindowCacheGetTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/WindowCacheGetTest.java
@@ -53,6 +53,8 @@
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
import org.eclipse.jgit.errors.CorruptObjectException;
@@ -66,9 +68,25 @@
import org.eclipse.jgit.util.MutableInteger;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+@RunWith(Parameterized.class)
public class WindowCacheGetTest extends SampleDataRepositoryTestCase {
private List<TestObject> toLoad;
+ private WindowCacheConfig cfg;
+ private boolean useStrongRefs;
+
+ @Parameters(name = "useStrongRefs={0}")
+ public static Collection<Object[]> data() {
+ return Arrays
+ .asList(new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } });
+ }
+
+ public WindowCacheGetTest(Boolean useStrongRef) {
+ this.useStrongRefs = useStrongRef.booleanValue();
+ }
@Override
@Before
@@ -93,11 +111,12 @@
}
}
assertEquals(96, toLoad.size());
+ cfg = new WindowCacheConfig();
+ cfg.setPackedGitUseStrongRefs(useStrongRefs);
}
@Test
public void testCache_Defaults() throws IOException {
- WindowCacheConfig cfg = new WindowCacheConfig();
cfg.install();
doCacheTests();
checkLimits(cfg);
@@ -122,7 +141,6 @@
@Test
public void testCache_TooFewFiles() throws IOException {
- final WindowCacheConfig cfg = new WindowCacheConfig();
cfg.setPackedGitOpenFiles(2);
cfg.install();
doCacheTests();
@@ -131,7 +149,6 @@
@Test
public void testCache_TooSmallLimit() throws IOException {
- final WindowCacheConfig cfg = new WindowCacheConfig();
cfg.setPackedGitWindowSize(4096);
cfg.setPackedGitLimit(4096);
cfg.install();
@@ -142,26 +159,31 @@
private static void checkLimits(WindowCacheConfig cfg) {
final WindowCache cache = WindowCache.getInstance();
WindowCacheStats s = cache.getStats();
- assertTrue(0 < s.getAverageLoadTime());
- assertTrue(0 < s.getOpenByteCount());
- assertTrue(0 < s.getOpenByteCount());
- assertTrue(0.0 < s.getAverageLoadTime());
- assertTrue(0 <= s.getEvictionCount());
- assertTrue(0 < s.getHitCount());
- assertTrue(0 < s.getHitRatio());
- assertTrue(1 > s.getHitRatio());
- assertTrue(0 < s.getLoadCount());
- assertTrue(0 <= s.getLoadFailureCount());
- assertTrue(0.0 <= s.getLoadFailureRatio());
- assertTrue(1 > s.getLoadFailureRatio());
- assertTrue(0 < s.getLoadSuccessCount());
- assertTrue(s.getOpenByteCount() <= cfg.getPackedGitLimit());
- assertTrue(s.getOpenFileCount() <= cfg.getPackedGitOpenFiles());
- assertTrue(0 <= s.getMissCount());
- assertTrue(0 <= s.getMissRatio());
- assertTrue(1 > s.getMissRatio());
- assertTrue(0 < s.getRequestCount());
- assertTrue(0 < s.getTotalLoadTime());
+ assertTrue("average load time should be > 0",
+ 0 < s.getAverageLoadTime());
+ assertTrue("open byte count should be > 0", 0 < s.getOpenByteCount());
+ assertTrue("eviction count should be >= 0", 0 <= s.getEvictionCount());
+ assertTrue("hit count should be > 0", 0 < s.getHitCount());
+ assertTrue("hit ratio should be > 0", 0 < s.getHitRatio());
+ assertTrue("hit ratio should be < 1", 1 > s.getHitRatio());
+ assertTrue("load count should be > 0", 0 < s.getLoadCount());
+ assertTrue("load failure count should be >= 0",
+ 0 <= s.getLoadFailureCount());
+ assertTrue("load failure ratio should be >= 0",
+ 0.0 <= s.getLoadFailureRatio());
+ assertTrue("load failure ratio should be < 1",
+ 1 > s.getLoadFailureRatio());
+ assertTrue("load success count should be > 0",
+ 0 < s.getLoadSuccessCount());
+ assertTrue("open byte count should be <= core.packedGitLimit",
+ s.getOpenByteCount() <= cfg.getPackedGitLimit());
+ assertTrue("open file count should be <= core.packedGitOpenFiles",
+ s.getOpenFileCount() <= cfg.getPackedGitOpenFiles());
+ assertTrue("miss success count should be >= 0", 0 <= s.getMissCount());
+ assertTrue("miss ratio should be > 0", 0 <= s.getMissRatio());
+ assertTrue("miss ratio should be < 1", 1 > s.getMissRatio());
+ assertTrue("request count should be > 0", 0 < s.getRequestCount());
+ assertTrue("total load time should be > 0", 0 < s.getTotalLoadTime());
}
private void doCacheTests() throws IOException {
diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters
index 58ec3cb..a2066ec 100644
--- a/org.eclipse.jgit/.settings/.api_filters
+++ b/org.eclipse.jgit/.settings/.api_filters
@@ -1,13 +1,5 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<component id="org.eclipse.jgit" version="2">
- <resource path="META-INF/MANIFEST.MF">
- <filter id="924844039">
- <message_arguments>
- <message_argument value="5.5.2"/>
- <message_argument value="5.5.0"/>
- </message_arguments>
- </filter>
- </resource>
<resource path="src/org/eclipse/jgit/dircache/DirCacheEntry.java" type="org.eclipse.jgit.dircache.DirCacheEntry">
<filter id="1142947843">
<message_arguments>
@@ -54,6 +46,36 @@
</filter>
<filter id="1142947843">
<message_arguments>
+ <message_argument value="5.1.13"/>
+ <message_argument value="CONFIG_KEY_PACKED_GIT_LIMIT"/>
+ </message_arguments>
+ </filter>
+ <filter id="1142947843">
+ <message_arguments>
+ <message_argument value="5.1.13"/>
+ <message_argument value="CONFIG_KEY_PACKED_GIT_MMAP"/>
+ </message_arguments>
+ </filter>
+ <filter id="1142947843">
+ <message_arguments>
+ <message_argument value="5.1.13"/>
+ <message_argument value="CONFIG_KEY_PACKED_GIT_OPENFILES"/>
+ </message_arguments>
+ </filter>
+ <filter id="1142947843">
+ <message_arguments>
+ <message_argument value="5.1.13"/>
+ <message_argument value="CONFIG_KEY_PACKED_GIT_USE_STRONGREFS"/>
+ </message_arguments>
+ </filter>
+ <filter id="1142947843">
+ <message_arguments>
+ <message_argument value="5.1.13"/>
+ <message_argument value="CONFIG_KEY_PACKED_GIT_WINDOWSIZE"/>
+ </message_arguments>
+ </filter>
+ <filter id="1142947843">
+ <message_arguments>
<message_argument value="5.1.9"/>
<message_argument value="CONFIG_FILESYSTEM_SECTION"/>
</message_arguments>
@@ -79,6 +101,20 @@
</message_arguments>
</filter>
</resource>
+ <resource path="src/org/eclipse/jgit/storage/file/WindowCacheConfig.java" type="org.eclipse.jgit.storage.file.WindowCacheConfig">
+ <filter id="1142947843">
+ <message_arguments>
+ <message_argument value="5.1.13"/>
+ <message_argument value="isPackedGitUseStrongRefs()"/>
+ </message_arguments>
+ </filter>
+ <filter id="1142947843">
+ <message_arguments>
+ <message_argument value="5.1.13"/>
+ <message_argument value="setPackedGitUseStrongRefs(boolean)"/>
+ </message_arguments>
+ </filter>
+ </resource>
<resource path="src/org/eclipse/jgit/storage/file/WindowCacheStats.java" type="org.eclipse.jgit.storage.file.WindowCacheStats">
<filter id="337809484">
<message_arguments>
diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
index b82512f..52359d1 100644
--- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
+++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
@@ -368,7 +368,7 @@
invalidPath=Invalid path: {0}
invalidPurgeFactor=Invalid purgeFactor {0}, values have to be in range between 0 and 1
invalidRedirectLocation=Invalid redirect location {0} -> {1}
-invalidRefAdvertisementLine=Invalid ref advertisement line: ''{1}''
+invalidRefAdvertisementLine=Invalid ref advertisement line: ''{0}''
invalidReflogRevision=Invalid reflog revision: {0}
invalidRefName=Invalid ref name: {0}
invalidReftableBlock=Invalid reftable block
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java
index 015866e..838de0b 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java
@@ -47,11 +47,16 @@
import java.io.IOException;
import java.lang.ref.ReferenceQueue;
import java.lang.ref.SoftReference;
+import java.util.Collections;
+import java.util.Map;
import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReferenceArray;
import java.util.concurrent.atomic.LongAdder;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
import org.eclipse.jgit.annotations.NonNull;
import org.eclipse.jgit.internal.JGitText;
@@ -85,9 +90,16 @@
* comprised of roughly 10% of the cache, and evicting the oldest accessed entry
* within that window.
* <p>
- * Entities created by the cache are held under SoftReferences, permitting the
+ * Entities created by the cache are held under SoftReferences if option
+ * {@code core.packedGitUseStrongRefs} is set to {@code false} in the git config
+ * (this is the default) or by calling
+ * {@link WindowCacheConfig#setPackedGitUseStrongRefs(boolean)}, permitting the
* Java runtime's garbage collector to evict entries when heap memory gets low.
* Most JREs implement a loose least recently used algorithm for this eviction.
+ * When this option is set to {@code true} strong references are used which
+ * means that Java gc cannot evict the WindowCache to reclaim memory. On the
+ * other hand this provides more predictable performance since the cache isn't
+ * flushed when used heap comes close to the maximum heap size.
* <p>
* The internal hash table does not expand at runtime, instead it is fixed in
* size at cache creation time. The internal lock table used to gate load
@@ -104,19 +116,19 @@
* for a given <code>(PackFile,position)</code> tuple.</li>
* <li>For every <code>load()</code> invocation there is exactly one
* {@link #createRef(PackFile, long, ByteWindow)} invocation to wrap a
- * SoftReference around the cached entity.</li>
+ * SoftReference or a StrongReference around the cached entity.</li>
* <li>For every Reference created by <code>createRef()</code> there will be
- * exactly one call to {@link #clear(Ref)} to cleanup any resources associated
+ * exactly one call to {@link #clear(PageRef)} to cleanup any resources associated
* with the (now expired) cached entity.</li>
* </ul>
* <p>
* Therefore, it is safe to perform resource accounting increments during the
* {@link #load(PackFile, long)} or
* {@link #createRef(PackFile, long, ByteWindow)} methods, and matching
- * decrements during {@link #clear(Ref)}. Implementors may need to override
+ * decrements during {@link #clear(PageRef)}. Implementors may need to override
* {@link #createRef(PackFile, long, ByteWindow)} in order to embed additional
* accounting information into an implementation specific
- * {@link org.eclipse.jgit.internal.storage.file.WindowCache.Ref} subclass, as
+ * {@link org.eclipse.jgit.internal.storage.file.WindowCache.PageRef} subclass, as
* the cached entity may have already been evicted by the JRE's garbage
* collector.
* <p>
@@ -176,18 +188,21 @@
/**
* Record files opened by cache
*
- * @param count
+ * @param delta
* delta of number of files opened by cache
*/
- void recordOpenFiles(int count);
+ void recordOpenFiles(int delta);
/**
* Record cached bytes
*
- * @param count
+ * @param pack
+ * pack file the bytes are read from
+ *
+ * @param delta
* delta of cached bytes
*/
- void recordOpenBytes(int count);
+ void recordOpenBytes(PackFile pack, int delta);
/**
* Returns a snapshot of this recorder's stats. Note that this may be an
@@ -209,6 +224,7 @@
private final LongAdder evictionCount;
private final LongAdder openFileCount;
private final LongAdder openByteCount;
+ private final Map<String, LongAdder> openByteCountPerRepository;
/**
* Constructs an instance with all counts initialized to zero.
@@ -222,6 +238,7 @@
evictionCount = new LongAdder();
openFileCount = new LongAdder();
openByteCount = new LongAdder();
+ openByteCountPerRepository = new ConcurrentHashMap<>();
}
@Override
@@ -252,13 +269,28 @@
}
@Override
- public void recordOpenFiles(int count) {
- openFileCount.add(count);
+ public void recordOpenFiles(int delta) {
+ openFileCount.add(delta);
}
@Override
- public void recordOpenBytes(int count) {
- openByteCount.add(count);
+ public void recordOpenBytes(PackFile pack, int delta) {
+ openByteCount.add(delta);
+ String repositoryId = repositoryId(pack);
+ LongAdder la = openByteCountPerRepository
+ .computeIfAbsent(repositoryId, k -> new LongAdder());
+ la.add(delta);
+ if (delta < 0) {
+ openByteCountPerRepository.computeIfPresent(repositoryId,
+ (k, v) -> v.longValue() == 0 ? null : v);
+ }
+ }
+
+ private static String repositoryId(PackFile pack) {
+ // use repository's gitdir since packfile doesn't know its
+ // repository
+ return pack.getPackFile().getParentFile().getParentFile()
+ .getParent();
}
@Override
@@ -315,6 +347,15 @@
totalLoadTime.reset();
evictionCount.reset();
}
+
+ @Override
+ public Map<String, Long> getOpenByteCountPerRepository() {
+ return Collections.unmodifiableMap(
+ openByteCountPerRepository.entrySet().stream()
+ .collect(Collectors.toMap(Map.Entry::getKey,
+ e -> Long.valueOf(e.getValue().sum()),
+ (u, v) -> v)));
+ }
}
private static final int bits(int newSize) {
@@ -390,8 +431,8 @@
cache.removeAll(pack);
}
- /** ReferenceQueue to cleanup released and garbage collected windows. */
- private final ReferenceQueue<ByteWindow> queue;
+ /** cleanup released and/or garbage collected windows. */
+ private final CleanupQueue queue;
/** Number of entries in {@link #table}. */
private final int tableSize;
@@ -425,6 +466,8 @@
private final StatsRecorderImpl mbean;
+ private boolean useStrongRefs;
+
private WindowCache(WindowCacheConfig cfg) {
tableSize = tableSize(cfg);
final int lockCount = lockCount(cfg);
@@ -433,7 +476,6 @@
if (lockCount < 1)
throw new IllegalArgumentException(JGitText.get().lockCountMustBeGreaterOrEqual1);
- queue = new ReferenceQueue<>();
clock = new AtomicLong(1);
table = new AtomicReferenceArray<>(tableSize);
locks = new Lock[lockCount];
@@ -455,6 +497,9 @@
mmap = cfg.isPackedGitMMAP();
windowSizeShift = bits(cfg.getPackedGitWindowSize());
windowSize = 1 << windowSizeShift;
+ useStrongRefs = cfg.isPackedGitUseStrongRefs();
+ queue = useStrongRefs ? new StrongCleanupQueue(this)
+ : new SoftCleanupQueue(this);
mbean = new StatsRecorderImpl();
statsRecorder = mbean;
@@ -503,16 +548,18 @@
}
}
- private Ref createRef(PackFile p, long o, ByteWindow v) {
- final Ref ref = new Ref(p, o, v, queue);
- statsRecorder.recordOpenBytes(ref.size);
+ private PageRef<ByteWindow> createRef(PackFile p, long o, ByteWindow v) {
+ final PageRef<ByteWindow> ref = useStrongRefs
+ ? new StrongRef(p, o, v, queue)
+ : new SoftRef(p, o, v, (SoftCleanupQueue) queue);
+ statsRecorder.recordOpenBytes(ref.getPack(), ref.getSize());
return ref;
}
- private void clear(Ref ref) {
- statsRecorder.recordOpenBytes(-ref.size);
+ private void clear(PageRef<ByteWindow> ref) {
+ statsRecorder.recordOpenBytes(ref.getPack(), -ref.getSize());
statsRecorder.recordEvictions(1);
- close(ref.pack);
+ close(ref.getPack());
}
private void close(PackFile pack) {
@@ -577,7 +624,7 @@
}
v = load(pack, position);
- final Ref ref = createRef(pack, position, v);
+ final PageRef<ByteWindow> ref = createRef(pack, position, v);
hit(ref);
for (;;) {
final Entry n = new Entry(clean(e2), ref);
@@ -601,8 +648,8 @@
private ByteWindow scan(Entry n, PackFile pack, long position) {
for (; n != null; n = n.next) {
- final Ref r = n.ref;
- if (r.pack == pack && r.position == position) {
+ final PageRef<ByteWindow> r = n.ref;
+ if (r.getPack() == pack && r.getPosition() == position) {
final ByteWindow v = r.get();
if (v != null) {
hit(r);
@@ -615,7 +662,7 @@
return null;
}
- private void hit(Ref r) {
+ private void hit(PageRef r) {
// We don't need to be 100% accurate here. Its sufficient that at least
// one thread performs the increment. Any other concurrent access at
// exactly the same time can simply use the same clock value.
@@ -625,7 +672,7 @@
//
final long c = clock.get();
clock.compareAndSet(c, c + 1);
- r.lastAccess = c;
+ r.setLastAccess(c);
}
private void evict() {
@@ -639,7 +686,8 @@
for (Entry e = table.get(ptr); e != null; e = e.next) {
if (e.dead)
continue;
- if (old == null || e.ref.lastAccess < old.ref.lastAccess) {
+ if (old == null || e.ref.getLastAccess() < old.ref
+ .getLastAccess()) {
old = e;
slot = ptr;
}
@@ -659,7 +707,7 @@
* <p>
* This is a last-ditch effort to clear out the cache, such as before it
* gets replaced by another cache that is configured differently. This
- * method tries to force every cached entry through {@link #clear(Ref)} to
+ * method tries to force every cached entry through {@link #clear(PageRef)} to
* ensure that resources are correctly accounted for and cleaned up by the
* subclass. A concurrent reader loading entries while this method is
* running may cause resource accounting failures.
@@ -692,7 +740,7 @@
final Entry e1 = table.get(s);
boolean hasDead = false;
for (Entry e = e1; e != null; e = e.next) {
- if (e.ref.pack == pack) {
+ if (e.ref.getPack() == pack) {
e.kill();
hasDead = true;
} else if (e.dead)
@@ -705,20 +753,7 @@
}
private void gc() {
- Ref r;
- while ((r = (Ref) queue.poll()) != null) {
- clear(r);
-
- final int s = slot(r.pack, r.position);
- final Entry e1 = table.get(s);
- for (Entry n = e1; n != null; n = n.next) {
- if (n.ref == r) {
- n.dead = true;
- table.compareAndSet(s, e1, clean(e1));
- break;
- }
- }
- }
+ queue.gc();
}
private int slot(PackFile pack, long position) {
@@ -731,7 +766,7 @@
private static Entry clean(Entry top) {
while (top != null && top.dead) {
- top.ref.enqueue();
+ top.ref.kill();
top = top.next;
}
if (top == null)
@@ -745,7 +780,7 @@
final Entry next;
/** The referenced object. */
- final Ref ref;
+ final PageRef<ByteWindow> ref;
/**
* Marked true when ref.get() returns null and the ref is dead.
@@ -756,34 +791,275 @@
*/
volatile boolean dead;
- Entry(Entry n, Ref r) {
+ Entry(Entry n, PageRef<ByteWindow> r) {
next = n;
ref = r;
}
final void kill() {
dead = true;
- ref.enqueue();
+ ref.kill();
}
}
+ private static interface PageRef<T> {
+ /**
+ * Returns this reference object's referent. If this reference object
+ * has been cleared, either by the program or by the garbage collector,
+ * then this method returns <code>null</code>.
+ *
+ * @return The object to which this reference refers, or
+ * <code>null</code> if this reference object has been cleared
+ */
+ T get();
+
+ /**
+ * Kill this ref
+ *
+ * @return <code>true</code> if this reference object was successfully
+ * killed; <code>false</code> if it was already killed
+ */
+ boolean kill();
+
+ /**
+ * Get the packfile the referenced cache page is allocated for
+ *
+ * @return the packfile the referenced cache page is allocated for
+ */
+ PackFile getPack();
+
+ /**
+ * Get the position of the referenced cache page in the packfile
+ *
+ * @return the position of the referenced cache page in the packfile
+ */
+ long getPosition();
+
+ /**
+ * Get size of cache page
+ *
+ * @return size of cache page
+ */
+ int getSize();
+
+ /**
+ * Get pseudo time of last access to this cache page
+ *
+ * @return pseudo time of last access to this cache page
+ */
+ long getLastAccess();
+
+ /**
+ * Set pseudo time of last access to this cache page
+ *
+ * @param time
+ * pseudo time of last access to this cache page
+ */
+ void setLastAccess(long time);
+
+ /**
+ * Whether this is a strong reference.
+ * @return {@code true} if this is a strong reference
+ */
+ boolean isStrongRef();
+ }
+
/** A soft reference wrapped around a cached object. */
- private static class Ref extends SoftReference<ByteWindow> {
- final PackFile pack;
+ private static class SoftRef extends SoftReference<ByteWindow>
+ implements PageRef<ByteWindow> {
+ private final PackFile pack;
- final long position;
+ private final long position;
- final int size;
+ private final int size;
- long lastAccess;
+ private long lastAccess;
- protected Ref(final PackFile pack, final long position,
- final ByteWindow v, final ReferenceQueue<ByteWindow> queue) {
+ protected SoftRef(final PackFile pack, final long position,
+ final ByteWindow v, final SoftCleanupQueue queue) {
super(v, queue);
this.pack = pack;
this.position = position;
this.size = v.size();
}
+
+ @Override
+ public PackFile getPack() {
+ return pack;
+ }
+
+ @Override
+ public long getPosition() {
+ return position;
+ }
+
+ @Override
+ public int getSize() {
+ return size;
+ }
+
+ @Override
+ public long getLastAccess() {
+ return lastAccess;
+ }
+
+ @Override
+ public void setLastAccess(long time) {
+ this.lastAccess = time;
+ }
+
+ @Override
+ public boolean kill() {
+ return enqueue();
+ }
+
+ @Override
+ public boolean isStrongRef() {
+ return false;
+ }
+ }
+
+ /** A strong reference wrapped around a cached object. */
+ private static class StrongRef implements PageRef<ByteWindow> {
+ private ByteWindow referent;
+
+ private final PackFile pack;
+
+ private final long position;
+
+ private final int size;
+
+ private long lastAccess;
+
+ private CleanupQueue queue;
+
+ protected StrongRef(final PackFile pack, final long position,
+ final ByteWindow v, final CleanupQueue queue) {
+ this.pack = pack;
+ this.position = position;
+ this.referent = v;
+ this.size = v.size();
+ this.queue = queue;
+ }
+
+ @Override
+ public PackFile getPack() {
+ return pack;
+ }
+
+ @Override
+ public long getPosition() {
+ return position;
+ }
+
+ @Override
+ public int getSize() {
+ return size;
+ }
+
+ @Override
+ public long getLastAccess() {
+ return lastAccess;
+ }
+
+ @Override
+ public void setLastAccess(long time) {
+ this.lastAccess = time;
+ }
+
+ @Override
+ public ByteWindow get() {
+ return referent;
+ }
+
+ @Override
+ public boolean kill() {
+ if (referent == null) {
+ return false;
+ }
+ referent = null;
+ return queue.enqueue(this);
+ }
+
+ @Override
+ public boolean isStrongRef() {
+ return true;
+ }
+ }
+
+ private static interface CleanupQueue {
+ boolean enqueue(PageRef<ByteWindow> r);
+ void gc();
+ }
+
+ private static class SoftCleanupQueue extends ReferenceQueue<ByteWindow>
+ implements CleanupQueue {
+ private final WindowCache wc;
+
+ SoftCleanupQueue(WindowCache cache) {
+ this.wc = cache;
+ }
+
+ @Override
+ public boolean enqueue(PageRef<ByteWindow> r) {
+ // no need to explicitly add soft references which are enqueued by
+ // the JVM
+ return false;
+ }
+
+ @Override
+ public void gc() {
+ SoftRef r;
+ while ((r = (SoftRef) poll()) != null) {
+ wc.clear(r);
+
+ final int s = wc.slot(r.getPack(), r.getPosition());
+ final Entry e1 = wc.table.get(s);
+ for (Entry n = e1; n != null; n = n.next) {
+ if (n.ref == r) {
+ n.dead = true;
+ wc.table.compareAndSet(s, e1, clean(e1));
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ private static class StrongCleanupQueue implements CleanupQueue {
+ private final WindowCache wc;
+
+ private final ConcurrentLinkedQueue<PageRef<ByteWindow>> queue = new ConcurrentLinkedQueue<>();
+
+ StrongCleanupQueue(WindowCache wc) {
+ this.wc = wc;
+ }
+
+ @Override
+ public boolean enqueue(PageRef<ByteWindow> r) {
+ if (queue.contains(r)) {
+ return false;
+ }
+ return queue.add(r);
+ }
+
+ @Override
+ public void gc() {
+ PageRef<ByteWindow> r;
+ while ((r = queue.poll()) != null) {
+ wc.clear(r);
+
+ final int s = wc.slot(r.getPack(), r.getPosition());
+ final Entry e1 = wc.table.get(s);
+ for (Entry n = e1; n != null; n = n.next) {
+ if (n.ref == r) {
+ n.dead = true;
+ wc.table.compareAndSet(s, e1, clean(e1));
+ break;
+ }
+ }
+ }
+ }
}
private static final class Lock {
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
index 068a8d8..078bf78 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
@@ -230,6 +230,36 @@
/** The "streamFileThreshold" key */
public static final String CONFIG_KEY_STREAM_FILE_TRESHOLD = "streamFileThreshold";
+ /**
+ * The "packedGitMmap" key
+ * @since 5.1.13
+ */
+ public static final String CONFIG_KEY_PACKED_GIT_MMAP = "packedgitmmap";
+
+ /**
+ * The "packedGitWindowSize" key
+ * @since 5.1.13
+ */
+ public static final String CONFIG_KEY_PACKED_GIT_WINDOWSIZE = "packedgitwindowsize";
+
+ /**
+ * The "packedGitLimit" key
+ * @since 5.1.13
+ */
+ public static final String CONFIG_KEY_PACKED_GIT_LIMIT = "packedgitlimit";
+
+ /**
+ * The "packedGitOpenFiles" key
+ * @since 5.1.13
+ */
+ public static final String CONFIG_KEY_PACKED_GIT_OPENFILES = "packedgitopenfiles";
+
+ /**
+ * The "packedGitUseStrongRefs" key
+ * @since 5.1.13
+ */
+ public static final String CONFIG_KEY_PACKED_GIT_USE_STRONGREFS = "packedgitusestrongrefs";
+
/** The "remote" key */
public static final String CONFIG_KEY_REMOTE = "remote";
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java
index ff49976..be19df8 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java
@@ -43,6 +43,15 @@
package org.eclipse.jgit.storage.file;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_DELTA_BASE_CACHE_LIMIT;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_LIMIT;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_MMAP;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_OPENFILES;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_WINDOWSIZE;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_FILE_TRESHOLD;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_USE_STRONGREFS;
+
import org.eclipse.jgit.internal.storage.file.WindowCache;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.pack.PackConfig;
@@ -61,6 +70,8 @@
private long packedGitLimit;
+ private boolean useStrongRefs;
+
private int packedGitWindowSize;
private boolean packedGitMMAP;
@@ -75,6 +86,7 @@
public WindowCacheConfig() {
packedGitOpenFiles = 128;
packedGitLimit = 10 * MB;
+ useStrongRefs = false;
packedGitWindowSize = 8 * KB;
packedGitMMAP = false;
deltaBaseCacheLimit = 10 * MB;
@@ -126,6 +138,31 @@
}
/**
+ * Get whether the window cache should use strong references or
+ * SoftReferences
+ *
+ * @return {@code true} if the window cache should use strong references,
+ * otherwise it will use {@link java.lang.ref.SoftReference}s
+ * @since 5.1.13
+ */
+ public boolean isPackedGitUseStrongRefs() {
+ return useStrongRefs;
+ }
+
+ /**
+ * Set if the cache should use strong refs or soft refs
+ *
+ * @param useStrongRefs
+ * if @{code true} the cache strongly references cache pages
+ * otherwise it uses {@link java.lang.ref.SoftReference}s which
+ * can be evicted by the Java gc if heap is almost full
+ * @since 5.1.13
+ */
+ public void setPackedGitUseStrongRefs(boolean useStrongRefs) {
+ this.useStrongRefs = useStrongRefs;
+ }
+
+ /**
* Get size in bytes of a single window mapped or read in from the pack
* file.
*
@@ -227,20 +264,23 @@
* @since 3.0
*/
public WindowCacheConfig fromConfig(Config rc) {
- setPackedGitOpenFiles(rc.getInt(
- "core", null, "packedgitopenfiles", getPackedGitOpenFiles())); //$NON-NLS-1$ //$NON-NLS-2$
- setPackedGitLimit(rc.getLong(
- "core", null, "packedgitlimit", getPackedGitLimit())); //$NON-NLS-1$ //$NON-NLS-2$
- setPackedGitWindowSize(rc.getInt(
- "core", null, "packedgitwindowsize", getPackedGitWindowSize())); //$NON-NLS-1$ //$NON-NLS-2$
- setPackedGitMMAP(rc.getBoolean(
- "core", null, "packedgitmmap", isPackedGitMMAP())); //$NON-NLS-1$ //$NON-NLS-2$
- setDeltaBaseCacheLimit(rc.getInt(
- "core", null, "deltabasecachelimit", getDeltaBaseCacheLimit())); //$NON-NLS-1$ //$NON-NLS-2$
+ setPackedGitUseStrongRefs(rc.getBoolean(CONFIG_CORE_SECTION,
+ CONFIG_KEY_PACKED_GIT_USE_STRONGREFS,
+ isPackedGitUseStrongRefs()));
+ setPackedGitOpenFiles(rc.getInt(CONFIG_CORE_SECTION, null,
+ CONFIG_KEY_PACKED_GIT_OPENFILES, getPackedGitOpenFiles()));
+ setPackedGitLimit(rc.getLong(CONFIG_CORE_SECTION, null,
+ CONFIG_KEY_PACKED_GIT_LIMIT, getPackedGitLimit()));
+ setPackedGitWindowSize(rc.getInt(CONFIG_CORE_SECTION, null,
+ CONFIG_KEY_PACKED_GIT_WINDOWSIZE, getPackedGitWindowSize()));
+ setPackedGitMMAP(rc.getBoolean(CONFIG_CORE_SECTION, null,
+ CONFIG_KEY_PACKED_GIT_MMAP, isPackedGitMMAP()));
+ setDeltaBaseCacheLimit(rc.getInt(CONFIG_CORE_SECTION, null,
+ CONFIG_KEY_DELTA_BASE_CACHE_LIMIT, getDeltaBaseCacheLimit()));
long maxMem = Runtime.getRuntime().maxMemory();
- long sft = rc.getLong(
- "core", null, "streamfilethreshold", getStreamFileThreshold()); //$NON-NLS-1$ //$NON-NLS-2$
+ long sft = rc.getLong(CONFIG_CORE_SECTION, null,
+ CONFIG_KEY_STREAM_FILE_TRESHOLD, getStreamFileThreshold());
sft = Math.min(sft, maxMem / 4); // don't use more than 1/4 of the heap
sft = Math.min(sft, Integer.MAX_VALUE); // cannot exceed array length
setStreamFileThreshold((int) sft);
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java
index b7f6394..65f8dae 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java
@@ -42,6 +42,8 @@
*/
package org.eclipse.jgit.storage.file;
+import java.util.Map;
+
import javax.management.MXBean;
import org.eclipse.jgit.internal.storage.file.WindowCache;
@@ -226,6 +228,13 @@
long getOpenByteCount();
/**
+ * Number of bytes cached per repository
+ *
+ * @return number of bytes cached per repository
+ */
+ Map<String, Long> getOpenByteCountPerRepository();
+
+ /**
* Reset counters. Does not reset open bytes and open files counters.
*/
void resetCounters();
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/AbstractTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/AbstractTreeIterator.java
index 335abe1..a3c93cf 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/AbstractTreeIterator.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/AbstractTreeIterator.java
@@ -239,12 +239,10 @@
path = p.path;
pathOffset = p.pathLen + 1;
- try {
- path[pathOffset - 1] = '/';
- } catch (ArrayIndexOutOfBoundsException e) {
+ if (pathOffset > path.length) {
growPath(p.pathLen);
- path[pathOffset - 1] = '/';
}
+ path[pathOffset - 1] = '/';
}
/**
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/CanonicalTreeParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/CanonicalTreeParser.java
index 0199688..b2d8fc3 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/CanonicalTreeParser.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/CanonicalTreeParser.java
@@ -387,14 +387,13 @@
tmp = pathOffset;
for (;; tmp++) {
c = raw[ptr++];
- if (c == 0)
+ if (c == 0) {
break;
- try {
- path[tmp] = c;
- } catch (ArrayIndexOutOfBoundsException e) {
- growPath(tmp);
- path[tmp] = c;
}
+ if (tmp >= path.length) {
+ growPath(tmp);
+ }
+ path[tmp] = c;
}
pathLen = tmp;
nextPtr = ptr + OBJECT_ID_LENGTH;
