show-caches: Improve memory reporting
The way memory was reported was almost useless. Change it to show
the actual values, and the equation that determines how these are
put together to form the current usage.
Include some additional data including server version, current time,
process uptime, active SSH connections, and tasks in the task queue.
The --show-jvm option will report additional data about the JVM,
and tell the caller where it is running.
Change-Id: I80af3cc1d00ef13c985ade3d5faaab792554bac6
diff --git a/Documentation/cmd-show-caches.txt b/Documentation/cmd-show-caches.txt
index 997f9f62..a841fc1 100644
--- a/Documentation/cmd-show-caches.txt
+++ b/Documentation/cmd-show-caches.txt
@@ -8,12 +8,23 @@
SYNOPSIS
--------
[verse]
-'ssh' -p <port> <host> 'gerrit show-caches'
+'ssh' -p <port> <host> 'gerrit show-caches' [--gc] [--show-jvm]
DESCRIPTION
-----------
Display statistics about the size and hit ratio of in-memory caches.
+OPTIONS
+-------
+--gc::
+ Request Java garbage collection before displaying information
+ about the Java memory heap.
+
+--show-jvm::
+ List the name and version of the Java virtual machine, host
+ operating system, and other details about the environment
+ that Gerrit Code Review is running in.
+
ACCESS
------
Caller must be a member of the privileged 'Administrators' group,
@@ -28,27 +39,33 @@
====
$ ssh -p 29418 review.example.com gerrit show-caches
+ Gerrit Code Review 2.2.2 now 10:03:34 PDT
+ uptime 1 min 39 sec
+
Name Max |Object Count | AvgGet |Hit Ratio |
Age | Disk Mem Cnt| |Disk Mem Agg |
-------------------------+--------------------+----------+--------------+
- accounts 90d | 295| | 99%|
- accounts_byemail 90d | 109| | 97%|
- D diff 90d | 2695 128 2707| 0.4ms | 11% 86% 98%|
- groups 90d | 94| | 80%|
- openid 5m | 30| 0.4ms | 9%|
- projects 90d | 188| | 99%|
- sshkeys 90d | 9| | 94%|
- D web_sessions 12h | 30 30| | 0% 99% 99%|
-
- JGit Buffer Cache:
- open files : 23
- loaded : 6.82 mb
- mem% : 2%
-
- JVM Heap:
- max : 880.00 mb
- inuse : 136.57 mb
- mem% : 44%
+ accounts 90d | 1| | 95%|
+ accounts_byemail 90d | | | |
+ accounts_byname 90d | 1| | |
+ adv_bases 10m | | | |
+ D diff 90d | 8 8| | |
+ D diff_intraline 90d | 1 1| | |
+ groups 90d | 19| | 0%|
+ groups_byext 90d | | | |
+ groups_byinclude 90d | 21| | 80%|
+ groups_byname 90d | | | |
+ groups_byuuid 90d | | | |
+ project_list 90d | | | |
+ projects 90d | 1| | 80%|
+ sshkeys 90d | 1| | 90%|
+ D web_sessions 12h | | | |
+
+ SSH: 1 users, oldest session started 782 ms ago
+ Tasks: 2 total = 1 running + 0 ready + 1 sleeping
+ Mem: 46.13m total = 16.17m used + 29.96m free + 0.00k buffers
+ 246.56m max
+ 0 open files, 6 cpus available, 23 threads
====
SEE ALSO
diff --git a/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/DefaultCommandModule.java b/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/DefaultCommandModule.java
index a7e3214..a369b86 100644
--- a/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/DefaultCommandModule.java
+++ b/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/DefaultCommandModule.java
@@ -14,6 +14,7 @@
package com.google.gerrit.sshd.commands;
+import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.sshd.CommandModule;
import com.google.gerrit.sshd.CommandName;
import com.google.gerrit.sshd.Commands;
@@ -58,5 +59,12 @@
command("gerrit-receive-pack").to(Commands.key(git, "receive-pack"));
command("suexec").to(SuExec.class);
+
+ install(new LifecycleModule() {
+ @Override
+ protected void configure() {
+ listener().to(ShowCaches.StartupListener.class);
+ }
+ });
}
}
diff --git a/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/ShowCaches.java b/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/ShowCaches.java
index 28676e5..7dadf83 100644
--- a/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/ShowCaches.java
+++ b/gerrit-sshd/src/main/java/com/google/gerrit/sshd/commands/ShowCaches.java
@@ -14,23 +14,71 @@
package com.google.gerrit.sshd.commands;
+import com.google.gerrit.common.Version;
+import com.google.gerrit.lifecycle.LifecycleListener;
import com.google.gerrit.server.IdentifiedUser;
+import com.google.gerrit.server.config.SitePath;
+import com.google.gerrit.server.git.WorkQueue;
+import com.google.gerrit.server.git.WorkQueue.Task;
import com.google.gerrit.sshd.BaseCommand;
+import com.google.gerrit.sshd.SshDaemon;
import com.google.inject.Inject;
import net.sf.ehcache.Ehcache;
import net.sf.ehcache.Statistics;
import net.sf.ehcache.config.CacheConfiguration;
+import org.apache.mina.core.service.IoAcceptor;
+import org.apache.mina.core.session.IoSession;
import org.apache.sshd.server.Environment;
import org.eclipse.jgit.storage.file.WindowCacheStatAccessor;
+import org.kohsuke.args4j.Option;
+import java.io.File;
+import java.io.IOException;
import java.io.PrintWriter;
+import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
+import java.lang.management.RuntimeMXBean;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.text.SimpleDateFormat;
+import java.util.Collection;
+import java.util.Date;
/** Show the current cache states. */
final class ShowCaches extends CacheCommand {
+ private static volatile long serverStarted;
+
+ static class StartupListener implements LifecycleListener {
+ @Override
+ public void start() {
+ serverStarted = System.currentTimeMillis();
+ }
+
+ @Override
+ public void stop() {
+ }
+ }
+
+ @Option(name = "--gc", usage = "perform Java GC before printing memory stats")
+ private boolean gc;
+
+ @Option(name = "--show-jvm", usage = "show details about the JVM")
+ private boolean showJVM;
+
@Inject
- IdentifiedUser currentUser;
+ private IdentifiedUser currentUser;
+
+ @Inject
+ private WorkQueue workQueue;
+
+ @Inject
+ private SshDaemon daemon;
+
+ @Inject
+ @SitePath
+ private File sitePath;
private PrintWriter p;
@@ -55,6 +103,18 @@
private void display() {
p = toPrintWriter(out);
+ Date now = new Date();
+ p.format(
+ "%-25s %-20s now %16s\n",
+ "Gerrit Code Review",
+ Version.getVersion() != null ? Version.getVersion() : "",
+ new SimpleDateFormat("HH:mm:ss zzz").format(now));
+ p.format(
+ "%-25s %-20s uptime %16s\n",
+ "", "",
+ uptime(now.getTime() - serverStarted));
+ p.print('\n');
+
p.print(String.format(//
"%1s %-18s %-4s|%-20s| %-5s |%-14s|\n" //
, "" //
@@ -77,8 +137,8 @@
, "Mem" //
, "Agg" //
));
- p.println("------------------"
- + "-------+--------------------+----------+--------------+");
+ p.print("------------------"
+ + "-------+--------------------+----------+--------------+\n");
for (final Ehcache cache : getAllCaches()) {
final CacheConfiguration cfg = cache.getCacheConfiguration();
final boolean useDisk = cfg.isDiskPersistent() || cfg.isOverflowToDisk();
@@ -111,49 +171,152 @@
));
}
}
- p.println();
+ p.print('\n');
+ if (gc) {
+ System.gc();
+ System.runFinalization();
+ System.gc();
+ }
+
+ sshSummary();
+ taskSummary();
+ memSummary();
+
+ if (showJVM) {
+ jvmSummary();
+ }
+
+ p.flush();
+ }
+
+ private void memSummary() {
final Runtime r = Runtime.getRuntime();
final long mMax = r.maxMemory();
final long mFree = r.freeMemory();
final long mTotal = r.totalMemory();
final long mInuse = mTotal - mFree;
+
+ final int jgitOpen = WindowCacheStatAccessor.getOpenFiles();
final long jgitBytes = WindowCacheStatAccessor.getOpenBytes();
- p.println("JGit Buffer Cache:");
- fItemCount("open files", WindowCacheStatAccessor.getOpenFiles());
- fByteCount("loaded", jgitBytes);
- fPercent("mem%", jgitBytes, mTotal);
- p.println();
-
- p.println("JVM Heap:");
- fByteCount("max", mMax);
- fByteCount("inuse", mInuse);
- fPercent("mem%", mInuse, mTotal);
- p.println();
-
- p.flush();
+ p.format("Mem: %s total = %s used + %s free + %s buffers\n",
+ bytes(mTotal),
+ bytes(mInuse - jgitBytes),
+ bytes(mFree),
+ bytes(jgitBytes));
+ p.format(" %s max\n", bytes(mMax));
+ p.format(" %8d open files, %8d cpus available, %8d threads\n",
+ jgitOpen,
+ r.availableProcessors(),
+ ManagementFactory.getThreadMXBean().getThreadCount());
+ p.print('\n');
}
- private void fItemCount(final String name, final long value) {
- p.println(String.format(" %1$-12s: %2$15d", name, value));
+ private void taskSummary() {
+ Collection<Task<?>> pending = workQueue.getTasks();
+ int tasksTotal = pending.size();
+ int tasksRunning = 0, tasksReady = 0, tasksSleeping = 0;
+ for (Task<?> task : pending) {
+ switch (task.getState()) {
+ case RUNNING: tasksRunning++; break;
+ case READY: tasksReady++; break;
+ case SLEEPING: tasksSleeping++; break;
+ }
+ }
+ p.format(
+ "Tasks: %4d total = %4d running + %4d ready + %4d sleeping\n",
+ tasksTotal,
+ tasksRunning,
+ tasksReady,
+ tasksSleeping);
}
- private void fByteCount(final String name, double value) {
- String suffix = "bytes";
+ private void sshSummary() {
+ IoAcceptor acceptor = daemon.getIoAcceptor();
+ if (acceptor == null) {
+ return;
+ }
+
+ long now = System.currentTimeMillis();
+ Collection<IoSession> list = acceptor.getManagedSessions().values();
+ long oldest = now;
+ double writeThroughput = 0.0;
+ for (IoSession s : list) {
+ oldest = Math.min(oldest, s.getCreationTime());
+ }
+
+ p.format(
+ "SSH: %4d users, oldest session started %s ago\n",
+ list.size(),
+ uptime(now - oldest));
+ }
+
+ private void jvmSummary() {
+ OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean();
+ RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean();
+ p.format("JVM: %s %s %s\n",
+ runtimeBean.getVmVendor(),
+ runtimeBean.getVmName(),
+ runtimeBean.getVmVersion());
+ p.format(" on %s %s %s\n", "",
+ osBean.getName(),
+ osBean.getVersion(),
+ osBean.getArch());
+ try {
+ p.format(" running as %s on %s\n",
+ System.getProperty("user.name"),
+ InetAddress.getLocalHost().getHostName());
+ } catch (UnknownHostException e) {
+ }
+ p.format(" cwd %s\n", path(new File(".").getAbsoluteFile().getParentFile()));
+ p.format(" site %s\n", path(sitePath));
+ }
+
+ private String path(File file) {
+ try {
+ return file.getCanonicalPath();
+ } catch (IOException err) {
+ return file.getAbsolutePath();
+ }
+ }
+
+ private String uptime(long uptimeMillis) {
+ if (uptimeMillis < 1000) {
+ return String.format("%3d ms", uptimeMillis);
+ }
+
+ long uptime = uptimeMillis / 1000L;
+
+ long min = uptime / 60;
+ if (min < 60) {
+ return String.format("%2d min %2d sec", min, uptime - min * 60);
+ }
+
+ long hr = uptime / 3600;
+ if (hr < 24) {
+ min = (uptime - hr * 3600) / 60;
+ return String.format("%2d hrs %2d min", hr, min);
+ }
+
+ long days = uptime / (24 * 3600);
+ hr = (uptime - (days * 24 * 3600)) / 3600;
+ return String.format("%4d days %2d hrs", days, hr);
+ }
+
+ private String bytes(double value) {
+ value /= 1024;
+ String suffix = "k";
+
if (value > 1024) {
value /= 1024;
- suffix = "kb";
+ suffix = "m";
}
if (value > 1024) {
value /= 1024;
- suffix = "mb";
+ suffix = "g";
}
- if (value > 1024) {
- value /= 1024;
- suffix = "gb";
- }
- p.println(String.format(" %1$-12s: %2$6.2f %3$s", name, value, suffix));
+ return String.format("%1$6.2f%2$s", value, suffix);
}
private String count(long cnt) {
@@ -211,9 +374,4 @@
final long pcent = (100 * value) / total;
return String.format("%3d%%", (int) pcent);
}
-
- private void fPercent(final String name, final long value, final long total) {
- final long pcent = 0 < total ? (100 * value) / total : 0;
- p.println(String.format(" %1$-12s: %2$3d%%", name, (int) pcent));
- }
}
