Add metric for sum of SHA1s

It is not possible to have a metric with a non-numeric value, so we
calculate the numeric representation of the SHA1. However, SHA1s are
too big to fit with a numeric data type, so a truncated version of the
numeric value is calculated.

The numerical calculation was taken from [1].

[1] http://www.java2s.com/example/java-utility-method/sha1/sha1hashint-string-text-d6c0e.html

Change-Id: I09bb9bda8f6e5dc655aa5b9db9bee120cc47ce51

multi-primary/localrefdb.groovy

diff --git a/multi-primary/localrefdb.groovy b/multi-primary/localrefdb.groovy
index b5d1632..fb73f89 100644
--- a/multi-primary/localrefdb.groovy
+++ b/multi-primary/localrefdb.groovy
@@ -29,15 +29,19 @@
 import java.util.concurrent.*
 import com.google.common.flogger.*
 import java.util.regex.Pattern
+import java.security.MessageDigest
 
 abstract class BaseSshCommand extends SshCommand {
-  void println(String msg) {
-    stdout.println msg
+  void println(String msg, boolean verbose = false) {
+    if (verbose) {
+      stdout.println msg
+    }
     stdout.flush()
   }
 
-  void error(String msg) { println("[ERROR] $msg") }
-  void warning(String msg) { println("[WARNING] $msg") }
+  void error(String msg) { println("[ERROR] $msg", true) }
+
+  void warning(String msg) { println("[WARNING] $msg", true) }
 }
 
 @Singleton
@@ -49,32 +53,51 @@
 
   CallbackMetric1<String, Integer> numRefsMetric
   final Map<String, Integer> projectsAndNumRefs = new ConcurrentHashMap()
+  CallbackMetric1<String, Integer> sha1AllRefsMetric
+  final Map<String, Integer> projectsAndSha1AllRefs = new ConcurrentHashMap()
 
-  void start() {
-    numRefsMetric =
-        metrics.newCallbackMetric(
-            "localrefdb/num_refs_per_project",
-            Integer.class,
-            new Description("Number of local refs").setGauge(),
-            Field.ofString("repository_name", { it.projectName } as BiConsumer)
-                .description("The name of the repository.")
-                .build())
-
+  void setupTrigger(CallbackMetric1<String, Integer> refsMetric, Map<String, Integer> projecsAndRefs) {
     metrics.newTrigger(
-        numRefsMetric, { ->
-      if (projectsAndNumRefs.isEmpty()) {
-        numRefsMetric.forceCreate("")
+        refsMetric, { ->
+      if (projecsAndRefs.isEmpty()) {
+        refsMetric.forceCreate("")
       } else {
-        projectsAndNumRefs.each { e ->
-          numRefsMetric.set(e.key, e.value)
+        projecsAndRefs.each { e ->
+          refsMetric.set(e.key, e.value)
         }
-        numRefsMetric.prune()
+        refsMetric.prune()
       }
     })
   }
 
+  CallbackMetric1<String, Integer> createCallbackMetric(String name, String description) {
+    metrics.newCallbackMetric(
+        name,
+        Integer.class,
+        new Description(description).setGauge(),
+        Field.ofString("repository_name", { it.projectName } as BiConsumer)
+            .description(description)
+            .build())
+  }
+
+  void start() {
+    numRefsMetric = createCallbackMetric("localrefdb/num_refs_per_project", "Number of local refs")
+
+    setupTrigger(numRefsMetric, projectsAndNumRefs)
+
+
+    sha1AllRefsMetric = createCallbackMetric("localrefdb/sha1_all_refs_per_project", "A SHA1 computed from combining all SHA1s of the repository.")
+
+    setupTrigger(sha1AllRefsMetric, projectsAndSha1AllRefs)
+  }
+
+  def listRefs(Repository repo) {
+    repo.refDatabase.refs.findAll { ref -> !(ref.name.startsWith("refs/users/.*")) && !ref.symbolic }
+  }
+
   void stop() {
     numRefsMetric.remove()
+    sha1AllRefsMetric.remove()
   }
 }
 
@@ -85,6 +108,9 @@
   @Argument(index = 0, usage = "Project name", metaVar = "PROJECTS", required = true)
   String[] projects
 
+  @Option(name = "--verbose", usage = "Display verbose logging")
+  boolean verbose = false
+
   @Inject
   GitRepositoryManager repoMgr
 
@@ -98,11 +124,10 @@
     projects.each { project ->
       try {
         def projectName = Project.nameKey(project)
-
         repoMgr.openRepository(projectName).with { repo ->
-          println "Counting refs of project $project ..."
-          def filteredRefs = repo.refDatabase.refs.findAll { ref -> !(ref.name.startsWith("refs/users/.*")) && !ref.symbolic }
-          println "Result: $project has ${filteredRefs.size()} refs"
+          println("Counting refs of project $project ...", verbose)
+          def filteredRefs = refdbMetrics.listRefs(repo)
+          println("Result: $project has ${filteredRefs.size()} refs", true)
           refdbMetrics.projectsAndNumRefs.put(sanitizeProjectName.sanitize(project), filteredRefs.size())
         }
       } catch (RepositoryNotFoundException e) {
@@ -112,6 +137,60 @@
   }
 }
 
+@CommandMetaData(name = "sha1-all-refs", description = "Combine all the local refs, excluding user edits, and publish the value as 'sha1_all_refs_per_project' metric")
+@RequiresCapability(GlobalCapability.ADMINISTRATE_SERVER)
+class Sha1AllRefs extends BaseSshCommand {
+
+  @Argument(index = 0, usage = "Project names", metaVar = "PROJECT", required = true)
+  String[] projects
+
+  @Option(name = "--verbose", usage = "Display verbose output")
+  boolean verbose = false
+
+  @Inject
+  GitRepositoryManager repoMgr
+
+  @Inject
+  RefDbMetrics refdbMetrics
+
+  SanitizeProjectName sanitizeProjectName = new SanitizeProjectName()
+
+  public void run() {
+    refdbMetrics.projectsAndSha1AllRefs.clear()
+    projects.each { project ->
+      try {
+        def projectName = Project.nameKey(project)
+
+        repoMgr.openRepository(projectName).with { repo ->
+          def startTime = System.currentTimeMillis()
+          println("Adding refs of project $project ...", verbose)
+          def filteredRefs = refdbMetrics.listRefs(repo)
+          println("Result: $project has ${filteredRefs.size()} refs", verbose)
+          def md = MessageDigest.getInstance("SHA-1")
+          def sortingStartTime = System.currentTimeMillis()
+          def sortedFilteredRefs = filteredRefs.sort { it.name }
+          println("Sorting refs took ${System.currentTimeMillis() - sortingStartTime} millis", verbose)
+          sortedFilteredRefs.each { ref -> md.update(ref.getObjectId().toString().getBytes("UTF-8")) }
+          def sha1SumBytes = md.digest()
+          println("MD Digest of sum of all SHA1 for project $project is: ${sha1SumBytes.encodeBase64().toString()}", true)
+          def sha1Sum = truncateHashToInt(sha1SumBytes)
+          println("Truncated Int representation of sum of all SHA1 for project $project is: $sha1Sum", verbose)
+          println("Whole operation too ${System.currentTimeMillis() - startTime} millis", verbose)
+          refdbMetrics.projectsAndSha1AllRefs.put(sanitizeProjectName.sanitize(project), sha1Sum)
+        }
+      } catch (RepositoryNotFoundException e) {
+        error "Project $project not found"
+      }
+    }
+  }
+
+  // Source http://www.java2s.com/example/java-utility-method/sha1/sha1hashint-string-text-d6c0e.html
+  static int truncateHashToInt(byte[] bytes) {
+    int offset = bytes[bytes.length - 1] & 0x0f;
+    return (bytes[offset] & (0x7f << 24)) | (bytes[offset + 1] & (0xff << 16)) | (bytes[offset + 2] & (0xff << 8)) | (bytes[offset + 3] & 0xff);
+  }
+}
+
 class MetricsModule extends LifecycleModule {
   protected void configure() {
     listener().to(RefDbMetrics)
@@ -121,6 +200,7 @@
 class LocalRefDbCommandModule extends PluginCommandModule {
   protected void configureCommands() {
     command(CountRefs)
+    command(Sha1AllRefs)
   }
 }