Remove stale packed-refs.lock files

When a packed-refs.lock is stale for a long period of
time (configurable) then do not just display a warning
and metric but also remove the lock file from the filesystem.

Change-Id: Icfb24e964be5659362d4d1cde52f6042ca4a8b82

admin/stale-packed-refs-1.0.groovy

diff --git a/admin/stale-packed-refs-1.0.groovy b/admin/stale-packed-refs-1.0.groovy
index 9065998..c3aecdc 100644
--- a/admin/stale-packed-refs-1.0.groovy
+++ b/admin/stale-packed-refs-1.0.groovy
@@ -8,6 +8,7 @@
 import com.google.gerrit.metrics.Description
 import com.google.gerrit.metrics.Field
 import com.google.gerrit.metrics.MetricMaker
+import com.google.gerrit.server.config.ConfigUtil
 import com.google.gerrit.server.config.PluginConfig
 import com.google.gerrit.server.config.PluginConfigFactory
 import com.google.gerrit.server.git.GitRepositoryManager
@@ -45,6 +46,7 @@
 
   private ScheduledFuture<?> scheduledCheckerTask
   private String[] projectPrefixes
+  private long removeAfter
 
   def CHECK_INTERVAL_SEC_DEFAULT = 10
 
@@ -59,10 +61,12 @@
     PluginConfig pluginConfig = configFactory.getFromGerritConfig(pluginName)
     def checkInterval = pluginConfig.getInt("checkIntervalSec", CHECK_INTERVAL_SEC_DEFAULT)
     projectPrefixes = pluginConfig.getStringList("projectPrefix")
+    def staleRemovalTime = pluginConfig.getString("removeAfter")
+    removeAfter = staleRemovalTime ? ConfigUtil.getTimeUnit(staleRemovalTime, Long.MAX_VALUE, TimeUnit.MILLISECONDS) : Long.MAX_VALUE
 
     scheduledCheckerTask = workQueue.getDefaultQueue().scheduleAtFixedRate({ checkProjects() }, checkInterval, checkInterval, TimeUnit.SECONDS)
     logger.atInfo().log("packed-refs.lock staleness checker started for %d projects (checkIntervalSec=%d, projectPrefix=%s)",
-        allProjectsToCheck().size(), checkInterval, Arrays.copyOf(projectPrefixes, projectPrefixes.length))
+     allProjectsToCheck().size(), checkInterval, Arrays.copyOf(projectPrefixes, projectPrefixes.length))
   }
 
   private def allProjectsToCheck() {
@@ -86,7 +90,7 @@
       allProjectsToCheck().each { String projectName ->
         repoMgr.openRepository(Project.nameKey(projectName)).with { Repository it ->
           try {
-            recordLockFileAgeMetric(it, projectName)
+            recordLockFileAgeMetricAndRemoveIfStale(it, projectName)
           } catch (Exception e) {
             logger.atSevere().withCause(e).log("Error whilst checking project %s", projectName)
           }
@@ -97,7 +101,7 @@
     }
   }
 
-  private void recordLockFileAgeMetric(Repository repo, String projectName) {
+  private void recordLockFileAgeMetricAndRemoveIfStale(Repository repo, String projectName) {
     def repoDir = repo.getDirectory()
     logger.atFine().log("Checking project %s ... ", projectName)
     File packedRefsLock = new File(repoDir, "packed-refs.lock")
@@ -110,7 +114,14 @@
     def packedRefsLockMillis = FileUtils.lastModified(packedRefsLock.getAbsolutePath())
     def lockFileAge = System.currentTimeMillis() - packedRefsLockMillis
     refdbMetrics.projectsAndLockFileAge.put(sanitizeProjectName.sanitize(projectName), lockFileAge)
-    logger.atFine().log("[%s] calculated age for lock file (creationMillis=%d)", projectName, lockFileAge)
+
+    if (lockFileAge > removeAfter) {
+      def deleteOk = packedRefsLock.delete()
+      logger.atWarning().log("[%s] %s stale lock file (creationMillis=%d, ageMillis=%d): ",
+              projectName, deleteOk ? "deleted" : "*FAILED* to delete", packedRefsLockMillis, lockFileAge)
+    } else {
+      logger.atFine().log("[%s] calculated age for lock file (creationMillis=%d, ageMillis=%d)", projectName, packedRefsLockMillis, lockFileAge)
+    }
   }
 }