shards: a progress message every 10s when loading

A zoekt-webserver does not start listening until it has loaded all
shards. In some cases (eg misconfigured IO) the startup time can be very
large. This commit adds a progress message so admins can understand what
is going on.

Change-Id: I997543ce4204f1de48f4071f635668b70c299bb4

shards/watcher.go

diff --git a/shards/watcher.go b/shards/watcher.go
index 4bdb8c1..4a7dc72 100644
--- a/shards/watcher.go
+++ b/shards/watcher.go
@@ -107,14 +107,30 @@
 		}
 	}
 
+	if len(toDrop) > 0 {
+		log.Printf("unloading %d shards", len(toDrop))
+	}
 	for _, t := range toDrop {
 		log.Printf("unloading: %s", t)
 		s.loader.drop(t)
 	}
 
+	if len(toLoad) == 0 {
+		return nil
+	}
+
+	log.Printf("loading %d shards", len(toLoad))
+
 	// Limit amount of concurrent shard loads.
 	throttle := make(chan struct{}, runtime.GOMAXPROCS(0))
-	for _, t := range toLoad {
+	lastProgress := time.Now()
+	for i, t := range toLoad {
+		// If taking a while to start-up occasionally give a progress message
+		if time.Since(lastProgress) > 10*time.Second {
+			log.Printf("still need to load %d shards...", len(toLoad)-i)
+			lastProgress = time.Now()
+		}
+
 		throttle <- struct{}{}
 		go func(k string) {
 			s.loader.load(k)