Schedule groups-backends in a queue
When loading groups from an auth backend (for example LDAP), Gerrit is
subject to its latency. This might be a problem because a slow backend,
or a very large number of groups, or both, might cause the warm-cache
command to last longer than the allowed timeouts imposed by Gerrit (or a
load balancer that fronts it).
Schedule the loading of the cache asynchronously over a new dedicated
queue named: "Groups-Backend-Cache-Warmer".
This allows to terminate the SSH command in a timely manner, while
loading the cache in the background.
The status of the cache loading can be observed via the `show-queue`
command as well as the `queue/groups_backend_cache_warmer/*` metrics.
Currently the size of the queue is not configurable and it is set to 8
threads.
Change-Id: Iccbb1f05bcf04c540a68e1d70cd6be9078a50815
diff --git a/admin/warm-cache-1.0.groovy b/admin/warm-cache-1.0.groovy
index 88a8148..c3c4892 100644
--- a/admin/warm-cache-1.0.groovy
+++ b/admin/warm-cache-1.0.groovy
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+import com.google.common.flogger.FluentLogger
import com.google.gerrit.common.data.GlobalCapability
import com.google.gerrit.sshd.*
import com.google.gerrit.extensions.annotations.*
@@ -20,9 +21,14 @@
import com.google.gerrit.server.IdentifiedUser
import com.google.gerrit.reviewdb.client.AccountGroup
import com.google.inject.*
-import org.kohsuke.args4j.*
+import com.google.gerrit.server.git.WorkQueue
+import org.apache.sshd.server.Environment
+
+import java.util.concurrent.ExecutorService
abstract class BaseSshCommand extends SshCommand {
+ protected static final FluentLogger logger = FluentLogger.forEnclosingClass()
+
void println(String msg) {
stdout.println msg
@@ -142,32 +148,70 @@
@Export("groups-backends")
@RequiresCapability(GlobalCapability.ADMINISTRATE_SERVER)
class WarmGroupsBackendsCache extends WarmAccountsCache {
+ private static final THREAD_POOL_SIZE = 8
+ private static final QUEUE_NAME = "Groups-Backend-Cache-Warmer"
@Inject
IdentifiedUser.GenericFactory userFactory
- public void run() {
- println "Loading groups ..."
+ @Inject WorkQueue queues
+
+ private static class GroupsBackendsTask implements Runnable {
+ IdentifiedUser user
+
+ GroupsBackendsTask(IdentifiedUser identifiedUser) {
+ user = identifiedUser
+ }
+
+ @Override
+ void run() {
+ def threadStart = System.currentTimeMillis()
+ def groupsUUIDs = user.getEffectiveGroups()?.getKnownGroups()
+ def threadElapsed = (System.currentTimeMillis() - threadStart)
+ logger.atInfo().log("Loaded %d groups for account %d in %s millis", groupsUUIDs.size(), user.getAccountId().get(), threadElapsed)
+ }
+
+ @Override
+ String toString() {
+ return "Warmup backend groups [accountId: ${user.getAccountId().get()}]"
+ }
+ }
+
+ ExecutorService executorService
+
+ private ExecutorService executor() {
+ def existingExecutor = queues.getExecutor(QUEUE_NAME)
+ if(existingExecutor != null) {
+ return existingExecutor
+ }
+ return queues.createQueue(THREAD_POOL_SIZE, QUEUE_NAME, true);
+ }
+
+ @Override
+ void start(Environment env) throws IOException {
+ super.start(env)
+ executorService = executor()
+ }
+
+ void run() {
+ println "Scheduling backend groups loading ..."
def start = System.currentTimeMillis()
- def loaded = 0
- def allGroupsUUIDs = new HashSet<AccountGroup.UUID>()
+ def scheduled = 0
def lastDisplay = 0
for (accountId in accounts.allIds()) {
- def user = userFactory.create(accountId)
- def groupsUUIDs = user?.getEffectiveGroups()?.getKnownGroups()
- if (groupsUUIDs != null) { allGroupsUUIDs.addAll(groupsUUIDs) }
+ scheduled++
+ executorService.submit(new GroupsBackendsTask(userFactory.create(accountId)))
- loaded = allGroupsUUIDs.size()
- if (loaded.intdiv(1000) > lastDisplay) {
- println "$loaded groups"
- lastDisplay = loaded.intdiv(1000)
+ if (scheduled.intdiv(1000) > lastDisplay) {
+ println "Scheduled loading of groups for $scheduled accounts"
+ lastDisplay = scheduled.intdiv(1000)
}
}
- def elapsed = (System.currentTimeMillis()-start)/1000
- println "$loaded groups loaded in $elapsed secs"
+ def elapsed = (System.currentTimeMillis() - start) / 1000
+ println "Scheduled loading of groups for $scheduled accounts in $elapsed secs"
}
}
