java/com/google/gerrit/server/account/GroupIncludeCacheImpl.java - gerrit.git - Git at Google

 // Copyright (C) 2011 The Android Open Source Project
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 // http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package com.google.gerrit.server.account;

 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;

 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import com.google.common.flogger.FluentLogger;
 import com.google.gerrit.entities.Account;
 import com.google.gerrit.entities.AccountGroup;
 import com.google.gerrit.entities.InternalGroup;
 import com.google.gerrit.proto.Protos;
 import com.google.gerrit.server.cache.CacheModule;
 import com.google.gerrit.server.cache.proto.Cache.AllExternalGroupsProto;
 import com.google.gerrit.server.cache.proto.Cache.AllExternalGroupsProto.ExternalGroupProto;
 import com.google.gerrit.server.cache.serialize.CacheSerializer;
 import com.google.gerrit.server.cache.serialize.StringCacheSerializer;
 import com.google.gerrit.server.group.db.Groups;
 import com.google.gerrit.server.logging.Metadata;
 import com.google.gerrit.server.logging.TraceContext;
 import com.google.gerrit.server.logging.TraceContext.TraceTimer;
 import com.google.gerrit.server.update.RetryHelper;
 import com.google.inject.Inject;
 import com.google.inject.Module;
 import com.google.inject.Singleton;
 import com.google.inject.TypeLiteral;
 import com.google.inject.name.Named;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;

 /** Tracks group inclusions in memory for efficient access. */
 @Singleton
 public class GroupIncludeCacheImpl implements GroupIncludeCache {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();

   private static final String PARENT_GROUPS_NAME = "groups_bysubgroup";
   private static final String GROUPS_WITH_MEMBER_NAME = "groups_bymember";
   private static final String EXTERNAL_NAME = "groups_external";
   private static final String PERSISTED_EXTERNAL_NAME = "groups_external_persisted";

   public static Module module() {
     return new CacheModule() {
       @Override
       protected void configure() {
         cache(
                 GROUPS_WITH_MEMBER_NAME,
                 Account.Id.class,
                 new TypeLiteral<ImmutableSet<AccountGroup.UUID>>() {})
             .loader(GroupsWithMemberLoader.class);

         cache(
                 PARENT_GROUPS_NAME,
                 AccountGroup.UUID.class,
                 new TypeLiteral<ImmutableSet<AccountGroup.UUID>>() {})
             .loader(ParentGroupsLoader.class);

         /**
          * Splitting the groups external cache into 2 caches: The first one is in memory, used to
          * serve the callers and has a single constant key "EXTERNAL_NAME". The second one is
          * persisted, its key represents the groups' state in NoteDb. The in-memory cache is used on
          * top of the persisted cache to enhance performance because the cache's value is used on
          * every request to Gerrit, potentially many times per request and the key computation can
          * become expensive.
          */
         cache(EXTERNAL_NAME, String.class, new TypeLiteral<ImmutableList<AccountGroup.UUID>>() {})
             .loader(AllExternalInMemoryLoader.class);

         persist(
                 PERSISTED_EXTERNAL_NAME,
                 String.class,
                 new TypeLiteral<ImmutableList<AccountGroup.UUID>>() {})
             .diskLimit(-1)
             .version(1)
             .maximumWeight(0)
             .keySerializer(StringCacheSerializer.INSTANCE)
             .valueSerializer(ExternalGroupsSerializer.INSTANCE);

         bind(GroupIncludeCacheImpl.class);
         bind(GroupIncludeCache.class).to(GroupIncludeCacheImpl.class);
       }
     };
   }

   private final LoadingCache<Account.Id, ImmutableSet<AccountGroup.UUID>> groupsWithMember;
   private final LoadingCache<AccountGroup.UUID, ImmutableSet<AccountGroup.UUID>> parentGroups;
   private final LoadingCache<String, ImmutableList<AccountGroup.UUID>> external;

   @Inject
   GroupIncludeCacheImpl(
       @Named(GROUPS_WITH_MEMBER_NAME)
           LoadingCache<Account.Id, ImmutableSet<AccountGroup.UUID>> groupsWithMember,
       @Named(PARENT_GROUPS_NAME)
           LoadingCache<AccountGroup.UUID, ImmutableSet<AccountGroup.UUID>> parentGroups,
       @Named(EXTERNAL_NAME) LoadingCache<String, ImmutableList<AccountGroup.UUID>> external) {
     this.groupsWithMember = groupsWithMember;
     this.parentGroups = parentGroups;
     this.external = external;
   }

   @Override
   public Collection<AccountGroup.UUID> getGroupsWithMember(Account.Id memberId) {
     try {
       return groupsWithMember.get(memberId);
     } catch (ExecutionException e) {
       logger.atWarning().withCause(e).log("Cannot load groups containing %s as member", memberId);
       return ImmutableSet.of();
     }
   }

   @Override
   public Collection<AccountGroup.UUID> parentGroupsOf(AccountGroup.UUID groupId) {
     try {
       return parentGroups.get(groupId);
     } catch (ExecutionException e) {
       logger.atWarning().withCause(e).log("Cannot load included groups");
       return Collections.emptySet();
     }
   }

   @Override
   public Collection<AccountGroup.UUID> parentGroupsOf(Set<AccountGroup.UUID> groupIds) {
     try {
       Set<AccountGroup.UUID> parents = new HashSet<>();
       parentGroups.getAll(groupIds).values().forEach(p -> parents.addAll(p));
       return parents;
     } catch (ExecutionException e) {
       logger.atWarning().withCause(e).log("Cannot load included groups");
       return Collections.emptySet();
     }
   }

   @Override
   public void evictGroupsWithMember(Account.Id memberId) {
     if (memberId != null) {
       logger.atFine().log("Evict groups with member %d", memberId.get());
       groupsWithMember.invalidate(memberId);
     }
   }

   @Override
   public void evictParentGroupsOf(AccountGroup.UUID groupId) {
     if (groupId != null) {
       logger.atFine().log("Evict parent groups of %s", groupId.get());
       parentGroups.invalidate(groupId);

       if (!groupId.isInternalGroup()) {
         logger.atFine().log("Evict external group %s", groupId.get());
         /**
          * No need to invalidate the persistent cache, because this eviction will change the state
          * of NoteDb causing the persistent cache's loader to use a new key that doesn't exist in
          * its cache.n
          */
         external.invalidate(EXTERNAL_NAME);
       }
     }
   }

   @Override
   public Collection<AccountGroup.UUID> allExternalMembers() {
     try {
       return external.get(EXTERNAL_NAME);
     } catch (ExecutionException e) {
       logger.atWarning().withCause(e).log("Cannot load set of non-internal groups");
       return ImmutableList.of();
     }
   }

   static class GroupsWithMemberLoader
       extends CacheLoader<Account.Id, ImmutableSet<AccountGroup.UUID>> {
     private final RetryHelper retryHelper;

     @Inject
     GroupsWithMemberLoader(RetryHelper retryHelper) {
       this.retryHelper = retryHelper;
     }

     @Override
     public ImmutableSet<AccountGroup.UUID> load(Account.Id memberId) {
       try (TraceTimer timer =
           TraceContext.newTimer(
               "Loading groups with member", Metadata.builder().accountId(memberId.get()).build())) {
         return retryHelper
             .groupIndexQuery(
                 "loadGroupWithMember",
                 q ->
                     q.byMember(memberId).stream()
                         .map(InternalGroup::getGroupUUID)
                         .collect(toImmutableSet()))
             .call();
       }
     }
   }

   static class ParentGroupsLoader
       extends CacheLoader<AccountGroup.UUID, ImmutableSet<AccountGroup.UUID>> {
     // Be conservative with batching: We don't want to exhaust the number of
     // results per page and maximum terms per query. Both are usually 1000+.
     private static final int MAX_BATCH_SIZE = 100;
     private final RetryHelper retryHelper;

     @Inject
     ParentGroupsLoader(RetryHelper retryHelper) {
       this.retryHelper = retryHelper;
     }

     @Override
     public ImmutableSet<AccountGroup.UUID> load(AccountGroup.UUID key) {
       try (TraceTimer timer =
           TraceContext.newTimer(
               "Loading parent groups", Metadata.builder().groupUuid(key.get()).build())) {
         return loadAll(ImmutableList.of(key)).get(key);
       }
     }

     @Override
     public Map<AccountGroup.UUID, ImmutableSet<AccountGroup.UUID>> loadAll(
         Iterable<? extends AccountGroup.UUID> keys) {
       int numKeys = Iterables.size(keys);
       Map<AccountGroup.UUID, ImmutableSet<AccountGroup.UUID>> result =
           Maps.newHashMapWithExpectedSize(numKeys);
       try (TraceTimer timer = TraceContext.newTimer("Loading " + numKeys + " parent groups")) {
         Map<AccountGroup.UUID, ImmutableSet<AccountGroup.UUID>> bySubgroups =
             retryHelper
                 .groupIndexQuery("loadParentGroups", q -> q.bySubgroups(ImmutableSet.copyOf(keys)))
                 .call();
         Iterables.partition(keys, MAX_BATCH_SIZE)
             .forEach(keyPartition -> result.putAll(bySubgroups));
         return result;
       }
     }
   }

   static class AllExternalInMemoryLoader
       extends CacheLoader<String, ImmutableList<AccountGroup.UUID>> {
     private final Cache<String, ImmutableList<AccountGroup.UUID>> persisted;
     private final GroupsSnapshotReader snapshotReader;
     private final Groups groups;

     @Inject
     AllExternalInMemoryLoader(
         @Named(PERSISTED_EXTERNAL_NAME) Cache<String, ImmutableList<AccountGroup.UUID>> persisted,
         GroupsSnapshotReader snapshotReader,
         Groups groups) {
       this.persisted = persisted;
       this.snapshotReader = snapshotReader;
       this.groups = groups;
     }

     @Override
     public ImmutableList<AccountGroup.UUID> load(String key) throws Exception {
       GroupsSnapshotReader.Snapshot snapshot = snapshotReader.getSnapshot();
       return persisted.get(
           snapshot.hash(),
           () -> {
             try (TraceTimer timer = TraceContext.newTimer("Loading all external groups")) {
               return groups.getExternalGroups(snapshot.groupsRefs()).collect(toImmutableList());
             }
           });
     }
   }

   public enum ExternalGroupsSerializer
       implements CacheSerializer<ImmutableList<AccountGroup.UUID>> {
     INSTANCE;

     @Override
     public byte[] serialize(ImmutableList<AccountGroup.UUID> object) {
       AllExternalGroupsProto.Builder allBuilder = AllExternalGroupsProto.newBuilder();
       object.stream()
           .map(group -> ExternalGroupProto.newBuilder().setGroupUuid(group.get()).build())
           .forEach(allBuilder::addExternalGroup);
       return Protos.toByteArray(allBuilder.build());
     }

     @Override
     public ImmutableList<AccountGroup.UUID> deserialize(byte[] in) {
       return Protos.parseUnchecked(AllExternalGroupsProto.parser(), in).getExternalGroupList()
           .stream()
           .map(groupProto -> AccountGroup.UUID.parse(groupProto.getGroupUuid()))
           .collect(toImmutableList());
     }
   }
 }
	// Copyright (C) 2011 The Android Open Source Project
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package com.google.gerrit.server.account;

	import static com.google.common.collect.ImmutableList.toImmutableList;
	import static com.google.common.collect.ImmutableSet.toImmutableSet;

	import com.google.common.cache.Cache;
	import com.google.common.cache.CacheLoader;
	import com.google.common.cache.LoadingCache;
	import com.google.common.collect.ImmutableList;
	import com.google.common.collect.ImmutableSet;
	import com.google.common.collect.Iterables;
	import com.google.common.collect.Maps;
	import com.google.common.flogger.FluentLogger;
	import com.google.gerrit.entities.Account;
	import com.google.gerrit.entities.AccountGroup;
	import com.google.gerrit.entities.InternalGroup;
	import com.google.gerrit.proto.Protos;
	import com.google.gerrit.server.cache.CacheModule;
	import com.google.gerrit.server.cache.proto.Cache.AllExternalGroupsProto;
	import com.google.gerrit.server.cache.proto.Cache.AllExternalGroupsProto.ExternalGroupProto;
	import com.google.gerrit.server.cache.serialize.CacheSerializer;
	import com.google.gerrit.server.cache.serialize.StringCacheSerializer;
	import com.google.gerrit.server.group.db.Groups;
	import com.google.gerrit.server.logging.Metadata;
	import com.google.gerrit.server.logging.TraceContext;
	import com.google.gerrit.server.logging.TraceContext.TraceTimer;
	import com.google.gerrit.server.update.RetryHelper;
	import com.google.inject.Inject;
	import com.google.inject.Module;
	import com.google.inject.Singleton;
	import com.google.inject.TypeLiteral;
	import com.google.inject.name.Named;
	import java.util.Collection;
	import java.util.Collections;
	import java.util.HashSet;
	import java.util.Map;
	import java.util.Set;
	import java.util.concurrent.ExecutionException;

	/** Tracks group inclusions in memory for efficient access. */
	@Singleton
	public class GroupIncludeCacheImpl implements GroupIncludeCache {
	private static final FluentLogger logger = FluentLogger.forEnclosingClass();

	private static final String PARENT_GROUPS_NAME = "groups_bysubgroup";
	private static final String GROUPS_WITH_MEMBER_NAME = "groups_bymember";
	private static final String EXTERNAL_NAME = "groups_external";
	private static final String PERSISTED_EXTERNAL_NAME = "groups_external_persisted";

	public static Module module() {
	return new CacheModule() {
	@Override
	protected void configure() {
	cache(
	GROUPS_WITH_MEMBER_NAME,
	Account.Id.class,
	new TypeLiteral<ImmutableSet<AccountGroup.UUID>>() {})
	.loader(GroupsWithMemberLoader.class);

	cache(
	PARENT_GROUPS_NAME,
	AccountGroup.UUID.class,
	new TypeLiteral<ImmutableSet<AccountGroup.UUID>>() {})
	.loader(ParentGroupsLoader.class);

	/**
	* Splitting the groups external cache into 2 caches: The first one is in memory, used to
	* serve the callers and has a single constant key "EXTERNAL_NAME". The second one is
	* persisted, its key represents the groups' state in NoteDb. The in-memory cache is used on
	* top of the persisted cache to enhance performance because the cache's value is used on
	* every request to Gerrit, potentially many times per request and the key computation can
	* become expensive.
	*/
	cache(EXTERNAL_NAME, String.class, new TypeLiteral<ImmutableList<AccountGroup.UUID>>() {})
	.loader(AllExternalInMemoryLoader.class);

	persist(
	PERSISTED_EXTERNAL_NAME,
	String.class,
	new TypeLiteral<ImmutableList<AccountGroup.UUID>>() {})
	.diskLimit(-1)
	.version(1)
	.maximumWeight(0)
	.keySerializer(StringCacheSerializer.INSTANCE)
	.valueSerializer(ExternalGroupsSerializer.INSTANCE);

	bind(GroupIncludeCacheImpl.class);
	bind(GroupIncludeCache.class).to(GroupIncludeCacheImpl.class);
	}
	};
	}

	private final LoadingCache<Account.Id, ImmutableSet<AccountGroup.UUID>> groupsWithMember;
	private final LoadingCache<AccountGroup.UUID, ImmutableSet<AccountGroup.UUID>> parentGroups;
	private final LoadingCache<String, ImmutableList<AccountGroup.UUID>> external;

	@Inject
	GroupIncludeCacheImpl(
	@Named(GROUPS_WITH_MEMBER_NAME)
	LoadingCache<Account.Id, ImmutableSet<AccountGroup.UUID>> groupsWithMember,
	@Named(PARENT_GROUPS_NAME)
	LoadingCache<AccountGroup.UUID, ImmutableSet<AccountGroup.UUID>> parentGroups,
	@Named(EXTERNAL_NAME) LoadingCache<String, ImmutableList<AccountGroup.UUID>> external) {
	this.groupsWithMember = groupsWithMember;
	this.parentGroups = parentGroups;
	this.external = external;
	}

	@Override
	public Collection<AccountGroup.UUID> getGroupsWithMember(Account.Id memberId) {
	try {
	return groupsWithMember.get(memberId);
	} catch (ExecutionException e) {
	logger.atWarning().withCause(e).log("Cannot load groups containing %s as member", memberId);
	return ImmutableSet.of();
	}
	}

	@Override
	public Collection<AccountGroup.UUID> parentGroupsOf(AccountGroup.UUID groupId) {
	try {
	return parentGroups.get(groupId);
	} catch (ExecutionException e) {
	logger.atWarning().withCause(e).log("Cannot load included groups");
	return Collections.emptySet();
	}
	}

	@Override
	public Collection<AccountGroup.UUID> parentGroupsOf(Set<AccountGroup.UUID> groupIds) {
	try {
	Set<AccountGroup.UUID> parents = new HashSet<>();
	parentGroups.getAll(groupIds).values().forEach(p -> parents.addAll(p));
	return parents;
	} catch (ExecutionException e) {
	logger.atWarning().withCause(e).log("Cannot load included groups");
	return Collections.emptySet();
	}
	}

	@Override
	public void evictGroupsWithMember(Account.Id memberId) {
	if (memberId != null) {
	logger.atFine().log("Evict groups with member %d", memberId.get());
	groupsWithMember.invalidate(memberId);
	}
	}

	@Override
	public void evictParentGroupsOf(AccountGroup.UUID groupId) {
	if (groupId != null) {
	logger.atFine().log("Evict parent groups of %s", groupId.get());
	parentGroups.invalidate(groupId);

	if (!groupId.isInternalGroup()) {
	logger.atFine().log("Evict external group %s", groupId.get());
	/**
	* No need to invalidate the persistent cache, because this eviction will change the state
	* of NoteDb causing the persistent cache's loader to use a new key that doesn't exist in
	* its cache.n
	*/
	external.invalidate(EXTERNAL_NAME);
	}
	}
	}

	@Override
	public Collection<AccountGroup.UUID> allExternalMembers() {
	try {
	return external.get(EXTERNAL_NAME);
	} catch (ExecutionException e) {
	logger.atWarning().withCause(e).log("Cannot load set of non-internal groups");
	return ImmutableList.of();
	}
	}

	static class GroupsWithMemberLoader
	extends CacheLoader<Account.Id, ImmutableSet<AccountGroup.UUID>> {
	private final RetryHelper retryHelper;

	@Inject
	GroupsWithMemberLoader(RetryHelper retryHelper) {
	this.retryHelper = retryHelper;
	}

	@Override
	public ImmutableSet<AccountGroup.UUID> load(Account.Id memberId) {
	try (TraceTimer timer =
	TraceContext.newTimer(
	"Loading groups with member", Metadata.builder().accountId(memberId.get()).build())) {
	return retryHelper
	.groupIndexQuery(
	"loadGroupWithMember",
	q ->
	q.byMember(memberId).stream()
	.map(InternalGroup::getGroupUUID)
	.collect(toImmutableSet()))
	.call();
	}
	}
	}

	static class ParentGroupsLoader
	extends CacheLoader<AccountGroup.UUID, ImmutableSet<AccountGroup.UUID>> {
	// Be conservative with batching: We don't want to exhaust the number of
	// results per page and maximum terms per query. Both are usually 1000+.
	private static final int MAX_BATCH_SIZE = 100;
	private final RetryHelper retryHelper;

	@Inject
	ParentGroupsLoader(RetryHelper retryHelper) {
	this.retryHelper = retryHelper;
	}

	@Override
	public ImmutableSet<AccountGroup.UUID> load(AccountGroup.UUID key) {
	try (TraceTimer timer =
	TraceContext.newTimer(
	"Loading parent groups", Metadata.builder().groupUuid(key.get()).build())) {
	return loadAll(ImmutableList.of(key)).get(key);
	}
	}

	@Override
	public Map<AccountGroup.UUID, ImmutableSet<AccountGroup.UUID>> loadAll(
	Iterable<? extends AccountGroup.UUID> keys) {
	int numKeys = Iterables.size(keys);
	Map<AccountGroup.UUID, ImmutableSet<AccountGroup.UUID>> result =
	Maps.newHashMapWithExpectedSize(numKeys);
	try (TraceTimer timer = TraceContext.newTimer("Loading " + numKeys + " parent groups")) {
	Map<AccountGroup.UUID, ImmutableSet<AccountGroup.UUID>> bySubgroups =
	retryHelper
	.groupIndexQuery("loadParentGroups", q -> q.bySubgroups(ImmutableSet.copyOf(keys)))
	.call();
	Iterables.partition(keys, MAX_BATCH_SIZE)
	.forEach(keyPartition -> result.putAll(bySubgroups));
	return result;
	}
	}
	}

	static class AllExternalInMemoryLoader
	extends CacheLoader<String, ImmutableList<AccountGroup.UUID>> {
	private final Cache<String, ImmutableList<AccountGroup.UUID>> persisted;
	private final GroupsSnapshotReader snapshotReader;
	private final Groups groups;

	@Inject
	AllExternalInMemoryLoader(
	@Named(PERSISTED_EXTERNAL_NAME) Cache<String, ImmutableList<AccountGroup.UUID>> persisted,
	GroupsSnapshotReader snapshotReader,
	Groups groups) {
	this.persisted = persisted;
	this.snapshotReader = snapshotReader;
	this.groups = groups;
	}

	@Override
	public ImmutableList<AccountGroup.UUID> load(String key) throws Exception {
	GroupsSnapshotReader.Snapshot snapshot = snapshotReader.getSnapshot();
	return persisted.get(
	snapshot.hash(),
	() -> {
	try (TraceTimer timer = TraceContext.newTimer("Loading all external groups")) {
	return groups.getExternalGroups(snapshot.groupsRefs()).collect(toImmutableList());
	}
	});
	}
	}

	public enum ExternalGroupsSerializer
	implements CacheSerializer<ImmutableList<AccountGroup.UUID>> {
	INSTANCE;

	@Override
	public byte[] serialize(ImmutableList<AccountGroup.UUID> object) {
	AllExternalGroupsProto.Builder allBuilder = AllExternalGroupsProto.newBuilder();
	object.stream()
	.map(group -> ExternalGroupProto.newBuilder().setGroupUuid(group.get()).build())
	.forEach(allBuilder::addExternalGroup);
	return Protos.toByteArray(allBuilder.build());
	}

	@Override
	public ImmutableList<AccountGroup.UUID> deserialize(byte[] in) {
	return Protos.parseUnchecked(AllExternalGroupsProto.parser(), in).getExternalGroupList()
	.stream()
	.map(groupProto -> AccountGroup.UUID.parse(groupProto.getGroupUuid()))
	.collect(toImmutableList());
	}
	}
	}