| // Copyright (C) 2019 The Android Open Source Project |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package com.google.gerrit.server.account.externalids; |
| |
| import com.google.common.base.CharMatcher; |
| import com.google.common.cache.Cache; |
| import com.google.common.cache.CacheLoader; |
| import com.google.common.collect.ImmutableSet; |
| import com.google.common.collect.ImmutableSetMultimap; |
| import com.google.common.flogger.FluentLogger; |
| import com.google.gerrit.entities.Account; |
| import com.google.gerrit.entities.RefNames; |
| import com.google.gerrit.metrics.Counter1; |
| import com.google.gerrit.metrics.Description; |
| import com.google.gerrit.metrics.Description.Units; |
| import com.google.gerrit.metrics.Field; |
| import com.google.gerrit.metrics.MetricMaker; |
| import com.google.gerrit.metrics.Timer0; |
| import com.google.gerrit.server.config.AllUsersName; |
| import com.google.gerrit.server.config.GerritServerConfig; |
| import com.google.gerrit.server.git.GitRepositoryManager; |
| import com.google.gerrit.server.logging.Metadata; |
| import com.google.gerrit.server.logging.TraceContext; |
| import com.google.gerrit.server.logging.TraceContext.TraceTimer; |
| import com.google.inject.Inject; |
| import com.google.inject.Provider; |
| import com.google.inject.Singleton; |
| import com.google.inject.name.Named; |
| import java.io.IOException; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.Map; |
| import java.util.Set; |
| import java.util.concurrent.TimeUnit; |
| import org.eclipse.jgit.errors.ConfigInvalidException; |
| import org.eclipse.jgit.lib.Config; |
| import org.eclipse.jgit.lib.ObjectId; |
| import org.eclipse.jgit.lib.ObjectReader; |
| import org.eclipse.jgit.lib.Ref; |
| import org.eclipse.jgit.lib.Repository; |
| import org.eclipse.jgit.revwalk.RevCommit; |
| import org.eclipse.jgit.revwalk.RevWalk; |
| import org.eclipse.jgit.treewalk.TreeWalk; |
| import org.eclipse.jgit.treewalk.filter.TreeFilter; |
| |
| /** Loads cache values for the external ID cache using either a full or a partial reload. */ |
| @Singleton |
| public class ExternalIdCacheLoader extends CacheLoader<ObjectId, AllExternalIds> { |
| private static final FluentLogger logger = FluentLogger.forEnclosingClass(); |
| |
| // Maximum number of prior states we inspect to find a base for differential. If no cached state |
| // is found within this number of parents, we fall back to reading everything from scratch. |
| private static final int MAX_HISTORY_LOOKBACK = 10; |
| |
| private final ExternalIdReader externalIdReader; |
| private final Provider<Cache<ObjectId, AllExternalIds>> externalIdCache; |
| private final GitRepositoryManager gitRepositoryManager; |
| private final AllUsersName allUsersName; |
| private final Counter1<Boolean> reloadCounter; |
| private final Timer0 reloadDifferential; |
| private final boolean enablePartialReloads; |
| private final boolean isPersistentCache; |
| |
| @Inject |
| ExternalIdCacheLoader( |
| GitRepositoryManager gitRepositoryManager, |
| AllUsersName allUsersName, |
| ExternalIdReader externalIdReader, |
| @Named(ExternalIdCacheImpl.CACHE_NAME) |
| Provider<Cache<ObjectId, AllExternalIds>> externalIdCache, |
| MetricMaker metricMaker, |
| @GerritServerConfig Config config) { |
| this.externalIdReader = externalIdReader; |
| this.externalIdCache = externalIdCache; |
| this.gitRepositoryManager = gitRepositoryManager; |
| this.allUsersName = allUsersName; |
| this.reloadCounter = |
| metricMaker.newCounter( |
| "notedb/external_id_cache_load_count", |
| new Description("Total number of external ID cache reloads from Git.") |
| .setRate() |
| .setUnit("updates"), |
| Field.ofBoolean("partial", Metadata.Builder::partial).build()); |
| this.reloadDifferential = |
| metricMaker.newTimer( |
| "notedb/external_id_partial_read_latency", |
| new Description( |
| "Latency for generating a new external ID cache state from a prior state.") |
| .setCumulative() |
| .setUnit(Units.MILLISECONDS)); |
| this.enablePartialReloads = |
| config.getBoolean("cache", ExternalIdCacheImpl.CACHE_NAME, "enablePartialReloads", true); |
| this.isPersistentCache = |
| config.getInt("cache", ExternalIdCacheImpl.CACHE_NAME, "diskLimit", 0) > 0; |
| } |
| |
| @Override |
| public AllExternalIds load(ObjectId notesRev) throws IOException, ConfigInvalidException { |
| if (!enablePartialReloads) { |
| logger.atInfo().log( |
| "Partial reloads of " |
| + ExternalIdCacheImpl.CACHE_NAME |
| + " disabled. Falling back to full reload."); |
| return reloadAllExternalIds(notesRev); |
| } |
| |
| // The requested value was not in the cache (hence, this loader was invoked). Therefore, try to |
| // create this entry from a past value using the minimal amount of Git operations possible to |
| // reduce latency. |
| // |
| // First, try to find the most recent state we have in the cache. Most of the time, this will be |
| // the state before the last update happened, but it can also date further back. We try a best |
| // effort approach and check the last 10 states. If nothing is found, we default to loading the |
| // value from scratch. |
| // |
| // If a prior state was found, we use Git to diff the trees and find modifications. This is |
| // faster than just loading the complete current tree and working off of that because of how the |
| // data is structured: NotesMaps use nested trees, so, for example, a NotesMap with 200k entries |
| // has two layers of nesting: 12/34/1234..99. TreeWalk is smart in skipping the traversal of |
| // identical subtrees. |
| // |
| // Once we know what files changed, we apply additions and removals to the previously cached |
| // state. |
| |
| try (Repository repo = gitRepositoryManager.openRepository(allUsersName); |
| RevWalk rw = new RevWalk(repo)) { |
| long start = System.nanoTime(); |
| Ref extIdRef = repo.exactRef(RefNames.REFS_EXTERNAL_IDS); |
| if (extIdRef == null) { |
| logger.atInfo().log( |
| RefNames.REFS_EXTERNAL_IDS + " not initialized, falling back to full reload."); |
| return reloadAllExternalIds(notesRev); |
| } |
| |
| RevCommit currentCommit = rw.parseCommit(extIdRef.getObjectId()); |
| rw.markStart(currentCommit); |
| RevCommit parentWithCacheValue; |
| AllExternalIds oldExternalIds = null; |
| int i = 0; |
| while ((parentWithCacheValue = rw.next()) != null |
| && i++ < MAX_HISTORY_LOOKBACK |
| && parentWithCacheValue.getParentCount() < 2) { |
| oldExternalIds = externalIdCache.get().getIfPresent(parentWithCacheValue.getId()); |
| if (oldExternalIds != null) { |
| // We found a previously cached state. |
| break; |
| } |
| } |
| if (oldExternalIds == null) { |
| if (isPersistentCache) { |
| // If there is no persistence, this is normal. Don't upset admins reading the logs. |
| logger.atWarning().log( |
| "Unable to find an old ExternalId cache state, falling back to full reload"); |
| } |
| return reloadAllExternalIds(notesRev); |
| } |
| |
| // Diff trees to recognize modifications |
| Set<ObjectId> removals = new HashSet<>(); // Set<Blob-Object-Id> |
| Map<ObjectId, ObjectId> additions = new HashMap<>(); // Map<Name-ObjectId, Blob-Object-Id> |
| try (TreeWalk treeWalk = new TreeWalk(repo)) { |
| treeWalk.setFilter(TreeFilter.ANY_DIFF); |
| treeWalk.setRecursive(true); |
| treeWalk.reset(parentWithCacheValue.getTree(), currentCommit.getTree()); |
| while (treeWalk.next()) { |
| String path = treeWalk.getPathString(); |
| ObjectId oldBlob = treeWalk.getObjectId(0); |
| ObjectId newBlob = treeWalk.getObjectId(1); |
| if (ObjectId.zeroId().equals(newBlob)) { |
| // Deletion |
| removals.add(oldBlob); |
| } else if (ObjectId.zeroId().equals(oldBlob)) { |
| // Addition |
| additions.put(fileNameToObjectId(path), newBlob); |
| } else { |
| // Modification |
| removals.add(oldBlob); |
| additions.put(fileNameToObjectId(path), newBlob); |
| } |
| } |
| } |
| |
| AllExternalIds allExternalIds = |
| buildAllExternalIds(repo, oldExternalIds, additions, removals); |
| reloadCounter.increment(true); |
| reloadDifferential.record(System.nanoTime() - start, TimeUnit.NANOSECONDS); |
| return allExternalIds; |
| } |
| } |
| |
| private static ObjectId fileNameToObjectId(String path) { |
| return ObjectId.fromString(CharMatcher.is('/').removeFrom(path)); |
| } |
| |
| /** |
| * Build a new {@link AllExternalIds} from an old state by applying additions and removals that |
| * were performed since then. |
| * |
| * <p>Removals are applied before additions. |
| * |
| * @param repo open repository |
| * @param oldExternalIds prior state that is used as base |
| * @param additions map of name to blob ID for each external ID that should be added |
| * @param removals set of name {@link ObjectId}s that should be removed |
| */ |
| private static AllExternalIds buildAllExternalIds( |
| Repository repo, |
| AllExternalIds oldExternalIds, |
| Map<ObjectId, ObjectId> additions, |
| Set<ObjectId> removals) |
| throws IOException { |
| ImmutableSetMultimap.Builder<Account.Id, ExternalId> byAccount = ImmutableSetMultimap.builder(); |
| ImmutableSetMultimap.Builder<String, ExternalId> byEmail = ImmutableSetMultimap.builder(); |
| |
| // Copy over old ExternalIds but exclude deleted ones |
| for (ExternalId externalId : oldExternalIds.byAccount().values()) { |
| if (removals.contains(externalId.blobId())) { |
| continue; |
| } |
| |
| byAccount.put(externalId.accountId(), externalId); |
| if (externalId.email() != null) { |
| byEmail.put(externalId.email(), externalId); |
| } |
| } |
| |
| // Add newly discovered ExternalIds |
| try (ObjectReader reader = repo.newObjectReader()) { |
| for (Map.Entry<ObjectId, ObjectId> nameToBlob : additions.entrySet()) { |
| ExternalId parsedExternalId; |
| try { |
| parsedExternalId = |
| ExternalId.parse( |
| nameToBlob.getKey().name(), |
| reader.open(nameToBlob.getValue()).getCachedBytes(), |
| nameToBlob.getValue()); |
| } catch (ConfigInvalidException | RuntimeException e) { |
| logger.atSevere().withCause(e).log( |
| "Ignoring invalid external ID note %s", nameToBlob.getKey().name()); |
| continue; |
| } |
| |
| byAccount.put(parsedExternalId.accountId(), parsedExternalId); |
| if (parsedExternalId.email() != null) { |
| byEmail.put(parsedExternalId.email(), parsedExternalId); |
| } |
| } |
| } |
| return new AutoValue_AllExternalIds(byAccount.build(), byEmail.build()); |
| } |
| |
| private AllExternalIds reloadAllExternalIds(ObjectId notesRev) |
| throws IOException, ConfigInvalidException { |
| try (TraceTimer ignored = |
| TraceContext.newTimer( |
| "Loading external IDs from scratch", |
| Metadata.builder().revision(notesRev.name()).build())) { |
| ImmutableSet<ExternalId> externalIds = externalIdReader.all(notesRev); |
| externalIds.forEach(ExternalId::checkThatBlobIdIsSet); |
| AllExternalIds allExternalIds = AllExternalIds.create(externalIds); |
| reloadCounter.increment(false); |
| return allExternalIds; |
| } |
| } |
| } |