src/main/java/com/googlesource/gerrit/plugins/remotegerritaccountcache/AccountCacheImpl.java

// Copyright (C) 2023 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.googlesource.gerrit.plugins.remotegerritaccountcache;

import static com.google.gerrit.server.account.AccountCacheImpl.AccountCacheModule.ACCOUNT_CACHE_MODULE;
import static com.google.gerrit.server.account.externalids.ExternalId.SCHEME_USERNAME;
import static com.google.gerrit.server.git.QueueProvider.QueueType.BATCH;

import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import com.google.common.flogger.FluentLogger;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.gerrit.entities.Account;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.exceptions.StorageException;
import com.google.gerrit.extensions.common.AccountDetailInfo;
import com.google.gerrit.extensions.common.AccountExternalIdInfo;
import com.google.gerrit.json.OutputFormat;
import com.google.gerrit.server.ModuleImpl;
import com.google.gerrit.server.account.AccountCache;
import com.google.gerrit.server.account.AccountConfig;
import com.google.gerrit.server.account.AccountState;
import com.google.gerrit.server.account.AccountsUpdate;
import com.google.gerrit.server.account.CachedAccountDetails;
import com.google.gerrit.server.account.externalids.ExternalId;
import com.google.gerrit.server.account.externalids.ExternalIdKeyFactory;
import com.google.gerrit.server.account.externalids.ExternalIds;
import com.google.gerrit.server.cache.CacheModule;
import com.google.gerrit.server.config.AllUsersName;
import com.google.gerrit.server.config.CachedPreferences;
import com.google.gerrit.server.config.DefaultPreferencesCache;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.index.IndexExecutor;
import com.google.gerrit.server.index.account.AccountIndexer;
import com.google.gerrit.server.util.time.TimeUtil;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import com.google.inject.AbstractModule;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import com.google.inject.name.Named;
import java.io.IOException;
import java.lang.reflect.Type;
import java.net.Authenticator;
import java.net.PasswordAuthentication;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;

public class AccountCacheImpl implements AccountCache {
  private static final FluentLogger logger = FluentLogger.forEnclosingClass();

  @ModuleImpl(name = ACCOUNT_CACHE_MODULE)
  public static class AccountCacheModule extends CacheModule {
    @Override
    protected void configure() {
      persist(BYID_AND_REV_NAME, CachedAccountDetails.Key.class, CachedAccountDetails.class)
          .version(2)
          .keySerializer(CachedAccountDetails.Key.Serializer.INSTANCE)
          .valueSerializer(CachedAccountDetails.Serializer.INSTANCE)
          .loader(Loader.class)
          .expireAfterWrite(Duration.ofDays(1))
          .refreshAfterWrite(Duration.ofHours(23));
    }
  }

  public static class AccountCacheBindingModule extends AbstractModule {
    @Override
    protected void configure() {
      bind(AccountCacheImpl.class);
      bind(AccountCache.class).to(AccountCacheImpl.class);
    }
  }

  protected static class Config {
    protected static final String SECTION = "remote-gerrit-account-cache";
    protected static final String REMOTE_GERRIT_BASE_URL = "remoteGerritBaseUrl";
    protected static final String HTTP_USERNAME = "httpUsername";
    protected static final String HTTP_PASSWORD = "httpPassword";

    protected final String remoteGerritBaseUrl;
    protected final String username;
    protected final String password;

    @Inject
    Config(@GerritServerConfig org.eclipse.jgit.lib.Config config) {
      remoteGerritBaseUrl = config.getString(SECTION, null, REMOTE_GERRIT_BASE_URL);
      username = config.getString(SECTION, null, HTTP_USERNAME);
      password = config.getString(SECTION, null, HTTP_PASSWORD);
    }

    URI getAccountDetailUri(Account.Id id) {
      return URI.create(String.format("%s/a/accounts/%s/detail", remoteGerritBaseUrl, id.get()));
    }

    URI getExternalIdsUri(Account.Id id) {
      return URI.create(
          String.format("%s/a/accounts/%s/external.ids", remoteGerritBaseUrl, id.get()));
    }
  }

  public static final String BYID_AND_REV_NAME = "accounts";
  private final ExternalIds externalIds;
  private final LoadingCache<CachedAccountDetails.Key, CachedAccountDetails> accountDetailsCache;
  private final GitRepositoryManager repoManager;
  private final AllUsersName allUsersName;
  private final DefaultPreferencesCache defaultPreferenceCache;
  private final ExternalIdKeyFactory externalIdKeyFactory;

  @Inject
  AccountCacheImpl(
      ExternalIds externalIds,
      @Named(BYID_AND_REV_NAME)
          LoadingCache<CachedAccountDetails.Key, CachedAccountDetails> accountDetailsCache,
      GitRepositoryManager repoManager,
      AllUsersName allUsersName,
      DefaultPreferencesCache defaultPreferenceCache,
      ExternalIdKeyFactory externalIdKeyFactory) {
    this.externalIds = externalIds;
    this.accountDetailsCache = accountDetailsCache;
    this.repoManager = repoManager;
    this.allUsersName = allUsersName;
    this.defaultPreferenceCache = defaultPreferenceCache;
    this.externalIdKeyFactory = externalIdKeyFactory;
  }

  @Override
  public AccountState getEvenIfMissing(Account.Id accountId) {
    return get(accountId).orElse(missing(accountId));
  }

  @Override
  public Optional<AccountState> get(Account.Id accountId) {
    return Optional.ofNullable(get(Collections.singleton(accountId)).getOrDefault(accountId, null));
  }

  @Override
  public AccountState getFromMetaId(Account.Id id, ObjectId metaId) {
    try {
      CachedAccountDetails.Key key = CachedAccountDetails.Key.create(id, metaId);

      CachedAccountDetails accountDetails = accountDetailsCache.get(key);
      return AccountState.forCachedAccount(accountDetails, CachedPreferences.EMPTY, externalIds);
    } catch (IOException | ExecutionException e) {
      throw new StorageException(e);
    }
  }

  @Override
  public Map<Account.Id, AccountState> get(Set<Account.Id> accountIds) {
    try {
      try (Repository allUsers = repoManager.openRepository(allUsersName)) {
        Set<CachedAccountDetails.Key> keys =
            Sets.newLinkedHashSetWithExpectedSize(accountIds.size());
        for (Account.Id id : accountIds) {
          Ref userRef = allUsers.exactRef(RefNames.refsUsers(id));
          if (userRef == null) {
            keys.add(CachedAccountDetails.Key.create(id, ObjectId.zeroId()));
          } else {
            keys.add(CachedAccountDetails.Key.create(id, userRef.getObjectId()));
          }
        }
        CachedPreferences defaultPreferences = defaultPreferenceCache.get();
        ImmutableMap.Builder<Account.Id, AccountState> result = ImmutableMap.builder();
        for (CachedAccountDetails.Key key : keys) {
          try {
            CachedAccountDetails cachedAccountDetails = accountDetailsCache.get(key);
            result.put(
                key.accountId(),
                AccountState.forCachedAccount(
                    cachedAccountDetails, defaultPreferences, externalIds));
          } catch (Exception e) {
            if (e instanceof ExecutionException
                && e.getCause() instanceof AccountNotFoundException) {
              continue;
            }
            throw e;
          }
        }
        return result.build();
      }
    } catch (Exception e) {
      throw new StorageException(e);
    }
  }

  @Override
  public Optional<AccountState> getByUsername(String username) {
    try {
      return externalIds
          .get(externalIdKeyFactory.create(SCHEME_USERNAME, username))
          .map(e -> get(e.accountId()))
          .orElseGet(Optional::empty);
    } catch (IOException e) {
      logger.atWarning().withCause(e).log("Cannot load AccountState for username %s", username);
      return Optional.empty();
    }
  }

  private AccountState missing(Account.Id accountId) {
    Account.Builder account = Account.builder(accountId, TimeUtil.now());
    account.setActive(false);
    return AccountState.forAccount(account.build());
  }

  @Singleton
  static class Loader extends CacheLoader<CachedAccountDetails.Key, CachedAccountDetails> {
    protected static final String UPDATE_MESSAGE = "Sync account via remote-gerrit-account-cache";

    private final Gson gson = OutputFormat.JSON.newGson();
    private final AccountsUpdate.AccountsUpdateLoader accountsUpdateFactory;
    private final Provider<AccountIndexer> accountIndexerProvider;
    private final ListeningExecutorService executor;
    private final ExternalIds externalIds;
    private final Config config;
    private final ExternalIdKeyFactory externalIdKeyFactory;
    private final GitRepositoryManager repoManager;
    private final AllUsersName allUsersName;
    private final HttpClient client;
    private final APIRateLimiter rateLimiter;

    @Inject
    Loader(
        @AccountsUpdate.AccountsUpdateLoader.NoReindex
            AccountsUpdate.AccountsUpdateLoader accountsUpdateFactory,
        Provider<AccountIndexer> accountIndexerProvider,
        @IndexExecutor(BATCH) ListeningExecutorService executor,
        ExternalIds externalIds,
        Config config,
        ExternalIdKeyFactory externalIdKeyFactory,
        GitRepositoryManager repoManager,
        AllUsersName allUsersName,
        APIRateLimiter rateLimiter) {
      this.accountsUpdateFactory = accountsUpdateFactory;
      this.accountIndexerProvider = accountIndexerProvider;
      this.executor = executor;
      this.externalIds = externalIds;
      this.config = config;
      this.externalIdKeyFactory = externalIdKeyFactory;
      this.repoManager = repoManager;
      this.allUsersName = allUsersName;
      this.rateLimiter = rateLimiter;
      this.client = getHttpClient();
    }

    @SuppressWarnings("unused")
    @Override
    public CachedAccountDetails load(CachedAccountDetails.Key key) throws Exception {
      AccountDetailInfo accountInfo = getAccountFromRemoteSite(key.accountId());
      if (accountInfo == null) {
        try (Repository repo = repoManager.openRepository(allUsersName)) {
          return getAccountDetailsFromNoteDb(repo, key);
        }
      }
      Account.Id accountId = Account.id(accountInfo._accountId);
      Collection<ExternalId> extIds = new ArrayList<>();
      Collection<ExternalId> oldExtIds = new ArrayList<>();
      try {
        extIds.addAll(getExternalIdsFromRemoteSite(accountId));
        oldExtIds.addAll(externalIds.byAccount(accountId));
      } catch (Exception ignore) {
      }

      AccountsUpdate.ConfigureStatelessDelta update =
          u ->
              u.setFullName(accountInfo.name)
                  .setDisplayName(accountInfo.displayName)
                  .setPreferredEmail(accountInfo.email)
                  .setStatus(accountInfo.status)
                  .deleteExternalIds(oldExtIds)
                  .addExternalIds(extIds);

      try (Repository repo = repoManager.openRepository(allUsersName)) {
        Ref userRef = repo.exactRef(RefNames.refsUsers(key.accountId()));
        if (userRef != null) {
          accountsUpdateFactory.createWithServerIdent().update(UPDATE_MESSAGE, accountId, update);
        } else {
          accountsUpdateFactory.createWithServerIdent().insert(UPDATE_MESSAGE, accountId, update);
        }
        logger.atInfo().log("Account updated: %s", accountId);
        Future<?> ignore =
            executor.submit(
                () -> {
                  try {
                    accountIndexerProvider.get().reindexIfStale(accountId);
                    logger.atInfo().log("Reindexing is done for account: %s", accountId);
                  } catch (Exception e) {
                    logger.atWarning().log(
                        "Reindexing for account: %s failed with error %s",
                        accountId, e.getMessage());
                  }
                });
        return getAccountDetailsFromNoteDb(repo, key);
      }
    }

    protected CachedAccountDetails getAccountDetailsFromNoteDb(
        Repository repo, CachedAccountDetails.Key key)
        throws ConfigInvalidException, IOException, AccountNotFoundException {
      Ref userRef = repo.exactRef(RefNames.refsUsers(key.accountId()));
      if (userRef == null) {
        throw new AccountNotFoundException(key.accountId() + " not found");
      }
      AccountConfig cfg =
          new AccountConfig(key.accountId(), allUsersName, repo).load(userRef.getObjectId());
      Account account =
          cfg.getLoadedAccount()
              .orElseThrow(() -> new AccountNotFoundException(key.accountId() + " not found"));
      return CachedAccountDetails.create(
          account, cfg.getProjectWatches(), cfg.asCachedPreferences());
    }

    protected AccountDetailInfo getAccountFromRemoteSite(Account.Id accountId) {
      rateLimiter.getLimiter().acquire();
      HttpRequest request =
          HttpRequest.newBuilder().uri(config.getAccountDetailUri(accountId)).GET().build();
      try {
        HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
        if (response.statusCode() == 200) {
          Type type = new TypeToken<AccountDetailInfo>() {}.getType();
          return gson.fromJson(response.body(), type);
        }
        logger.atSevere().log(
            "Failed to fetch account from remote Gerrit %s, %s", accountId, response.body());
      } catch (IOException | InterruptedException e) {
        logger.atSevere().withCause(e).log("Failed to fetch account from remote Gerrit");
      }
      return null;
    }

    public List<ExternalId> getExternalIdsFromRemoteSite(Account.Id accountId)
        throws AccountNotFoundInRemoteGerritException {
      rateLimiter.getLimiter().acquire();
      HttpRequest request =
          HttpRequest.newBuilder().uri(config.getExternalIdsUri(accountId)).GET().build();

      try {
        HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
        if (response.statusCode() == 200) {
          Type type = new TypeToken<List<AccountExternalIdInfo>>() {}.getType();
          List<AccountExternalIdInfo> accountExternalIdInfos = gson.fromJson(response.body(), type);
          return accountExternalIdInfos.stream()
              .map(
                  i ->
                      ExternalId.create(
                          externalIdKeyFactory.parse(i.identity),
                          accountId,
                          i.emailAddress,
                          null,
                          null))
              .collect(Collectors.toList());
        }
        logger.atSevere().log(
            "Failed to fetch remote ids for account %s, %s", accountId, response.body());
      } catch (IOException | InterruptedException e) {
        logger.atSevere().withCause(e).log("Failed to fetch remote ids for account %s", accountId);
      }
      throw new AccountNotFoundInRemoteGerritException(
          String.format("%s not found in remote Gerrit", accountId));
    }

    protected HttpClient getHttpClient() {
      return HttpClient.newBuilder()
          .authenticator(
              new Authenticator() {
                @Override
                protected PasswordAuthentication getPasswordAuthentication() {
                  return new PasswordAuthentication(config.username, config.password.toCharArray());
                }
              })
          .build();
    }
  }

  /** Signals that the account was not found in the primary storage. */
  private static class AccountNotFoundException extends Exception {
    private static final long serialVersionUID = 1L;

    public AccountNotFoundException(String message) {
      super(message);
    }
  }

  /** Signals that the account was not found in the remote Gerrit. */
  private static class AccountNotFoundInRemoteGerritException extends Exception {
    private static final long serialVersionUID = 1L;

    public AccountNotFoundInRemoteGerritException(String message) {
      super(message);
    }
  }
}