java/com/google/gerrit/server/patch/PatchScriptFactory.java

// Copyright (C) 2009 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.gerrit.server.patch;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;

import com.google.common.collect.ImmutableList;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.common.data.PatchScript;
import com.google.gerrit.entities.Change;
import com.google.gerrit.entities.PatchSet;
import com.google.gerrit.entities.Project;
import com.google.gerrit.exceptions.StorageException;
import com.google.gerrit.extensions.client.DiffPreferencesInfo;
import com.google.gerrit.extensions.client.DiffPreferencesInfo.Whitespace;
import com.google.gerrit.extensions.restapi.AuthException;
import com.google.gerrit.metrics.Counter1;
import com.google.gerrit.metrics.Description;
import com.google.gerrit.metrics.Field;
import com.google.gerrit.metrics.MetricMaker;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.PatchSetUtil;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.edit.ChangeEdit;
import com.google.gerrit.server.edit.ChangeEditUtil;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.LargeObjectException;
import com.google.gerrit.server.logging.Metadata;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.patch.PatchScriptBuilder.IntraLineDiffCalculatorResult;
import com.google.gerrit.server.patch.filediff.FileDiffOutput;
import com.google.gerrit.server.permissions.ChangePermission;
import com.google.gerrit.server.permissions.PermissionBackend;
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.gerrit.server.project.InvalidChangeOperationException;
import com.google.gerrit.server.project.NoSuchChangeException;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectState;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.assistedinject.AssistedInject;
import java.io.IOException;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.eclipse.jgit.diff.Edit;
import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;

public class PatchScriptFactory implements Callable<PatchScript> {

  private static final FluentLogger logger = FluentLogger.forEnclosingClass();

  public interface Factory {

    PatchScriptFactory create(
        ChangeNotes notes,
        String fileName,
        @Assisted("patchSetA") PatchSet.Id patchSetA,
        @Assisted("patchSetB") PatchSet.Id patchSetB,
        DiffPreferencesInfo diffPrefs,
        CurrentUser currentUser);

    PatchScriptFactory create(
        ChangeNotes notes,
        String fileName,
        int parentNum,
        PatchSet.Id patchSetB,
        DiffPreferencesInfo diffPrefs,
        CurrentUser currentUser);
  }

  /** These metrics are temporary for launching the new redesigned diff cache. */
  @Singleton
  static class Metrics {
    final Counter1<String> diffs;
    static final String MATCH = "match";
    static final String MISMATCH = "mismatch";
    static final String ERROR = "error";

    @Inject
    Metrics(MetricMaker metricMaker) {
      diffs =
          metricMaker.newCounter(
              "diff/get_diff/dark_launch",
              new Description(
                      "Total number of matching, non-matching, or error in diffs in the old and new diff cache implementations.")
                  .setRate()
                  .setUnit("count"),
              Field.ofString("type", Metadata.Builder::eventType).build());
    }
  }

  private final GitRepositoryManager repoManager;
  private final PatchSetUtil psUtil;
  private final Provider<PatchScriptBuilder> builderFactory;
  private final PatchListCache patchListCache;
  private final Metrics metrics;
  private final ExecutorService executor;

  private final String fileName;
  @Nullable private final PatchSet.Id psa;
  private final int parentNum;
  private final PatchSet.Id psb;
  private final DiffPreferencesInfo diffPrefs;
  private final CurrentUser currentUser;

  private final ChangeEditUtil editReader;
  private final PermissionBackend permissionBackend;
  private final ProjectCache projectCache;
  private final DiffOperations diffOperations;

  private final Change.Id changeId;

  private ChangeNotes notes;

  private final boolean runNewDiffCache;

  @AssistedInject
  PatchScriptFactory(
      GitRepositoryManager grm,
      PatchSetUtil psUtil,
      Provider<PatchScriptBuilder> builderFactory,
      PatchListCache patchListCache,
      ChangeEditUtil editReader,
      PermissionBackend permissionBackend,
      ProjectCache projectCache,
      DiffOperations diffOperations,
      Metrics metrics,
      @DiffExecutor ExecutorService executor,
      @GerritServerConfig Config cfg,
      @Assisted ChangeNotes notes,
      @Assisted String fileName,
      @Assisted("patchSetA") @Nullable PatchSet.Id patchSetA,
      @Assisted("patchSetB") PatchSet.Id patchSetB,
      @Assisted DiffPreferencesInfo diffPrefs,
      @Assisted CurrentUser currentUser) {
    this.repoManager = grm;
    this.psUtil = psUtil;
    this.builderFactory = builderFactory;
    this.patchListCache = patchListCache;
    this.notes = notes;
    this.editReader = editReader;
    this.permissionBackend = permissionBackend;
    this.projectCache = projectCache;
    this.diffOperations = diffOperations;
    this.metrics = metrics;
    this.executor = executor;

    this.fileName = fileName;
    this.psa = patchSetA;
    this.parentNum = -1;
    this.psb = patchSetB;
    this.diffPrefs = diffPrefs;
    this.currentUser = currentUser;

    this.runNewDiffCache = cfg.getBoolean("cache", "diff_cache", "runNewDiffCache_GetDiff", false);

    changeId = patchSetB.changeId();
  }

  @AssistedInject
  PatchScriptFactory(
      GitRepositoryManager grm,
      PatchSetUtil psUtil,
      Provider<PatchScriptBuilder> builderFactory,
      PatchListCache patchListCache,
      ChangeEditUtil editReader,
      PermissionBackend permissionBackend,
      ProjectCache projectCache,
      DiffOperations diffOperations,
      Metrics metrics,
      @DiffExecutor ExecutorService executor,
      @GerritServerConfig Config cfg,
      @Assisted ChangeNotes notes,
      @Assisted String fileName,
      @Assisted int parentNum,
      @Assisted PatchSet.Id patchSetB,
      @Assisted DiffPreferencesInfo diffPrefs,
      @Assisted CurrentUser currentUser) {
    this.repoManager = grm;
    this.psUtil = psUtil;
    this.builderFactory = builderFactory;
    this.patchListCache = patchListCache;
    this.notes = notes;
    this.editReader = editReader;
    this.permissionBackend = permissionBackend;
    this.projectCache = projectCache;
    this.diffOperations = diffOperations;
    this.metrics = metrics;
    this.executor = executor;

    this.fileName = fileName;
    this.psa = null;
    this.parentNum = parentNum;
    this.psb = patchSetB;
    this.diffPrefs = diffPrefs;
    this.currentUser = currentUser;

    this.runNewDiffCache = cfg.getBoolean("cache", "diff_cache", "runNewDiffCache_GetDiff", false);

    changeId = patchSetB.changeId();
    checkArgument(parentNum >= 0, "parentNum must be >= 0");
  }

  @Override
  public PatchScript call()
      throws LargeObjectException, AuthException, InvalidChangeOperationException, IOException,
          PermissionBackendException {

    try {
      permissionBackend.user(currentUser).change(notes).check(ChangePermission.READ);
    } catch (AuthException e) {
      throw new NoSuchChangeException(changeId, e);
    }

    if (!projectCache
        .get(notes.getProjectName())
        .map(ProjectState::statePermitsRead)
        .orElse(false)) {
      throw new NoSuchChangeException(changeId);
    }

    try (Repository git = repoManager.openRepository(notes.getProjectName())) {
      try {
        validatePatchSetId(psa);
        validatePatchSetId(psb);

        ObjectId aId = getAId().orElse(null);
        ObjectId bId = getBId().orElse(null);
        if (bId == null) {
          // Change edit: create synthetic PatchSet corresponding to the edit.
          Optional<ChangeEdit> edit = editReader.byChange(notes);
          if (!edit.isPresent()) {
            throw new NoSuchChangeException(notes.getChangeId());
          }
          bId = edit.get().getEditCommit();
        }
        if (runNewDiffCache) {
          PatchScript patchScript = getPatchScriptWithNewDiffCache(git, aId, bId);
          // TODO(ghareeb): remove the async run. This is temporarily used to keep sanity checking
          // the results while rolling out the new diff cache.
          runOldDiffCacheAsyncAndExportMetrics(git, aId, bId, patchScript);
          return patchScript;
        } else {
          return getPatchScriptWithOldDiffCache(git, aId, bId);
        }
      } catch (PatchListNotAvailableException e) {
        throw new NoSuchChangeException(changeId, e);
      } catch (DiffNotAvailableException e) {
        throw new StorageException(e);
      } catch (IOException e) {
        logger.atSevere().withCause(e).log("File content unavailable");
        throw new NoSuchChangeException(changeId, e);
      } catch (org.eclipse.jgit.errors.LargeObjectException err) {
        throw new LargeObjectException("File content is too large", err);
      }
    } catch (RepositoryNotFoundException e) {
      logger.atSevere().withCause(e).log("Repository %s not found", notes.getProjectName());
      throw new NoSuchChangeException(changeId, e);
    } catch (IOException e) {
      logger.atSevere().withCause(e).log("Cannot open repository %s", notes.getProjectName());
      throw new NoSuchChangeException(changeId, e);
    }
  }

  private void runOldDiffCacheAsyncAndExportMetrics(
      Repository git, ObjectId aId, ObjectId bId, PatchScript expected) {
    @SuppressWarnings("unused")
    Future<?> possiblyIgnoredError =
        executor.submit(
            () -> {
              try {
                PatchScript patchScript = getPatchScriptWithOldDiffCache(git, aId, bId);
                if (areEqualPatchscripts(patchScript, expected)) {
                  metrics.diffs.increment(Metrics.MATCH);
                } else {
                  metrics.diffs.increment(Metrics.MISMATCH);
                  logger.atWarning().atMostEvery(10, TimeUnit.SECONDS).log(
                      "Mismatching diff for change %s, old commit ID: %s, new commit ID: %s, file name: %s.",
                      changeId.toString(), aId, bId, fileName);
                }
              } catch (PatchListNotAvailableException | IOException e) {
                metrics.diffs.increment(Metrics.ERROR);
                logger.atSevere().atMostEvery(10, TimeUnit.SECONDS).log(
                    String.format(
                            "Error computing new diff for change %s, old commit ID: %s, new commit ID: %s.\n",
                            changeId.toString(), aId, bId)
                        + ExceptionUtils.getStackTrace(e));
              }
            });
  }

  private PatchScript getPatchScriptWithOldDiffCache(Repository git, ObjectId aId, ObjectId bId)
      throws IOException, PatchListNotAvailableException {
    PatchScriptBuilder patchScriptBuilder = newBuilder();
    PatchList list = listFor(keyFor(aId, bId, diffPrefs.ignoreWhitespace));
    PatchListEntry content = list.get(fileName);
    return patchScriptBuilder.toPatchScriptOld(git, list, content);
  }

  private PatchScript getPatchScriptWithNewDiffCache(Repository git, ObjectId aId, ObjectId bId)
      throws IOException, DiffNotAvailableException {
    FileDiffOutput fileDiffOutput =
        aId == null
            ? diffOperations.getModifiedFileAgainstParent(
                notes.getProjectName(),
                bId,
                parentNum == -1 ? null : parentNum + 1,
                fileName,
                diffPrefs.ignoreWhitespace)
            : diffOperations.getModifiedFile(
                notes.getProjectName(), aId, bId, fileName, diffPrefs.ignoreWhitespace);
    return newBuilder().toPatchScriptNew(git, fileDiffOutput);
  }

  /**
   * The comparison is not exhaustive but is using the most important fields. Comparing all fields
   * will require some work in {@link PatchScript} to, e.g., convert it to autovalue. This
   * comparison method shall give a strong signal that both patchscripts are almost identical.
   */
  private static boolean areEqualPatchscripts(PatchScript ps1, PatchScript ps2) {
    boolean equal = true;
    if (!ps1.getChangeType().equals(ps2.getChangeType())) {
      equal = false;
      logger.atWarning().log(
          "Mismatching change type: old = %s, new = %s.", ps1.getChangeType(), ps2.getChangeType());
    }
    if (!ps1.getPatchHeader().equals(ps2.getPatchHeader())) {
      equal = false;
      logger.atWarning().log(
          "Mismatching patch header: old = %s, new = %s.",
          ps1.getPatchHeader(), ps2.getPatchHeader());
    }
    if (!Objects.equals(ps1.getOldName(), ps2.getOldName())) {
      equal = false;
      logger.atWarning().log(
          "Mismatching old name: old = %s, new = %s.", ps1.getOldName(), ps2.getOldName());
    }
    if (!Objects.equals(ps1.getNewName(), ps2.getNewName())) {
      equal = false;
      logger.atWarning().log(
          "Mismatching new name: old = %s, new = %s.", ps1.getNewName(), ps2.getNewName());
    }
    if (!ps1.getEdits().containsAll(ps2.getEdits())) {
      equal = false;
      logger.atWarning().log(
          "Mismatching edits: old = %s, new = %s.", ps1.getEdits(), ps2.getEdits());
    }
    if (!ps2.getEdits().containsAll(ps1.getEdits())) {
      equal = false;
      logger.atWarning().log(
          "Mismatching edits: old = %s, new = %s.", ps1.getEdits(), ps2.getEdits());
    }
    if (!ps1.getEditsDueToRebase().equals(ps2.getEditsDueToRebase())) {
      equal = false;
      logger.atWarning().log(
          "Mismatching edits due to rebase: old = %s, new = %s.",
          ps1.getEditsDueToRebase(), ps2.getEditsDueToRebase());
    }
    if (!ps1.getA().equals(ps2.getA())) {
      equal = false;
      logger.atWarning().log("Mismatching sparse file content in old commit.");
    }
    if (!ps1.getB().equals(ps2.getB())) {
      equal = false;
      logger.atWarning().log("Mismatching sparse file content in new commit.");
    }
    return equal;
  }

  private Optional<ObjectId> getAId() {
    if (psa == null) {
      return Optional.empty();
    }
    checkState(parentNum < 0, "expected no parentNum when psa is present");
    checkArgument(psa.get() != 0, "edit not supported for left side");
    return Optional.of(getCommitId(psa));
  }

  private Optional<ObjectId> getBId() {
    if (psb.get() == 0) {
      // Change edit
      return Optional.empty();
    }
    return Optional.of(getCommitId(psb));
  }

  private PatchListKey keyFor(ObjectId aId, ObjectId bId, Whitespace whitespace) {
    if (parentNum < 0) {
      return PatchListKey.againstCommit(aId, bId, whitespace);
    }
    return PatchListKey.againstParentNum(parentNum + 1, bId, whitespace);
  }

  private PatchList listFor(PatchListKey key) throws PatchListNotAvailableException {
    return patchListCache.get(key, notes.getProjectName());
  }

  private PatchScriptBuilder newBuilder() {
    final PatchScriptBuilder b = builderFactory.get();
    b.setDiffPrefs(diffPrefs);
    if (diffPrefs.intralineDifference) {
      b.setIntraLineDiffCalculator(
          new IntraLineDiffCalculator(patchListCache, notes.getProjectName(), diffPrefs));
    }
    return b;
  }

  private ObjectId getCommitId(PatchSet.Id psId) {
    PatchSet ps = psUtil.get(notes, psId);
    if (ps == null) {
      throw new NoSuchChangeException(psId.changeId());
    }
    return ps.commitId();
  }

  private void validatePatchSetId(PatchSet.Id psId) throws NoSuchChangeException {
    if (psId == null) { // OK, means use base;
    } else if (changeId.equals(psId.changeId())) { // OK, same change;
    } else {
      throw new NoSuchChangeException(changeId);
    }
  }

  private static class IntraLineDiffCalculator
      implements PatchScriptBuilder.IntraLineDiffCalculator {

    private final PatchListCache patchListCache;
    private final Project.NameKey projectKey;
    private final DiffPreferencesInfo diffPrefs;

    IntraLineDiffCalculator(
        PatchListCache patchListCache, Project.NameKey projectKey, DiffPreferencesInfo diffPrefs) {
      this.patchListCache = patchListCache;
      this.projectKey = projectKey;
      this.diffPrefs = diffPrefs;
    }

    @Override
    public IntraLineDiffCalculatorResult calculateIntraLineDiff(
        ImmutableList<Edit> edits,
        Set<Edit> editsDueToRebase,
        ObjectId aId,
        ObjectId bId,
        Text aSrc,
        Text bSrc,
        ObjectId bTreeId,
        String bPath) {
      IntraLineDiff d =
          patchListCache.getIntraLineDiff(
              IntraLineDiffKey.create(aId, bId, diffPrefs.ignoreWhitespace),
              IntraLineDiffArgs.create(
                  aSrc, bSrc, edits, editsDueToRebase, projectKey, bTreeId, bPath));
      if (d == null) {
        return IntraLineDiffCalculatorResult.FAILURE;
      }
      switch (d.getStatus()) {
        case EDIT_LIST:
          return IntraLineDiffCalculatorResult.success(d.getEdits());

        case ERROR:
          return IntraLineDiffCalculatorResult.FAILURE;

        case TIMEOUT:
          return IntraLineDiffCalculatorResult.TIMEOUT;

        case DISABLED:
        default:
          return IntraLineDiffCalculatorResult.NO_RESULT;
      }
    }
  }
}