java/com/google/gerrit/server/DeleteZombieComments.java - gerrit/ - Git at Google

 // Copyright (C) 2023 The Android Open Source Project
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 // http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package com.google.gerrit.server;

 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;

 import com.google.auto.value.AutoValue;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Sets;
 import com.google.common.flogger.FluentLogger;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
 import com.google.gerrit.common.Nullable;
 import com.google.gerrit.entities.Account;
 import com.google.gerrit.entities.Change;
 import com.google.gerrit.entities.HumanComment;
 import com.google.gerrit.entities.Project;
 import com.google.gerrit.entities.RefNames;
 import com.google.gerrit.server.git.GitRepositoryManager;
 import com.google.gerrit.server.notedb.ChangeNotes;
 import java.io.IOException;
 import java.sql.Timestamp;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Consumer;
 import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.lib.Repository;

 /**
  * This class can be used to clean zombie draft comments. More context in <a
  * href="https://gerrit-review.googlesource.com/c/gerrit/+/246233">
  * https://gerrit-review.googlesource.com/c/gerrit/+/246233 </a>
  *
  * <p>The implementation has two cases for detecting zombie drafts:
  *
  * <ul>
  *   <li>An earlier bug in the deletion of draft comments caused some draft refs to remain empty but
  *       not get deleted.
  *   <li>Inspecting all draft-comments. Check for each draft if there exists a published comment
  *       with the same UUID. These comments are called zombie drafts. If the program is run in
  *       {@link DeleteZombieComments#dryRun} mode, the zombie draft IDs will only be logged for
  *       tracking, otherwise they will also be deleted.
  * </uL>
  */
 public abstract class DeleteZombieComments<KeyT> implements AutoCloseable {
   @AutoValue
   abstract static class ChangeUserIDsPair {
     abstract Change.Id changeId();

     abstract Account.Id accountId();

     static ChangeUserIDsPair create(Change.Id changeId, Account.Id accountId) {
       return new AutoValue_DeleteZombieComments_ChangeUserIDsPair(changeId, accountId);
     }
   }

   private static final FluentLogger logger = FluentLogger.forEnclosingClass();

   private final int cleanupPercentage;
   protected final boolean dryRun;
   @Nullable private final Consumer<String> uiConsumer;
   @Nullable private final GitRepositoryManager repoManager;
   @Nullable private final DraftCommentsReader draftCommentsReader;
   @Nullable private final ChangeNotes.Factory changeNotesFactory;
   @Nullable private final CommentsUtil commentsUtil;

   private Map<Change.Id, Project.NameKey> changeProjectMap = new HashMap<>();
   private Map<Change.Id, ChangeNotes> changeNotes = new HashMap<>();

   protected DeleteZombieComments(
       Integer cleanupPercentage,
       boolean dryRun,
       Consumer<String> uiConsumer,
       GitRepositoryManager repoManager,
       DraftCommentsReader draftCommentsReader,
       ChangeNotes.Factory changeNotesFactory,
       CommentsUtil commentsUtil) {
     this.cleanupPercentage = cleanupPercentage == null ? 100 : cleanupPercentage;
     this.dryRun = dryRun;
     this.uiConsumer = uiConsumer;
     this.repoManager = repoManager;
     this.draftCommentsReader = draftCommentsReader;
     this.changeNotesFactory = changeNotesFactory;
     this.commentsUtil = commentsUtil;
   }

   /** Deletes all draft comments. Returns the number of zombie draft comments that were deleted. */
   @CanIgnoreReturnValue
   public int execute() throws IOException {
     setup();
     ListMultimap<KeyT, HumanComment> alreadyPublished = listDraftCommentsThatAreAlsoPublished();
     if (!dryRun) {
       deleteZombieDrafts(alreadyPublished);
     }

     List<KeyT> emptyDrafts = filterByCleanupPercentage(listEmptyDrafts(), "empty");
     if (!dryRun) {
       deleteEmptyDraftsByKey(emptyDrafts);
     } else {
       logInfo(
           String.format(
               "Running in dry run mode. Skipping deletion."
                   + "\nStats (with %d cleanup-percentage):"
                   + "\nEmpty drafts = %d"
                   + "\nAlready published drafts (zombies) = %d",
               cleanupPercentage, emptyDrafts.size(), alreadyPublished.size()));
     }
     return emptyDrafts.size() + alreadyPublished.size();
   }

   @VisibleForTesting
   public abstract void setup() throws IOException;

   @Override
   public abstract void close() throws IOException;

   protected abstract List<KeyT> listAllDrafts() throws IOException;

   protected abstract List<KeyT> listEmptyDrafts() throws IOException;

   protected abstract void deleteEmptyDraftsByKey(Collection<KeyT> keys) throws IOException;

   protected abstract void deleteZombieDrafts(ListMultimap<KeyT, HumanComment> drafts)
       throws IOException;

   protected abstract Change.Id getChangeId(KeyT key);

   protected abstract Account.Id getAccountId(KeyT key);

   protected abstract String loggable(KeyT key);

   protected ChangeNotes getChangeNotes(Change.Id changeId) {
     if (changeNotes.containsKey(changeId)) {
       return changeNotes.get(changeId);
     }
     checkState(
         changeProjectMap.containsKey(changeId),
         "Cannot get a project associated with change ID " + changeId);
     ChangeNotes notes = changeNotesFactory.createChecked(changeProjectMap.get(changeId), changeId);
     changeNotes.put(changeId, notes);
     return notes;
   }

   private List<KeyT> filterByCleanupPercentage(List<KeyT> drafts, String reason) {
     if (cleanupPercentage >= 100) {
       logInfo(
           String.format(
               "Cleanup percentage = %d" + "\nNumber of drafts to be cleaned for %s = %d",
               cleanupPercentage, reason, drafts.size()));
       return drafts;
     }
     ImmutableList<KeyT> res =
         drafts.stream()
             .filter(key -> getChangeId(key).get() % 100 < cleanupPercentage)
             .collect(toImmutableList());
     logInfo(
         String.format(
             "Cleanup percentage = %d"
                 + "\nOriginal number of drafts for %s = %d"
                 + "\nNumber of drafts to be processed for %s = %d",
             cleanupPercentage, reason, drafts.size(), reason, res.size()));
     return res;
   }

   @VisibleForTesting
   public ListMultimap<KeyT, HumanComment> listDraftCommentsThatAreAlsoPublished()
       throws IOException {
     List<KeyT> draftKeys = filterByCleanupPercentage(listAllDrafts(), "all-drafts");
     changeProjectMap.putAll(mapChangesWithDraftsToProjects(draftKeys));

     ListMultimap<KeyT, HumanComment> zombieDrafts = ArrayListMultimap.create();
     Set<ChangeUserIDsPair> visitedSet = new HashSet<>();
     for (KeyT key : draftKeys) {
       try {
         Change.Id changeId = getChangeId(key);
         Account.Id accountId = getAccountId(key);
         ChangeUserIDsPair changeUserIDsPair = ChangeUserIDsPair.create(changeId, accountId);
         if (!visitedSet.add(changeUserIDsPair)) {
           continue;
         }
         if (!changeProjectMap.containsKey(changeId)) {
           logger.atWarning().log(
               "Could not find a project associated with change ID %s. Skipping draft [%s]",
               changeId, loggable(key));
           continue;
         }
         List<HumanComment> drafts =
             draftCommentsReader.getDraftsByChangeAndDraftAuthor(changeId, accountId);
         ChangeNotes notes = getChangeNotes(changeId);
         List<HumanComment> published = commentsUtil.publishedHumanCommentsByChange(notes);
         ImmutableSet<String> publishedIds = toUuid(published);
         ImmutableList<HumanComment> zombieDraftsForChangeAndAuthor =
             drafts.stream()
                 .filter(draft -> publishedIds.contains(draft.key.uuid))
                 .collect(toImmutableList());
         zombieDraftsForChangeAndAuthor.forEach(
             zombieDraft ->
                 logger.atWarning().log(
                     "Draft comment with uuid '%s' of change %s, account %s, written on %s,"
                         + " is a zombie draft that is already published.",
                     zombieDraft.key.uuid, changeId, accountId, zombieDraft.writtenOn));
         zombieDrafts.putAll(key, zombieDraftsForChangeAndAuthor);
       } catch (RuntimeException e) {
         logger.atWarning().withCause(e).log("Failed to process draft [%s]", loggable(key));
       }
     }

     if (!zombieDrafts.isEmpty()) {
       Timestamp earliestZombieTs = null;
       Timestamp latestZombieTs = null;
       for (HumanComment zombieDraft : zombieDrafts.values()) {
         earliestZombieTs = getEarlierTs(earliestZombieTs, zombieDraft.writtenOn);
         latestZombieTs = getLaterTs(latestZombieTs, zombieDraft.writtenOn);
       }
       logger.atWarning().log(
           "Detected %d zombie drafts that were already published (earliest at %s, latest at %s).",
           zombieDrafts.size(), earliestZombieTs, latestZombieTs);
     }
     return zombieDrafts;
   }

   /**
    * Map each change ID to its associated project.
    *
    * <p>When doing a ref scan of draft refs
    * "refs/draft-comments/$change_id_short/$change_id/$user_id" we don't know which project this
    * draft comment is associated with. The project name is needed to load published comments for the
    * change, hence we map each change ID to its project here by scanning through the change meta ref
    * of the change ID in all projects.
    */
   private Map<Change.Id, Project.NameKey> mapChangesWithDraftsToProjects(List<KeyT> drafts) {
     ImmutableSet<Change.Id> changeIds =
         drafts.stream().map(key -> getChangeId(key)).collect(ImmutableSet.toImmutableSet());
     Map<Change.Id, Project.NameKey> result = new HashMap<>();
     for (Project.NameKey project : repoManager.list()) {
       try (Repository repo = repoManager.openRepository(project)) {
         Sets.SetView<Change.Id> unmappedChangeIds = Sets.difference(changeIds, result.keySet());
         for (Change.Id changeId : unmappedChangeIds) {
           Ref ref = repo.getRefDatabase().exactRef(RefNames.changeMetaRef(changeId));
           if (ref != null) {
             result.put(changeId, project);
           }
         }
       } catch (Exception e) {
         logger.atWarning().withCause(e).log("Failed to open repository for project '%s'.", project);
       }
       if (changeIds.size() == result.size()) {
         // We do not need to scan the remaining repositories
         break;
       }
     }
     if (result.size() != changeIds.size()) {
       logger.atWarning().log(
           "Failed to associate the following change Ids to a project: %s",
           Sets.difference(changeIds, result.keySet()));
     }
     return result;
   }

   protected void logInfo(String message) {
     logger.atInfo().log("%s", message);
     uiConsumer.accept(message);
   }

   /** Map the list of input comments to their UUIDs. */
   private ImmutableSet<String> toUuid(List<HumanComment> in) {
     return in.stream().map(c -> c.key.uuid).collect(toImmutableSet());
   }

   private Timestamp getEarlierTs(@Nullable Timestamp t1, Timestamp t2) {
     if (t1 == null) {
       return t2;
     }
     return t1.before(t2) ? t1 : t2;
   }

   private Timestamp getLaterTs(@Nullable Timestamp t1, Timestamp t2) {
     if (t1 == null) {
       return t2;
     }
     return t1.after(t2) ? t1 : t2;
   }
 }
	// Copyright (C) 2023 The Android Open Source Project
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package com.google.gerrit.server;

	import static com.google.common.base.Preconditions.checkState;
	import static com.google.common.collect.ImmutableList.toImmutableList;
	import static com.google.common.collect.ImmutableSet.toImmutableSet;

	import com.google.auto.value.AutoValue;
	import com.google.common.annotations.VisibleForTesting;
	import com.google.common.collect.ArrayListMultimap;
	import com.google.common.collect.ImmutableList;
	import com.google.common.collect.ImmutableSet;
	import com.google.common.collect.ListMultimap;
	import com.google.common.collect.Sets;
	import com.google.common.flogger.FluentLogger;
	import com.google.errorprone.annotations.CanIgnoreReturnValue;
	import com.google.gerrit.common.Nullable;
	import com.google.gerrit.entities.Account;
	import com.google.gerrit.entities.Change;
	import com.google.gerrit.entities.HumanComment;
	import com.google.gerrit.entities.Project;
	import com.google.gerrit.entities.RefNames;
	import com.google.gerrit.server.git.GitRepositoryManager;
	import com.google.gerrit.server.notedb.ChangeNotes;
	import java.io.IOException;
	import java.sql.Timestamp;
	import java.util.Collection;
	import java.util.HashMap;
	import java.util.HashSet;
	import java.util.List;
	import java.util.Map;
	import java.util.Set;
	import java.util.function.Consumer;
	import org.eclipse.jgit.lib.Ref;
	import org.eclipse.jgit.lib.Repository;

	/**
	* This class can be used to clean zombie draft comments. More context in <a
	* href="https://gerrit-review.googlesource.com/c/gerrit/+/246233">
	* https://gerrit-review.googlesource.com/c/gerrit/+/246233 </a>
	*
	* <p>The implementation has two cases for detecting zombie drafts:
	*
	* <ul>
	* <li>An earlier bug in the deletion of draft comments caused some draft refs to remain empty but
	* not get deleted.
	* <li>Inspecting all draft-comments. Check for each draft if there exists a published comment
	* with the same UUID. These comments are called zombie drafts. If the program is run in
	* {@link DeleteZombieComments#dryRun} mode, the zombie draft IDs will only be logged for
	* tracking, otherwise they will also be deleted.
	* </uL>
	*/
	public abstract class DeleteZombieComments<KeyT> implements AutoCloseable {
	@AutoValue
	abstract static class ChangeUserIDsPair {
	abstract Change.Id changeId();

	abstract Account.Id accountId();

	static ChangeUserIDsPair create(Change.Id changeId, Account.Id accountId) {
	return new AutoValue_DeleteZombieComments_ChangeUserIDsPair(changeId, accountId);
	}
	}

	private static final FluentLogger logger = FluentLogger.forEnclosingClass();

	private final int cleanupPercentage;
	protected final boolean dryRun;
	@Nullable private final Consumer<String> uiConsumer;
	@Nullable private final GitRepositoryManager repoManager;
	@Nullable private final DraftCommentsReader draftCommentsReader;
	@Nullable private final ChangeNotes.Factory changeNotesFactory;
	@Nullable private final CommentsUtil commentsUtil;

	private Map<Change.Id, Project.NameKey> changeProjectMap = new HashMap<>();
	private Map<Change.Id, ChangeNotes> changeNotes = new HashMap<>();

	protected DeleteZombieComments(
	Integer cleanupPercentage,
	boolean dryRun,
	Consumer<String> uiConsumer,
	GitRepositoryManager repoManager,
	DraftCommentsReader draftCommentsReader,
	ChangeNotes.Factory changeNotesFactory,
	CommentsUtil commentsUtil) {
	this.cleanupPercentage = cleanupPercentage == null ? 100 : cleanupPercentage;
	this.dryRun = dryRun;
	this.uiConsumer = uiConsumer;
	this.repoManager = repoManager;
	this.draftCommentsReader = draftCommentsReader;
	this.changeNotesFactory = changeNotesFactory;
	this.commentsUtil = commentsUtil;
	}

	/** Deletes all draft comments. Returns the number of zombie draft comments that were deleted. */
	@CanIgnoreReturnValue
	public int execute() throws IOException {
	setup();
	ListMultimap<KeyT, HumanComment> alreadyPublished = listDraftCommentsThatAreAlsoPublished();
	if (!dryRun) {
	deleteZombieDrafts(alreadyPublished);
	}

	List<KeyT> emptyDrafts = filterByCleanupPercentage(listEmptyDrafts(), "empty");
	if (!dryRun) {
	deleteEmptyDraftsByKey(emptyDrafts);
	} else {
	logInfo(
	String.format(
	"Running in dry run mode. Skipping deletion."
	+ "\nStats (with %d cleanup-percentage):"
	+ "\nEmpty drafts = %d"
	+ "\nAlready published drafts (zombies) = %d",
	cleanupPercentage, emptyDrafts.size(), alreadyPublished.size()));
	}
	return emptyDrafts.size() + alreadyPublished.size();
	}

	@VisibleForTesting
	public abstract void setup() throws IOException;

	@Override
	public abstract void close() throws IOException;

	protected abstract List<KeyT> listAllDrafts() throws IOException;

	protected abstract List<KeyT> listEmptyDrafts() throws IOException;

	protected abstract void deleteEmptyDraftsByKey(Collection<KeyT> keys) throws IOException;

	protected abstract void deleteZombieDrafts(ListMultimap<KeyT, HumanComment> drafts)
	throws IOException;

	protected abstract Change.Id getChangeId(KeyT key);

	protected abstract Account.Id getAccountId(KeyT key);

	protected abstract String loggable(KeyT key);

	protected ChangeNotes getChangeNotes(Change.Id changeId) {
	if (changeNotes.containsKey(changeId)) {
	return changeNotes.get(changeId);
	}
	checkState(
	changeProjectMap.containsKey(changeId),
	"Cannot get a project associated with change ID " + changeId);
	ChangeNotes notes = changeNotesFactory.createChecked(changeProjectMap.get(changeId), changeId);
	changeNotes.put(changeId, notes);
	return notes;
	}

	private List<KeyT> filterByCleanupPercentage(List<KeyT> drafts, String reason) {
	if (cleanupPercentage >= 100) {
	logInfo(
	String.format(
	"Cleanup percentage = %d" + "\nNumber of drafts to be cleaned for %s = %d",
	cleanupPercentage, reason, drafts.size()));
	return drafts;
	}
	ImmutableList<KeyT> res =
	drafts.stream()
	.filter(key -> getChangeId(key).get() % 100 < cleanupPercentage)
	.collect(toImmutableList());
	logInfo(
	String.format(
	"Cleanup percentage = %d"
	+ "\nOriginal number of drafts for %s = %d"
	+ "\nNumber of drafts to be processed for %s = %d",
	cleanupPercentage, reason, drafts.size(), reason, res.size()));
	return res;
	}

	@VisibleForTesting
	public ListMultimap<KeyT, HumanComment> listDraftCommentsThatAreAlsoPublished()
	throws IOException {
	List<KeyT> draftKeys = filterByCleanupPercentage(listAllDrafts(), "all-drafts");
	changeProjectMap.putAll(mapChangesWithDraftsToProjects(draftKeys));

	ListMultimap<KeyT, HumanComment> zombieDrafts = ArrayListMultimap.create();
	Set<ChangeUserIDsPair> visitedSet = new HashSet<>();
	for (KeyT key : draftKeys) {
	try {
	Change.Id changeId = getChangeId(key);
	Account.Id accountId = getAccountId(key);
	ChangeUserIDsPair changeUserIDsPair = ChangeUserIDsPair.create(changeId, accountId);
	if (!visitedSet.add(changeUserIDsPair)) {
	continue;
	}
	if (!changeProjectMap.containsKey(changeId)) {
	logger.atWarning().log(
	"Could not find a project associated with change ID %s. Skipping draft [%s]",
	changeId, loggable(key));
	continue;
	}
	List<HumanComment> drafts =
	draftCommentsReader.getDraftsByChangeAndDraftAuthor(changeId, accountId);
	ChangeNotes notes = getChangeNotes(changeId);
	List<HumanComment> published = commentsUtil.publishedHumanCommentsByChange(notes);
	ImmutableSet<String> publishedIds = toUuid(published);
	ImmutableList<HumanComment> zombieDraftsForChangeAndAuthor =
	drafts.stream()
	.filter(draft -> publishedIds.contains(draft.key.uuid))
	.collect(toImmutableList());
	zombieDraftsForChangeAndAuthor.forEach(
	zombieDraft ->
	logger.atWarning().log(
	"Draft comment with uuid '%s' of change %s, account %s, written on %s,"
	+ " is a zombie draft that is already published.",
	zombieDraft.key.uuid, changeId, accountId, zombieDraft.writtenOn));
	zombieDrafts.putAll(key, zombieDraftsForChangeAndAuthor);
	} catch (RuntimeException e) {
	logger.atWarning().withCause(e).log("Failed to process draft [%s]", loggable(key));
	}
	}

	if (!zombieDrafts.isEmpty()) {
	Timestamp earliestZombieTs = null;
	Timestamp latestZombieTs = null;
	for (HumanComment zombieDraft : zombieDrafts.values()) {
	earliestZombieTs = getEarlierTs(earliestZombieTs, zombieDraft.writtenOn);
	latestZombieTs = getLaterTs(latestZombieTs, zombieDraft.writtenOn);
	}
	logger.atWarning().log(
	"Detected %d zombie drafts that were already published (earliest at %s, latest at %s).",
	zombieDrafts.size(), earliestZombieTs, latestZombieTs);
	}
	return zombieDrafts;
	}

	/**
	* Map each change ID to its associated project.
	*
	* <p>When doing a ref scan of draft refs
	* "refs/draft-comments/$change_id_short/$change_id/$user_id" we don't know which project this
	* draft comment is associated with. The project name is needed to load published comments for the
	* change, hence we map each change ID to its project here by scanning through the change meta ref
	* of the change ID in all projects.
	*/
	private Map<Change.Id, Project.NameKey> mapChangesWithDraftsToProjects(List<KeyT> drafts) {
	ImmutableSet<Change.Id> changeIds =
	drafts.stream().map(key -> getChangeId(key)).collect(ImmutableSet.toImmutableSet());
	Map<Change.Id, Project.NameKey> result = new HashMap<>();
	for (Project.NameKey project : repoManager.list()) {
	try (Repository repo = repoManager.openRepository(project)) {
	Sets.SetView<Change.Id> unmappedChangeIds = Sets.difference(changeIds, result.keySet());
	for (Change.Id changeId : unmappedChangeIds) {
	Ref ref = repo.getRefDatabase().exactRef(RefNames.changeMetaRef(changeId));
	if (ref != null) {
	result.put(changeId, project);
	}
	}
	} catch (Exception e) {
	logger.atWarning().withCause(e).log("Failed to open repository for project '%s'.", project);
	}
	if (changeIds.size() == result.size()) {
	// We do not need to scan the remaining repositories
	break;
	}
	}
	if (result.size() != changeIds.size()) {
	logger.atWarning().log(
	"Failed to associate the following change Ids to a project: %s",
	Sets.difference(changeIds, result.keySet()));
	}
	return result;
	}

	protected void logInfo(String message) {
	logger.atInfo().log("%s", message);
	uiConsumer.accept(message);
	}

	/** Map the list of input comments to their UUIDs. */
	private ImmutableSet<String> toUuid(List<HumanComment> in) {
	return in.stream().map(c -> c.key.uuid).collect(toImmutableSet());
	}

	private Timestamp getEarlierTs(@Nullable Timestamp t1, Timestamp t2) {
	if (t1 == null) {
	return t2;
	}
	return t1.before(t2) ? t1 : t2;
	}

	private Timestamp getLaterTs(@Nullable Timestamp t1, Timestamp t2) {
	if (t1 == null) {
	return t2;
	}
	return t1.after(t2) ? t1 : t2;
	}
	}